def check_permissions(self, node, upgraded):
# use an exclusive connection to ensure we only talk to the specified node
klaus = self.patient_exclusive_cql_connection(node, user='******', password='******', timeout=20)
# klaus is a superuser, so should be able to list all permissions
# the output of LIST PERMISSIONS changes slightly with #7653 adding
# a new role column to results, so we need to tailor our check
# based on whether the node has been upgraded or not
if not upgraded:
assert_all(klaus,
'LIST ALL PERMISSIONS',
[['michael', '<table ks.cf1>', 'MODIFY'],
['michael', '<table ks.cf2>', 'SELECT']])
else:
assert_all(klaus,
'LIST ALL PERMISSIONS',
[['michael', 'michael', '<table ks.cf1>', 'MODIFY'],
['michael', 'michael', '<table ks.cf2>', 'SELECT']])
klaus.shutdown()
michael = self.patient_exclusive_cql_connection(node, user='******', password='******')
michael.execute('INSERT INTO ks.cf1 (id, val) VALUES (0,0)')
michael.execute('SELECT * FROM ks.cf2')
assert_invalid(michael,
'SELECT * FROM ks.cf1',
'User michael has no SELECT permission on <table ks.cf1> or any of its parents',
Unauthorized)
michael.shutdown()
def ttl_is_replicated_test(self):
"""
Test that the ttl setting is replicated properly on all nodes
"""
self.prepare(default_time_to_live=5)
cursor1 = self.patient_exclusive_cql_connection(self.node1)
cursor2 = self.patient_exclusive_cql_connection(self.node2)
cursor1.execute("USE ks;")
cursor2.execute("USE ks;")
query = SimpleStatement(
"INSERT INTO ttl_table (key, col1) VALUES (1, 1);",
consistency_level=ConsistencyLevel.ALL
)
cursor1.execute(query)
assert_all(
cursor1,
"SELECT * FROM ttl_table;",
[[1, 1, None, None]],
cl=ConsistencyLevel.ALL
)
ttl_cursor1 = cursor1.execute('SELECT ttl(col1) FROM ttl_table;')
ttl_cursor2 = cursor2.execute('SELECT ttl(col1) FROM ttl_table;')
assert_almost_equal(ttl_cursor1[0][0], ttl_cursor2[0][0], error=0.05)
time.sleep(7)
assert_none(cursor1, "SELECT * FROM ttl_table;", cl=ConsistencyLevel.ALL)
def ttl_is_replicated_test(self):
"""
Test that the ttl setting is replicated properly on all nodes
"""
self.prepare(default_time_to_live=5)
session1 = self.patient_exclusive_cql_connection(self.node1)
session2 = self.patient_exclusive_cql_connection(self.node2)
session1.execute("USE ks;")
session2.execute("USE ks;")
query = SimpleStatement(
"INSERT INTO ttl_table (key, col1) VALUES (1, 1);",
consistency_level=ConsistencyLevel.ALL
)
session1.execute(query)
assert_all(
session1,
"SELECT * FROM ttl_table;",
[[1, 1, None, None]],
cl=ConsistencyLevel.ALL
)
ttl_session1 = session1.execute('SELECT ttl(col1) FROM ttl_table;')
ttl_session2 = session2.execute('SELECT ttl(col1) FROM ttl_table;')
# since the two queries are not executed simultaneously, the remaining
# TTLs can differ by one second
self.assertLessEqual(abs(ttl_session1[0][0] - ttl_session2[0][0]), 1)
time.sleep(7)
assert_none(session1, "SELECT * FROM ttl_table;", cl=ConsistencyLevel.ALL)
def drop_column_compaction_test(self):
session = self.prepare()
session.execute("USE ks")
session.execute(
"CREATE TABLE cf (key int PRIMARY KEY, c1 int, c2 int)")
# insert some data.
session.execute("INSERT INTO cf (key, c1, c2) VALUES (0, 1, 2)")
session.execute("INSERT INTO cf (key, c1, c2) VALUES (1, 2, 3)")
session.execute("INSERT INTO cf (key, c1, c2) VALUES (2, 3, 4)")
# drop and readd c1.
session.execute("ALTER TABLE cf DROP c1")
session.execute("ALTER TABLE cf ADD c1 int")
# add another row.
session.execute("INSERT INTO cf (key, c1, c2) VALUES (3, 4, 5)")
node = self.cluster.nodelist()[0]
node.flush()
node.compact()
# test that c1 values have been compacted away.
session = self.patient_cql_connection(node)
assert_all(session,
"SELECT c1 FROM ks.cf", [[None], [None], [None], [4]],
ignore_order=True)
def ttl_is_replicated_test(self):
"""
Test that the ttl setting is replicated properly on all nodes
"""
self.prepare(default_time_to_live=5)
session1 = self.patient_exclusive_cql_connection(self.node1)
session2 = self.patient_exclusive_cql_connection(self.node2)
session1.execute("USE ks;")
session2.execute("USE ks;")
query = SimpleStatement(
"INSERT INTO ttl_table (key, col1) VALUES (1, 1);",
consistency_level=ConsistencyLevel.ALL)
session1.execute(query)
assert_all(session1,
"SELECT * FROM ttl_table;", [[1, 1, None, None]],
cl=ConsistencyLevel.ALL)
ttl_session1 = session1.execute('SELECT ttl(col1) FROM ttl_table;')
ttl_session2 = session2.execute('SELECT ttl(col1) FROM ttl_table;')
# since the two queries are not executed simultaneously, the remaining
# TTLs can differ by one second
self.assertLessEqual(abs(ttl_session1[0][0] - ttl_session2[0][0]), 1)
time.sleep(7)
assert_none(session1,
"SELECT * FROM ttl_table;",
cl=ConsistencyLevel.ALL)
def upgrade_with_range_tombstones_test(self):
"""
Checks sstable including range tombstone can be read after upgrade.
@jira_ticket CASSANDRA-10360
"""
ROWS = 100
session = self._setup_cluster()
session.execute('CREATE TABLE t (k int, t1 int, t2 int, PRIMARY KEY (k, t1, t2))')
for n in range(ROWS):
session.execute("INSERT INTO t(k, t1, t2) VALUES (0, 0, %d)" % n)
session.execute("DELETE FROM t WHERE k=0 AND t1=0")
for n in range(0, ROWS, 2):
session.execute("INSERT INTO t(k, t1, t2) VALUES (0, 0, %d)" % n)
session = self._do_upgrade()
assert_all(session, "SELECT * FROM t WHERE k = 0", [[0, 0, n] for n in range(0, ROWS, 2)])
self.cluster.compact()
def replace_with_reset_resume_state_test(self):
"""Test replace with resetting bootstrap progress"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
node1.stress(['write', 'n=100K', 'no-warmup', '-schema', 'replication(factor=3)'])
session = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE)
initial_data = rows_to_list(session.execute(query))
node3.stop(gently=False)
# kill node1 in the middle of streaming to let it fail
t = InterruptBootstrap(node1)
t.start()
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4', cluster=cluster, auto_bootstrap=True, thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000), jmx_port='7400', remote_debug_port='0',
initial_token=None, binary_interface=('127.0.0.4', 9042))
# keep timeout low so that test won't hang
node4.set_configuration_options(values={'streaming_socket_timeout_in_ms': 1000})
cluster.add(node4, False)
try:
node4.start(jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"], wait_other_notice=False)
except NodeError:
pass # node doesn't start as expected
t.join()
node1.start()
# restart node4 bootstrap with resetting bootstrap state
node4.stop()
mark = node4.mark_log()
node4.start(jvm_args=[
"-Dcassandra.replace_address_first_boot=127.0.0.3",
"-Dcassandra.reset_bootstrap_progress=true"
])
# check if we reset bootstrap state
node4.watch_log_for("Resetting bootstrap progress to start fresh", from_mark=mark)
# wait for node3 ready to query
node4.watch_log_for("Listening for thrift clients...", from_mark=mark)
# check if 2nd bootstrap succeeded
assert_bootstrap_state(self, node4, 'COMPLETED')
# query should work again
debug("Stopping old nodes")
node1.stop(gently=False, wait_other_notice=True)
node2.stop(gently=False, wait_other_notice=True)
debug("Verifying data on new node.")
session = self.patient_exclusive_cql_connection(node4)
assert_all(session, 'SELECT * from {} LIMIT 1'.format(stress_table),
expected=initial_data,
cl=ConsistencyLevel.ONE)
def drop_column_queries_test(self):
session = self.prepare()
session.execute("USE ks")
session.execute("CREATE TABLE cf (key int PRIMARY KEY, c1 int, c2 int)")
session.execute("CREATE INDEX ON cf(c2)")
# insert some data.
session.execute("INSERT INTO cf (key, c1, c2) VALUES (0, 1, 2)")
session.execute("INSERT INTO cf (key, c1, c2) VALUES (1, 2, 3)")
session.execute("INSERT INTO cf (key, c1, c2) VALUES (2, 3, 4)")
# drop and readd c1.
session.execute("ALTER TABLE cf DROP c1")
session.execute("ALTER TABLE cf ADD c1 int")
# add another row.
session.execute("INSERT INTO cf (key, c1, c2) VALUES (3, 4, 5)")
# test that old (pre-drop) c1 values aren't returned and new ones are.
assert_all(session, "SELECT c1 FROM cf", [[None], [None], [None], [4]], ignore_order=True)
assert_all(session, "SELECT * FROM cf", [[0, None, 2], [1, None, 3], [2, None, 4], [3, 4, 5]], ignore_order=True)
assert_one(session, "SELECT c1 FROM cf WHERE key = 0", [None])
assert_one(session, "SELECT c1 FROM cf WHERE key = 3", [4])
assert_one(session, "SELECT * FROM cf WHERE c2 = 2", [0, None, 2])
assert_one(session, "SELECT * FROM cf WHERE c2 = 5", [3, 4, 5])
def collection_map_ttl_test(self):
"""
Test that ttl has a granularity of elements using a map collection.
"""
self.prepare(default_time_to_live=6)
self.session1.execute("ALTER TABLE ttl_table ADD mymap map<int, int>;"
"")
start = time.time()
self.session1.execute("""
INSERT INTO ttl_table (key, col1, mymap) VALUES (%d, %d, %s);
""" % (1, 1, '{1:1,2:2,3:3,4:4,5:5}'))
self.session1.execute("""
UPDATE ttl_table USING TTL 2 SET mymap[1] = 42, mymap[5] = 42 WHERE key=1;
""")
assert_all(self.session1, "SELECT * FROM ttl_table;", [[
1, 1, None, None,
OrderedDict([(1, 42), (2, 2), (3, 3), (4, 4), (5, 42)])
]])
self.smart_sleep(start, 4)
assert_all(self.session1, "SELECT * FROM ttl_table;",
[[1, 1, None, None,
OrderedDict([(2, 2), (3, 3), (4, 4)])]])
self.smart_sleep(start, 8)
assert_row_count(self.session1, 'ttl_table', 0)
def upgrade_with_range_tombstones_test(self):
"""
Checks sstable including range tombstone can be read after upgrade.
@jira_ticket CASSANDRA-10360
"""
ROWS = 100
session = self._setup_cluster()
session.execute(
'CREATE TABLE t (k int, t1 int, t2 int, PRIMARY KEY (k, t1, t2))')
for n in range(ROWS):
session.execute(
"INSERT INTO t(k, t1, t2) VALUES (0, 0, {})".format(n))
session.execute("DELETE FROM t WHERE k=0 AND t1=0")
for n in range(0, ROWS, 2):
session.execute(
"INSERT INTO t(k, t1, t2) VALUES (0, 0, {})".format(n))
session = self._do_upgrade()
assert_all(session, "SELECT * FROM t WHERE k = 0",
[[0, 0, n] for n in range(0, ROWS, 2)])
self.cluster.compact()
def collection_map_ttl_test(self):
"""
Test that ttl has a granularity of elements using a map collection.
"""
self.prepare(default_time_to_live=6)
self.session1.execute("ALTER TABLE ttl_table ADD mymap map<int, int>;""")
start = time.time()
self.session1.execute("""
INSERT INTO ttl_table (key, col1, mymap) VALUES (%d, %d, %s);
""" % (1, 1, '{1:1,2:2,3:3,4:4,5:5}'))
self.session1.execute("""
UPDATE ttl_table USING TTL 2 SET mymap[1] = 42, mymap[5] = 42 WHERE key=1;
""")
assert_all(
self.session1,
"SELECT * FROM ttl_table;",
[[1, 1, None, None, OrderedDict([(1, 42), (2, 2), (3, 3), (4, 4), (5, 42)])]]
)
self.smart_sleep(start, 4)
assert_all(
self.session1,
"SELECT * FROM ttl_table;",
[[1, 1, None, None, OrderedDict([(2, 2), (3, 3), (4, 4)])]]
)
self.smart_sleep(start, 8)
assert_row_count(self.session1, 'ttl_table', 0)
def collection_set_ttl_test(self):
"""
Test that ttl has a granularity of elements using a set collection.
"""
self.prepare(default_time_to_live=10)
self.session1.execute("ALTER TABLE ttl_table ADD myset set<int>;""")
start = time.time()
self.session1.execute("""
INSERT INTO ttl_table (key, col1, myset) VALUES (%d, %d, %s);
""" % (1, 1, '{1,2,3,4,5}'))
self.session1.execute("""
UPDATE ttl_table USING TTL 3 SET myset = myset + {42} WHERE key=1;
""")
assert_all(
self.session1,
"SELECT * FROM ttl_table;",
[[1, 1, None, None, sortedset([1, 2, 3, 4, 5, 42])]]
)
self.smart_sleep(start, 5)
assert_all(
self.session1,
"SELECT * FROM ttl_table;",
[[1, 1, None, None, sortedset([1, 2, 3, 4, 5])]]
)
self.smart_sleep(start, 12)
assert_row_count(self.session1, 'ttl_table', 0)
def resumable_replace_test(self):
"""
Test resumable bootstrap while replacing node. Feature introduced in
2.2 with ticket https://issues.apache.org/jira/browse/CASSANDRA-8838
@jira_ticket https://issues.apache.org/jira/browse/CASSANDRA-8838
"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
node1.stress(['write', 'n=100K', 'no-warmup', '-schema', 'replication(factor=3)'])
session = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE)
initial_data = rows_to_list(session.execute(query))
node3.stop(gently=False)
# kill node1 in the middle of streaming to let it fail
t = InterruptBootstrap(node1)
t.start()
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4', cluster=cluster, auto_bootstrap=True, thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000), jmx_port='7400', remote_debug_port='0',
initial_token=None, binary_interface=('127.0.0.4', 9042))
# keep timeout low so that test won't hang
node4.set_configuration_options(values={'streaming_socket_timeout_in_ms': 1000})
cluster.add(node4, False)
try:
node4.start(jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"], wait_other_notice=False)
except NodeError:
pass # node doesn't start as expected
t.join()
# bring back node1 and invoke nodetool bootstrap to resume bootstrapping
node1.start()
node4.nodetool('bootstrap resume')
# check if we skipped already retrieved ranges
node4.watch_log_for("already available. Skipping streaming.")
# wait for node3 ready to query
node4.watch_log_for("Listening for thrift clients...")
# check if 2nd bootstrap succeeded
assert_bootstrap_state(self, node4, 'COMPLETED')
# query should work again
debug("Stopping old nodes")
node1.stop(gently=False, wait_other_notice=True)
node2.stop(gently=False, wait_other_notice=True)
debug("Verifying data on new node.")
session = self.patient_exclusive_cql_connection(node4)
assert_all(session, 'SELECT * from {} LIMIT 1'.format(stress_table),
expected=initial_data,
cl=ConsistencyLevel.ONE)
def assertions_test(self):
# assert_exception_test
mock_session = Mock(
**
{'execute.side_effect': AlreadyExists("Dummy exception message.")})
assert_exception(mock_session, "DUMMY QUERY", expected=AlreadyExists)
# assert_unavailable_test
mock_session = Mock(**{
'execute.side_effect':
Unavailable("Dummy Unavailabile message.")
})
assert_unavailable(mock_session.execute)
# assert_invalid_test
mock_session = Mock(**{
'execute.side_effect':
InvalidRequest("Dummy InvalidRequest message.")
})
assert_invalid(mock_session, "DUMMY QUERY")
# assert_unauthorized_test
mock_session = Mock(**{
'execute.side_effect':
Unauthorized("Dummy Unauthorized message.")
})
assert_unauthorized(mock_session, "DUMMY QUERY", None)
# assert_one_test
mock_session = Mock()
mock_session.execute = Mock(return_value=[[1, 1]])
assert_one(mock_session, "SELECT * FROM test", [1, 1])
# assert_none_test
mock_session = Mock()
mock_session.execute = Mock(return_value=[])
assert_none(mock_session, "SELECT * FROM test")
# assert_all_test
mock_session = Mock()
mock_session.execute = Mock(return_value=[[i, i]
for i in range(0, 10)])
assert_all(mock_session,
"SELECT k, v FROM test", [[i, i] for i in range(0, 10)],
ignore_order=True)
# assert_almost_equal_test
assert_almost_equal(1, 1.1, 1.2, 1.9, error=1.0)
# assert_row_count_test
mock_session = Mock()
mock_session.execute = Mock(return_value=[[1]])
assert_row_count(mock_session, 'test', 1)
# assert_length_equal_test
check = [1, 2, 3, 4]
assert_length_equal(check, 4)
def unlogged_batch_accepts_regular_mutations_test(self):
""" Test that unlogged batch accepts regular mutations """
session = self.prepare()
session.execute("""
BEGIN UNLOGGED BATCH
INSERT INTO users (id, firstname, lastname) VALUES (0, 'Jack', 'Sparrow')
INSERT INTO users (id, firstname, lastname) VALUES (2, 'Elizabeth', 'Swann')
APPLY BATCH
""")
assert_all(session, "SELECT * FROM users", [[0, u'Jack', u'Sparrow'], [2, u'Elizabeth', u'Swann']])
def logged_batch_accepts_regular_mutations_test(self):
""" Test that logged batch accepts regular mutations """
session = self.prepare()
session.execute("""
BEGIN BATCH
INSERT INTO users (id, firstname, lastname) VALUES (0, 'Jack', 'Sparrow')
INSERT INTO users (id, firstname, lastname) VALUES (1, 'Will', 'Turner')
APPLY BATCH
""")
assert_all(session, "SELECT * FROM users", [[1, u'Will', u'Turner'], [0, u'Jack', u'Sparrow']])
def counter_batch_accepts_counter_mutations_test(self):
""" Test that counter batch accepts counter mutations """
session = self.prepare()
session.execute("""
BEGIN COUNTER BATCH
UPDATE clicks SET total = total + 1 WHERE userid = 1 and url = 'http://foo.com'
UPDATE clicks SET total = total + 1 WHERE userid = 1 and url = 'http://bar.com'
UPDATE clicks SET total = total + 1 WHERE userid = 2 and url = 'http://baz.com'
APPLY BATCH
""")
assert_all(session, "SELECT total FROM clicks", [[1], [1], [1]])
def batch_uses_proper_timestamp_test(self):
""" Test that each statement will be executed with provided BATCH timestamp """
session = self.prepare()
session.execute("""
BEGIN BATCH USING TIMESTAMP 1111111111111111
INSERT INTO users (id, firstname, lastname) VALUES (0, 'Jack', 'Sparrow')
INSERT INTO users (id, firstname, lastname) VALUES (1, 'Will', 'Turner')
APPLY BATCH
""")
query = "SELECT id, writetime(firstname), writetime(lastname) FROM users"
assert_all(session, query, [[1, 1111111111111111, 1111111111111111], [0, 1111111111111111, 1111111111111111]])
def logged_batch_doesnt_throw_uae_test(self):
""" Test that logged batch DOES NOT throw UAE if there are at least 2 live nodes """
session = self.prepare(nodes=3)
self.cluster.nodelist()[-1].stop(wait_other_notice=True)
query = SimpleStatement("""
BEGIN BATCH
INSERT INTO users (id, firstname, lastname) VALUES (0, 'Jack', 'Sparrow')
INSERT INTO users (id, firstname, lastname) VALUES (1, 'Will', 'Turner')
APPLY BATCH
""", consistency_level=ConsistencyLevel.ANY)
session.execute(query)
assert_all(session, "SELECT * FROM users", [[1, u'Will', u'Turner'], [0, u'Jack', u'Sparrow']])
def update_single_column_ttl_test(self):
""" Test that specifying a TTL on a single column works """
self.prepare()
self.session1.execute("""
INSERT INTO ttl_table (key, col1, col2, col3) VALUES (%d, %d, %d, %d);
""" % (1, 1, 1, 1))
start = time.time()
self.session1.execute("UPDATE ttl_table USING TTL 3 set col1=42 where key=%s;" % (1,))
assert_all(self.session1, "SELECT * FROM ttl_table;", [[1, 42, 1, 1]])
self.smart_sleep(start, 5)
assert_all(self.session1, "SELECT * FROM ttl_table;", [[1, None, 1, 1]])
def remove_column_ttl_test(self):
"""
Test that removing a column ttl works.
"""
self.prepare()
start = time.time()
self.session1.execute("""
INSERT INTO ttl_table (key, col1, col2, col3) VALUES (%d, %d, %d, %d) USING TTL 2;
""" % (1, 1, 1, 1))
self.session1.execute("UPDATE ttl_table set col1=42 where key=%s;" % (1,))
self.smart_sleep(start, 4)
assert_all(self.session1, "SELECT * FROM ttl_table;", [[1, 42, None, None]])
def upgrade_with_statics_test(self):
"""
Validates we can read legacy sstables with static columns.
"""
PARTITIONS = 1
ROWS = 10
session = self._setup_cluster()
session.execute('CREATE TABLE t (k int, s1 int static, s2 int static, t int, v1 int, v2 int, PRIMARY KEY (k, t))')
for n in range(PARTITIONS):
for r in range(ROWS):
session.execute("INSERT INTO t(k, s1, s2, t, v1, v2) VALUES (%d, %d, %d, %d, %d, %d)" % (n, r, r + 1, r, r, r + 1))
session = self._do_upgrade()
for n in range(PARTITIONS):
assert_all(session,
"SELECT * FROM t WHERE k = %d" % (n),
[[n, v, ROWS - 1, ROWS, v, v + 1] for v in range(ROWS)])
assert_all(session,
"SELECT * FROM t WHERE k = %d ORDER BY t DESC" % (n),
[[n, v, ROWS - 1, ROWS, v, v + 1] for v in range(ROWS - 1, -1, -1)])
self.cluster.compact()
for n in range(PARTITIONS):
assert_all(session,
"SELECT * FROM t WHERE k = %d" % (n),
[[n, v, ROWS - 1, ROWS, v, v + 1] for v in range(ROWS)])
assert_all(session,
"SELECT * FROM t WHERE k = %d ORDER BY t DESC" % (n),
[[n, v, ROWS - 1, ROWS, v, v + 1] for v in range(ROWS - 1, -1, -1)])
def table_test(self):
"""
Smoke test that basic table operations work:
- create 2 tables, one with and one without COMPACT STORAGE
- ALTER the table without COMPACT STORAGE, adding a column
For each of those tables:
- insert 10 values
- SELECT * and assert the values are there
- TRUNCATE the table
- SELECT * and assert there are no values
- DROP the table
- SELECT * and assert the statement raises an InvalidRequest
# TODO run SELECTs to make sure each statement works
"""
session = self.prepare()
ks_meta = UpdatingKeyspaceMetadataWrapper(session.cluster, ks_name='ks')
session.execute("CREATE TABLE test1 (k int PRIMARY KEY, v1 int)")
self.assertIn('test1', ks_meta.tables)
session.execute("CREATE TABLE test2 (k int, c1 int, v1 int, PRIMARY KEY (k, c1)) WITH COMPACT STORAGE")
self.assertIn('test2', ks_meta.tables)
t1_meta = UpdatingTableMetadataWrapper(session.cluster, ks_name='ks', table_name='test1')
session.execute("ALTER TABLE test1 ADD v2 int")
self.assertIn('v2', t1_meta.columns)
for i in range(0, 10):
session.execute("INSERT INTO test1 (k, v1, v2) VALUES ({i}, {i}, {i})".format(i=i))
session.execute("INSERT INTO test2 (k, c1, v1) VALUES ({i}, {i}, {i})".format(i=i))
assert_all(session, "SELECT * FROM test1", [[i, i, i] for i in range(0, 10)], ignore_order=True)
assert_all(session, "SELECT * FROM test2", [[i, i, i] for i in range(0, 10)], ignore_order=True)
session.execute("TRUNCATE test1")
session.execute("TRUNCATE test2")
assert_none(session, "SELECT * FROM test1")
assert_none(session, "SELECT * FROM test2")
session.execute("DROP TABLE test1")
self.assertNotIn('test1', ks_meta.tables)
session.execute("DROP TABLE test2")
self.assertNotIn('test2', ks_meta.tables)
def update_single_column_ttl_test(self):
""" Test that specifying a TTL on a single column works """
self.prepare()
self.session1.execute("""
INSERT INTO ttl_table (key, col1, col2, col3) VALUES (%d, %d, %d, %d);
""" % (1, 1, 1, 1))
start = time.time()
self.session1.execute(
"UPDATE ttl_table USING TTL 3 set col1=42 where key=%s;" % (1, ))
assert_all(self.session1, "SELECT * FROM ttl_table;", [[1, 42, 1, 1]])
self.smart_sleep(start, 5)
assert_all(self.session1, "SELECT * FROM ttl_table;",
[[1, None, 1, 1]])
def update_multiple_columns_ttl_test(self):
""" Test that specifying a TTL on multiple columns works """
self.prepare()
self.cursor1.execute("""
INSERT INTO ttl_table (key, col1, col2, col3) VALUES (%d, %d, %d, %d);
""" % (1, 1, 1, 1))
start = time.time()
self.cursor1.execute("""
UPDATE ttl_table USING TTL 2 set col1=42, col2=42, col3=42 where key=%s;
""" % (1,))
assert_all(self.cursor1, "SELECT * FROM ttl_table;", [[1, 42, 42, 42]])
self.smart_sleep(start, 4)
assert_all(self.cursor1, "SELECT * FROM ttl_table;", [[1, None, None, None]])
def ttl_is_respected_on_repair_test(self):
""" Test that ttl is respected on repair """
self.prepare()
self.session1.execute("""
ALTER KEYSPACE ks WITH REPLICATION =
{'class' : 'SimpleStrategy', 'replication_factor' : 1};
""")
self.session1.execute("""
INSERT INTO ttl_table (key, col1) VALUES (1, 1) USING TTL 5;
""")
self.session1.execute("""
INSERT INTO ttl_table (key, col1) VALUES (2, 2) USING TTL 1000;
""")
assert_all(
self.session1,
"SELECT * FROM ttl_table;",
[[1, 1, None, None], [2, 2, None, None]]
)
time.sleep(7)
self.node1.stop()
session2 = self.patient_exclusive_cql_connection(self.node2)
session2.execute("USE ks;")
assert_unavailable(session2.execute, "SELECT * FROM ttl_table;")
self.node1.start(wait_for_binary_proto=True)
self.session1 = self.patient_exclusive_cql_connection(self.node1)
self.session1.execute("USE ks;")
self.session1.execute("""
ALTER KEYSPACE ks WITH REPLICATION =
{'class' : 'SimpleStrategy', 'replication_factor' : 2};
""")
self.node1.repair(['ks'])
ttl_start = time.time()
ttl_session1 = self.session1.execute('SELECT ttl(col1) FROM ttl_table;')
self.node1.stop()
assert_row_count(session2, 'ttl_table', 1)
assert_all(
session2,
"SELECT * FROM ttl_table;",
[[2, 2, None, None]]
)
# Check that the TTL on both server are the same
ttl_session2 = session2.execute('SELECT ttl(col1) FROM ttl_table;')
ttl_session1 = ttl_session1[0][0] - (time.time() - ttl_start)
assert_almost_equal(ttl_session1, ttl_session2[0][0], error=0.005)
def remove_column_ttl_test(self):
"""
Test that removing a column ttl works.
"""
self.prepare()
start = time.time()
self.session1.execute("""
INSERT INTO ttl_table (key, col1, col2, col3) VALUES (%d, %d, %d, %d) USING TTL 2;
""" % (1, 1, 1, 1))
self.session1.execute("UPDATE ttl_table set col1=42 where key=%s;" %
(1, ))
self.smart_sleep(start, 4)
assert_all(self.session1, "SELECT * FROM ttl_table;",
[[1, 42, None, None]])
def set_ttl_to_zero_to_default_ttl_test(self):
"""
Test that we can remove the default ttl by setting the ttl explicitly to zero.
CASSANDRA-11207
"""
self.prepare(default_time_to_live=2)
start = time.time()
self.session1.execute("INSERT INTO ttl_table (key, col1, col2, col3) VALUES ({}, {}, {}, {});".format(1, 1, 1, 1))
self.session1.execute("INSERT INTO ttl_table (key, col1, col2, col3) VALUES ({}, {}, {}, {});".format(2, 1, 1, 1))
self.session1.execute("UPDATE ttl_table using ttl 0 set col1=42 where key={};".format(1))
self.session1.execute("UPDATE ttl_table using ttl 3 set col1=42 where key={};".format(2))
self.smart_sleep(start, 5)
# The first row should be deleted, using ttl 0 should fallback to default_time_to_live
assert_all(self.session1, "SELECT * FROM ttl_table;", [[1, 42, None, None]])
def update_column_ttl_with_default_ttl_test2(self):
"""
Test that specifying a column ttl works when a default ttl is set.
This test specify a higher column ttl than the default ttl.
"""
self.prepare(default_time_to_live=2)
start = time.time()
self.session1.execute("""
INSERT INTO ttl_table (key, col1, col2, col3) VALUES (%d, %d, %d, %d);
""" % (1, 1, 1, 1))
self.session1.execute("UPDATE ttl_table USING TTL 6 set col1=42 where key=%s;" % (1,))
self.smart_sleep(start, 4)
assert_all(self.session1, "SELECT * FROM ttl_table;", [[1, 42, None, None]])
self.smart_sleep(start, 8)
assert_row_count(self.session1, 'ttl_table', 0)
def update_column_ttl_with_default_ttl_test(self):
"""
Test that specifying a column ttl works when a default ttl is set.
This test specify a lower ttl for the column than the default ttl.
"""
self.prepare(default_time_to_live=8)
start = time.time()
self.cursor1.execute("""
INSERT INTO ttl_table (key, col1, col2, col3) VALUES (%d, %d, %d, %d);
""" % (1, 1, 1, 1))
self.cursor1.execute("UPDATE ttl_table USING TTL 3 set col1=42 where key=%s;" % (1,))
assert_all(self.cursor1, "SELECT * FROM ttl_table;", [[1, 42, 1, 1]])
self.smart_sleep(start, 5)
assert_all(self.cursor1, "SELECT * FROM ttl_table;", [[1, None, 1, 1]])
self.smart_sleep(start, 10)
assert_row_count(self.cursor1, 'ttl_table', 0)
def update_column_ttl_with_default_ttl_test2(self):
"""
Test that specifying a column ttl works when a default ttl is set.
This test specify a higher column ttl than the default ttl.
"""
self.prepare(default_time_to_live=2)
start = time.time()
self.session1.execute("""
INSERT INTO ttl_table (key, col1, col2, col3) VALUES (%d, %d, %d, %d);
""" % (1, 1, 1, 1))
self.session1.execute(
"UPDATE ttl_table USING TTL 6 set col1=42 where key=%s;" % (1, ))
self.smart_sleep(start, 4)
assert_all(self.session1, "SELECT * FROM ttl_table;",
[[1, 42, None, None]])
self.smart_sleep(start, 8)
assert_row_count(self.session1, 'ttl_table', 0)
def assertions_test(self):
# assert_exception_test
mock_session = Mock(**{'execute.side_effect': AlreadyExists("Dummy exception message.")})
assert_exception(mock_session, "DUMMY QUERY", expected=AlreadyExists)
# assert_unavailable_test
mock_session = Mock(**{'execute.side_effect': Unavailable("Dummy Unavailabile message.")})
assert_unavailable(mock_session.execute)
# assert_invalid_test
mock_session = Mock(**{'execute.side_effect': InvalidRequest("Dummy InvalidRequest message.")})
assert_invalid(mock_session, "DUMMY QUERY")
# assert_unauthorized_test
mock_session = Mock(**{'execute.side_effect': Unauthorized("Dummy Unauthorized message.")})
assert_unauthorized(mock_session, "DUMMY QUERY", None)
# assert_one_test
mock_session = Mock()
mock_session.execute = Mock(return_value=[[1, 1]])
assert_one(mock_session, "SELECT * FROM test", [1, 1])
# assert_none_test
mock_session = Mock()
mock_session.execute = Mock(return_value=[])
assert_none(mock_session, "SELECT * FROM test")
# assert_all_test
mock_session = Mock()
mock_session.execute = Mock(return_value=[[i, i] for i in range(0, 10)])
assert_all(mock_session, "SELECT k, v FROM test", [[i, i] for i in range(0, 10)], ignore_order=True)
# assert_almost_equal_test
assert_almost_equal(1, 1.1, 1.2, 1.9, error=1.0)
# assert_row_count_test
mock_session = Mock()
mock_session.execute = Mock(return_value=[[1]])
assert_row_count(mock_session, 'test', 1)
# assert_length_equal_test
check = [1, 2, 3, 4]
assert_length_equal(check, 4)
def ttl_is_respected_on_repair_test(self):
""" Test that ttl is respected on repair """
self.prepare()
self.session1.execute("""
ALTER KEYSPACE ks WITH REPLICATION =
{'class' : 'SimpleStrategy', 'replication_factor' : 1};
""")
self.session1.execute("""
INSERT INTO ttl_table (key, col1) VALUES (1, 1) USING TTL 5;
""")
self.session1.execute("""
INSERT INTO ttl_table (key, col1) VALUES (2, 2) USING TTL 1000;
""")
assert_all(self.session1, "SELECT * FROM ttl_table;",
[[1, 1, None, None], [2, 2, None, None]])
time.sleep(7)
self.node1.stop()
session2 = self.patient_exclusive_cql_connection(self.node2)
session2.execute("USE ks;")
assert_unavailable(session2.execute, "SELECT * FROM ttl_table;")
self.node1.start(wait_for_binary_proto=True)
self.session1 = self.patient_exclusive_cql_connection(self.node1)
self.session1.execute("USE ks;")
self.session1.execute("""
ALTER KEYSPACE ks WITH REPLICATION =
{'class' : 'SimpleStrategy', 'replication_factor' : 2};
""")
self.node1.repair(['ks'])
ttl_start = time.time()
ttl_session1 = self.session1.execute(
'SELECT ttl(col1) FROM ttl_table;')
self.node1.stop()
assert_row_count(session2, 'ttl_table', 1)
assert_all(session2, "SELECT * FROM ttl_table;", [[2, 2, None, None]])
# Check that the TTL on both server are the same
ttl_session2 = session2.execute('SELECT ttl(col1) FROM ttl_table;')
ttl_session1 = ttl_session1[0][0] - (time.time() - ttl_start)
assert_almost_equal(ttl_session1, ttl_session2[0][0], error=0.005)
def collection_list_ttl_test(self):
"""
Test that ttl has a granularity of elements using a list collection.
"""
self.prepare(default_time_to_live=10)
self.session1.execute("ALTER TABLE ttl_table ADD mylist list<int>;""")
start = time.time()
self.session1.execute("""
INSERT INTO ttl_table (key, col1, mylist) VALUES (%d, %d, %s);
""" % (1, 1, [1, 2, 3, 4, 5]))
self.session1.execute("""
UPDATE ttl_table USING TTL 5 SET mylist[0] = 42, mylist[4] = 42 WHERE key=1;
""")
assert_all(self.session1, "SELECT * FROM ttl_table;", [[1, 1, None, None, [42, 2, 3, 4, 42]]])
self.smart_sleep(start, 7)
assert_all(self.session1, "SELECT * FROM ttl_table;", [[1, 1, None, None, [2, 3, 4]]])
self.smart_sleep(start, 12)
assert_row_count(self.session1, 'ttl_table', 0)
def remove_column_ttl_with_default_ttl_test(self):
"""
Test that we cannot remove a column ttl when a default ttl is set.
"""
self.prepare(default_time_to_live=2)
start = time.time()
self.session1.execute("""
INSERT INTO ttl_table (key, col1, col2, col3) VALUES (%d, %d, %d, %d);
""" % (1, 1, 1, 1))
self.session1.execute("""
INSERT INTO ttl_table (key, col1, col2, col3) VALUES (%d, %d, %d, %d);
""" % (2, 1, 1, 1))
self.session1.execute("UPDATE ttl_table using ttl 0 set col1=42 where key=%s;" % (1,))
self.session1.execute("UPDATE ttl_table using ttl 8 set col1=42 where key=%s;" % (2,))
self.smart_sleep(start, 5)
# The first row should be deleted, using ttl 0 should fallback to default_time_to_live
assert_all(self.session1, "SELECT * FROM ttl_table;", [[2, 42, None, None]])
self.smart_sleep(start, 10)
assert_row_count(self.session1, 'ttl_table', 0)
def statements_test(self):
"""
Smoke test SELECT and UPDATE statements:
- create a table
- insert 20 rows into the table
- run SELECT COUNT queries and assert they return the correct values
- bare and with IN and equality conditions
- run SELECT * queries with = conditions
- run UPDATE queries
- SELECT * and assert the UPDATEd values are there
- DELETE with a = condition
- SELECT the deleted values and make sure nothing is returned
# TODO run SELECTs to make sure each statement works
"""
session = self.prepare()
session.execute(
"CREATE TABLE test7 (kind text, time int, v1 int, v2 int, PRIMARY KEY(kind, time) )"
)
for i in range(0, 10):
session.execute(
"INSERT INTO test7 (kind, time, v1, v2) VALUES ('ev1', {i}, {i}, {i})"
.format(i=i))
session.execute(
"INSERT INTO test7 (kind, time, v1, v2) VALUES ('ev2', {i}, {i}, {i})"
.format(i=i))
assert_one(session, "SELECT COUNT(*) FROM test7 WHERE kind = 'ev1'",
[10])
assert_one(session,
"SELECT COUNT(*) FROM test7 WHERE kind IN ('ev1', 'ev2')",
[20])
assert_one(
session,
"SELECT COUNT(*) FROM test7 WHERE kind IN ('ev1', 'ev2') AND time=0",
[2])
assert_all(session, "SELECT * FROM test7 WHERE kind = 'ev1'",
[['ev1', i, i, i] for i in range(0, 10)])
assert_all(session, "SELECT * FROM test7 WHERE kind = 'ev2'",
[['ev2', i, i, i] for i in range(0, 10)])
for i in range(0, 10):
session.execute(
"UPDATE test7 SET v1 = 0, v2 = 0 where kind = 'ev1' AND time={i}"
.format(i=i))
assert_all(session, "SELECT * FROM test7 WHERE kind = 'ev1'",
[['ev1', i, 0, 0] for i in range(0, 10)])
session.execute("DELETE FROM test7 WHERE kind = 'ev1'")
assert_none(session, "SELECT * FROM test7 WHERE kind = 'ev1'")
assert_one(session, "SELECT COUNT(*) FROM test7 WHERE kind = 'ev1'",
[0])
def ttl_is_respected_on_delayed_replication_test(self):
""" Test that ttl is respected on delayed replication """
self.prepare()
self.node2.stop()
self.cursor1.execute("""
INSERT INTO ttl_table (key, col1) VALUES (1, 1) USING TTL 5;
""")
self.cursor1.execute("""
INSERT INTO ttl_table (key, col1) VALUES (2, 2) USING TTL 60;
""")
assert_all(
self.cursor1,
"SELECT * FROM ttl_table;",
[[1, 1, None, None], [2, 2, None, None]]
)
time.sleep(7)
self.node1.stop()
self.node2.start(wait_for_binary_proto=True)
cursor2 = self.patient_exclusive_cql_connection(self.node2)
cursor2.execute("USE ks;")
assert_row_count(cursor2, 'ttl_table', 0) # should be 0 since node1 is down, no replica yet
self.node1.start(wait_for_binary_proto=True)
self.cursor1 = self.patient_exclusive_cql_connection(self.node1)
self.cursor1.execute("USE ks;")
self.node1.cleanup()
# Check that the expired data has not been replicated
assert_row_count(cursor2, 'ttl_table', 1)
assert_all(
cursor2,
"SELECT * FROM ttl_table;",
[[2, 2, None, None]],
cl=ConsistencyLevel.ALL
)
# Check that the TTL on both server are the same
ttl_cursor1 = self.cursor1.execute('SELECT ttl(col1) FROM ttl_table;')
ttl_cursor2 = cursor2.execute('SELECT ttl(col1) FROM ttl_table;')
assert_almost_equal(ttl_cursor1[0][0], ttl_cursor2[0][0], error=0.1)
def collection_list_ttl_test(self):
"""
Test that ttl has a granularity of elements using a list collection.
"""
self.prepare(default_time_to_live=10)
self.session1.execute("ALTER TABLE ttl_table ADD mylist list<int>;" "")
start = time.time()
self.session1.execute("""
INSERT INTO ttl_table (key, col1, mylist) VALUES (%d, %d, %s);
""" % (1, 1, [1, 2, 3, 4, 5]))
self.session1.execute("""
UPDATE ttl_table USING TTL 5 SET mylist[0] = 42, mylist[4] = 42 WHERE key=1;
""")
assert_all(self.session1, "SELECT * FROM ttl_table;",
[[1, 1, None, None, [42, 2, 3, 4, 42]]])
self.smart_sleep(start, 7)
assert_all(self.session1, "SELECT * FROM ttl_table;",
[[1, 1, None, None, [2, 3, 4]]])
self.smart_sleep(start, 12)
assert_row_count(self.session1, 'ttl_table', 0)
def many_columns_test(self):
"""
Test for tables with thousands of columns.
For CASSANDRA-11621.
"""
session = self.prepare()
width = 5000
cluster = self.cluster
session.execute("CREATE TABLE very_wide_table (pk int PRIMARY KEY, " +
",".join(map(lambda i: "c_{} int".format(i), range(width))) +
")")
session.execute("INSERT INTO very_wide_table (pk, " +
",".join(map(lambda i: "c_{}".format(i), range(width))) +
") VALUES (100," +
",".join(map(lambda i: str(i), range(width))) +
")")
assert_all(session, "SELECT " +
",".join(map(lambda i: "c_{}".format(i), range(width))) +
" FROM very_wide_table", [[i for i in range(width)]])
def drop_column_queries_test(self):
session = self.prepare()
session.execute("USE ks")
session.execute(
"CREATE TABLE cf (key int PRIMARY KEY, c1 int, c2 int)")
session.execute("CREATE INDEX ON cf(c2)")
# insert some data.
session.execute("INSERT INTO cf (key, c1, c2) VALUES (0, 1, 2)")
session.execute("INSERT INTO cf (key, c1, c2) VALUES (1, 2, 3)")
session.execute("INSERT INTO cf (key, c1, c2) VALUES (2, 3, 4)")
# drop and readd c1.
session.execute("ALTER TABLE cf DROP c1")
session.execute("ALTER TABLE cf ADD c1 int")
# add another row.
session.execute("INSERT INTO cf (key, c1, c2) VALUES (3, 4, 5)")
# test that old (pre-drop) c1 values aren't returned and new ones are.
assert_all(session,
"SELECT c1 FROM cf", [[None], [None], [None], [4]],
ignore_order=True)
assert_all(session,
"SELECT * FROM cf",
[[0, None, 2], [1, None, 3], [2, None, 4], [3, 4, 5]],
ignore_order=True)
assert_one(session, "SELECT c1 FROM cf WHERE key = 0", [None])
assert_one(session, "SELECT c1 FROM cf WHERE key = 3", [4])
assert_one(session, "SELECT * FROM cf WHERE c2 = 2", [0, None, 2])
assert_one(session, "SELECT * FROM cf WHERE c2 = 5", [3, 4, 5])
def many_columns_test(self):
"""
Test for tables with thousands of columns.
For CASSANDRA-11621.
"""
session = self.prepare()
width = 5000
cluster = self.cluster
session.execute(
"CREATE TABLE very_wide_table (pk int PRIMARY KEY, " +
",".join(map(lambda i: "c_{} int".format(i), range(width))) + ")")
session.execute(
"INSERT INTO very_wide_table (pk, " +
",".join(map(lambda i: "c_{}".format(i), range(width))) +
") VALUES (100," + ",".join(map(lambda i: str(i), range(width))) +
")")
assert_all(
session, "SELECT " +
",".join(map(lambda i: "c_{}".format(i), range(width))) +
" FROM very_wide_table", [[i for i in range(width)]])
def drop_column_compaction_test(self):
session = self.prepare()
session.execute("USE ks")
session.execute("CREATE TABLE cf (key int PRIMARY KEY, c1 int, c2 int)")
# insert some data.
session.execute("INSERT INTO cf (key, c1, c2) VALUES (0, 1, 2)")
session.execute("INSERT INTO cf (key, c1, c2) VALUES (1, 2, 3)")
session.execute("INSERT INTO cf (key, c1, c2) VALUES (2, 3, 4)")
# drop and readd c1.
session.execute("ALTER TABLE cf DROP c1")
session.execute("ALTER TABLE cf ADD c1 int")
# add another row.
session.execute("INSERT INTO cf (key, c1, c2) VALUES (3, 4, 5)")
node = self.cluster.nodelist()[0]
node.flush()
node.compact()
# test that c1 values have been compacted away.
session = self.patient_cql_connection(node)
assert_all(session, "SELECT c1 FROM ks.cf", [[None], [None], [None], [4]], ignore_order=True)
def collection_set_ttl_test(self):
"""
Test that ttl has a granularity of elements using a set collection.
"""
self.prepare(default_time_to_live=10)
self.session1.execute("ALTER TABLE ttl_table ADD myset set<int>;" "")
start = time.time()
self.session1.execute("""
INSERT INTO ttl_table (key, col1, myset) VALUES (%d, %d, %s);
""" % (1, 1, '{1,2,3,4,5}'))
self.session1.execute("""
UPDATE ttl_table USING TTL 3 SET myset = myset + {42} WHERE key=1;
""")
assert_all(self.session1, "SELECT * FROM ttl_table;",
[[1, 1, None, None,
sortedset([1, 2, 3, 4, 5, 42])]])
self.smart_sleep(start, 5)
assert_all(
self.session1, "SELECT * FROM ttl_table;",
[[1, 1, None, None, sortedset([1, 2, 3, 4, 5])]])
self.smart_sleep(start, 12)
assert_row_count(self.session1, 'ttl_table', 0)
def remove_column_ttl_with_default_ttl_test(self):
"""
Test that we cannot remove a column ttl when a default ttl is set.
"""
self.prepare(default_time_to_live=2)
start = time.time()
self.session1.execute("""
INSERT INTO ttl_table (key, col1, col2, col3) VALUES (%d, %d, %d, %d);
""" % (1, 1, 1, 1))
self.session1.execute("""
INSERT INTO ttl_table (key, col1, col2, col3) VALUES (%d, %d, %d, %d);
""" % (2, 1, 1, 1))
self.session1.execute(
"UPDATE ttl_table using ttl 0 set col1=42 where key=%s;" % (1, ))
self.session1.execute(
"UPDATE ttl_table using ttl 8 set col1=42 where key=%s;" % (2, ))
self.smart_sleep(start, 5)
# The first row should be deleted, using ttl 0 should fallback to default_time_to_live
assert_all(self.session1, "SELECT * FROM ttl_table;",
[[2, 42, None, None]])
self.smart_sleep(start, 10)
assert_row_count(self.session1, 'ttl_table', 0)
def set_ttl_to_zero_to_default_ttl_test(self):
"""
Test that we can remove the default ttl by setting the ttl explicitly to zero.
CASSANDRA-11207
"""
self.prepare(default_time_to_live=2)
start = time.time()
self.session1.execute(
"INSERT INTO ttl_table (key, col1, col2, col3) VALUES ({}, {}, {}, {});"
.format(1, 1, 1, 1))
self.session1.execute(
"INSERT INTO ttl_table (key, col1, col2, col3) VALUES ({}, {}, {}, {});"
.format(2, 1, 1, 1))
self.session1.execute(
"UPDATE ttl_table using ttl 0 set col1=42 where key={};".format(1))
self.session1.execute(
"UPDATE ttl_table using ttl 3 set col1=42 where key={};".format(2))
self.smart_sleep(start, 5)
# The first row should be deleted, using ttl 0 should fallback to default_time_to_live
assert_all(self.session1, "SELECT * FROM ttl_table;",
[[1, 42, None, None]])
def upgrade_with_statics_test(self):
"""
Validates we can read legacy sstables with static columns.
"""
PARTITIONS = 1
ROWS = 10
session = self._setup_cluster()
session.execute(
'CREATE TABLE t (k int, s1 int static, s2 int static, t int, v1 int, v2 int, PRIMARY KEY (k, t))'
)
for n in range(PARTITIONS):
for r in range(ROWS):
session.execute(
"INSERT INTO t(k, s1, s2, t, v1, v2) VALUES ({}, {}, {}, {}, {}, {})"
.format(n, r, r + 1, r, r, r + 1))
session = self._do_upgrade()
for n in range(PARTITIONS):
assert_all(session, "SELECT * FROM t WHERE k = {}".format(n),
[[n, v, ROWS - 1, ROWS, v, v + 1] for v in range(ROWS)])
assert_all(
session,
"SELECT * FROM t WHERE k = {} ORDER BY t DESC".format(n),
[[n, v, ROWS - 1, ROWS, v, v + 1]
for v in range(ROWS - 1, -1, -1)])
self.cluster.compact()
for n in range(PARTITIONS):
assert_all(session, "SELECT * FROM t WHERE k = {}".format(n),
[[n, v, ROWS - 1, ROWS, v, v + 1] for v in range(ROWS)])
assert_all(
session,
"SELECT * FROM t WHERE k = {} ORDER BY t DESC".format(n),
[[n, v, ROWS - 1, ROWS, v, v + 1]
for v in range(ROWS - 1, -1, -1)])
def upgrade_with_index_test(self):
"""
Checks a simple index can still be read after upgrade.
"""
PARTITIONS = 2
ROWS = 4
session = self._setup_cluster()
session.execute(
'CREATE TABLE t (k int, t int, v1 int, v2 int, PRIMARY KEY (k, t))'
)
session.execute('CREATE INDEX ON t(v1)')
for p in range(PARTITIONS):
for r in range(ROWS):
session.execute(
"INSERT INTO t(k, t, v1, v2) VALUES ({}, {}, {}, {})".
format(p, r, r % 2, r * 2))
self.cluster.flush()
assert_all(session,
"SELECT * FROM t WHERE v1 = 0",
[[p, r, 0, r * 2] for p in range(PARTITIONS)
for r in range(ROWS) if r % 2 == 0],
ignore_order=True)
session = self._do_upgrade()
assert_all(session,
"SELECT * FROM t WHERE v1 = 0",
[[p, r, 0, r * 2] for p in range(PARTITIONS)
for r in range(ROWS) if r % 2 == 0],
ignore_order=True)
self.cluster.compact()
assert_all(session,
"SELECT * FROM t WHERE v1 = 0",
[[p, r, 0, r * 2] for p in range(PARTITIONS)
for r in range(ROWS) if r % 2 == 0],
ignore_order=True)
def statements_test(self):
"""
Smoke test SELECT and UPDATE statements:
- create a table
- insert 20 rows into the table
- run SELECT COUNT queries and assert they return the correct values
- bare and with IN and equality conditions
- run SELECT * queries with = conditions
- run UPDATE queries
- SELECT * and assert the UPDATEd values are there
- DELETE with a = condition
- SELECT the deleted values and make sure nothing is returned
# TODO run SELECTs to make sure each statement works
"""
session = self.prepare()
session.execute("CREATE TABLE test7 (kind text, time int, v1 int, v2 int, PRIMARY KEY(kind, time) )")
for i in range(0, 10):
session.execute("INSERT INTO test7 (kind, time, v1, v2) VALUES ('ev1', {i}, {i}, {i})".format(i=i))
session.execute("INSERT INTO test7 (kind, time, v1, v2) VALUES ('ev2', {i}, {i}, {i})".format(i=i))
assert_one(session, "SELECT COUNT(*) FROM test7 WHERE kind = 'ev1'", [10])
assert_one(session, "SELECT COUNT(*) FROM test7 WHERE kind IN ('ev1', 'ev2')", [20])
assert_one(session, "SELECT COUNT(*) FROM test7 WHERE kind IN ('ev1', 'ev2') AND time=0", [2])
assert_all(session, "SELECT * FROM test7 WHERE kind = 'ev1'", [['ev1', i, i, i] for i in range(0, 10)])
assert_all(session, "SELECT * FROM test7 WHERE kind = 'ev2'", [['ev2', i, i, i] for i in range(0, 10)])
for i in range(0, 10):
session.execute("UPDATE test7 SET v1 = 0, v2 = 0 where kind = 'ev1' AND time={i}".format(i=i))
assert_all(session, "SELECT * FROM test7 WHERE kind = 'ev1'", [['ev1', i, 0, 0] for i in range(0, 10)])
session.execute("DELETE FROM test7 WHERE kind = 'ev1'")
assert_none(session, "SELECT * FROM test7 WHERE kind = 'ev1'")
assert_one(session, "SELECT COUNT(*) FROM test7 WHERE kind = 'ev1'", [0])
def ttl_is_respected_on_delayed_replication_test(self):
""" Test that ttl is respected on delayed replication """
self.prepare()
self.node2.stop()
self.session1.execute("""
INSERT INTO ttl_table (key, col1) VALUES (1, 1) USING TTL 5;
""")
self.session1.execute("""
INSERT INTO ttl_table (key, col1) VALUES (2, 2) USING TTL 1000;
""")
assert_all(self.session1, "SELECT * FROM ttl_table;",
[[1, 1, None, None], [2, 2, None, None]])
time.sleep(7)
self.node1.stop()
self.node2.start(wait_for_binary_proto=True)
session2 = self.patient_exclusive_cql_connection(self.node2)
session2.execute("USE ks;")
assert_row_count(session2, 'ttl_table',
0) # should be 0 since node1 is down, no replica yet
self.node1.start(wait_for_binary_proto=True)
self.session1 = self.patient_exclusive_cql_connection(self.node1)
self.session1.execute("USE ks;")
self.node1.cleanup()
assert_all(session2,
"SELECT count(*) FROM ttl_table", [[1]],
cl=ConsistencyLevel.ALL)
assert_all(session2,
"SELECT * FROM ttl_table;", [[2, 2, None, None]],
cl=ConsistencyLevel.ALL)
# Check that the TTL on both server are the same
ttl_session1 = self.session1.execute(
'SELECT ttl(col1) FROM ttl_table;')
ttl_session2 = session2.execute('SELECT ttl(col1) FROM ttl_table;')
self.assertLessEqual(abs(ttl_session1[0][0] - ttl_session2[0][0]), 1)
def resumable_replace_test(self):
"""
Test resumable bootstrap while replacing node. Feature introduced in
2.2 with ticket https://issues.apache.org/jira/browse/CASSANDRA-8838
@jira_ticket https://issues.apache.org/jira/browse/CASSANDRA-8838
"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
node1.stress([
'write', 'n=100K', 'no-warmup', '-schema', 'replication(factor=3)'
])
session = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table,
consistency_level=ConsistencyLevel.THREE)
initial_data = rows_to_list(session.execute(query))
node3.stop(gently=False)
# kill node1 in the middle of streaming to let it fail
t = InterruptBootstrap(node1)
t.start()
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4',
cluster=cluster,
auto_bootstrap=True,
thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000),
jmx_port='7400',
remote_debug_port='0',
initial_token=None,
binary_interface=('127.0.0.4', 9042))
# keep timeout low so that test won't hang
node4.set_configuration_options(
values={'streaming_socket_timeout_in_ms': 1000})
cluster.add(node4, False)
try:
node4.start(
jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"],
wait_other_notice=False)
except NodeError:
pass # node doesn't start as expected
t.join()
# bring back node1 and invoke nodetool bootstrap to resume bootstrapping
node1.start()
node4.nodetool('bootstrap resume')
# check if we skipped already retrieved ranges
node4.watch_log_for("already available. Skipping streaming.")
# wait for node3 ready to query
node4.watch_log_for("Listening for thrift clients...")
# check if 2nd bootstrap succeeded
assert_bootstrap_state(self, node4, 'COMPLETED')
# query should work again
debug("Stopping old nodes")
node1.stop(gently=False, wait_other_notice=True)
node2.stop(gently=False, wait_other_notice=True)
debug("Verifying data on new node.")
session = self.patient_exclusive_cql_connection(node4)
assert_all(session,
'SELECT * from {} LIMIT 1'.format(stress_table),
expected=initial_data,
cl=ConsistencyLevel.ONE)
def multi_dc_replace_with_rf1_test(self):
"""
Test that multi-dc replace works when rf=1 on each dc
"""
cluster = self.cluster
cluster.populate([1, 1])
cluster.start()
node1, node2 = cluster.nodelist()
node1 = cluster.nodes['node1']
yaml_config = """
# Create the keyspace and table
keyspace: keyspace1
keyspace_definition: |
CREATE KEYSPACE keyspace1 WITH replication = {'class': 'NetworkTopologyStrategy', 'dc1': 1, 'dc2': 1};
table: users
table_definition:
CREATE TABLE users (
username text,
first_name text,
last_name text,
email text,
PRIMARY KEY(username)
) WITH compaction = {'class':'SizeTieredCompactionStrategy'};
insert:
partitions: fixed(1)
batchtype: UNLOGGED
queries:
read:
cql: select * from users where username = ?
fields: samerow
"""
with tempfile.NamedTemporaryFile(mode='w+') as stress_config:
stress_config.write(yaml_config)
stress_config.flush()
node1.stress([
'user', 'profile=' + stress_config.name, 'n=10k', 'no-warmup',
'ops(insert=1)', '-rate', 'threads=50'
])
session = self.patient_cql_connection(node1)
# change system_auth keyspace to 2 (default is 1) to avoid
# "Unable to find sufficient sources for streaming" warning
if cluster.cassandra_version() >= '2.2.0':
session.execute("""
ALTER KEYSPACE system_auth
WITH replication = {'class':'SimpleStrategy', 'replication_factor':2};
""")
# Save initial data
stress_table = 'keyspace1.users'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table,
consistency_level=ConsistencyLevel.TWO)
initial_data = rows_to_list(session.execute(query))
# stop node to replace
debug("Stopping node to replace.")
node2.stop(wait_other_notice=True)
node3 = new_node(cluster, data_center='dc2')
node3.start(replace_address='127.0.0.2', wait_for_binary_proto=True)
assert_bootstrap_state(self, node3, 'COMPLETED')
# Check that keyspace was replicated from dc1 to dc2
self.assertFalse(
node3.grep_log(
"Unable to find sufficient sources for streaming range"))
# query should work again with node1 stopped
node1.stop(wait_other_notice=True)
debug("Verifying data on new node.")
session = self.patient_exclusive_cql_connection(node3)
assert_all(session,
'SELECT * from {} LIMIT 1'.format(stress_table),
expected=initial_data,
cl=ConsistencyLevel.LOCAL_ONE)
def upgrade_with_clustered_table(self, compact_storage=False):
PARTITIONS = 2
ROWS = 1000
session = self._setup_cluster()
session.execute(
'CREATE TABLE t (k int, t int, v int, PRIMARY KEY (k, t))' +
(' WITH COMPACT STORAGE' if compact_storage else ''))
for n in range(PARTITIONS):
for r in range(ROWS):
session.execute(
"INSERT INTO t(k, t, v) VALUES ({n}, {r}, {r})".format(
n=n, r=r))
session = self._do_upgrade()
for n in range(PARTITIONS):
assert_all(session, "SELECT * FROM t WHERE k = {}".format(n),
[[n, v, v] for v in range(ROWS)])
assert_all(
session,
"SELECT * FROM t WHERE k = {} ORDER BY t DESC".format(n),
[[n, v, v] for v in range(ROWS - 1, -1, -1)])
# Querying a "large" slice
start = ROWS / 10
end = ROWS - 1 - (ROWS / 10)
assert_all(
session,
"SELECT * FROM t WHERE k = {n} AND t >= {start} AND t < {end}".
format(n=n, start=start,
end=end), [[n, v, v] for v in range(start, end)])
assert_all(
session,
"SELECT * FROM t WHERE k = {n} AND t >= {start} AND t < {end} ORDER BY t DESC"
.format(n=n, start=start, end=end),
[[n, v, v] for v in range(end - 1, start - 1, -1)])
# Querying a "small" slice
start = ROWS / 2
end = ROWS / 2 + 5
assert_all(
session,
"SELECT * FROM t WHERE k = {n} AND t >= {start} AND t < {end}".
format(n=n, start=start,
end=end), [[n, v, v] for v in range(start, end)])
assert_all(
session,
"SELECT * FROM t WHERE k = {n} AND t >= {start} AND t < {end} ORDER BY t DESC"
.format(n=n, start=start, end=end),
[[n, v, v] for v in range(end - 1, start - 1, -1)])
self.cluster.compact()
for n in range(PARTITIONS):
assert_all(session, "SELECT * FROM t WHERE k = {}".format(n),
[[n, v, v] for v in range(ROWS)])
assert_all(
session,
"SELECT * FROM t WHERE k = {} ORDER BY t DESC".format(n),
[[n, v, v] for v in range(ROWS - 1, -1, -1)])
# Querying a "large" slice
start = ROWS / 10
end = ROWS - 1 - (ROWS / 10)
assert_all(
session,
"SELECT * FROM t WHERE k = {n} AND t >= {start} AND t < {end}".
format(n=n, start=start,
end=end), [[n, v, v] for v in range(start, end)])
assert_all(
session,
"SELECT * FROM t WHERE k = {n} AND t >= {start} AND t < {end} ORDER BY t DESC"
.format(n=n, start=start, end=end),
[[n, v, v] for v in range(end - 1, start - 1, -1)])
# Querying a "small" slice
start = ROWS / 2
end = ROWS / 2 + 5
assert_all(
session,
"SELECT * FROM t WHERE k = {n} AND t >= {start} AND t < {end}".
format(n=n, start=start,
end=end), [[n, v, v] for v in range(start, end)])
assert_all(
session,
"SELECT * FROM t WHERE k = {n} AND t >= {start} AND t < {end} ORDER BY t DESC"
.format(n=n, start=start, end=end),
[[n, v, v] for v in range(end - 1, start - 1, -1)])
def replace_with_reset_resume_state_test(self):
"""Test replace with resetting bootstrap progress"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
node1.stress([
'write', 'n=100K', 'no-warmup', '-schema', 'replication(factor=3)'
])
session = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table,
consistency_level=ConsistencyLevel.THREE)
initial_data = rows_to_list(session.execute(query))
node3.stop(gently=False)
# kill node1 in the middle of streaming to let it fail
t = InterruptBootstrap(node1)
t.start()
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4',
cluster=cluster,
auto_bootstrap=True,
thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000),
jmx_port='7400',
remote_debug_port='0',
initial_token=None,
binary_interface=('127.0.0.4', 9042))
# keep timeout low so that test won't hang
node4.set_configuration_options(
values={'streaming_socket_timeout_in_ms': 1000})
cluster.add(node4, False)
try:
node4.start(
jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"],
wait_other_notice=False)
except NodeError:
pass # node doesn't start as expected
t.join()
node1.start()
# restart node4 bootstrap with resetting bootstrap state
node4.stop()
mark = node4.mark_log()
node4.start(jvm_args=[
"-Dcassandra.replace_address_first_boot=127.0.0.3",
"-Dcassandra.reset_bootstrap_progress=true"
])
# check if we reset bootstrap state
node4.watch_log_for("Resetting bootstrap progress to start fresh",
from_mark=mark)
# wait for node3 ready to query
node4.watch_log_for("Listening for thrift clients...", from_mark=mark)
# check if 2nd bootstrap succeeded
assert_bootstrap_state(self, node4, 'COMPLETED')
# query should work again
debug("Stopping old nodes")
node1.stop(gently=False, wait_other_notice=True)
node2.stop(gently=False, wait_other_notice=True)
debug("Verifying data on new node.")
session = self.patient_exclusive_cql_connection(node4)
assert_all(session,
'SELECT * from {} LIMIT 1'.format(stress_table),
expected=initial_data,
cl=ConsistencyLevel.ONE)
def _replace_node_test(self, gently):
"""
Check that the replace address function correctly replaces a node that has failed in a cluster.
Create a cluster, cause a node to fail, and bring up a new node with the replace_address parameter.
Check that tokens are migrated and that data is replicated properly.
"""
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
if DISABLE_VNODES:
num_tokens = 1
else:
# a little hacky but grep_log returns the whole line...
num_tokens = int(node3.get_conf_option('num_tokens'))
debug("testing with num_tokens: {}".format(num_tokens))
debug("Inserting Data...")
node1.stress([
'write', 'n=10K', 'no-warmup', '-schema', 'replication(factor=3)'
])
session = self.patient_cql_connection(node1)
session.default_timeout = 45
stress_table = 'keyspace1.standard1'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table,
consistency_level=ConsistencyLevel.THREE)
initial_data = rows_to_list(session.execute(query))
# stop node, query should not work with consistency 3
debug("Stopping node 3.")
node3.stop(gently=gently, wait_other_notice=True)
debug("Testing node stoppage (query should fail).")
with self.assertRaises(NodeUnavailable):
try:
query = SimpleStatement(
'select * from %s LIMIT 1' % stress_table,
consistency_level=ConsistencyLevel.THREE)
session.execute(query)
except (Unavailable, ReadTimeout):
raise NodeUnavailable("Node could not be queried.")
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4',
cluster=cluster,
auto_bootstrap=True,
thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000),
jmx_port='7400',
remote_debug_port='0',
initial_token=None,
binary_interface=('127.0.0.4', 9042))
cluster.add(node4, False)
node4.start(replace_address='127.0.0.3', wait_for_binary_proto=True)
debug("Verifying tokens migrated sucessfully")
moved_tokens = node4.grep_log(
"Token .* changing ownership from /127.0.0.3 to /127.0.0.4")
debug("number of moved tokens: {}".format(len(moved_tokens)))
self.assertEqual(len(moved_tokens), num_tokens)
# check that restarting node 3 doesn't work
debug("Try to restart node 3 (should fail)")
node3.start(wait_other_notice=False)
collision_log = node1.grep_log(
"between /127.0.0.3 and /127.0.0.4; /127.0.0.4 is the new owner")
debug(collision_log)
self.assertEqual(len(collision_log), 1)
node3.stop(gently=False)
# query should work again
debug("Stopping old nodes")
node1.stop(gently=False, wait_other_notice=True)
node2.stop(gently=False, wait_other_notice=True)
debug("Verifying data on new node.")
session = self.patient_exclusive_cql_connection(node4)
assert_all(session,
'SELECT * from {} LIMIT 1'.format(stress_table),
expected=initial_data,
cl=ConsistencyLevel.ONE)
