def test_storage_service_keyspace_scrub(cql, this_dc, rest_api):
with new_test_keyspace(
cql,
f"WITH REPLICATION = {{ 'class' : 'NetworkTopologyStrategy', '{this_dc}' : 1 }}"
) as keyspace:
with new_test_table(cql, keyspace, "a int, PRIMARY KEY (a)") as t0:
with new_test_table(cql, keyspace, "a int, PRIMARY KEY (a)") as t1:
test_tables = [t0.split('.')[1], t1.split('.')[1]]
resp = rest_api.send(
"GET", f"storage_service/keyspace_scrub/{keyspace}")
resp.raise_for_status()
resp = rest_api.send(
"GET", f"storage_service/keyspace_scrub/{keyspace}",
{"cf": f"{test_tables[1]}"})
resp.raise_for_status()
resp = rest_api.send(
"GET", f"storage_service/keyspace_scrub/{keyspace}",
{"cf": f"{test_tables[0]},{test_tables[1]}"})
resp.raise_for_status()
# non-existing table
resp = rest_api.send(
"POST", f"storage_service/keyspace_scrub/{keyspace}",
{"cf": f"{test_tables[0]},XXX"})
assert resp.status_code == requests.codes.not_found
def test_compaction_manager_stop_keyspace_compaction_tables(cql, this_dc, rest_api):
keyspace = new_keyspace(cql, this_dc)
with new_test_table(cql, keyspace, "a int, PRIMARY KEY (a)") as t0:
test_tables = [t0 if not '.' in t0 else t0.split('.')[1]]
resp = rest_api.send("POST", f"compaction_manager/stop_keyspace_compaction/{keyspace}", { "tables": f"{test_tables[0]}", "type": "COMPACTION" })
resp.raise_for_status()
# non-existing table
resp = rest_api.send("POST", f"compaction_manager/stop_keyspace_compaction/{keyspace}", { "tables": "XXX", "type": "COMPACTION" })
try:
resp.raise_for_status()
pytest.fail("Failed to raise exception")
except requests.HTTPError as e:
expected_status_code = requests.codes.bad_request
assert resp.status_code == expected_status_code, e
# multiple tables
with new_test_table(cql, keyspace, "b int, PRIMARY KEY (b)") as t1:
test_tables += [t1 if not '.' in t1 else t1.split('.')[1]]
resp = rest_api.send("POST", f"compaction_manager/stop_keyspace_compaction/{keyspace}", { "tables": f"{test_tables[0]},{test_tables[1]}", "type": "COMPACTION" })
resp.raise_for_status()
# mixed existing and non-existing tables
resp = rest_api.send("POST", f"compaction_manager/stop_keyspace_compaction/{keyspace}", { "tables": f"{test_tables[1]},XXX", "type": "COMPACTION" })
try:
resp.raise_for_status()
pytest.fail("Failed to raise exception")
except requests.HTTPError as e:
expected_status_code = requests.codes.bad_request
assert resp.status_code == expected_status_code, e
cql.execute(f"DROP KEYSPACE {keyspace}")
def test_limit_attribute_length_key_bad(dynamodb):
too_long_name = random_string(256)
with pytest.raises(ClientError, match='ValidationException.*length'):
with new_test_table(dynamodb,
KeySchema=[{
'AttributeName': too_long_name,
'KeyType': 'HASH'
}],
AttributeDefinitions=[{
'AttributeName': too_long_name,
'AttributeType': 'S'
}]) as table:
pass
with pytest.raises(ClientError, match='ValidationException.*length'):
with new_test_table(dynamodb,
KeySchema=[
{
'AttributeName': 'x',
'KeyType': 'HASH',
'AttributeName': too_long_name,
'KeyType': 'RANGE'
},
],
AttributeDefinitions=[{
'AttributeName': too_long_name,
'AttributeType': 'S'
}, {
'AttributeName': 'x',
'AttributeType': 'S'
}]) as table:
pass
def do_test_filter_UDT_restriction(cql, test_keyspace, frozen):
# Single-integer UDT, should be comparable like a normal integer:
with new_type(cql, test_keyspace, "(a int)") as typ:
ftyp = f"frozen<{typ}>" if frozen else typ
schema = f"pk int, ck int, x {ftyp}, PRIMARY KEY (pk, ck)"
with new_test_table(cql, test_keyspace, schema) as table:
stmt = cql.prepare(f"INSERT INTO {table} (pk, ck, x) VALUES (?, ?, ?)")
for i in range(5):
cql.execute(stmt, [1, i, user_type("a", i*2)])
stmt = cql.prepare(f"SELECT ck FROM {table} WHERE x = ? ALLOW FILTERING")
assert [(2,)] == list(cql.execute(stmt, [user_type("a", 4)]))
assert [] == list(cql.execute(stmt, [user_type("a", 3)]))
stmt = cql.prepare(f"SELECT ck FROM {table} WHERE x < ? ALLOW FILTERING")
assert [(0,), (1,)] == list(cql.execute(stmt, [user_type("a", 4)]))
assert [] == list(cql.execute(stmt, [user_type("a", -1)]))
# UDT with two integers. EQ operator is obvious, LT is lexicographical
with new_type(cql, test_keyspace, "(a int, b int)") as typ:
ftyp = f"frozen<{typ}>" if frozen else typ
schema = f"pk int, ck int, x {ftyp}, PRIMARY KEY (pk, ck)"
with new_test_table(cql, test_keyspace, schema) as table:
stmt = cql.prepare(f"INSERT INTO {table} (pk, ck, x) VALUES (?, ?, ?)")
for i in range(5):
cql.execute(stmt, [1, i, user_type("a", i*2, "b", i*3)])
stmt = cql.prepare(f"SELECT ck FROM {table} WHERE x = ? ALLOW FILTERING")
assert [(2,)] == list(cql.execute(stmt, [user_type("a", 4, "b", 6)]))
assert [] == list(cql.execute(stmt, [user_type("a", 4, "b", 5)]))
stmt = cql.prepare(f"SELECT ck FROM {table} WHERE x < ? ALLOW FILTERING")
assert [(0,), (1,)] == list(cql.execute(stmt, [user_type("a", 4, "b", 6)]))
assert [(0,), (1,), (2,)] == list(cql.execute(stmt, [user_type("a", 4, "b", 7)]))
assert [] == list(cql.execute(stmt, [user_type("a", -1, "b", 7)]))
def test_cdc_log_entries_use_cdc_streams(scylla_only, cql, test_keyspace):
'''Test that the stream IDs chosen for CDC log entries come from the CDC generation
whose streams are listed in the streams description table. Since this test is executed
on a single-node cluster, there is only one generation.'''
schema = "pk int primary key"
extra = " with cdc = {'enabled': true}"
with new_test_table(cql, test_keyspace, schema, extra) as table:
stmt = cql.prepare(
f"insert into {table} (pk) values (?) using timeout 5m")
for i in range(100):
cql.execute(stmt, [i])
log_stream_ids = set(r[0] for r in cql.execute(
f'select "cdc$stream_id" from {table}_scylla_cdc_log'))
# There should be exactly one generation, so we just select the streams
streams_desc = cql.execute(
SimpleStatement(
'select streams from system_distributed.cdc_streams_descriptions_v2',
consistency_level=ConsistencyLevel.ONE))
stream_ids = set()
for entry in streams_desc:
stream_ids.update(entry.streams)
assert (log_stream_ids.issubset(stream_ids))
def test_storage_service_auto_compaction_keyspace(cql, this_dc, rest_api):
keyspace = new_keyspace(cql, this_dc)
# test empty keyspace
resp = rest_api.send("DELETE",
f"storage_service/auto_compaction/{keyspace}")
resp.raise_for_status()
resp = rest_api.send("POST", f"storage_service/auto_compaction/{keyspace}")
resp.raise_for_status()
# test non-empty keyspace
with new_test_table(cql, keyspace, "a int, PRIMARY KEY (a)") as t:
resp = rest_api.send("DELETE",
f"storage_service/auto_compaction/{keyspace}")
resp.raise_for_status()
resp = rest_api.send("POST",
f"storage_service/auto_compaction/{keyspace}")
resp.raise_for_status()
# non-existing keyspace
resp = rest_api.send("POST", f"storage_service/auto_compaction/XXX")
assert resp.status_code == requests.codes.bad_request
cql.execute(f"DROP KEYSPACE {keyspace}")
def test_limit_attribute_length_key_good(dynamodb):
long_name1 = random_string(255)
long_name2 = random_string(255)
with new_test_table(dynamodb,
KeySchema=[{
'AttributeName': long_name1,
'KeyType': 'HASH'
}, {
'AttributeName': long_name2,
'KeyType': 'RANGE'
}],
AttributeDefinitions=[{
'AttributeName': long_name1,
'AttributeType': 'S'
}, {
'AttributeName': long_name2,
'AttributeType': 'S'
}]) as table:
table.put_item(Item={long_name1: 'hi', long_name2: 'ho', 'another': 2})
assert table.get_item(Key={
long_name1: 'hi',
long_name2: 'ho'
},
ConsistentRead=True)['Item'] == {
long_name1: 'hi',
long_name2: 'ho',
'another': 2
}
def test_storage_service_snapshot_mv_si(cql, this_dc, rest_api):
resp = rest_api.send("GET", "storage_service/snapshots")
resp.raise_for_status()
with new_test_keyspace(
cql,
f"WITH REPLICATION = {{ 'class' : 'NetworkTopologyStrategy', '{this_dc}' : 1 }}"
) as keyspace:
schema = 'p int, v text, primary key (p)'
with new_test_table(cql, keyspace, schema) as table:
with new_materialized_view(
cql, table, '*', 'v, p',
'v is not null and p is not null') as mv:
try:
with new_test_snapshot(rest_api, keyspace,
mv.split('.')[1]) as snap:
pytest.fail(
f"Snapshot of materialized view {mv} should have failed"
)
except requests.HTTPError:
pass
with new_secondary_index(cql, table, 'v') as si:
try:
with new_test_snapshot(rest_api, keyspace,
si.split('.')[1]) as snap:
pytest.fail(
f"Snapshot of secondary index {si} should have failed"
)
except requests.HTTPError:
pass
def test_index_weird_chars_in_col_name(cql, test_keyspace):
with tempfile.TemporaryDirectory() as tmpdir:
# When issue #3403 exists, column name ../../...../tmpdir/x_yz will
# cause Scylla to create the new index in tmpdir!
# Of course a more sinister attacker can cause Scylla to create
# directories anywhere in the file system - or to crash if the
# directory creation fails - e.g., if magic_path ends in
# /dev/null/hello, and /dev/null is not a directory
magic_path = '/..' * 20 + tmpdir + '/x_yz'
schema = f'pk int PRIMARY KEY, "{magic_path}" int'
with new_test_table(cql, test_keyspace, schema) as table:
cql.execute(f'CREATE INDEX ON {table}("{magic_path}")')
# Creating the index should not have miraculously created
# something in tmpdir! If it has, we have issue #3403.
assert os.listdir(tmpdir) == []
# Check that the expected index name was chosen - based on
# only the alphanumeric/underscore characters of the column name.
ks, cf = table.split('.')
index_name = list(
cql.execute(
f"SELECT index_name FROM system_schema.indexes WHERE keyspace_name = '{ks}' AND table_name = '{cf}'"
))[0].index_name
iswordchar = lambda x: str.isalnum(x) or x == '_'
cleaned_up_column_name = ''.join(filter(iswordchar, magic_path))
assert index_name == cf + '_' + cleaned_up_column_name + '_idx'
def test_filtering_null_map_with_subscript(cql, test_keyspace):
schema = 'p text primary key, m map<int, int>'
with new_test_table(cql, test_keyspace, schema) as table:
cql.execute(f"INSERT INTO {table} (p) VALUES ('dog')")
assert list(
cql.execute(f"SELECT p FROM {table} WHERE m[2] = 3 ALLOW FILTERING"
)) == []
def test_filtering_contiguous_nonmatching_partition_range(cql, test_keyspace):
# The bug depends on the amount of data being scanned passing some
# page size limit, so it doesn't matter if the reproducer has a lot of
# small rows or fewer long rows - and inserting fewer long rows is
# significantly faster.
count = 100
long = 'x' * 60000
with new_test_table(cql, test_keyspace,
"p int, c text, v int, PRIMARY KEY (p, c)") as table:
stmt = cql.prepare(
f"INSERT INTO {table} (p, c, v) VALUES (?, '{long}', ?)")
for i in range(count):
cql.execute(stmt, [i, i])
# We want the filter to match only at the end the scan - but we don't
# know the partition order (we don't want the test to depend on the
# partitioner). So we first figure out a partition near the end (at
# some high token), and use that in the filter.
p, v = list(
cql.execute(
f"SELECT p, v FROM {table} WHERE TOKEN(p) > 8000000000000000000 LIMIT 1"
))[0]
assert list(
cql.execute(
f"SELECT p FROM {table} WHERE v={v} ALLOW FILTERING")) == [
(p, )
]
def test_validation_blob_as_int_len(cql, test_keyspace):
types = [
('i', 'int', 4),
('b', 'bigint', 8),
('s', 'smallint', 2),
('t', 'tinyint', 1),
]
types_def = ','.join([f'{x[0]} {x[1]}' for x in types])
with new_test_table(cql, test_keyspace,
f'k int primary key, {types_def}') as table:
k = unique_key_int()
for var, typ, length in types:
# Check that a blob with exactly length bytes is fine, one with one
# less or one more is rejected as an invalid request:
cql.execute(
f"INSERT INTO {table} (k, {var}) VALUES ({k}, blobAs{typ}(0x{'00'*length}))"
)
assert 0 == getattr(
cql.execute(f"SELECT {var} FROM {table} WHERE k = {k}").one(),
var)
with pytest.raises(InvalidRequest, match='is not a valid binary'):
cql.execute(
f"INSERT INTO {table} (k, {var}) VALUES ({k}, blobAs{typ}(0x{'00'*(length+1)}))"
)
if length - 1 != 0:
with pytest.raises(InvalidRequest,
match='is not a valid binary'):
cql.execute(
f"INSERT INTO {table} (k, {var}) VALUES ({k}, blobAs{typ}(0x{'00'*(length-1)}))"
)
def test_empty_string_for_nonstring_partition_key2(cql, test_keyspace):
schema = 'p inet primary key, v int'
with new_test_table(cql, test_keyspace, schema) as table:
# Cassandra returns a "Key may not be empty" but even better would
# be to report earlier that an empty string cannot be converted to
# a number.
with pytest.raises(InvalidRequest):
cql.execute(f"INSERT INTO {table} (p,v) VALUES ('', 3)")
def test_lsi_query_select(dynamodb):
with new_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' }, { 'AttributeName': 'c', 'KeyType': 'RANGE' } ],
AttributeDefinitions=[
{ 'AttributeName': 'p', 'AttributeType': 'S' },
{ 'AttributeName': 'c', 'AttributeType': 'S' },
{ 'AttributeName': 'b', 'AttributeType': 'S' },
],
LocalSecondaryIndexes=[
{ 'IndexName': 'hello',
'KeySchema': [
{ 'AttributeName': 'p', 'KeyType': 'HASH' },
{ 'AttributeName': 'b', 'KeyType': 'RANGE' }
],
'Projection': { 'ProjectionType': 'INCLUDE',
'NonKeyAttributes': ['a'] }
}
]) as table:
items = [{'p': random_string(), 'c': random_string(), 'b': random_string(), 'a': random_string(), 'x': random_string()} for i in range(10)]
with table.batch_writer() as batch:
for item in items:
batch.put_item(item)
p = items[0]['p']
b = items[0]['b']
# Although in LSI all attributes are available (as we'll check
# below) the default Select is ALL_PROJECTED_ATTRIBUTES, and
# returns just the projected attributes (in this case all key
# attributes in either base or LSI, and 'a' - but not 'x'):
expected_items = [{'p': z['p'], 'c': z['c'], 'b': z['b'], 'a': z['a']} for z in items if z['b'] == b]
assert_index_query(table, 'hello', expected_items,
KeyConditions={'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}})
assert_index_query(table, 'hello', expected_items,
Select='ALL_PROJECTED_ATTRIBUTES',
KeyConditions={'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}})
# Unlike in GSI, in LSI Select=ALL_ATTRIBUTES *is* allowed even
# when only a subset of the attributes being projected:
expected_items = [z for z in items if z['b'] == b]
assert_index_query(table, 'hello', expected_items,
Select='ALL_ATTRIBUTES',
KeyConditions={'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}})
# Also in LSI, SPECIFIC_ATTRIBUTES (with AttributesToGet /
# ProjectionExpression) is allowed for any attribute, projected
# or not projected. Let's try 'a' (projected) and 'x' (not projected):
expected_items = [{'a': z['a'], 'x': z['x']} for z in items if z['b'] == b]
assert_index_query(table, 'hello', expected_items,
Select='SPECIFIC_ATTRIBUTES',
AttributesToGet=['a', 'x'],
KeyConditions={'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}})
# Select=COUNT is also allowed, and as expected returns no content.
assert not 'Items' in table.query(ConsistentRead=False,
IndexName='hello',
Select='COUNT',
KeyConditions={'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}})
def test_multi_column_with_regular_index(cql, test_keyspace):
"""Reproduces #9085."""
with new_test_table(cql, test_keyspace, 'p int, c1 int, c2 int, r int, primary key(p,c1,c2)') as tbl:
cql.execute(f'CREATE INDEX ON {tbl}(r)')
cql.execute(f'INSERT INTO {tbl}(p, c1, c2, r) VALUES (1, 1, 1, 0)')
cql.execute(f'INSERT INTO {tbl}(p, c1, c2, r) VALUES (1, 1, 2, 1)')
cql.execute(f'INSERT INTO {tbl}(p, c1, c2, r) VALUES (1, 2, 1, 0)')
assert_rows(cql.execute(f'SELECT c1 FROM {tbl} WHERE (c1,c2)<(2,0) AND r=0 ALLOW FILTERING'), [1])
assert_rows(cql.execute(f'SELECT c1 FROM {tbl} WHERE p=1 AND (c1,c2)<(2,0) AND r=0 ALLOW FILTERING'), [1])
def test_filter_like_on_desc_column(cql, test_keyspace, cassandra_bug):
with new_test_table(cql,
test_keyspace,
"a int, b text, primary key(a, b)",
extra="with clustering order by (b desc)") as table:
cql.execute(f"INSERT INTO {table} (a, b) VALUES (1, 'one')")
res = cql.execute(
f"SELECT b FROM {table} WHERE b LIKE '%%%' ALLOW FILTERING")
assert res.one().b == "one"
