def test_storage_service_keyspace_scrub(cql, this_dc, rest_api): with new_test_keyspace( cql, f"WITH REPLICATION = {{ 'class' : 'NetworkTopologyStrategy', '{this_dc}' : 1 }}" ) as keyspace: with new_test_table(cql, keyspace, "a int, PRIMARY KEY (a)") as t0: with new_test_table(cql, keyspace, "a int, PRIMARY KEY (a)") as t1: test_tables = [t0.split('.')[1], t1.split('.')[1]] resp = rest_api.send( "GET", f"storage_service/keyspace_scrub/{keyspace}") resp.raise_for_status() resp = rest_api.send( "GET", f"storage_service/keyspace_scrub/{keyspace}", {"cf": f"{test_tables[1]}"}) resp.raise_for_status() resp = rest_api.send( "GET", f"storage_service/keyspace_scrub/{keyspace}", {"cf": f"{test_tables[0]},{test_tables[1]}"}) resp.raise_for_status() # non-existing table resp = rest_api.send( "POST", f"storage_service/keyspace_scrub/{keyspace}", {"cf": f"{test_tables[0]},XXX"}) assert resp.status_code == requests.codes.not_found
def test_compaction_manager_stop_keyspace_compaction_tables(cql, this_dc, rest_api): keyspace = new_keyspace(cql, this_dc) with new_test_table(cql, keyspace, "a int, PRIMARY KEY (a)") as t0: test_tables = [t0 if not '.' in t0 else t0.split('.')[1]] resp = rest_api.send("POST", f"compaction_manager/stop_keyspace_compaction/{keyspace}", { "tables": f"{test_tables[0]}", "type": "COMPACTION" }) resp.raise_for_status() # non-existing table resp = rest_api.send("POST", f"compaction_manager/stop_keyspace_compaction/{keyspace}", { "tables": "XXX", "type": "COMPACTION" }) try: resp.raise_for_status() pytest.fail("Failed to raise exception") except requests.HTTPError as e: expected_status_code = requests.codes.bad_request assert resp.status_code == expected_status_code, e # multiple tables with new_test_table(cql, keyspace, "b int, PRIMARY KEY (b)") as t1: test_tables += [t1 if not '.' in t1 else t1.split('.')[1]] resp = rest_api.send("POST", f"compaction_manager/stop_keyspace_compaction/{keyspace}", { "tables": f"{test_tables[0]},{test_tables[1]}", "type": "COMPACTION" }) resp.raise_for_status() # mixed existing and non-existing tables resp = rest_api.send("POST", f"compaction_manager/stop_keyspace_compaction/{keyspace}", { "tables": f"{test_tables[1]},XXX", "type": "COMPACTION" }) try: resp.raise_for_status() pytest.fail("Failed to raise exception") except requests.HTTPError as e: expected_status_code = requests.codes.bad_request assert resp.status_code == expected_status_code, e cql.execute(f"DROP KEYSPACE {keyspace}")
def test_limit_attribute_length_key_bad(dynamodb): too_long_name = random_string(256) with pytest.raises(ClientError, match='ValidationException.*length'): with new_test_table(dynamodb, KeySchema=[{ 'AttributeName': too_long_name, 'KeyType': 'HASH' }], AttributeDefinitions=[{ 'AttributeName': too_long_name, 'AttributeType': 'S' }]) as table: pass with pytest.raises(ClientError, match='ValidationException.*length'): with new_test_table(dynamodb, KeySchema=[ { 'AttributeName': 'x', 'KeyType': 'HASH', 'AttributeName': too_long_name, 'KeyType': 'RANGE' }, ], AttributeDefinitions=[{ 'AttributeName': too_long_name, 'AttributeType': 'S' }, { 'AttributeName': 'x', 'AttributeType': 'S' }]) as table: pass
def do_test_filter_UDT_restriction(cql, test_keyspace, frozen): # Single-integer UDT, should be comparable like a normal integer: with new_type(cql, test_keyspace, "(a int)") as typ: ftyp = f"frozen<{typ}>" if frozen else typ schema = f"pk int, ck int, x {ftyp}, PRIMARY KEY (pk, ck)" with new_test_table(cql, test_keyspace, schema) as table: stmt = cql.prepare(f"INSERT INTO {table} (pk, ck, x) VALUES (?, ?, ?)") for i in range(5): cql.execute(stmt, [1, i, user_type("a", i*2)]) stmt = cql.prepare(f"SELECT ck FROM {table} WHERE x = ? ALLOW FILTERING") assert [(2,)] == list(cql.execute(stmt, [user_type("a", 4)])) assert [] == list(cql.execute(stmt, [user_type("a", 3)])) stmt = cql.prepare(f"SELECT ck FROM {table} WHERE x < ? ALLOW FILTERING") assert [(0,), (1,)] == list(cql.execute(stmt, [user_type("a", 4)])) assert [] == list(cql.execute(stmt, [user_type("a", -1)])) # UDT with two integers. EQ operator is obvious, LT is lexicographical with new_type(cql, test_keyspace, "(a int, b int)") as typ: ftyp = f"frozen<{typ}>" if frozen else typ schema = f"pk int, ck int, x {ftyp}, PRIMARY KEY (pk, ck)" with new_test_table(cql, test_keyspace, schema) as table: stmt = cql.prepare(f"INSERT INTO {table} (pk, ck, x) VALUES (?, ?, ?)") for i in range(5): cql.execute(stmt, [1, i, user_type("a", i*2, "b", i*3)]) stmt = cql.prepare(f"SELECT ck FROM {table} WHERE x = ? ALLOW FILTERING") assert [(2,)] == list(cql.execute(stmt, [user_type("a", 4, "b", 6)])) assert [] == list(cql.execute(stmt, [user_type("a", 4, "b", 5)])) stmt = cql.prepare(f"SELECT ck FROM {table} WHERE x < ? ALLOW FILTERING") assert [(0,), (1,)] == list(cql.execute(stmt, [user_type("a", 4, "b", 6)])) assert [(0,), (1,), (2,)] == list(cql.execute(stmt, [user_type("a", 4, "b", 7)])) assert [] == list(cql.execute(stmt, [user_type("a", -1, "b", 7)]))
def test_cdc_log_entries_use_cdc_streams(scylla_only, cql, test_keyspace): '''Test that the stream IDs chosen for CDC log entries come from the CDC generation whose streams are listed in the streams description table. Since this test is executed on a single-node cluster, there is only one generation.''' schema = "pk int primary key" extra = " with cdc = {'enabled': true}" with new_test_table(cql, test_keyspace, schema, extra) as table: stmt = cql.prepare( f"insert into {table} (pk) values (?) using timeout 5m") for i in range(100): cql.execute(stmt, [i]) log_stream_ids = set(r[0] for r in cql.execute( f'select "cdc$stream_id" from {table}_scylla_cdc_log')) # There should be exactly one generation, so we just select the streams streams_desc = cql.execute( SimpleStatement( 'select streams from system_distributed.cdc_streams_descriptions_v2', consistency_level=ConsistencyLevel.ONE)) stream_ids = set() for entry in streams_desc: stream_ids.update(entry.streams) assert (log_stream_ids.issubset(stream_ids))
def test_storage_service_auto_compaction_keyspace(cql, this_dc, rest_api): keyspace = new_keyspace(cql, this_dc) # test empty keyspace resp = rest_api.send("DELETE", f"storage_service/auto_compaction/{keyspace}") resp.raise_for_status() resp = rest_api.send("POST", f"storage_service/auto_compaction/{keyspace}") resp.raise_for_status() # test non-empty keyspace with new_test_table(cql, keyspace, "a int, PRIMARY KEY (a)") as t: resp = rest_api.send("DELETE", f"storage_service/auto_compaction/{keyspace}") resp.raise_for_status() resp = rest_api.send("POST", f"storage_service/auto_compaction/{keyspace}") resp.raise_for_status() # non-existing keyspace resp = rest_api.send("POST", f"storage_service/auto_compaction/XXX") assert resp.status_code == requests.codes.bad_request cql.execute(f"DROP KEYSPACE {keyspace}")
def test_limit_attribute_length_key_good(dynamodb): long_name1 = random_string(255) long_name2 = random_string(255) with new_test_table(dynamodb, KeySchema=[{ 'AttributeName': long_name1, 'KeyType': 'HASH' }, { 'AttributeName': long_name2, 'KeyType': 'RANGE' }], AttributeDefinitions=[{ 'AttributeName': long_name1, 'AttributeType': 'S' }, { 'AttributeName': long_name2, 'AttributeType': 'S' }]) as table: table.put_item(Item={long_name1: 'hi', long_name2: 'ho', 'another': 2}) assert table.get_item(Key={ long_name1: 'hi', long_name2: 'ho' }, ConsistentRead=True)['Item'] == { long_name1: 'hi', long_name2: 'ho', 'another': 2 }
def test_storage_service_snapshot_mv_si(cql, this_dc, rest_api): resp = rest_api.send("GET", "storage_service/snapshots") resp.raise_for_status() with new_test_keyspace( cql, f"WITH REPLICATION = {{ 'class' : 'NetworkTopologyStrategy', '{this_dc}' : 1 }}" ) as keyspace: schema = 'p int, v text, primary key (p)' with new_test_table(cql, keyspace, schema) as table: with new_materialized_view( cql, table, '*', 'v, p', 'v is not null and p is not null') as mv: try: with new_test_snapshot(rest_api, keyspace, mv.split('.')[1]) as snap: pytest.fail( f"Snapshot of materialized view {mv} should have failed" ) except requests.HTTPError: pass with new_secondary_index(cql, table, 'v') as si: try: with new_test_snapshot(rest_api, keyspace, si.split('.')[1]) as snap: pytest.fail( f"Snapshot of secondary index {si} should have failed" ) except requests.HTTPError: pass
def test_index_weird_chars_in_col_name(cql, test_keyspace): with tempfile.TemporaryDirectory() as tmpdir: # When issue #3403 exists, column name ../../...../tmpdir/x_yz will # cause Scylla to create the new index in tmpdir! # Of course a more sinister attacker can cause Scylla to create # directories anywhere in the file system - or to crash if the # directory creation fails - e.g., if magic_path ends in # /dev/null/hello, and /dev/null is not a directory magic_path = '/..' * 20 + tmpdir + '/x_yz' schema = f'pk int PRIMARY KEY, "{magic_path}" int' with new_test_table(cql, test_keyspace, schema) as table: cql.execute(f'CREATE INDEX ON {table}("{magic_path}")') # Creating the index should not have miraculously created # something in tmpdir! If it has, we have issue #3403. assert os.listdir(tmpdir) == [] # Check that the expected index name was chosen - based on # only the alphanumeric/underscore characters of the column name. ks, cf = table.split('.') index_name = list( cql.execute( f"SELECT index_name FROM system_schema.indexes WHERE keyspace_name = '{ks}' AND table_name = '{cf}'" ))[0].index_name iswordchar = lambda x: str.isalnum(x) or x == '_' cleaned_up_column_name = ''.join(filter(iswordchar, magic_path)) assert index_name == cf + '_' + cleaned_up_column_name + '_idx'
def test_filtering_null_map_with_subscript(cql, test_keyspace): schema = 'p text primary key, m map<int, int>' with new_test_table(cql, test_keyspace, schema) as table: cql.execute(f"INSERT INTO {table} (p) VALUES ('dog')") assert list( cql.execute(f"SELECT p FROM {table} WHERE m[2] = 3 ALLOW FILTERING" )) == []
def test_filtering_contiguous_nonmatching_partition_range(cql, test_keyspace): # The bug depends on the amount of data being scanned passing some # page size limit, so it doesn't matter if the reproducer has a lot of # small rows or fewer long rows - and inserting fewer long rows is # significantly faster. count = 100 long = 'x' * 60000 with new_test_table(cql, test_keyspace, "p int, c text, v int, PRIMARY KEY (p, c)") as table: stmt = cql.prepare( f"INSERT INTO {table} (p, c, v) VALUES (?, '{long}', ?)") for i in range(count): cql.execute(stmt, [i, i]) # We want the filter to match only at the end the scan - but we don't # know the partition order (we don't want the test to depend on the # partitioner). So we first figure out a partition near the end (at # some high token), and use that in the filter. p, v = list( cql.execute( f"SELECT p, v FROM {table} WHERE TOKEN(p) > 8000000000000000000 LIMIT 1" ))[0] assert list( cql.execute( f"SELECT p FROM {table} WHERE v={v} ALLOW FILTERING")) == [ (p, ) ]
def test_validation_blob_as_int_len(cql, test_keyspace): types = [ ('i', 'int', 4), ('b', 'bigint', 8), ('s', 'smallint', 2), ('t', 'tinyint', 1), ] types_def = ','.join([f'{x[0]} {x[1]}' for x in types]) with new_test_table(cql, test_keyspace, f'k int primary key, {types_def}') as table: k = unique_key_int() for var, typ, length in types: # Check that a blob with exactly length bytes is fine, one with one # less or one more is rejected as an invalid request: cql.execute( f"INSERT INTO {table} (k, {var}) VALUES ({k}, blobAs{typ}(0x{'00'*length}))" ) assert 0 == getattr( cql.execute(f"SELECT {var} FROM {table} WHERE k = {k}").one(), var) with pytest.raises(InvalidRequest, match='is not a valid binary'): cql.execute( f"INSERT INTO {table} (k, {var}) VALUES ({k}, blobAs{typ}(0x{'00'*(length+1)}))" ) if length - 1 != 0: with pytest.raises(InvalidRequest, match='is not a valid binary'): cql.execute( f"INSERT INTO {table} (k, {var}) VALUES ({k}, blobAs{typ}(0x{'00'*(length-1)}))" )
def test_empty_string_for_nonstring_partition_key2(cql, test_keyspace): schema = 'p inet primary key, v int' with new_test_table(cql, test_keyspace, schema) as table: # Cassandra returns a "Key may not be empty" but even better would # be to report earlier that an empty string cannot be converted to # a number. with pytest.raises(InvalidRequest): cql.execute(f"INSERT INTO {table} (p,v) VALUES ('', 3)")
def test_lsi_query_select(dynamodb): with new_test_table(dynamodb, KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' }, { 'AttributeName': 'c', 'KeyType': 'RANGE' } ], AttributeDefinitions=[ { 'AttributeName': 'p', 'AttributeType': 'S' }, { 'AttributeName': 'c', 'AttributeType': 'S' }, { 'AttributeName': 'b', 'AttributeType': 'S' }, ], LocalSecondaryIndexes=[ { 'IndexName': 'hello', 'KeySchema': [ { 'AttributeName': 'p', 'KeyType': 'HASH' }, { 'AttributeName': 'b', 'KeyType': 'RANGE' } ], 'Projection': { 'ProjectionType': 'INCLUDE', 'NonKeyAttributes': ['a'] } } ]) as table: items = [{'p': random_string(), 'c': random_string(), 'b': random_string(), 'a': random_string(), 'x': random_string()} for i in range(10)] with table.batch_writer() as batch: for item in items: batch.put_item(item) p = items[0]['p'] b = items[0]['b'] # Although in LSI all attributes are available (as we'll check # below) the default Select is ALL_PROJECTED_ATTRIBUTES, and # returns just the projected attributes (in this case all key # attributes in either base or LSI, and 'a' - but not 'x'): expected_items = [{'p': z['p'], 'c': z['c'], 'b': z['b'], 'a': z['a']} for z in items if z['b'] == b] assert_index_query(table, 'hello', expected_items, KeyConditions={'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'}, 'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}}) assert_index_query(table, 'hello', expected_items, Select='ALL_PROJECTED_ATTRIBUTES', KeyConditions={'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'}, 'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}}) # Unlike in GSI, in LSI Select=ALL_ATTRIBUTES *is* allowed even # when only a subset of the attributes being projected: expected_items = [z for z in items if z['b'] == b] assert_index_query(table, 'hello', expected_items, Select='ALL_ATTRIBUTES', KeyConditions={'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'}, 'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}}) # Also in LSI, SPECIFIC_ATTRIBUTES (with AttributesToGet / # ProjectionExpression) is allowed for any attribute, projected # or not projected. Let's try 'a' (projected) and 'x' (not projected): expected_items = [{'a': z['a'], 'x': z['x']} for z in items if z['b'] == b] assert_index_query(table, 'hello', expected_items, Select='SPECIFIC_ATTRIBUTES', AttributesToGet=['a', 'x'], KeyConditions={'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'}, 'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}}) # Select=COUNT is also allowed, and as expected returns no content. assert not 'Items' in table.query(ConsistentRead=False, IndexName='hello', Select='COUNT', KeyConditions={'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'}, 'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}})
def test_multi_column_with_regular_index(cql, test_keyspace): """Reproduces #9085.""" with new_test_table(cql, test_keyspace, 'p int, c1 int, c2 int, r int, primary key(p,c1,c2)') as tbl: cql.execute(f'CREATE INDEX ON {tbl}(r)') cql.execute(f'INSERT INTO {tbl}(p, c1, c2, r) VALUES (1, 1, 1, 0)') cql.execute(f'INSERT INTO {tbl}(p, c1, c2, r) VALUES (1, 1, 2, 1)') cql.execute(f'INSERT INTO {tbl}(p, c1, c2, r) VALUES (1, 2, 1, 0)') assert_rows(cql.execute(f'SELECT c1 FROM {tbl} WHERE (c1,c2)<(2,0) AND r=0 ALLOW FILTERING'), [1]) assert_rows(cql.execute(f'SELECT c1 FROM {tbl} WHERE p=1 AND (c1,c2)<(2,0) AND r=0 ALLOW FILTERING'), [1])
def test_filter_like_on_desc_column(cql, test_keyspace, cassandra_bug): with new_test_table(cql, test_keyspace, "a int, b text, primary key(a, b)", extra="with clustering order by (b desc)") as table: cql.execute(f"INSERT INTO {table} (a, b) VALUES (1, 'one')") res = cql.execute( f"SELECT b FROM {table} WHERE b LIKE '%%%' ALLOW FILTERING") assert res.one().b == "one"