def test_longest_running_query(self): long_running_pod, long_running_svc, _, _ = alerts.random_pod_choice_for_callbacks( chi) # 600s trigger + 2*30s - double prometheus scraping interval clickhouse.query( chi["metadata"]["name"], "SELECT now(),sleepEachRow(1),number FROM system.numbers LIMIT 660", host=long_running_svc, timeout=670) with Then("check ClickHouseLongestRunningQuery firing"): fired = alerts.wait_alert_state("ClickHouseLongestRunningQuery", "firing", True, labels={"hostname": long_running_svc}, time_range='30s') assert fired, error( "can't get ClickHouseLongestRunningQuery alert in firing state") with Then("check ClickHouseLongestRunningQuery gone away"): resolved = alerts.wait_alert_state( "ClickHouseLongestRunningQuery", "firing", False, labels={"hostname": long_running_svc}) assert resolved, error( "can't check ClickHouseLongestRunningQuery alert is gone away")
def test_017(): kubectl.create_and_check(config="configs/test-017-multi-version.yaml", check={ "pod_count": 4, "do_not_delete": 1, }) chi = "test-017-multi-version" queries = [ "CREATE TABLE test_max (epoch Int32, offset SimpleAggregateFunction(max, Int64)) ENGINE = AggregatingMergeTree() ORDER BY epoch", "insert into test_max select 0, 3650487030+number from numbers(5) settings max_block_size=1", "insert into test_max select 0, 5898217176+number from numbers(5)", "insert into test_max select 0, 5898217176+number from numbers(10) settings max_block_size=1", "OPTIMIZE TABLE test_max FINAL" ] for q in queries: print(f"{q}") test_query = "select min(offset), max(offset) from test_max" print(f"{test_query}") for shard in range(4): host = f"chi-{chi}-default-{shard}-0" for q in queries: clickhouse.query(chi, host=host, sql=q) out = clickhouse.query(chi, host=host, sql=test_query) ver = clickhouse.query(chi, host=host, sql="select version()") print(f"version: {ver}, result: {out}") kubectl.delete_chi(chi)
def test_replicas_max_abosulute_delay(): stop_replica_pod, stop_replica_svc, insert_pod, insert_svc = random_pod_choice_for_callbacks() create_table_on_cluster('all-replicated', 'default.test_repl', '(event_time DateTime, test UInt64) ENGINE ReplicatedMergeTree(\'/clickhouse/tables/{installation}-{shard}/{uuid}/test_repl\', \'{replica}\') ORDER BY tuple()') prometheus_scrape_interval = 15 def restart_clickhouse_and_insert_to_replicated_table(): with When(f"stop replica fetches on {stop_replica_svc}"): sql = "SYSTEM STOP FETCHES default.test_repl" kubectl.launch( f"exec -n {kubectl.namespace} {stop_replica_pod} -c clickhouse -- clickhouse-client -q \"{sql}\"", ok_to_fail=True, ) sql = "INSERT INTO default.test_repl SELECT now(), number FROM numbers(100000)" kubectl.launch( f"exec -n {kubectl.namespace} {insert_pod} -c clickhouse -- clickhouse-client -q \"{sql}\"", ) with Then("check ClickHouseReplicasMaxAbsoluteDelay firing"): fired = wait_alert_state("ClickHouseReplicasMaxAbsoluteDelay", "firing", True, labels={"hostname": stop_replica_svc}, time_range='60s', sleep_time=prometheus_scrape_interval * 2, callback=restart_clickhouse_and_insert_to_replicated_table) assert fired, error("can't get ClickHouseReadonlyReplica alert in firing state") clickhouse.query( chi["metadata"]["name"], "SYSTEM START FETCHES; SYSTEM RESTART REPLICAS; SYSTEM SYNC REPLICA default.test_repl", host=stop_replica_svc, timeout=240 ) with Then("check ClickHouseReplicasMaxAbsoluteDelay gone away"): resolved = wait_alert_state("ClickHouseReplicasMaxAbsoluteDelay", "firing", False, labels={"hostname": stop_replica_svc}) assert resolved, error("can't check ClickHouseReplicasMaxAbsoluteDelay alert is gone away") drop_table_on_cluster('all-replicated', 'default.test_repl')
def create_distributed_table_on_cluster(cluster_name='all-sharded', distr_table='default.test_distr', local_table='default.test', fields_definition='(event_time DateTime, test UInt64)', local_engine='ENGINE MergeTree() ORDER BY tuple()', distr_engine='ENGINE Distributed("all-sharded",default, test, test)'): create_table_on_cluster(cluster_name, local_table, fields_definition + ' ' + local_engine) create_distr_sql = f'CREATE TABLE {distr_table} ON CLUSTER \\\"{cluster_name}\\\" {fields_definition} {distr_engine}' clickhouse.query(chi["metadata"]["name"], create_distr_sql, timeout=120)
def insert_many_parts_to_clickhouse(): stop_merges = "SYSTEM STOP MERGES default.test;" min_block = "SET max_block_size=1; SET max_insert_block_size=1; SET min_insert_block_size_rows=1;" with When(f"Insert to MergeTree table {parts_limits} parts"): r = parts_limits sql = stop_merges + min_block + f"INSERT INTO default.test(event_time, test) SELECT now(),number FROM system.numbers LIMIT {r};" clickhouse.query(chi_name, sql, host=selected_svc, ns=kubectl.namespace) # @TODO we need only one query after resolve https://github.com/ClickHouse/ClickHouse/issues/11384 and switch to 21.3+ sql = min_block + "INSERT INTO default.test(event_time, test) SELECT now(), number FROM system.numbers LIMIT 1;" clickhouse.query_with_error(chi_name, sql, host=selected_svc, ns=kubectl.namespace) with Then( f"wait prometheus_scrape_interval={prometheus_scrape_interval}*2 sec" ): time.sleep(prometheus_scrape_interval * 2) sql = min_block + "INSERT INTO default.test(event_time, test) SELECT now(), number FROM system.numbers LIMIT 1;" clickhouse.query_with_error(chi_name, sql, host=selected_svc, ns=kubectl.namespace)
def test_015(): kubectl.create_and_check(config="configs/test-015-host-network.yaml", check={ "pod_count": 2, "do_not_delete": 1, }) time.sleep(30) with Then("Query from one server to another one should work"): out = clickhouse.query( "test-015-host-network", host="chi-test-015-host-network-default-0-0", port="10000", sql= "SELECT * FROM remote('chi-test-015-host-network-default-0-1', system.one)" ) print("remote out=") print(out) with Then("Distributed query should work"): out = clickhouse.query( "test-015-host-network", host="chi-test-015-host-network-default-0-0", port="10000", sql= "SELECT count() FROM cluster('all-sharded', system.one) settings receive_timeout=10" ) print("cluster out=") print(out) assert out == "2" kubectl.delete_chi("test-015-host-network")
def prepare_table_for_backup(backup_pod, rows=1000): backup_name = f'test_backup_{time.strftime("%Y-%m-%d_%H%M%S")}' clickhouse.query( chi['metadata']['name'], "CREATE TABLE IF NOT EXISTS default.test_backup ON CLUSTER 'all-sharded' (i UInt64) ENGINE MergeTree() ORDER BY tuple();" f"INSERT INTO default.test_backup SELECT number FROM numbers({rows})", pod=backup_pod) return backup_name
def run_queries_with_priority(): sql = "" for i in range(50): sql += f"SET priority={i % 20};SELECT uniq(number) FROM numbers(20000000):" cmd = f"echo \\\"{sql} SELECT 1\\\" | xargs -i'{{}}' --no-run-if-empty -d ':' -P 20 clickhouse-client --time -m -n -q \\\"{{}}\\\"" kubectl.launch(f"exec {priority_pod} -- bash -c \"{cmd}\"", timeout=120) clickhouse.query( chi["metadata"]["name"], "SELECT event_time, CurrentMetric_QueryPreempted FROM system.metric_log WHERE CurrentMetric_QueryPreempted > 0", host=priority_svc, )
def attach_all_parts(): detached_parts = clickhouse.query( chi["metadata"]["name"], "SELECT name FROM system.detached_parts WHERE database='default' AND table='test' AND reason=''", pod=detached_pod) all_parts = "" for part in detached_parts.splitlines(): all_parts += f"ALTER TABLE default.test ATTACH PART '{part}';" if all_parts.strip() != "": clickhouse.query(chi["metadata"]["name"], all_parts, pod=detached_pod)
def create_part_and_detach(): clickhouse.query( chi["metadata"]["name"], "INSERT INTO default.test SELECT now(), number FROM numbers(100)", pod=detached_pod) part_name = clickhouse.query( chi["metadata"]["name"], sql= "SELECT name FROM system.parts WHERE database='default' AND table='test' ORDER BY modification_time DESC LIMIT 1", pod=detached_pod) clickhouse.query(chi["metadata"]["name"], f"ALTER TABLE default.test DETACH PART '{part_name}'", pod=detached_pod)
def test_020(config="configs/test-020-multi-volume.yaml"): chi = manifest.get_chi_name(util.get_full_path(config)) kubectl.create_and_check( config=config, check={ "pod_count": 1, "pod_volumes": { "/var/lib/clickhouse", "/var/lib/clickhouse2", }, "do_not_delete": 1, }) with When("Create a table and insert 1 row"): clickhouse.query(chi, "create table test_disks(a Int8) Engine = MergeTree() order by a") clickhouse.query(chi, "insert into test_disks values (1)") with Then("Data should be placed on default disk"): out = clickhouse.query(chi, "select disk_name from system.parts where table='test_disks'") assert out == 'default' with When("alter table test_disks move partition tuple() to disk 'disk2'"): clickhouse.query(chi, "alter table test_disks move partition tuple() to disk 'disk2'") with Then("Data should be placed on disk2"): out = clickhouse.query(chi, "select disk_name from system.parts where table='test_disks'") assert out == 'disk2' kubectl.delete_chi(chi)
def test_backup_size(self): decrease_pod, _, increase_pod, _ = alerts.random_pod_choice_for_callbacks( chi) backup_cases = { decrease_pod: { 'rows': (10000, 1000), 'decrease': True, }, increase_pod: { 'rows': (1000, 10000), 'decrease': False, }, } for backup_pod in backup_cases: decrease = backup_cases[backup_pod]['decrease'] for backup_rows in backup_cases[backup_pod]['rows']: backup_name = prepare_table_for_backup(backup_pod, rows=backup_rows) exec_on_backup_container( backup_pod, f'curl -X POST -sL "http://127.0.0.1:7171/backup/create?name={backup_name}"' ) wait_backup_command_status(backup_pod, f'create {backup_name}', expected_status='success') if decrease: clickhouse.query(chi['metadata']['name'], f"TRUNCATE TABLE default.test_backup", pod=backup_pod) time.sleep(15) fired = alerts.wait_alert_state( "ClickHouseBackupSizeChanged", "firing", expected_state=True, sleep_time=settings.prometheus_scrape_interval, labels={"pod_name": backup_pod}, time_range='60s') assert fired, error( f"can't get ClickHouseBackupSizeChanged alert in firing state, decrease={decrease}" ) with Then("check ClickHouseBackupSizeChanged gone away"): resolved = alerts.wait_alert_state("ClickHouseBackupSizeChanged", "firing", expected_state=False, labels={"pod_name": backup_pod}) assert resolved, error( f"can't get ClickHouseBackupSizeChanged alert is gone away, decrease={decrease}" )
def reboot_clickhouse_and_distributed_exection(): # we need 70 delayed files for catch insert_sql = 'INSERT INTO default.test_distr(event_time, test) SELECT now(), number FROM system.numbers LIMIT 10000' select_sql = 'SELECT count() FROM default.test_distr' with Then("reboot clickhouse-server pod"): kubectl.launch( f"exec -n {kubectl.namespace} {restarted_pod} -c clickhouse -- kill 1", ok_to_fail=True, ) with And("Insert to distributed table"): clickhouse.query(chi["metadata"]["name"], insert_sql, host=delayed_pod, ns=kubectl.namespace) with And("Select from distributed table"): clickhouse.query_with_error(chi["metadata"]["name"], select_sql, host=delayed_pod, ns=kubectl.namespace)
def table(): table = request.args['table'] fields = clickhouse.query('DESC TABLE ' + table) # first column: name # second: type res = {'fields': [[f[0], f[1]] for f in fields]} return jsonify(res)
def drop_table(): sql = "DROP TABLE IF EXISTS {db}.{name}"\ .format(db=settings.CLICKHOUSE['db_export'], name=settings.BASE_TABLE) res = clickhouse.query(sql) if (res['returncode'] != 0): logger.error('There is an error: %s', repr(res['err'])) sys.exit(-1)
def check_table(): sql = """EXISTS TABLE {db}.{name}"""\ .format(db=settings.CLICKHOUSE['db_export'], name=settings.BASE_TABLE) res = clickhouse.query(sql) if (res['returncode'] == 0): if str(res['out'].splitlines()).strip('[]') == "'1'": return True else: return False else: logger.error('There is an error: %s', repr(res['err'])) sys.exit(-1)
def test_ch_002(self): kubectl.create_and_check( "configs/test-ch-002-row-level.yaml", { "apply_templates": {"templates/tpl-clickhouse-20.3.yaml"}, "do_not_delete": 1, }) chi = "test-ch-002-row-level" create_table = """create table test (d Date default today(), team LowCardinality(String), user String) Engine = MergeTree() PARTITION BY d ORDER BY d;""" with When("Create test table"): clickhouse.query(chi, create_table) with And("Insert some data"): clickhouse.query( chi, "INSERT INTO test(team, user) values('team1', 'user1'),('team2', 'user2'),('team3', 'user3'),('team4', 'user4')" ) with Then( "Make another query for different users. It should be restricted to corresponding team by row-level security" ): for user in ['user1', 'user2', 'user3', 'user4']: out = clickhouse.query(chi, "select user from test", user=user) assert out == user with Then( "Make a count() query for different users. It should be restricted to corresponding team by row-level security" ): for user in ['user1', 'user2', 'user3', 'user4']: out = clickhouse.query(chi, "select count() from test", user=user) assert out == "1" kubectl.delete_chi(chi)
def get_tables(): """Get available table names from clickhouse that contains log data""" sql = "select name from system.tables where database='{db}' and name like 'log_%_%_%'"\ .format(db=settings.CLICKHOUSE['db_export']) res = clickhouse.query(sql) if (res['returncode'] == 0): tables = res['out'].splitlines() tables.sort() # Do not take the most recent table, because it can be still in use return tables[:-1] else: logger.error('There is an error: %s', repr(res['err'])) sys.exit(-1)
def create_table(): sql = """CREATE TABLE {db}.{name} ( date Date, type_id UInt16, item_id UInt32, tube_id UInt32, country String, num_views AggregateFunction(count)) ENGINE = AggregatingMergeTree(date, (date, type_id, tube_id, country, item_id), 8192)"""\ .format(db=settings.CLICKHOUSE['db_export'], name=settings.BASE_TABLE) res = clickhouse.query(sql) if (res['returncode'] != 0 and res['returncode'] != 57): logger.error('There is an error: %s', repr(res['err'])) sys.exit(-1)
def wait_clickhouse_cluster_ready(chi): with Given("All expected pods present in system.clusters"): all_pods_ready = False while all_pods_ready is False: all_pods_ready = True for pod in chi['status']['pods']: cluster_response = clickhouse.query( chi["metadata"]["name"], "SYSTEM RELOAD CONFIG; SELECT host_name FROM system.clusters WHERE cluster='all-sharded'", pod=pod ) for host in chi['status']['fqdns']: svc_short_name = host.replace(f'.{settings.test_namespace}.svc.cluster.local', '') if svc_short_name not in cluster_response: with Then("Not ready, sleep 5 seconds"): all_pods_ready = False time.sleep(5)
def test_distributed_files_to_insert(): delayed_pod, delayed_svc, restarted_pod, restarted_svc = random_pod_choice_for_callbacks() create_distributed_table_on_cluster() insert_sql = 'INSERT INTO default.test_distr(event_time, test) SELECT now(), number FROM system.numbers LIMIT 1000' clickhouse.query( chi["metadata"]["name"], 'SYSTEM STOP DISTRIBUTED SENDS default.test_distr', pod=delayed_pod, ns=kubectl.namespace ) files_to_insert_from_metrics = 0 files_to_insert_from_disk = 0 tries = 0 # we need more than 50 delayed files for catch while files_to_insert_from_disk <= 55 and files_to_insert_from_metrics <= 55 and tries < 500: kubectl.launch( f"exec -n {kubectl.namespace} {restarted_pod} -c clickhouse -- kill 1", ok_to_fail=True, ) clickhouse.query(chi["metadata"]["name"], insert_sql, pod=delayed_pod, host=delayed_pod, ns=kubectl.namespace) files_to_insert_from_metrics = clickhouse.query( chi["metadata"]["name"], "SELECT value FROM system.metrics WHERE metric='DistributedFilesToInsert'", pod=delayed_pod, ns=kubectl.namespace ) files_to_insert_from_metrics = int(files_to_insert_from_metrics) files_to_insert_from_disk = int(kubectl.launch( f"exec -n {kubectl.namespace} {delayed_pod} -c clickhouse -- bash -c 'ls -la /var/lib/clickhouse/data/default/test_distr/*/*.bin 2>/dev/null | wc -l'", ok_to_fail=False, )) with When("reboot clickhouse-server pod"): fired = wait_alert_state("ClickHouseDistributedFilesToInsertHigh", "firing", True, labels={"hostname": delayed_svc, "chi": chi["metadata"]["name"]}) assert fired, error("can't get ClickHouseDistributedFilesToInsertHigh alert in firing state") kubectl.wait_pod_status(restarted_pod, "Running", ns=kubectl.namespace) clickhouse.query( chi["metadata"]["name"], 'SYSTEM START DISTRIBUTED SENDS default.test_distr', pod=delayed_pod, ns=kubectl.namespace ) with Then("check ClickHouseDistributedFilesToInsertHigh gone away"): resolved = wait_alert_state("ClickHouseDistributedFilesToInsertHigh", "firing", False, labels={"hostname": delayed_svc}) assert resolved, error("can't check ClickHouseDistributedFilesToInsertHigh alert is gone away") drop_distributed_table_on_cluster()
def test_zookeeper_rescale(self): with When('create replicated table'): clickhouse.create_table_on_cluster( chi, 'all-sharded', 'default.zk_repl', '(id UInt64) ENGINE=ReplicatedMergeTree(\'/clickhouse/tables/default.zk_repl/{shard}\',\'{replica}\') ORDER BY (id)' ) with Then('insert data x1'): clickhouse.query( chi['metadata']['name'], 'INSERT INTO default.zk_repl SELECT number FROM numbers(1000)', pod="chi-test-cluster-for-zk-default-0-0-0") with Then('scale up zookeeper to 3 nodes'): util.require_zookeeper('zookeeper-3-nodes-1GB-for-tests-only.yaml', force_install=True) kubectl.wait_pod_status('zookeeper-0', 'Running', settings.test_namespace) kubectl.wait_pod_status('zookeeper-1', 'Running', settings.test_namespace) kubectl.wait_pod_status('zookeeper-2', 'Running', settings.test_namespace) with Then('insert data x2'): clickhouse.query( chi['metadata']['name'], 'INSERT INTO default.zk_repl SELECT number*2 FROM numbers(1000)', pod="chi-test-cluster-for-zk-default-0-1-0") with Then('scale down zookeeper to 1 nodes'): util.require_zookeeper('zookeeper-1-node-1GB-for-tests-only.yaml', force_install=True) kubectl.wait_pod_status('zookeeper-0', 'Running', settings.test_namespace) with Then('insert data x2'): clickhouse.query( chi['metadata']['name'], 'INSERT INTO default.zk_repl SELECT number*3 FROM numbers(1000)', pod="chi-test-cluster-for-zk-default-0-0-0") assert clickhouse.query(chi['metadata']['name'], 'SELECT count() FROM default.zk_repl', pod="chi-test-cluster-for-zk-default-0-1-0" ) == '3000', "Invalid rows after 3x1000 inserts" clickhouse.drop_table_on_cluster(chi, 'all-sharded', 'default.zk_repl')
def test_019(config="configs/test-019-retain-volume.yaml"): require_zookeeper() chi = manifest.get_chi_name(util.get_full_path(config)) kubectl.create_and_check( config=config, check={ "pod_count": 1, "do_not_delete": 1, }) create_non_replicated_table = "create table t1 Engine = Log as select 1 as a" create_replicated_table = """ create table t2 Engine = ReplicatedMergeTree('/clickhouse/{installation}/{cluster}/tables/{shard}/{database}/{table}', '{replica}') partition by tuple() order by a as select 1 as a""".replace('\r', '').replace('\n', '') with Given("ClickHouse has some data in place"): clickhouse.query(chi, sql=create_non_replicated_table) clickhouse.query(chi, sql=create_replicated_table) with When("CHI with retained volume is deleted"): pvc_count = kubectl.get_count("pvc") pv_count = kubectl.get_count("pv") kubectl.delete_chi(chi) with Then("PVC should be retained"): assert kubectl.get_count("pvc") == pvc_count assert kubectl.get_count("pv") == pv_count with When("Re-create CHI"): kubectl.create_and_check( config=config, check={ "pod_count": 1, "do_not_delete": 1, }) with Then("PVC should be re-mounted"): with And("Non-replicated table should have data"): out = clickhouse.query(chi, sql="select a from t1") assert out == "1" with And("Replicated table should have data"): out = clickhouse.query(chi, sql="select a from t2") assert out == "1" kubectl.delete_chi(chi)
def test_delayed_and_rejected_insert_and_max_part_count_for_partition_and_low_inserted_rows_per_query(): create_table_on_cluster() delayed_pod, delayed_svc, rejected_pod, rejected_svc = random_pod_choice_for_callbacks() prometheus_scrape_interval = 15 # default values in system.merge_tree_settings parts_to_throw_insert = 300 parts_to_delay_insert = 150 chi_name = chi["metadata"]["name"] parts_limits = parts_to_delay_insert selected_svc = delayed_svc def insert_many_parts_to_clickhouse(): stop_merges = "SYSTEM STOP MERGES default.test;" min_block = "SET max_block_size=1; SET max_insert_block_size=1; SET min_insert_block_size_rows=1;" with When(f"Insert to MergeTree table {parts_limits} parts"): r = parts_limits sql = stop_merges + min_block + \ "INSERT INTO default.test(event_time, test) SELECT now(), number FROM system.numbers LIMIT %d;" % r clickhouse.query(chi_name, sql, host=selected_svc, ns=kubectl.namespace) # @TODO we need only one query after resolve https://github.com/ClickHouse/ClickHouse/issues/11384 sql = min_block + "INSERT INTO default.test(event_time, test) SELECT now(), number FROM system.numbers LIMIT 1;" clickhouse.query_with_error(chi_name, sql, host=selected_svc, ns=kubectl.namespace) with And(f"wait prometheus_scrape_interval={prometheus_scrape_interval}*2 sec"): time.sleep(prometheus_scrape_interval * 2) sql = min_block + "INSERT INTO default.test(event_time, test) SELECT now(), number FROM system.numbers LIMIT 1;" clickhouse.query_with_error(chi_name, sql, host=selected_svc, ns=kubectl.namespace) insert_many_parts_to_clickhouse() with Then("check ClickHouseDelayedInsertThrottling firing"): fired = wait_alert_state("ClickHouseDelayedInsertThrottling", "firing", True, labels={"hostname": delayed_svc}, time_range="30s", sleep_time=5) assert fired, error("can't get ClickHouseDelayedInsertThrottling alert in firing state") with Then("check ClickHouseMaxPartCountForPartition firing"): fired = wait_alert_state("ClickHouseMaxPartCountForPartition", "firing", True, labels={"hostname": delayed_svc}, time_range="45s", sleep_time=5) assert fired, error("can't get ClickHouseMaxPartCountForPartition alert in firing state") with Then("check ClickHouseLowInsertedRowsPerQuery firing"): fired = wait_alert_state("ClickHouseLowInsertedRowsPerQuery", "firing", True, labels={"hostname": delayed_svc}, time_range="60s", sleep_time=5) assert fired, error("can't get ClickHouseLowInsertedRowsPerQuery alert in firing state") clickhouse.query(chi_name, "SYSTEM START MERGES default.test", host=selected_svc, ns=kubectl.namespace) with Then("check ClickHouseDelayedInsertThrottling gone away"): resolved = wait_alert_state("ClickHouseDelayedInsertThrottling", "firing", False, labels={"hostname": delayed_svc}, sleep_time=5) assert resolved, error("can't check ClickHouseDelayedInsertThrottling alert is gone away") with Then("check ClickHouseMaxPartCountForPartition gone away"): resolved = wait_alert_state("ClickHouseMaxPartCountForPartition", "firing", False, labels={"hostname": delayed_svc}, sleep_time=5) assert resolved, error("can't check ClickHouseMaxPartCountForPartition alert is gone away") with Then("check ClickHouseLowInsertedRowsPerQuery gone away"): resolved = wait_alert_state("ClickHouseLowInsertedRowsPerQuery", "firing", False, labels={"hostname": delayed_svc}, sleep_time=5) assert resolved, error("can't check ClickHouseLowInsertedRowsPerQuery alert is gone away") parts_limits = parts_to_throw_insert selected_svc = rejected_svc insert_many_parts_to_clickhouse() with Then("check ClickHouseRejectedInsert firing"): fired = wait_alert_state("ClickHouseRejectedInsert", "firing", True, labels={"hostname": rejected_svc}, time_range="30s", sleep_time=5) assert fired, error("can't get ClickHouseRejectedInsert alert in firing state") with Then("check ClickHouseRejectedInsert gone away"): resolved = wait_alert_state("ClickHouseRejectedInsert", "firing", False, labels={"hostname": rejected_svc}, sleep_time=5) assert resolved, error("can't check ClickHouseRejectedInsert alert is gone away") clickhouse.query(chi_name, "SYSTEM START MERGES default.test", host=selected_svc, ns=kubectl.namespace) drop_table_on_cluster()
def test_013(): config = "configs/test-013-add-shards-1.yaml" chi = manifest.get_chi_name(util.get_full_path(config)) cluster = "default" kubectl.create_and_check(config=config, check={ "apply_templates": { settings.clickhouse_template, }, "object_counts": { "statefulset": 1, "pod": 1, "service": 2, }, "do_not_delete": 1, }) start_time = kubectl.get_field("pod", f"chi-{chi}-{cluster}-0-0-0", ".status.startTime") schema_objects = [ 'test_local', 'test_distr', 'events-distr', ] with Then("Create local and distributed tables"): clickhouse.query( chi, "CREATE TABLE test_local Engine = Log as SELECT * FROM system.one") clickhouse.query( chi, "CREATE TABLE test_distr as test_local Engine = Distributed('default', default, test_local)" ) clickhouse.query(chi, "CREATE DATABASE \\\"test-db\\\"") clickhouse.query( chi, "CREATE TABLE \\\"test-db\\\".\\\"events-distr\\\" as system.events " "ENGINE = Distributed('all-sharded', system, events)") with Then("Add shards"): kubectl.create_and_check(config="configs/test-013-add-shards-2.yaml", check={ "object_counts": { "statefulset": 3, "pod": 3, "service": 4, }, "do_not_delete": 1, }) # Give some time for replication to catch up time.sleep(10) with Then("Unaffected pod should not be restarted"): new_start_time = kubectl.get_field("pod", f"chi-{chi}-{cluster}-0-0-0", ".status.startTime") assert start_time == new_start_time with And("Schema objects should be migrated to new shards"): for obj in schema_objects: out = clickhouse.query( chi, f"SELECT count() FROM system.tables WHERE name = '{obj}'", host=f"chi-{chi}-{cluster}-1-0") assert out == "1" out = clickhouse.query( chi, f"SELECT count() FROM system.tables WHERE name = '{obj}'", host=f"chi-{chi}-{cluster}-2-0") assert out == "1" with When("Remove shards"): kubectl.create_and_check(config=config, check={ "object_counts": { "statefulset": 1, "pod": 1, "service": 2, }, "do_not_delete": 1, }) time.sleep(10) with Then("Unaffected pod should not be restarted"): new_start_time = kubectl.get_field("pod", f"chi-{chi}-{cluster}-0-0-0", ".status.startTime") assert start_time == new_start_time kubectl.delete_chi(chi)
def test_ch_001(self): util.require_zookeeper() chit_data = manifest.get_chit_data( util.get_full_path("templates/tpl-clickhouse-19.11.yaml")) kubectl.launch(f"delete chit {chit_data['metadata']['name']}", ns=settings.test_namespace) kubectl.create_and_check( "configs/test-ch-001-insert-quorum.yaml", { "apply_templates": {"templates/tpl-clickhouse-20.8.yaml"}, "pod_count": 2, "do_not_delete": 1, }) chi = manifest.get_chi_name( util.get_full_path("configs/test-ch-001-insert-quorum.yaml")) chi_data = kubectl.get("chi", ns=settings.test_namespace, name=chi) util.wait_clickhouse_cluster_ready(chi_data) host0 = "chi-test-ch-001-insert-quorum-default-0-0" host1 = "chi-test-ch-001-insert-quorum-default-0-1" create_table = """ create table t1 on cluster default (a Int8, d Date default today()) Engine = ReplicatedMergeTree('/clickhouse/tables/{table}', '{replica}') partition by d order by a TTL d + interval 5 second SETTINGS merge_with_ttl_timeout=5""".replace('\r', '').replace('\n', '') create_mv_table2 = """ create table t2 on cluster default (a Int8) Engine = ReplicatedMergeTree('/clickhouse/tables/{table}', '{replica}') partition by tuple() order by a""".replace('\r', '').replace('\n', '') create_mv_table3 = """ create table t3 on cluster default (a Int8) Engine = ReplicatedMergeTree('/clickhouse/tables/{table}', '{replica}') partition by tuple() order by a""".replace('\r', '').replace('\n', '') create_mv2 = "create materialized view t_mv2 on cluster default to t2 as select a from t1" create_mv3 = "create materialized view t_mv3 on cluster default to t3 as select a from t1" with Given("Tables t1, t2, t3 and MVs t1->t2, t1-t3 are created"): clickhouse.query(chi, create_table) clickhouse.query(chi, create_mv_table2) clickhouse.query(chi, create_mv_table3) clickhouse.query(chi, create_mv2) clickhouse.query(chi, create_mv3) with When("Add a row to an old partition"): clickhouse.query(chi, "insert into t1(a,d) values(6, today()-1)", host=host0) with When("Stop fetches for t1 at replica1"): clickhouse.query(chi, "system stop fetches default.t1", host=host1) with Then("Wait 10 seconds and the data should be dropped by TTL"): time.sleep(10) out = clickhouse.query(chi, "select count() from t1 where a=6", host=host0) assert out == "0" with When("Resume fetches for t1 at replica1"): clickhouse.query(chi, "system start fetches default.t1", host=host1) time.sleep(5) with Then("Inserts should resume"): clickhouse.query(chi, "insert into t1(a) values(7)", host=host0) clickhouse.query(chi, "insert into t1(a) values(1)") with When("Stop fetches for t2 at replica1"): clickhouse.query(chi, "system stop fetches default.t2", host=host1) with Then("Insert should fail since it can not reach the quorum"): out = clickhouse.query_with_error( chi, "insert into t1(a) values(2)", host=host0) assert "Timeout while waiting for quorum" in out # kubectl(f"exec {host0}-0 -n test -- cp /var/lib//clickhouse/data/default/t2/all_1_1_0/a.mrk2 /var/lib//clickhouse/data/default/t2/all_1_1_0/a.bin") # with Then("Corrupt data part in t2"): # kubectl(f"exec {host0}-0 -n test -- sed -i \"s/b/c/\" /var/lib/clickhouse/data/default/t2/all_1_1_0/a.bin") with When("Resume fetches for t2 at replica1"): clickhouse.query(chi, "system start fetches default.t2", host=host1) i = 0 while "2" != clickhouse.query( chi, "select active_replicas from system.replicas where database='default' and table='t1'", pod=host0) and i < 10: with Then("Not ready, wait 5 seconds"): time.sleep(5) i += 1 with Then( "Inserts should fail with an error regarding not satisfied quorum" ): out = clickhouse.query_with_error( chi, "insert into t1(a) values(3)", host=host0) assert "Quorum for previous write has not been satisfied yet" in out with And("Second insert of the same block should pass"): clickhouse.query(chi, "insert into t1(a) values(3)", host=host0) with And("Insert of the new block should fail"): out = clickhouse.query_with_error( chi, "insert into t1(a) values(4)", host=host0) assert "Quorum for previous write has not been satisfied yet" in out with And( "Second insert of the same block with 'deduplicate_blocks_in_dependent_materialized_views' setting should fail" ): out = clickhouse.query_with_error( chi, "set deduplicate_blocks_in_dependent_materialized_views=1; insert into t1(a) values(5)", host=host0) assert "Quorum for previous write has not been satisfied yet" in out out = clickhouse.query_with_error( chi, "select t1.a t1_a, t2.a t2_a from t1 left outer join t2 using (a) order by t1_a settings join_use_nulls=1" ) print(out)
def drop_distributed_table_on_cluster(cluster_name='all-sharded', distr_table='default.test_distr', local_table='default.test'): drop_distr_sql = f'DROP TABLE {distr_table} ON CLUSTER \\\"{cluster_name}\\\"' clickhouse.query(chi["metadata"]["name"], drop_distr_sql, timeout=120) drop_table_on_cluster(cluster_name, local_table)
def create_table_on_cluster(cluster_name='all-sharded', table='default.test', create_definition='(event_time DateTime, test UInt64) ENGINE MergeTree() ORDER BY tuple()'): create_local_sql = f'CREATE TABLE {table} ON CLUSTER \\\"{cluster_name}\\\" {create_definition}' clickhouse.query(chi["metadata"]["name"], create_local_sql, timeout=120)
def drop_table_on_cluster(cluster_name='all-sharded', table='default.test'): drop_local_sql = f'DROP TABLE {table} ON CLUSTER \\\"{cluster_name}\\\" SYNC' clickhouse.query(chi["metadata"]["name"], drop_local_sql, timeout=120)
def test_014(): require_zookeeper() create_table = """ CREATE TABLE test_local(a Int8) Engine = ReplicatedMergeTree('/clickhouse/{installation}/{cluster}/tables/{shard}/{database}/{table}', '{replica}') PARTITION BY tuple() ORDER BY a """.replace('\r', '').replace('\n', '') config = "configs/test-014-replication-1.yaml" chi = manifest.get_chi_name(util.get_full_path(config)) cluster = "default" kubectl.create_and_check( config=config, check={ "apply_templates": { settings.clickhouse_template, "templates/tpl-persistent-volume-100Mi.yaml", }, "object_counts": { "statefulset": 2, "pod": 2, "service": 3, }, "do_not_delete": 1, }) start_time = kubectl.get_field("pod", f"chi-{chi}-{cluster}-0-0-0", ".status.startTime") schema_objects = ['test_local', 'test_view', 'test_mv', 'a_view'] with Given("Create schema objects"): clickhouse.query(chi, create_table, host=f"chi-{chi}-{cluster}-0-0") clickhouse.query(chi, "CREATE VIEW test_view as SELECT * from test_local", host=f"chi-{chi}-{cluster}-0-0") clickhouse.query(chi, "CREATE VIEW a_view as SELECT * from test_view", host=f"chi-{chi}-{cluster}-0-0") clickhouse.query( chi, "CREATE MATERIALIZED VIEW test_mv Engine = Log as SELECT * from test_local", host=f"chi-{chi}-{cluster}-0-0") clickhouse.query( chi, "CREATE DICTIONARY test_dict (a Int8, b Int8) PRIMARY KEY a SOURCE(CLICKHOUSE(host 'localhost' port 9000 table 'test_local' user 'default')) LAYOUT(FLAT()) LIFETIME(0)", host=f"chi-{chi}-{cluster}-0-0") with Given( "Replicated table is created on a first replica and data is inserted" ): clickhouse.query(chi, "INSERT INTO test_local values(1)", host=f"chi-{chi}-{cluster}-0-0") with When("Table is created on the second replica"): clickhouse.query(chi, create_table, host=f"chi-{chi}-{cluster}-0-1") # Give some time for replication to catch up time.sleep(10) with Then("Data should be replicated"): out = clickhouse.query(chi, "SELECT a FROM test_local", host=f"chi-{chi}-{cluster}-0-1") assert out == "1" with When("Add one more replica"): kubectl.create_and_check(config="configs/test-014-replication-2.yaml", check={ "pod_count": 3, "do_not_delete": 1, }) # Give some time for replication to catch up time.sleep(10) new_start_time = kubectl.get_field("pod", f"chi-{chi}-{cluster}-0-0-0", ".status.startTime") assert start_time == new_start_time with Then("Schema objects should be migrated to the new replica"): for obj in schema_objects: out = clickhouse.query( chi, f"SELECT count() FROM system.tables WHERE name = '{obj}'", host=f"chi-{chi}-{cluster}-0-2") assert out == "1" # Check dictionary out = clickhouse.query( chi, f"SELECT count() FROM system.dictionaries WHERE name = 'test_dict'", host=f"chi-{chi}-{cluster}-0-2") assert out == "1" with And("Replicated table should have the data"): out = clickhouse.query(chi, "SELECT a FROM test_local", host=f"chi-{chi}-{cluster}-0-2") assert out == "1" with When("Remove replica"): kubectl.create_and_check(config=config, check={ "pod_count": 1, "do_not_delete": 1, }) new_start_time = kubectl.get_field("pod", f"chi-{chi}-{cluster}-0-0-0", ".status.startTime") assert start_time == new_start_time with Then("Replica needs to be removed from the Zookeeper as well"): out = clickhouse.query( chi, "SELECT count() FROM system.replicas WHERE table='test_local'") assert out == "1" with When("Restart Zookeeper pod"): with Then("Delete Zookeeper pod"): kubectl.launch("delete pod zookeeper-0") time.sleep(1) with Then( "Insert into the table while there is no Zookeeper -- table should be in readonly mode" ): out = clickhouse.query_with_error( chi, "INSERT INTO test_local values(2)") assert "Table is in readonly mode" in out with Then("Wait for Zookeeper pod to come back"): kubectl.wait_object("pod", "zookeeper-0") kubectl.wait_pod_status("zookeeper-0", "Running") with Then( "Wait for ClickHouse to reconnect to Zookeeper and switch to read-write mode" ): time.sleep(30) # with Then("Restart clickhouse pods"): # kubectl("delete pod chi-test-014-replication-default-0-0-0") # kubectl("delete pod chi-test-014-replication-default-0-1-0") with Then("Table should be back to normal"): clickhouse.query(chi, "INSERT INTO test_local values(3)") kubectl.delete_chi("test-014-replication")
def test_016(): chi = "test-016-settings" kubectl.create_and_check(config="configs/test-016-settings-01.yaml", check={ "apply_templates": { settings.clickhouse_template, }, "pod_count": 1, "do_not_delete": 1, }) with Then("Custom macro 'layer' should be available"): out = clickhouse.query( chi, sql="select substitution from system.macros where macro='layer'") assert out == "01" with And("Custom macro 'test' should be available"): out = clickhouse.query( chi, sql="select substitution from system.macros where macro='test'") assert out == "test" with And("dictGet() should work"): out = clickhouse.query(chi, sql="select dictGet('one', 'one', toUInt64(0))") assert out == "0" with And("query_log should be disabled"): clickhouse.query(chi, sql="system flush logs") out = clickhouse.query_with_error( chi, sql="select count() from system.query_log") assert "doesn't exist" in out with And("max_memory_usage should be 7000000000"): out = clickhouse.query( chi, sql= "select value from system.settings where name='max_memory_usage'") assert out == "7000000000" with And("test_usersd user should be available"): clickhouse.query(chi, sql="select version()", user="******") with And("user1 user should be available"): clickhouse.query(chi, sql="select version()", user="******", pwd="qwerty") with And("system.clusters should be empty due to remote_servers override"): out = clickhouse.query(chi, sql="select count() from system.clusters") assert out == "0" with When("Update usersd settings"): start_time = kubectl.get_field("pod", f"chi-{chi}-default-0-0-0", ".status.startTime") kubectl.create_and_check(config="configs/test-016-settings-02.yaml", check={ "do_not_delete": 1, }) with Then("Wait for configmap changes to apply"): config_map_applied_num = "0" i = 1 while config_map_applied_num == "0" and i < 10: config_map_applied_num = kubectl.launch( f"exec chi-{chi}-default-0-0-0 -- bash -c \"grep test_norestart /etc/clickhouse-server/users.d/my_users.xml | wc -l\"" ) if config_map_applied_num != "0": break with And(f"not applied, wait {15 * i}s"): time.sleep(15 * i) i += 1 assert config_map_applied_num != "0", "ConfigMap should be applied" version = "" with Then("test_norestart user should be available"): version = clickhouse.query(chi, sql="select version()", user="******") with And("user1 user should not be available"): version_user1 = clickhouse.query_with_error(chi, sql="select version()", user="******", pwd="qwerty") assert version != version_user1 with And("user2 user should be available"): version_user2 = clickhouse.query(chi, sql="select version()", user="******", pwd="qwerty") assert version == version_user2 with And("ClickHouse should not be restarted"): new_start_time = kubectl.get_field("pod", f"chi-{chi}-default-0-0-0", ".status.startTime") assert start_time == new_start_time kubectl.delete_chi("test-016-settings")
def query(): query = request.args['query'] res = clickhouse.query(query) return render_template('bi.html', query=query, result=res[0][0])