def forwards_local(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.SESSIONS,
table_name="sessions_raw_local",
columns=raw_columns,
engine=table_engines.MergeTree(
storage_set=StorageSetKey.SESSIONS,
order_by=
"(org_id, project_id, release, environment, started)",
partition_by="(toMonday(started))",
settings={"index_granularity": "16384"},
),
),
operations.CreateTable(
storage_set=StorageSetKey.SESSIONS,
table_name="sessions_hourly_local",
columns=aggregate_columns_v1,
engine=table_engines.AggregatingMergeTree(
storage_set=StorageSetKey.SESSIONS,
order_by=
"(org_id, project_id, release, environment, started)",
partition_by="(toMonday(started))",
settings={"index_granularity": "256"},
),
),
create_matview_v1,
]
def forwards_local(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.SESSIONS,
table_name="sessions_raw_local",
columns=raw_columns,
engine=table_engines.MergeTree(
storage_set=StorageSetKey.SESSIONS,
order_by=
"(org_id, project_id, release, environment, started)",
partition_by="(toMonday(started))",
settings={"index_granularity": "16384"},
),
),
operations.CreateTable(
storage_set=StorageSetKey.SESSIONS,
table_name="sessions_hourly_local",
columns=aggregate_columns,
engine=table_engines.AggregatingMergeTree(
storage_set=StorageSetKey.SESSIONS,
order_by=
"(org_id, project_id, release, environment, started)",
partition_by="(toMonday(started))",
settings={"index_granularity": "256"},
),
),
operations.CreateMaterializedView(
storage_set=StorageSetKey.SESSIONS,
view_name="sessions_hourly_mv_local",
destination_table_name="sessions_hourly_local",
columns=aggregate_columns,
query=f"""
SELECT
org_id,
project_id,
toStartOfHour(started) as started,
release,
environment,
quantilesIfState(0.5, 0.9)(
duration,
duration <> {MAX_UINT32} AND status == 1
) as duration_quantiles,
countIfState(session_id, seq == 0) as sessions,
uniqIfState(distinct_id, distinct_id != '{NIL_UUID}') as users,
countIfState(session_id, status == 2) as sessions_crashed,
countIfState(session_id, status == 3) as sessions_abnormal,
uniqIfState(session_id, errors > 0) as sessions_errored,
uniqIfState(distinct_id, status == 2) as users_crashed,
uniqIfState(distinct_id, status == 3) as users_abnormal,
uniqIfState(distinct_id, errors > 0) as users_errored
FROM
sessions_raw_local
GROUP BY
org_id, project_id, started, release, environment
""",
),
]
def forwards_local(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.QUERYLOG,
table_name="querylog_local",
columns=columns,
engine=table_engines.MergeTree(
storage_set=StorageSetKey.QUERYLOG,
order_by="(toStartOfDay(timestamp), request_id)",
partition_by="(toMonday(timestamp))",
sample_by="request_id",
),
)
]
def forwards_local(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.METRICS,
table_name=self.local_table_name,
columns=self.column_list,
engine=table_engines.MergeTree(
storage_set=StorageSetKey.METRICS,
order_by=
"(use_case_id, metric_type, org_id, project_id, metric_id, timestamp)",
partition_by="(toStartOfDay(timestamp))",
ttl="timestamp + toIntervalDay(7)",
),
)
]
def test_zookeeper_path_override() -> None:
orig_path = "/clickhouse/tables/events/all/default/test_table"
override_path = "/clickhouse/tables/test_table"
engine = table_engines.MergeTree(storage_set=StorageSetKey.EVENTS,
order_by="timestamp",
unsharded=True)
assert (
engine.get_sql(multi_node_cluster, "test_table") ==
f"ReplicatedMergeTree('{orig_path}', '{{replica}}') ORDER BY timestamp"
)
settings.CLICKHOUSE_ZOOKEEPER_OVERRIDE = {orig_path: override_path}
assert (
engine.get_sql(multi_node_cluster, "test_table") ==
f"ReplicatedMergeTree('{override_path}', '{{replica}}') ORDER BY timestamp"
)
settings.CLICKHOUSE_ZOOKEEPER_OVERRIDE = {}
def forwards_local(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.OUTCOMES,
table_name="outcomes_raw_local",
columns=raw_columns,
engine=table_engines.MergeTree(
storage_set=StorageSetKey.OUTCOMES,
order_by="(org_id, project_id, timestamp)",
partition_by="(toMonday(timestamp))",
settings={"index_granularity": "16384"},
),
),
operations.CreateTable(
storage_set=StorageSetKey.OUTCOMES,
table_name="outcomes_hourly_local",
columns=hourly_columns,
engine=table_engines.SummingMergeTree(
storage_set=StorageSetKey.OUTCOMES,
order_by=
"(org_id, project_id, key_id, outcome, reason, timestamp)",
partition_by="(toMonday(timestamp))",
settings={"index_granularity": "256"},
),
),
operations.CreateMaterializedView(
storage_set=StorageSetKey.OUTCOMES,
view_name="outcomes_mv_hourly_local",
destination_table_name="outcomes_hourly_local",
columns=materialized_view_columns,
query="""
SELECT
org_id,
project_id,
ifNull(key_id, 0) AS key_id,
toStartOfHour(timestamp) AS timestamp,
outcome,
ifNull(reason, 'none') AS reason,
count() AS times_seen
FROM outcomes_raw_local
GROUP BY org_id, project_id, key_id, timestamp, outcome, reason
""",
),
]
def get_forward_bucket_table_local(
table_name: str,
value_cols: Sequence[Column[Modifiers]],
) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.METRICS,
table_name=table_name,
columns=[
*PRE_VALUE_BUCKETS_COLUMNS,
*value_cols,
*POST_VALUES_BUCKETS_COLUMNS,
],
engine=table_engines.MergeTree(
storage_set=StorageSetKey.METRICS,
order_by="(org_id, project_id, metric_id, timestamp)",
partition_by="toMonday(timestamp)",
ttl="timestamp + toIntervalDay(14)",
),
)
]
multi_node_cluster = ClickhouseCluster(
host="host_2",
port=9000,
user="******",
password="",
database="default",
http_port=8123,
storage_sets={"events"},
single_node=False,
cluster_name="cluster_1",
distributed_cluster_name="dist_hosts",
)
merge_test_cases = [
pytest.param(
table_engines.MergeTree(storage_set=StorageSetKey.EVENTS,
order_by="timestamp"),
"MergeTree() ORDER BY timestamp",
"ReplicatedMergeTree('/clickhouse/tables/events/{shard}/test_table', '{replica}') ORDER BY timestamp",
id="Merge tree",
),
pytest.param(
table_engines.MergeTree(
storage_set=StorageSetKey.TRANSACTIONS,
order_by="date",
settings={"index_granularity": "256"},
),
"MergeTree() ORDER BY date SETTINGS index_granularity=256",
"ReplicatedMergeTree('/clickhouse/tables/transactions/{shard}/test_table', '{replica}') ORDER BY date SETTINGS index_granularity=256",
id="Merge tree with settings",
),
pytest.param(