def forwards_dist(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.SESSIONS,
table_name="sessions_raw_dist",
columns=raw_columns,
engine=table_engines.Distributed(
local_table_name="sessions_raw_local", sharding_key="org_id",
),
),
operations.CreateTable(
storage_set=StorageSetKey.SESSIONS,
table_name="sessions_hourly_dist",
columns=aggregate_columns,
engine=table_engines.Distributed(
local_table_name="sessions_hourly_local", sharding_key="org_id",
),
),
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.CDC,
table_name="groupassignee_dist",
columns=columns,
engine=table_engines.Distributed(
local_table_name="groupassignee_local", sharding_key=None,
),
)
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.GENERIC_METRICS_SETS,
table_name="generic_metric_sets_aggregated_dist",
engine=table_engines.Distributed(
local_table_name=self.local_table_name, sharding_key=None
),
columns=self.columns,
)
]
def forwards_dist(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.EVENTS,
table_name="errors_dist",
columns=columns,
engine=table_engines.Distributed(
local_table_name="errors_local", sharding_key="event_hash",
),
)
]
def forwards_dist(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.QUERYLOG,
table_name="querylog_dist",
columns=columns,
engine=table_engines.Distributed(
local_table_name="querylog_local", sharding_key=None,
),
)
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.GENERIC_METRICS_SETS,
table_name=self.dist_table_name,
engine=table_engines.Distributed(
local_table_name=self.local_table_name,
sharding_key="cityHash64(timeseries_id)",
),
columns=self.columns,
)
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.PROFILES,
table_name="profiles_dist",
columns=columns,
engine=table_engines.Distributed(
local_table_name="profiles_local",
sharding_key="cityHash64(profile_id)",
),
)
]
def backwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.EVENTS,
table_name="sentry_dist",
columns=columns,
engine=table_engines.Distributed(
local_table_name="sentry_local",
sharding_key="cityHash64(toString(event_id))",
),
),
operations.CreateTable(
storage_set=StorageSetKey.EVENTS_RO,
table_name="sentry_dist_ro",
columns=columns,
engine=table_engines.Distributed(
local_table_name="sentry_local",
sharding_key="cityHash64(toString(event_id))",
),
),
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.REPLAYS,
table_name="replays_dist",
columns=raw_columns,
engine=table_engines.Distributed(
local_table_name="replays_local",
sharding_key="cityHash64(toString(replay_id))",
),
),
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.METRICS,
table_name=self.dist_table_name,
columns=self.column_list,
engine=table_engines.Distributed(
local_table_name=self.local_table_name,
sharding_key=None,
),
)
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.OUTCOMES,
table_name="outcomes_raw_dist",
columns=raw_columns,
engine=table_engines.Distributed(
local_table_name="outcomes_raw_local",
sharding_key="org_id",
),
),
operations.CreateTable(
storage_set=StorageSetKey.OUTCOMES,
table_name="outcomes_hourly_dist",
columns=hourly_columns,
engine=table_engines.Distributed(
local_table_name="outcomes_hourly_local",
sharding_key="org_id",
),
),
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_dist",
columns=columns,
engine=table_engines.Distributed(
local_table_name="transactions_local",
sharding_key="cityHash64(span_id)",
),
)
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.EVENTS,
table_name="errors_dist_new",
columns=columns,
engine=table_engines.Distributed(
local_table_name="errors_local",
sharding_key=sample_expr,
),
),
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="errors_dist_new",
column=Column(
"_tags_hash_map",
Array(UInt(64),
Modifiers(materialized=TAGS_HASH_MAP_COLUMN)),
),
after="tags",
),
operations.DropTable(
storage_set=StorageSetKey.EVENTS,
table_name="errors_dist",
),
operations.RenameTable(
storage_set=StorageSetKey.EVENTS,
old_table_name="errors_dist_new",
new_table_name="errors_dist",
),
operations.CreateTable(
storage_set=StorageSetKey.EVENTS_RO,
table_name="errors_dist_ro",
columns=columns,
engine=table_engines.Distributed(
local_table_name="errors_local",
sharding_key=sample_expr,
),
),
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.METRICS,
table_name=self.dist_table_name,
columns=self.aggregated_cols,
engine=table_engines.Distributed(
local_table_name=self.table_name, sharding_key=None),
),
operations.AddColumn(
storage_set=StorageSetKey.METRICS,
table_name=self.dist_table_name,
column=Column(
"_tags_hash",
Array(UInt(64),
Modifiers(materialized=INT_TAGS_HASH_MAP_COLUMN)),
),
after="tags.value",
),
]
def get_forward_bucket_table_dist(
local_table_name: str,
dist_table_name: str,
value_cols: Sequence[Column[Modifiers]],
) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.METRICS,
table_name=dist_table_name,
columns=[
*PRE_VALUE_BUCKETS_COLUMNS,
*value_cols,
*POST_VALUES_BUCKETS_COLUMNS,
],
engine=table_engines.Distributed(
local_table_name=local_table_name,
sharding_key=None,
),
),
]
def forwards_dist(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="spans_experimental_dist",
columns=columns,
engine=table_engines.Distributed(
local_table_name="spans_experimental_local",
sharding_key="cityHash64(transaction_span_id)",
),
),
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="spans_experimental_dist",
column=Column(
"_tags_hash_map",
Materialized(Array(UInt(64)), TAGS_HASH_MAP_COLUMN),
),
after="tags.value",
),
]
def forwards_dist(self) -> Sequence[operations.Operation]:
# We removed the materialized for the dist table DDL.
def strip_materialized(columns: Sequence[Column]) -> None:
for col in columns:
if isinstance(col.type, Materialized):
col.type = col.type.inner_type
dist_columns = deepcopy(columns)
strip_materialized(dist_columns)
return [
operations.CreateTable(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_dist",
columns=dist_columns,
engine=table_engines.Distributed(
local_table_name="transactions_local",
sharding_key="cityHash64(span_id)",
),
)
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
# We removed the materialized for the dist table DDL.
def strip_materialized(columns: Sequence[Column[Modifiers]]) -> None:
for col in columns:
if col.type.has_modifier(Materialized):
modifiers = col.type.get_modifiers()
assert modifiers is not None
col.type = col.type.set_modifiers(
replace(modifiers, materialized=None))
dist_columns = deepcopy(columns)
strip_materialized(dist_columns)
return [
operations.CreateTable(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_dist",
columns=dist_columns,
engine=table_engines.Distributed(
local_table_name="transactions_local",
sharding_key="cityHash64(span_id)",
),
)
]
def get_forward_migrations_dist(
dist_table_name: str,
local_table_name: str,
aggregation_col_schema: Sequence[Column[Modifiers]],
) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.METRICS,
table_name=dist_table_name,
columns=[*COMMON_AGGR_COLUMNS, *aggregation_col_schema],
engine=table_engines.Distributed(local_table_name=local_table_name,
sharding_key=None),
),
operations.AddColumn(
storage_set=StorageSetKey.METRICS,
table_name=dist_table_name,
column=Column(
"_tags_hash",
Array(UInt(64),
Modifiers(materialized=INT_TAGS_HASH_MAP_COLUMN)),
),
after="tags.value",
),
]
"MergeTree() ORDER BY timestamp",
"ReplicatedMergeTree('/clickhouse/tables/events/all/test_table', '{replica}') ORDER BY timestamp",
id="Unsharded merge tree",
),
]
@pytest.mark.parametrize("engine, single_node_sql, multi_node_sql",
merge_test_cases)
def test_merge_table(engine: table_engines.TableEngine, single_node_sql: str,
multi_node_sql: str) -> None:
assert engine.get_sql(single_node_cluster, "test_table") == single_node_sql
assert engine.get_sql(multi_node_cluster, "test_table") == multi_node_sql
dist_test_cases = [
pytest.param(
table_engines.Distributed(local_table_name="test_table_local",
sharding_key="event_id"),
"Distributed(cluster_1, default, test_table_local, event_id)",
id="Disributed",
)
]
@pytest.mark.parametrize("engine, sql", dist_test_cases)
def test_distributed(engine: table_engines.TableEngine, sql: str) -> None:
with pytest.raises(AssertionError):
engine.get_sql(single_node_cluster, "test_table")
assert engine.get_sql(multi_node_cluster, "test_table") == sql