def forwards_local(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="spans_experimental_local",
columns=columns,
engine=table_engines.ReplacingMergeTree(
storage_set=StorageSetKey.TRANSACTIONS,
version_column="deleted",
order_by=
("(project_id, toStartOfDay(finish_ts), transaction_name, "
"cityHash64(transaction_span_id), op, cityHash64(trace_id), "
"cityHash64(span_id))"),
partition_by="(toMonday(finish_ts))",
sample_by="cityHash64(span_id)",
ttl="finish_ts + toIntervalDay(retention_days)",
settings={"index_granularity": "8192"},
),
),
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="spans_experimental_local",
column=Column(
"_tags_hash_map",
Materialized(Array(UInt(64)), TAGS_HASH_MAP_COLUMN),
),
after="tags.value",
),
]
def backwards_local(self) -> Sequence[operations.SqlOperation]:
sample_expr = "cityHash64(toString(event_id))"
return [
operations.CreateTable(
storage_set=StorageSetKey.EVENTS,
table_name="sentry_local",
columns=columns,
engine=table_engines.ReplacingMergeTree(
storage_set=StorageSetKey.EVENTS,
version_column="deleted",
order_by="(project_id, toStartOfDay(timestamp), %s)" % sample_expr,
partition_by="(toMonday(timestamp), if(equals(retention_days, 30), 30, 90))",
sample_by=sample_expr,
),
),
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="sentry_local",
column=Column(
"_tags_hash_map",
Array(UInt(64), Modifiers(materialized=TAGS_HASH_MAP_COLUMN)),
),
after="_tags_flattened",
),
]
def forwards_local(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.CDC,
table_name="groupassignee_local",
columns=columns,
engine=table_engines.ReplacingMergeTree(
storage_set=StorageSetKey.CDC,
version_column="offset",
order_by="(project_id, group_id)",
unsharded=True,
),
)
]
def forwards_local(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.EVENTS,
table_name="groupedmessage_local",
columns=columns,
engine=table_engines.ReplacingMergeTree(
storage_set=StorageSetKey.EVENTS,
version_column="offset",
order_by="(project_id, id)",
sample_by="id",
unsharded=True,
),
)
]
def forwards_local(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
columns=columns,
engine=table_engines.ReplacingMergeTree(
storage_set=StorageSetKey.TRANSACTIONS,
version_column="deleted",
order_by="(project_id, toStartOfDay(finish_ts), transaction_name, cityHash64(span_id))",
partition_by="(retention_days, toMonday(finish_ts))",
sample_by="cityHash64(span_id)",
ttl="finish_ts + toIntervalDay(retention_days)",
settings={"index_granularity": "8192"},
),
)
]
def forwards_local(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.PROFILES,
table_name="profiles_local",
columns=columns,
engine=table_engines.ReplacingMergeTree(
order_by=
"(organization_id, project_id, toStartOfDay(received), cityHash64(profile_id))",
partition_by="(retention_days, toMonday(received))",
sample_by="cityHash64(profile_id)",
settings={"index_granularity": "8192"},
storage_set=StorageSetKey.PROFILES,
ttl="received + toIntervalDay(retention_days)",
),
)
]
def forwards_local(self) -> Sequence[operations.Operation]:
sample_expr = "cityHash64(toString(event_id))"
return [
operations.CreateTable(
storage_set=StorageSetKey.EVENTS,
table_name="sentry_local",
columns=columns,
engine=table_engines.ReplacingMergeTree(
storage_set=StorageSetKey.EVENTS,
version_column="deleted",
order_by="(project_id, toStartOfDay(timestamp), %s)" % sample_expr,
partition_by="(toMonday(timestamp), if(equals(retention_days, 30), 30, 90))",
sample_by=sample_expr,
),
),
]
def forwards_local(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.EVENTS,
table_name="errors_local",
columns=columns,
engine=table_engines.ReplacingMergeTree(
storage_set=StorageSetKey.EVENTS,
version_column="deleted",
order_by="(org_id, project_id, toStartOfDay(timestamp), primary_hash_hex, event_hash)",
partition_by="(toMonday(timestamp), if(retention_days = 30, 30, 90))",
sample_by="event_hash",
ttl="timestamp + toIntervalDay(retention_days)",
settings={"index_granularity": "8192"},
),
)
]
def forwards_local(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.EVENTS,
table_name="errors_local_new",
columns=columns,
engine=table_engines.ReplacingMergeTree(
storage_set=StorageSetKey.EVENTS,
version_column="deleted",
order_by=
"(project_id, toStartOfDay(timestamp), primary_hash, %s)" %
sample_expr,
partition_by="(retention_days, toMonday(timestamp))",
sample_by=sample_expr,
ttl="timestamp + toIntervalDay(retention_days)",
settings={"index_granularity": "8192"},
),
),
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="errors_local_new",
column=Column(
"_tags_hash_map",
Array(UInt(64),
Modifiers(materialized=TAGS_HASH_MAP_COLUMN)),
),
after="tags",
),
operations.DropTable(
storage_set=StorageSetKey.EVENTS,
table_name="errors_local",
),
operations.RenameTable(
storage_set=StorageSetKey.EVENTS,
old_table_name="errors_local_new",
new_table_name="errors_local",
),
]
def forwards_local(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.REPLAYS,
table_name="replays_local",
columns=raw_columns,
engine=table_engines.ReplacingMergeTree(
storage_set=StorageSetKey.REPLAYS,
order_by=
"(project_id, toStartOfDay(timestamp), cityHash64(replay_id), sequence_id)",
partition_by="(retention_days, toMonday(timestamp))",
settings={"index_granularity": "8192"},
ttl="timestamp + toIntervalDay(retention_days)",
),
),
operations.AddIndex(
storage_set=StorageSetKey.REPLAYS,
table_name="replays_local",
index_name="bf_trace_ids_hashed",
index_expression="_trace_ids_hashed",
index_type="bloom_filter()",
granularity=1,
),
]
),
pytest.param(
table_engines.MergeTree(
storage_set=StorageSetKey.TRANSACTIONS,
order_by="date",
settings={"index_granularity": "256"},
),
"MergeTree() ORDER BY date SETTINGS index_granularity=256",
"ReplicatedMergeTree('/clickhouse/tables/transactions/{shard}/test_table', '{replica}') ORDER BY date SETTINGS index_granularity=256",
id="Merge tree with settings",
),
pytest.param(
table_engines.ReplacingMergeTree(
storage_set=StorageSetKey.EVENTS,
version_column="timestamp",
order_by="timestamp",
partition_by="(toMonday(timestamp))",
sample_by="id",
ttl="timestamp + INTERVAL 1 MONTH",
),
"ReplacingMergeTree(timestamp) ORDER BY timestamp PARTITION BY (toMonday(timestamp)) SAMPLE BY id TTL timestamp + INTERVAL 1 MONTH",
"ReplicatedReplacingMergeTree('/clickhouse/tables/events/{shard}/test_table', '{replica}', timestamp) ORDER BY timestamp PARTITION BY (toMonday(timestamp)) SAMPLE BY id TTL timestamp + INTERVAL 1 MONTH",
id="Replicated merge tree with partition, sample, ttl clauses",
),
pytest.param(
table_engines.MergeTree(storage_set=StorageSetKey.EVENTS,
order_by="timestamp",
unsharded=True),
"MergeTree() ORDER BY timestamp",
"ReplicatedMergeTree('/clickhouse/tables/events/all/test_table', '{replica}') ORDER BY timestamp",
id="Unsharded merge tree",
),
