def forwards_local(self) -> Sequence[operations.Operation]: return [ operations.CreateTable( storage_set=StorageSetKey.TRANSACTIONS, table_name="spans_experimental_local", columns=columns, engine=table_engines.ReplacingMergeTree( storage_set=StorageSetKey.TRANSACTIONS, version_column="deleted", order_by= ("(project_id, toStartOfDay(finish_ts), transaction_name, " "cityHash64(transaction_span_id), op, cityHash64(trace_id), " "cityHash64(span_id))"), partition_by="(toMonday(finish_ts))", sample_by="cityHash64(span_id)", ttl="finish_ts + toIntervalDay(retention_days)", settings={"index_granularity": "8192"}, ), ), operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name="spans_experimental_local", column=Column( "_tags_hash_map", Materialized(Array(UInt(64)), TAGS_HASH_MAP_COLUMN), ), after="tags.value", ), ]
def backwards_local(self) -> Sequence[operations.SqlOperation]: sample_expr = "cityHash64(toString(event_id))" return [ operations.CreateTable( storage_set=StorageSetKey.EVENTS, table_name="sentry_local", columns=columns, engine=table_engines.ReplacingMergeTree( storage_set=StorageSetKey.EVENTS, version_column="deleted", order_by="(project_id, toStartOfDay(timestamp), %s)" % sample_expr, partition_by="(toMonday(timestamp), if(equals(retention_days, 30), 30, 90))", sample_by=sample_expr, ), ), operations.AddColumn( storage_set=StorageSetKey.EVENTS, table_name="sentry_local", column=Column( "_tags_hash_map", Array(UInt(64), Modifiers(materialized=TAGS_HASH_MAP_COLUMN)), ), after="_tags_flattened", ), ]
def forwards_local(self) -> Sequence[operations.SqlOperation]: return [ operations.CreateTable( storage_set=StorageSetKey.CDC, table_name="groupassignee_local", columns=columns, engine=table_engines.ReplacingMergeTree( storage_set=StorageSetKey.CDC, version_column="offset", order_by="(project_id, group_id)", unsharded=True, ), ) ]
def forwards_local(self) -> Sequence[operations.Operation]: return [ operations.CreateTable( storage_set=StorageSetKey.EVENTS, table_name="groupedmessage_local", columns=columns, engine=table_engines.ReplacingMergeTree( storage_set=StorageSetKey.EVENTS, version_column="offset", order_by="(project_id, id)", sample_by="id", unsharded=True, ), ) ]
def forwards_local(self) -> Sequence[operations.Operation]: return [ operations.CreateTable( storage_set=StorageSetKey.TRANSACTIONS, table_name="transactions_local", columns=columns, engine=table_engines.ReplacingMergeTree( storage_set=StorageSetKey.TRANSACTIONS, version_column="deleted", order_by="(project_id, toStartOfDay(finish_ts), transaction_name, cityHash64(span_id))", partition_by="(retention_days, toMonday(finish_ts))", sample_by="cityHash64(span_id)", ttl="finish_ts + toIntervalDay(retention_days)", settings={"index_granularity": "8192"}, ), ) ]
def forwards_local(self) -> Sequence[operations.SqlOperation]: return [ operations.CreateTable( storage_set=StorageSetKey.PROFILES, table_name="profiles_local", columns=columns, engine=table_engines.ReplacingMergeTree( order_by= "(organization_id, project_id, toStartOfDay(received), cityHash64(profile_id))", partition_by="(retention_days, toMonday(received))", sample_by="cityHash64(profile_id)", settings={"index_granularity": "8192"}, storage_set=StorageSetKey.PROFILES, ttl="received + toIntervalDay(retention_days)", ), ) ]
def forwards_local(self) -> Sequence[operations.Operation]: sample_expr = "cityHash64(toString(event_id))" return [ operations.CreateTable( storage_set=StorageSetKey.EVENTS, table_name="sentry_local", columns=columns, engine=table_engines.ReplacingMergeTree( storage_set=StorageSetKey.EVENTS, version_column="deleted", order_by="(project_id, toStartOfDay(timestamp), %s)" % sample_expr, partition_by="(toMonday(timestamp), if(equals(retention_days, 30), 30, 90))", sample_by=sample_expr, ), ), ]
def forwards_local(self) -> Sequence[operations.Operation]: return [ operations.CreateTable( storage_set=StorageSetKey.EVENTS, table_name="errors_local", columns=columns, engine=table_engines.ReplacingMergeTree( storage_set=StorageSetKey.EVENTS, version_column="deleted", order_by="(org_id, project_id, toStartOfDay(timestamp), primary_hash_hex, event_hash)", partition_by="(toMonday(timestamp), if(retention_days = 30, 30, 90))", sample_by="event_hash", ttl="timestamp + toIntervalDay(retention_days)", settings={"index_granularity": "8192"}, ), ) ]
def forwards_local(self) -> Sequence[operations.Operation]: return [ operations.CreateTable( storage_set=StorageSetKey.EVENTS, table_name="errors_local_new", columns=columns, engine=table_engines.ReplacingMergeTree( storage_set=StorageSetKey.EVENTS, version_column="deleted", order_by= "(project_id, toStartOfDay(timestamp), primary_hash, %s)" % sample_expr, partition_by="(retention_days, toMonday(timestamp))", sample_by=sample_expr, ttl="timestamp + toIntervalDay(retention_days)", settings={"index_granularity": "8192"}, ), ), operations.AddColumn( storage_set=StorageSetKey.EVENTS, table_name="errors_local_new", column=Column( "_tags_hash_map", Array(UInt(64), Modifiers(materialized=TAGS_HASH_MAP_COLUMN)), ), after="tags", ), operations.DropTable( storage_set=StorageSetKey.EVENTS, table_name="errors_local", ), operations.RenameTable( storage_set=StorageSetKey.EVENTS, old_table_name="errors_local_new", new_table_name="errors_local", ), ]
def forwards_local(self) -> Sequence[operations.SqlOperation]: return [ operations.CreateTable( storage_set=StorageSetKey.REPLAYS, table_name="replays_local", columns=raw_columns, engine=table_engines.ReplacingMergeTree( storage_set=StorageSetKey.REPLAYS, order_by= "(project_id, toStartOfDay(timestamp), cityHash64(replay_id), sequence_id)", partition_by="(retention_days, toMonday(timestamp))", settings={"index_granularity": "8192"}, ttl="timestamp + toIntervalDay(retention_days)", ), ), operations.AddIndex( storage_set=StorageSetKey.REPLAYS, table_name="replays_local", index_name="bf_trace_ids_hashed", index_expression="_trace_ids_hashed", index_type="bloom_filter()", granularity=1, ), ]
), pytest.param( table_engines.MergeTree( storage_set=StorageSetKey.TRANSACTIONS, order_by="date", settings={"index_granularity": "256"}, ), "MergeTree() ORDER BY date SETTINGS index_granularity=256", "ReplicatedMergeTree('/clickhouse/tables/transactions/{shard}/test_table', '{replica}') ORDER BY date SETTINGS index_granularity=256", id="Merge tree with settings", ), pytest.param( table_engines.ReplacingMergeTree( storage_set=StorageSetKey.EVENTS, version_column="timestamp", order_by="timestamp", partition_by="(toMonday(timestamp))", sample_by="id", ttl="timestamp + INTERVAL 1 MONTH", ), "ReplacingMergeTree(timestamp) ORDER BY timestamp PARTITION BY (toMonday(timestamp)) SAMPLE BY id TTL timestamp + INTERVAL 1 MONTH", "ReplicatedReplacingMergeTree('/clickhouse/tables/events/{shard}/test_table', '{replica}', timestamp) ORDER BY timestamp PARTITION BY (toMonday(timestamp)) SAMPLE BY id TTL timestamp + INTERVAL 1 MONTH", id="Replicated merge tree with partition, sample, ttl clauses", ), pytest.param( table_engines.MergeTree(storage_set=StorageSetKey.EVENTS, order_by="timestamp", unsharded=True), "MergeTree() ORDER BY timestamp", "ReplicatedMergeTree('/clickhouse/tables/events/all/test_table', '{replica}') ORDER BY timestamp", id="Unsharded merge tree", ),