def __backward_migrations( self, table_name: str ) -> Sequence[operations.SqlOperation]: return [ operations.ModifyColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name=table_name, column=Column( "title", String( Modifiers(low_cardinality=True, materialized="transaction_name") ), ), ), operations.ModifyColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name=table_name, column=Column( "message", String( Modifiers(low_cardinality=True, materialized="transaction_name") ), ), ), ]
def __backwards_migrations( self, table_name: str) -> Sequence[operations.SqlOperation]: return [ operations.ModifyColumn( storage_set=StorageSetKey.DISCOVER, table_name=table_name, column=Column("title", String(Modifiers(nullable=True))), ), operations.ModifyColumn( storage_set=StorageSetKey.DISCOVER, table_name=table_name, column=Column("message", String(Modifiers(nullable=True))), ), ]
def backwards_local(self) -> Sequence[operations.SqlOperation]: sample_expr = "cityHash64(toString(event_id))" return [ operations.CreateTable( storage_set=StorageSetKey.EVENTS, table_name="sentry_local", columns=columns, engine=table_engines.ReplacingMergeTree( storage_set=StorageSetKey.EVENTS, version_column="deleted", order_by="(project_id, toStartOfDay(timestamp), %s)" % sample_expr, partition_by="(toMonday(timestamp), if(equals(retention_days, 30), 30, 90))", sample_by=sample_expr, ), ), operations.AddColumn( storage_set=StorageSetKey.EVENTS, table_name="sentry_local", column=Column( "_tags_hash_map", Array(UInt(64), Modifiers(materialized=TAGS_HASH_MAP_COLUMN)), ), after="_tags_flattened", ), ]
def get_forward_view_migration_polymorphic_table_v3( source_table_name: str, table_name: str, mv_name: str, aggregation_col_schema: Sequence[Column[Modifiers]], aggregation_states: str, metric_type: str, target_mat_version: int, appended_where_clause: str = "", ) -> operations.SqlOperation: aggregated_cols = [ Column("use_case_id", String(Modifiers(low_cardinality=True))), *COMMON_AGGR_COLUMNS, *aggregation_col_schema, ] return operations.CreateMaterializedView( storage_set=StorageSetKey.METRICS, view_name=mv_name, destination_table_name=table_name, columns=aggregated_cols, query=MATVIEW_STATEMENT_POLYMORPHIC_TABLE_V3 % { "metric_type": metric_type, "raw_table_name": source_table_name, "aggregation_states": aggregation_states, "target_mat_version": target_mat_version, "appended_where_clause": appended_where_clause, "materialization_version": target_mat_version, }, )
def forwards_local(self) -> Sequence[operations.SqlOperation]: return [ operations.CreateTable( storage_set=StorageSetKey.TRANSACTIONS, table_name="spans_experimental_local", columns=columns, engine=table_engines.ReplacingMergeTree( storage_set=StorageSetKey.TRANSACTIONS, version_column="deleted", order_by= ("(project_id, toStartOfDay(finish_ts), transaction_name, " "cityHash64(transaction_span_id), op, cityHash64(trace_id), " "cityHash64(span_id))"), partition_by="(toMonday(finish_ts))", sample_by="cityHash64(span_id)", ttl="finish_ts + toIntervalDay(retention_days)", settings={"index_granularity": "8192"}, ), ), operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name="spans_experimental_local", column=Column( "_tags_hash_map", Array(UInt(64), Modifiers(materialized=TAGS_HASH_MAP_COLUMN)), ), after="tags.value", ), ]
def __forward_migrations( self, table_name: str) -> Sequence[operations.Operation]: return [ operations.ModifyColumn( StorageSetKey.QUERYLOG, table_name, Column("status", String(Modifiers(low_cardinality=True))), ), operations.ModifyColumn( StorageSetKey.QUERYLOG, table_name, Column( "clickhouse_queries.status", Array(String(Modifiers(low_cardinality=True))), ), ), ]
def __forward_migrations( self, table_name: str ) -> Sequence[operations.SqlOperation]: return [ operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name=table_name, column=Column( "type", String( Modifiers(low_cardinality=True, materialized="'transaction'") ), ), after="deleted", ), operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name=table_name, column=Column( "message", String( Modifiers(low_cardinality=True, materialized="transaction_name") ), ), after="type", ), operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name=table_name, column=Column( "title", String( Modifiers(low_cardinality=True, materialized="transaction_name") ), ), after="message", ), operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name=table_name, column=Column( "timestamp", DateTime(Modifiers(materialized="finish_ts")) ), after="title", ), ]
def get_forward_migrations_local( source_table_name: str, table_name: str, mv_name: str, aggregation_col_schema: Sequence[Column[Modifiers]], aggregation_states: str, granularity: int, ) -> Sequence[operations.SqlOperation]: aggregated_cols = [*COMMON_AGGR_COLUMNS, *aggregation_col_schema] return [ operations.CreateTable( storage_set=StorageSetKey.METRICS, table_name=table_name, columns=aggregated_cols, engine=table_engines.AggregatingMergeTree( storage_set=StorageSetKey.METRICS, order_by= "(org_id, project_id, metric_id, granularity, timestamp, tags.key, tags.value)", partition_by="(retention_days, toMonday(timestamp))", settings={"index_granularity": "256"}, ), ), operations.AddColumn( storage_set=StorageSetKey.METRICS, table_name=table_name, column=Column( "_tags_hash", Array(UInt(64), Modifiers(materialized=INT_TAGS_HASH_MAP_COLUMN)), ), after="tags.value", ), operations.AddIndex( storage_set=StorageSetKey.METRICS, table_name=table_name, index_name="bf_tags_hash", index_expression="_tags_hash", index_type="bloom_filter()", granularity=1, ), operations.AddIndex( storage_set=StorageSetKey.METRICS, table_name=table_name, index_name="bf_tags_key_hash", index_expression="tags.key", index_type="bloom_filter()", granularity=1, ), ] + [ get_forward_view_migration_local( source_table_name, table_name, mv_name, aggregation_col_schema, aggregation_states, granularity, ) ]
def __backwards_migrations( self, table_name: str) -> Sequence[operations.Operation]: return [ operations.ModifyColumn( storage_set=StorageSetKey.DISCOVER, table_name=table_name, column=Column("user", String(Modifiers(low_cardinality=True))), ) ]
def __forward_migrations( self, table_name: str) -> Sequence[operations.SqlOperation]: return [ operations.ModifyColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name=table_name, column=Column("trace_id", UUID(Modifiers(nullable=True))), ), ]
def forwards_dist(self) -> Sequence[operations.Operation]: return [ operations.AddColumn( storage_set=StorageSetKey.EVENTS, table_name="errors_dist", column=Column( "http_method", String(Modifiers(nullable=True, low_cardinality=True)), ), after="sdk_version", ), operations.AddColumn( storage_set=StorageSetKey.EVENTS, table_name="errors_dist", column=Column("http_referer", String(Modifiers(nullable=True))), after="http_method", ), ]
def __forward_migrations( self, table_name: str) -> Sequence[operations.SqlOperation]: return [ operations.AddColumn( storage_set=StorageSetKey.DISCOVER, table_name=table_name, column=Column("trace_id", UUID(Modifiers(nullable=True))), after="contexts", ), ]
def forwards_local(self) -> Sequence[operations.Operation]: return [ operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name="transactions_local", column=Column( "http_method", String(Modifiers(nullable=True, low_cardinality=True)), ), after="sdk_version", ), operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name="transactions_local", column=Column("http_referer", String(Modifiers(nullable=True))), after="http_method", ), ]
def __forward_migrations( self, table_name: str) -> Sequence[operations.Operation]: return [ operations.ModifyColumn( storage_set=StorageSetKey.EVENTS, table_name=table_name, column=Column( "level", String(Modifiers(low_cardinality=True, nullable=True))), ) ]
def forwards_local(self) -> Sequence[operations.Operation]: return [ operations.AddColumn( storage_set=StorageSetKey.EVENTS, table_name="errors_local", column=Column( "_tags_hash_map", Array(UInt(64), Modifiers(materialized=TAGS_HASH_MAP_COLUMN)), ), after="_tags_flattened", ), ]
def __forward_migrations( self, table_name: str ) -> Sequence[operations.SqlOperation]: return [ operations.ModifyColumn( storage_set=StorageSetKey.DISCOVER, table_name=table_name, column=Column( "transaction_name", String(Modifiers(low_cardinality=True)) ), ), ]
def forwards_dist(self) -> Sequence[operations.SqlOperation]: return [ operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name="transactions_dist", column=Column( "_tags_hash_map", Array(UInt(64), Modifiers(materialized=TAGS_HASH_MAP_COLUMN)), ), after="_tags_flattened", ), ]
def forwards_dist(self) -> Sequence[operations.SqlOperation]: return [ operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name="transactions_dist", column=Column( "span_op_breakdowns", Nested([ ("key", String(Modifiers(low_cardinality=True))), ("value", Float(64)), ]), ), after="measurements.value", ), ]
def forwards_local(self) -> Sequence[operations.SqlOperation]: return [ operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name="transactions_local", column=Column( "measurements", Nested([ ("key", String(Modifiers(low_cardinality=True))), ("value", Float(64)), ]), ), after="_contexts_flattened", ), ]
def forwards_local(self) -> Sequence[operations.SqlOperation]: return [ operations.CreateTable( storage_set=StorageSetKey.METRICS, table_name=self.table_name, columns=self.aggregated_cols, engine=table_engines.AggregatingMergeTree( storage_set=StorageSetKey.METRICS, order_by= "(use_case_id, org_id, project_id, metric_id, granularity, timestamp, tags.key, tags.value, retention_days)", primary_key= "(use_case_id, org_id, project_id, metric_id, granularity, timestamp)", partition_by="(retention_days, toMonday(timestamp))", settings={"index_granularity": self.granularity}, ttl="timestamp + toIntervalDay(retention_days)", ), ), operations.AddColumn( storage_set=StorageSetKey.METRICS, table_name=self.table_name, column=Column( "_tags_hash", Array(UInt(64), Modifiers(materialized=INT_TAGS_HASH_MAP_COLUMN)), ), after="tags.value", ), operations.AddIndex( storage_set=StorageSetKey.METRICS, table_name=self.table_name, index_name="bf_tags_hash", index_expression="_tags_hash", index_type="bloom_filter()", granularity=1, ), operations.AddIndex( storage_set=StorageSetKey.METRICS, table_name=self.table_name, index_name="bf_tags_key_hash", index_expression="tags.key", index_type="bloom_filter()", granularity=1, ), ]
def forwards_dist(self) -> Sequence[operations.SqlOperation]: return [ operations.CreateTable( storage_set=StorageSetKey.METRICS, table_name=self.dist_table_name, columns=self.aggregated_cols, engine=table_engines.Distributed( local_table_name=self.table_name, sharding_key=None), ), operations.AddColumn( storage_set=StorageSetKey.METRICS, table_name=self.dist_table_name, column=Column( "_tags_hash", Array(UInt(64), Modifiers(materialized=INT_TAGS_HASH_MAP_COLUMN)), ), after="tags.value", ), ]
def forwards_dist(self) -> Sequence[operations.Operation]: return [ operations.CreateTable( storage_set=StorageSetKey.TRANSACTIONS, table_name="spans_experimental_dist", columns=columns, engine=table_engines.Distributed( local_table_name="spans_experimental_local", sharding_key="cityHash64(transaction_span_id)", ), ), operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name="spans_experimental_dist", column=Column( "_tags_hash_map", Array(UInt(64), Modifiers(materialized=TAGS_HASH_MAP_COLUMN)), ), after="tags.value", ), ]
def forwards_dist(self) -> Sequence[operations.SqlOperation]: return [ operations.CreateTable( storage_set=StorageSetKey.EVENTS, table_name="errors_dist_new", columns=columns, engine=table_engines.Distributed( local_table_name="errors_local", sharding_key=sample_expr, ), ), operations.AddColumn( storage_set=StorageSetKey.EVENTS, table_name="errors_dist_new", column=Column( "_tags_hash_map", Array(UInt(64), Modifiers(materialized=TAGS_HASH_MAP_COLUMN)), ), after="tags", ), operations.DropTable( storage_set=StorageSetKey.EVENTS, table_name="errors_dist", ), operations.RenameTable( storage_set=StorageSetKey.EVENTS, old_table_name="errors_dist_new", new_table_name="errors_dist", ), operations.CreateTable( storage_set=StorageSetKey.EVENTS_RO, table_name="errors_dist_ro", columns=columns, engine=table_engines.Distributed( local_table_name="errors_local", sharding_key=sample_expr, ), ), ]
def forwards_local(self) -> Sequence[operations.Operation]: return [ operations.CreateTable( storage_set=StorageSetKey.EVENTS, table_name="errors_local_new", columns=columns, engine=table_engines.ReplacingMergeTree( storage_set=StorageSetKey.EVENTS, version_column="deleted", order_by= "(project_id, toStartOfDay(timestamp), primary_hash, %s)" % sample_expr, partition_by="(retention_days, toMonday(timestamp))", sample_by=sample_expr, ttl="timestamp + toIntervalDay(retention_days)", settings={"index_granularity": "8192"}, ), ), operations.AddColumn( storage_set=StorageSetKey.EVENTS, table_name="errors_local_new", column=Column( "_tags_hash_map", Array(UInt(64), Modifiers(materialized=TAGS_HASH_MAP_COLUMN)), ), after="tags", ), operations.DropTable( storage_set=StorageSetKey.EVENTS, table_name="errors_local", ), operations.RenameTable( storage_set=StorageSetKey.EVENTS, old_table_name="errors_local_new", new_table_name="errors_local", ), ]
def get_forward_migrations_dist( dist_table_name: str, local_table_name: str, aggregation_col_schema: Sequence[Column[Modifiers]], ) -> Sequence[operations.SqlOperation]: return [ operations.CreateTable( storage_set=StorageSetKey.METRICS, table_name=dist_table_name, columns=[*COMMON_AGGR_COLUMNS, *aggregation_col_schema], engine=table_engines.Distributed(local_table_name=local_table_name, sharding_key=None), ), operations.AddColumn( storage_set=StorageSetKey.METRICS, table_name=dist_table_name, column=Column( "_tags_hash", Array(UInt(64), Modifiers(materialized=INT_TAGS_HASH_MAP_COLUMN)), ), after="tags.value", ), ]
from typing import Sequence from snuba.clickhouse.columns import Column, DateTime, UInt from snuba.clusters.storage_sets import StorageSetKey from snuba.migrations import migration, operations, table_engines from snuba.migrations.columns import MigrationModifiers as Modifiers columns: Sequence[Column[Modifiers]] = [ # Kafka topic offset Column("offset", UInt(64)), Column("record_deleted", UInt(8)), # PG columns Column("project_id", UInt(64)), Column("group_id", UInt(64)), Column("date_added", DateTime(Modifiers(nullable=True))), Column("user_id", UInt(64, Modifiers(nullable=True))), Column("team_id", UInt(64, Modifiers(nullable=True))), ] class Migration(migration.ClickhouseNodeMigration): blocking = False def forwards_local(self) -> Sequence[operations.SqlOperation]: return [ operations.CreateTable( storage_set=StorageSetKey.CDC, table_name="groupassignee_local", columns=columns, engine=table_engines.ReplacingMergeTree( storage_set=StorageSetKey.CDC,
from typing import Sequence from snuba.clickhouse.columns import Column, DateTime, Enum, String, UInt from snuba.clusters.storage_sets import StorageSetKey from snuba.migrations import migration, operations from snuba.migrations.columns import MigrationModifiers as Modifiers from snuba.migrations.context import Context from snuba.migrations.status import Status from snuba.migrations.table_engines import Distributed, ReplacingMergeTree columns: Sequence[Column[Modifiers]] = [ Column("group", String()), Column("migration_id", String()), Column("timestamp", DateTime()), Column("status", Enum([("completed", 0), ("in_progress", 1), ("not_started", 2)]),), Column("version", UInt(64, Modifiers(default="1"))), ] class Migration(migration.Migration): """ This migration extends Migration instead of MultiStepMigration since it is responsible for bootstrapping the migration system itself. It skips setting the in progress status in the forwards method and the not started status in the backwards method. Since the migration table doesn't exist yet, we can't write any statuses until this migration is completed. """ blocking = False def __forwards_local(self) -> Sequence[operations.Operation]:
Column, DateTime, String, UInt, ) from snuba.clusters.storage_sets import StorageSetKey from snuba.migrations import migration, operations from snuba.migrations.columns import MigrationModifiers as Modifiers from snuba.processor import MAX_UINT32, NIL_UUID from .matview import create_matview_v1 aggregate_columns: Sequence[Column[Modifiers]] = [ Column("org_id", UInt(64)), Column("project_id", UInt(64)), Column("started", DateTime()), Column("release", String(Modifiers(low_cardinality=True))), Column("environment", String(Modifiers(low_cardinality=True))), Column("user_agent", String(Modifiers(low_cardinality=True))), Column("os", String(Modifiers(low_cardinality=True))), # durations Column( "duration_quantiles", AggregateFunction("quantilesIf(0.5, 0.9)", [UInt(32), UInt(8)]), ), Column("duration_avg", AggregateFunction("avgIf", [UInt(32), UInt(8)])), # sessions: Column("sessions", AggregateFunction("countIf", [UUID(), UInt(8)])), Column("sessions_preaggr", AggregateFunction("sumIf", [UInt(32), UInt(8)])),
from typing import Sequence from snuba.clickhouse.columns import UUID, Column, DateTime, String, UInt from snuba.clusters.storage_sets import StorageSetKey from snuba.migrations import migration, operations, table_engines from snuba.migrations.columns import MigrationModifiers as Modifiers raw_columns = [ Column("org_id", UInt(64)), Column("project_id", UInt(64)), Column("key_id", UInt(64, Modifiers(nullable=True))), Column("timestamp", DateTime()), Column("outcome", UInt(8)), Column("reason", String(Modifiers(nullable=True, low_cardinality=True))), Column("event_id", UUID(Modifiers(nullable=True))), ] hourly_columns = [ Column("org_id", UInt(64)), Column("project_id", UInt(64)), Column("key_id", UInt(64)), Column("timestamp", DateTime()), Column("outcome", UInt(8)), Column("reason", String(Modifiers(low_cardinality=True))), Column("times_seen", UInt(64)), ] materialized_view_columns = [ Column("org_id", UInt(64)), Column("project_id", UInt(64)), Column("key_id", UInt(64)),
String, UInt, ) from snuba.clusters.storage_sets import StorageSetKey from snuba.migrations import migration, operations, table_engines from snuba.migrations.columns import Materialized from snuba.migrations.columns import MigrationModifiers as Modifiers UNKNOWN_SPAN_STATUS = 2 columns: List[Column[Modifiers]] = [ Column("project_id", UInt(64)), Column("event_id", UUID()), Column("trace_id", UUID()), Column("span_id", UInt(64)), Column("transaction_name", String(Modifiers(low_cardinality=True))), Column( "transaction_hash", UInt(64, Modifiers(materialized="cityHash64(transaction_name)")), ), Column("transaction_op", String(Modifiers(low_cardinality=True))), Column("transaction_status", UInt(8, Modifiers(default=str(UNKNOWN_SPAN_STATUS)))), Column("start_ts", DateTime()), Column("start_ms", UInt(16)), Column("finish_ts", DateTime()), Column("finish_ms", UInt(16)), Column("duration", UInt(32)), Column("platform", String(Modifiers(low_cardinality=True))), Column("environment", String(Modifiers(nullable=True, low_cardinality=True))),