def forwards_dist(self) -> Sequence[operations.SqlOperation]: return [ operations.AddColumn( storage_set=StorageSetKey.OUTCOMES, table_name="outcomes_raw_dist", column=Column("quantity", UInt(32)), after=None, ), operations.AddColumn( storage_set=StorageSetKey.OUTCOMES, table_name="outcomes_raw_dist", column=Column("category", UInt(8)), after=None, ), operations.AddColumn( storage_set=StorageSetKey.OUTCOMES, table_name="outcomes_hourly_dist", column=Column("quantity", UInt(64)), after=None, ), operations.AddColumn( storage_set=StorageSetKey.OUTCOMES, table_name="outcomes_hourly_dist", column=Column("category", UInt(8)), after=None, ), ]
def forwards_local(self) -> Sequence[operations.SqlOperation]: return [ operations.AddColumn( storage_set=StorageSetKey.OUTCOMES, table_name="outcomes_raw_local", column=Column("quantity", UInt(32)), after="reason", ), operations.AddColumn( storage_set=StorageSetKey.OUTCOMES, table_name="outcomes_raw_local", column=Column("category", UInt(8)), after="timestamp", ), operations.AddColumn( storage_set=StorageSetKey.OUTCOMES, table_name="outcomes_hourly_local", column=Column("quantity", UInt(64)), after="reason", ), operations.RunSql( storage_set=StorageSetKey.OUTCOMES, statement=""" ALTER TABLE outcomes_hourly_local ADD COLUMN IF NOT EXISTS category UInt8 AFTER timestamp, MODIFY ORDER BY (org_id, project_id, key_id, outcome, reason, timestamp, category); """, ), ]
def backwards_dist(self) -> Sequence[operations.SqlOperation]: return [ operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name="transactions_dist", column=Column("_contexts_flattened", String()), ), operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name="transactions_dist", column=Column("_tags_flattened", String()), ), ]
def forwards_dist(self) -> Sequence[operations.SqlOperation]: return [ operations.AddColumn( storage_set=StorageSetKey.EVENTS, table_name="errors_dist", column=Column("hierarchical_hashes", Array(UUID())), after="primary_hash", ), operations.AddColumn( storage_set=StorageSetKey.EVENTS, table_name="sentry_dist", column=Column("hierarchical_hashes", Array(FixedString(32)),), after="primary_hash", ), ]
def forwards_local(self) -> Sequence[operations.Operation]: return [ operations.CreateTable( storage_set=StorageSetKey.TRANSACTIONS, table_name="spans_experimental_local", columns=columns, engine=table_engines.ReplacingMergeTree( storage_set=StorageSetKey.TRANSACTIONS, version_column="deleted", order_by= ("(project_id, toStartOfDay(finish_ts), transaction_name, " "cityHash64(transaction_span_id), op, cityHash64(trace_id), " "cityHash64(span_id))"), partition_by="(toMonday(finish_ts))", sample_by="cityHash64(span_id)", ttl="finish_ts + toIntervalDay(retention_days)", settings={"index_granularity": "8192"}, ), ), operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name="spans_experimental_local", column=Column( "_tags_hash_map", Materialized(Array(UInt(64)), TAGS_HASH_MAP_COLUMN), ), after="tags.value", ), ]
def __forward_migrations( self, table_name: str) -> Sequence[operations.SqlOperation]: return [ operations.AddColumn( storage_set=StorageSetKey.DISCOVER, table_name=table_name, column=Column("_tags_hash_map", Array(UInt(64))), after="tags", ), operations.AddColumn( storage_set=StorageSetKey.DISCOVER, table_name=table_name, column=Column("deleted", UInt(8)), after="contexts", ), ]
def forwards_local(self) -> Sequence[operations.Operation]: return [ operations.AddColumn( storage_set=StorageSetKey.EVENTS, table_name="errors_local", column=Column("http_method", LowCardinality(Nullable(String()))), after="sdk_version", ), operations.AddColumn( storage_set=StorageSetKey.EVENTS, table_name="errors_local", column=Column("http_referer", Nullable(String())), after="http_method", ), ]
def forwards_dist(self) -> Sequence[operations.Operation]: return [ operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name="transactions_dist", column=Column("http_method", LowCardinality(Nullable(String()))), after="sdk_version", ), operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name="transactions_dist", column=Column("http_referer", Nullable(String())), after="http_method", ), ]
def forwards_dist(self) -> Sequence[operations.SqlOperation]: return [ operations.AddColumn( storage_set=StorageSetKey.SESSIONS, table_name="sessions_raw_dist", column=column, after=after, ) for [column, after] in new_raw_columns ] + [ operations.AddColumn( storage_set=StorageSetKey.SESSIONS, table_name="sessions_hourly_dist", column=column, after=after, ) for [column, after] in new_dest_columns ]
def backwards_local(self) -> Sequence[operations.SqlOperation]: sample_expr = "cityHash64(toString(event_id))" return [ operations.CreateTable( storage_set=StorageSetKey.EVENTS, table_name="sentry_local", columns=columns, engine=table_engines.ReplacingMergeTree( storage_set=StorageSetKey.EVENTS, version_column="deleted", order_by="(project_id, toStartOfDay(timestamp), %s)" % sample_expr, partition_by="(toMonday(timestamp), if(equals(retention_days, 30), 30, 90))", sample_by=sample_expr, ), ), operations.AddColumn( storage_set=StorageSetKey.EVENTS, table_name="sentry_local", column=Column( "_tags_hash_map", Array(UInt(64), Modifiers(materialized=TAGS_HASH_MAP_COLUMN)), ), after="_tags_flattened", ), ]
def forwards_dist(self) -> Sequence[operations.SqlOperation]: return [ operations.AddColumn( storage_set=StorageSetKey.GENERIC_METRICS_SETS, table_name=self.table_name, column=self.new_column, ) ]
def __forward_migrations( self, table_name: str ) -> Sequence[operations.SqlOperation]: return [ operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name=table_name, column=Column( "type", String( Modifiers(low_cardinality=True, materialized="'transaction'") ), ), after="deleted", ), operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name=table_name, column=Column( "message", String( Modifiers(low_cardinality=True, materialized="transaction_name") ), ), after="type", ), operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name=table_name, column=Column( "title", String( Modifiers(low_cardinality=True, materialized="transaction_name") ), ), after="message", ), operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name=table_name, column=Column( "timestamp", DateTime(Modifiers(materialized="finish_ts")) ), after="title", ), ]
def get_forward_migrations_local( source_table_name: str, table_name: str, mv_name: str, aggregation_col_schema: Sequence[Column[Modifiers]], aggregation_states: str, granularity: int, ) -> Sequence[operations.SqlOperation]: aggregated_cols = [*COMMON_AGGR_COLUMNS, *aggregation_col_schema] return [ operations.CreateTable( storage_set=StorageSetKey.METRICS, table_name=table_name, columns=aggregated_cols, engine=table_engines.AggregatingMergeTree( storage_set=StorageSetKey.METRICS, order_by= "(org_id, project_id, metric_id, granularity, timestamp, tags.key, tags.value)", partition_by="(retention_days, toMonday(timestamp))", settings={"index_granularity": "256"}, ), ), operations.AddColumn( storage_set=StorageSetKey.METRICS, table_name=table_name, column=Column( "_tags_hash", Array(UInt(64), Modifiers(materialized=INT_TAGS_HASH_MAP_COLUMN)), ), after="tags.value", ), operations.AddIndex( storage_set=StorageSetKey.METRICS, table_name=table_name, index_name="bf_tags_hash", index_expression="_tags_hash", index_type="bloom_filter()", granularity=1, ), operations.AddIndex( storage_set=StorageSetKey.METRICS, table_name=table_name, index_name="bf_tags_key_hash", index_expression="tags.key", index_type="bloom_filter()", granularity=1, ), ] + [ get_forward_view_migration_local( source_table_name, table_name, mv_name, aggregation_col_schema, aggregation_states, granularity, ) ]
def forwards_dist(self) -> Sequence[operations.SqlOperation]: return [ operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name="transactions_dist", column=Column("spans.exclusive_time_32", Array(Float(32))), after="spans.group", ), ]
def forwards_local(self) -> Sequence[operations.SqlOperation]: return [ operations.AddColumn( storage_set=StorageSetKey.EVENTS, table_name="errors_local", column=Column("message_timestamp", DateTime()), after="offset", ) ]
def forwards_dist(self) -> Sequence[operations.Operation]: return [ operations.AddColumn( storage_set=StorageSetKey.EVENTS, table_name="errors_dist", column=Column( "http_method", String(Modifiers(nullable=True, low_cardinality=True)), ), after="sdk_version", ), operations.AddColumn( storage_set=StorageSetKey.EVENTS, table_name="errors_dist", column=Column("http_referer", String(Modifiers(nullable=True))), after="http_method", ), ]
def forwards_dist(self) -> Sequence[operations.Operation]: return [ operations.AddColumn( storage_set=StorageSetKey.EVENTS, table_name="errors_dist", column=Column("_tags_hash_map", Array(UInt(64)),), after="_tags_flattened", ), ]
def fix_order_by() -> None: cluster = get_cluster(StorageSetKey.EVENTS) if not cluster.is_single_node(): return clickhouse = cluster.get_query_connection(ClickhouseClientSettings.MIGRATE) database = cluster.get_database() new_primary_key = "project_id, id" old_primary_key = "id" ((curr_primary_key,),) = clickhouse.execute( f"SELECT primary_key FROM system.tables WHERE name = '{TABLE_NAME}' AND database = '{database}'" ) assert curr_primary_key in [ new_primary_key, old_primary_key, ], "Groupmessage table has invalid primary key" if curr_primary_key != old_primary_key: return # Add the project_id column add_column_sql = operations.AddColumn( storage_set=StorageSetKey.EVENTS, table_name=TABLE_NAME, column=Column("project_id", UInt(64)), after="record_deleted", ).format_sql() clickhouse.execute(add_column_sql) # There shouldn't be any data in the table yet assert ( clickhouse.execute(f"SELECT COUNT() FROM {TABLE_NAME} FINAL;")[0][0] == 0 ), f"{TABLE_NAME} is not empty" new_order_by = f"ORDER BY ({new_primary_key})" old_order_by = f"ORDER BY {old_primary_key}" ((curr_create_table_statement,),) = clickhouse.execute( f"SHOW CREATE TABLE {database}.{TABLE_NAME}" ) new_create_table_statement = curr_create_table_statement.replace( TABLE_NAME, TABLE_NAME_NEW ).replace(old_order_by, new_order_by) clickhouse.execute(new_create_table_statement) clickhouse.execute(f"RENAME TABLE {TABLE_NAME} TO {TABLE_NAME_OLD};") clickhouse.execute(f"RENAME TABLE {TABLE_NAME_NEW} TO {TABLE_NAME};") clickhouse.execute(f"DROP TABLE {TABLE_NAME_OLD};")
def __forward_migrations( self, table_name: str) -> Sequence[operations.SqlOperation]: return [ operations.AddColumn( storage_set=StorageSetKey.DISCOVER, table_name=table_name, column=Column("trace_id", UUID(Modifiers(nullable=True))), after="contexts", ), ]
def forwards_local(self) -> Sequence[operations.Operation]: return [ operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name="transactions_local", column=Column( "http_method", String(Modifiers(nullable=True, low_cardinality=True)), ), after="sdk_version", ), operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name="transactions_local", column=Column("http_referer", String(Modifiers(nullable=True))), after="http_method", ), ]
def forwards_local(self) -> Sequence[operations.Operation]: return [ operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name="transactions_local", column=Column( "_tags_hash_map", Materialized(Array(UInt(64)), TAGS_HASH_MAP_COLUMN), ), after="_tags_flattened", ), ]
def forwards_local(self) -> Sequence[operations.Operation]: return [ operations.AddColumn( storage_set=StorageSetKey.EVENTS, table_name="errors_local", column=Column( "_tags_hash_map", Array(UInt(64), Modifiers(materialized=TAGS_HASH_MAP_COLUMN)), ), after="_tags_flattened", ), ]
def forwards_dist(self) -> Sequence[operations.Operation]: return [ operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name="transactions_dist", column=Column( "measurements", Nested([("key", LowCardinality(String())), ("value", Float(64))]), ), after="_contexts_flattened", ), ]
def forwards_dist(self) -> Sequence[operations.SqlOperation]: return [ operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name="transactions_dist", column=Column( "_tags_hash_map", Array(UInt(64)), ), after="_tags_flattened", ), ]
def __forward_migrations( self, table_name: str) -> Sequence[operations.SqlOperation]: return [ operations.AddColumn( storage_set=StorageSetKey.METRICS, table_name=table_name, column=Column( "histogram_buckets", AggregateFunction("histogram(250)", [Float(64)]), ), after="count", ) ]
def forwards_local(self) -> Sequence[operations.SqlOperation]: return [ operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name="transactions_local", column=Column( "measurements", Nested([ ("key", String(Modifiers(low_cardinality=True))), ("value", Float(64)), ]), ), after="_contexts_flattened", ), ]
def forwards_dist(self) -> Sequence[operations.SqlOperation]: return [ operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name="transactions_dist", column=Column( "span_op_breakdowns", Nested([ ("key", String(Modifiers(low_cardinality=True))), ("value", Float(64)), ]), ), after="measurements.value", ), ]
def forwards_local(self) -> Sequence[operations.SqlOperation]: return [ operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name="transactions_local", column=Column("spans.exclusive_time_32", Array(Float(32))), after="spans.group", ), operations.ModifyColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name="transactions_local", column=Column("spans.exclusive_time_32", Array(Float(32))), ttl_month=("finish_ts", 1), ), ]
def forwards_local(self) -> Sequence[operations.SqlOperation]: return [ operations.CreateTable( storage_set=StorageSetKey.METRICS, table_name=self.table_name, columns=self.aggregated_cols, engine=table_engines.AggregatingMergeTree( storage_set=StorageSetKey.METRICS, order_by= "(use_case_id, org_id, project_id, metric_id, granularity, timestamp, tags.key, tags.value, retention_days)", primary_key= "(use_case_id, org_id, project_id, metric_id, granularity, timestamp)", partition_by="(retention_days, toMonday(timestamp))", settings={"index_granularity": self.granularity}, ttl="timestamp + toIntervalDay(retention_days)", ), ), operations.AddColumn( storage_set=StorageSetKey.METRICS, table_name=self.table_name, column=Column( "_tags_hash", Array(UInt(64), Modifiers(materialized=INT_TAGS_HASH_MAP_COLUMN)), ), after="tags.value", ), operations.AddIndex( storage_set=StorageSetKey.METRICS, table_name=self.table_name, index_name="bf_tags_hash", index_expression="_tags_hash", index_type="bloom_filter()", granularity=1, ), operations.AddIndex( storage_set=StorageSetKey.METRICS, table_name=self.table_name, index_name="bf_tags_key_hash", index_expression="tags.key", index_type="bloom_filter()", granularity=1, ), ]
def forwards_dist(self) -> Sequence[operations.SqlOperation]: return [ operations.CreateTable( storage_set=StorageSetKey.METRICS, table_name=self.dist_table_name, columns=self.aggregated_cols, engine=table_engines.Distributed( local_table_name=self.table_name, sharding_key=None), ), operations.AddColumn( storage_set=StorageSetKey.METRICS, table_name=self.dist_table_name, column=Column( "_tags_hash", Array(UInt(64), Modifiers(materialized=INT_TAGS_HASH_MAP_COLUMN)), ), after="tags.value", ), ]