def __backward_migrations(
self, table_name: str
) -> Sequence[operations.SqlOperation]:
return [
operations.ModifyColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name=table_name,
column=Column(
"title",
String(
Modifiers(low_cardinality=True, materialized="transaction_name")
),
),
),
operations.ModifyColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name=table_name,
column=Column(
"message",
String(
Modifiers(low_cardinality=True, materialized="transaction_name")
),
),
),
]
def __backwards_migrations(
self, table_name: str) -> Sequence[operations.SqlOperation]:
return [
operations.ModifyColumn(
storage_set=StorageSetKey.DISCOVER,
table_name=table_name,
column=Column("title", String(Modifiers(nullable=True))),
),
operations.ModifyColumn(
storage_set=StorageSetKey.DISCOVER,
table_name=table_name,
column=Column("message", String(Modifiers(nullable=True))),
),
]
def backwards_local(self) -> Sequence[operations.SqlOperation]:
sample_expr = "cityHash64(toString(event_id))"
return [
operations.CreateTable(
storage_set=StorageSetKey.EVENTS,
table_name="sentry_local",
columns=columns,
engine=table_engines.ReplacingMergeTree(
storage_set=StorageSetKey.EVENTS,
version_column="deleted",
order_by="(project_id, toStartOfDay(timestamp), %s)" % sample_expr,
partition_by="(toMonday(timestamp), if(equals(retention_days, 30), 30, 90))",
sample_by=sample_expr,
),
),
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="sentry_local",
column=Column(
"_tags_hash_map",
Array(UInt(64), Modifiers(materialized=TAGS_HASH_MAP_COLUMN)),
),
after="_tags_flattened",
),
]
def get_forward_view_migration_polymorphic_table_v3(
source_table_name: str,
table_name: str,
mv_name: str,
aggregation_col_schema: Sequence[Column[Modifiers]],
aggregation_states: str,
metric_type: str,
target_mat_version: int,
appended_where_clause: str = "",
) -> operations.SqlOperation:
aggregated_cols = [
Column("use_case_id", String(Modifiers(low_cardinality=True))),
*COMMON_AGGR_COLUMNS,
*aggregation_col_schema,
]
return operations.CreateMaterializedView(
storage_set=StorageSetKey.METRICS,
view_name=mv_name,
destination_table_name=table_name,
columns=aggregated_cols,
query=MATVIEW_STATEMENT_POLYMORPHIC_TABLE_V3 % {
"metric_type": metric_type,
"raw_table_name": source_table_name,
"aggregation_states": aggregation_states,
"target_mat_version": target_mat_version,
"appended_where_clause": appended_where_clause,
"materialization_version": target_mat_version,
},
)
def forwards_local(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="spans_experimental_local",
columns=columns,
engine=table_engines.ReplacingMergeTree(
storage_set=StorageSetKey.TRANSACTIONS,
version_column="deleted",
order_by=
("(project_id, toStartOfDay(finish_ts), transaction_name, "
"cityHash64(transaction_span_id), op, cityHash64(trace_id), "
"cityHash64(span_id))"),
partition_by="(toMonday(finish_ts))",
sample_by="cityHash64(span_id)",
ttl="finish_ts + toIntervalDay(retention_days)",
settings={"index_granularity": "8192"},
),
),
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="spans_experimental_local",
column=Column(
"_tags_hash_map",
Array(UInt(64),
Modifiers(materialized=TAGS_HASH_MAP_COLUMN)),
),
after="tags.value",
),
]
def __forward_migrations(
self, table_name: str) -> Sequence[operations.Operation]:
return [
operations.ModifyColumn(
StorageSetKey.QUERYLOG,
table_name,
Column("status", String(Modifiers(low_cardinality=True))),
),
operations.ModifyColumn(
StorageSetKey.QUERYLOG,
table_name,
Column(
"clickhouse_queries.status",
Array(String(Modifiers(low_cardinality=True))),
),
),
]
def __forward_migrations(
self, table_name: str
) -> Sequence[operations.SqlOperation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name=table_name,
column=Column(
"type",
String(
Modifiers(low_cardinality=True, materialized="'transaction'")
),
),
after="deleted",
),
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name=table_name,
column=Column(
"message",
String(
Modifiers(low_cardinality=True, materialized="transaction_name")
),
),
after="type",
),
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name=table_name,
column=Column(
"title",
String(
Modifiers(low_cardinality=True, materialized="transaction_name")
),
),
after="message",
),
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name=table_name,
column=Column(
"timestamp", DateTime(Modifiers(materialized="finish_ts"))
),
after="title",
),
]
def get_forward_migrations_local(
source_table_name: str,
table_name: str,
mv_name: str,
aggregation_col_schema: Sequence[Column[Modifiers]],
aggregation_states: str,
granularity: int,
) -> Sequence[operations.SqlOperation]:
aggregated_cols = [*COMMON_AGGR_COLUMNS, *aggregation_col_schema]
return [
operations.CreateTable(
storage_set=StorageSetKey.METRICS,
table_name=table_name,
columns=aggregated_cols,
engine=table_engines.AggregatingMergeTree(
storage_set=StorageSetKey.METRICS,
order_by=
"(org_id, project_id, metric_id, granularity, timestamp, tags.key, tags.value)",
partition_by="(retention_days, toMonday(timestamp))",
settings={"index_granularity": "256"},
),
),
operations.AddColumn(
storage_set=StorageSetKey.METRICS,
table_name=table_name,
column=Column(
"_tags_hash",
Array(UInt(64),
Modifiers(materialized=INT_TAGS_HASH_MAP_COLUMN)),
),
after="tags.value",
),
operations.AddIndex(
storage_set=StorageSetKey.METRICS,
table_name=table_name,
index_name="bf_tags_hash",
index_expression="_tags_hash",
index_type="bloom_filter()",
granularity=1,
),
operations.AddIndex(
storage_set=StorageSetKey.METRICS,
table_name=table_name,
index_name="bf_tags_key_hash",
index_expression="tags.key",
index_type="bloom_filter()",
granularity=1,
),
] + [
get_forward_view_migration_local(
source_table_name,
table_name,
mv_name,
aggregation_col_schema,
aggregation_states,
granularity,
)
]
def __backwards_migrations(
self, table_name: str) -> Sequence[operations.Operation]:
return [
operations.ModifyColumn(
storage_set=StorageSetKey.DISCOVER,
table_name=table_name,
column=Column("user", String(Modifiers(low_cardinality=True))),
)
]
def __forward_migrations(
self, table_name: str) -> Sequence[operations.SqlOperation]:
return [
operations.ModifyColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name=table_name,
column=Column("trace_id", UUID(Modifiers(nullable=True))),
),
]
def forwards_dist(self) -> Sequence[operations.Operation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="errors_dist",
column=Column(
"http_method",
String(Modifiers(nullable=True, low_cardinality=True)),
),
after="sdk_version",
),
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="errors_dist",
column=Column("http_referer", String(Modifiers(nullable=True))),
after="http_method",
),
]
def __forward_migrations(
self, table_name: str) -> Sequence[operations.SqlOperation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.DISCOVER,
table_name=table_name,
column=Column("trace_id", UUID(Modifiers(nullable=True))),
after="contexts",
),
]
def forwards_local(self) -> Sequence[operations.Operation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
column=Column(
"http_method",
String(Modifiers(nullable=True, low_cardinality=True)),
),
after="sdk_version",
),
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
column=Column("http_referer",
String(Modifiers(nullable=True))),
after="http_method",
),
]
def __forward_migrations(
self, table_name: str) -> Sequence[operations.Operation]:
return [
operations.ModifyColumn(
storage_set=StorageSetKey.EVENTS,
table_name=table_name,
column=Column(
"level",
String(Modifiers(low_cardinality=True, nullable=True))),
)
]
def forwards_local(self) -> Sequence[operations.Operation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="errors_local",
column=Column(
"_tags_hash_map",
Array(UInt(64), Modifiers(materialized=TAGS_HASH_MAP_COLUMN)),
),
after="_tags_flattened",
),
]
def __forward_migrations(
self, table_name: str
) -> Sequence[operations.SqlOperation]:
return [
operations.ModifyColumn(
storage_set=StorageSetKey.DISCOVER,
table_name=table_name,
column=Column(
"transaction_name", String(Modifiers(low_cardinality=True))
),
),
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_dist",
column=Column(
"_tags_hash_map",
Array(UInt(64),
Modifiers(materialized=TAGS_HASH_MAP_COLUMN)),
),
after="_tags_flattened",
),
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_dist",
column=Column(
"span_op_breakdowns",
Nested([
("key", String(Modifiers(low_cardinality=True))),
("value", Float(64)),
]),
),
after="measurements.value",
),
]
def forwards_local(self) -> Sequence[operations.SqlOperation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
column=Column(
"measurements",
Nested([
("key", String(Modifiers(low_cardinality=True))),
("value", Float(64)),
]),
),
after="_contexts_flattened",
),
]
def forwards_local(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.METRICS,
table_name=self.table_name,
columns=self.aggregated_cols,
engine=table_engines.AggregatingMergeTree(
storage_set=StorageSetKey.METRICS,
order_by=
"(use_case_id, org_id, project_id, metric_id, granularity, timestamp, tags.key, tags.value, retention_days)",
primary_key=
"(use_case_id, org_id, project_id, metric_id, granularity, timestamp)",
partition_by="(retention_days, toMonday(timestamp))",
settings={"index_granularity": self.granularity},
ttl="timestamp + toIntervalDay(retention_days)",
),
),
operations.AddColumn(
storage_set=StorageSetKey.METRICS,
table_name=self.table_name,
column=Column(
"_tags_hash",
Array(UInt(64),
Modifiers(materialized=INT_TAGS_HASH_MAP_COLUMN)),
),
after="tags.value",
),
operations.AddIndex(
storage_set=StorageSetKey.METRICS,
table_name=self.table_name,
index_name="bf_tags_hash",
index_expression="_tags_hash",
index_type="bloom_filter()",
granularity=1,
),
operations.AddIndex(
storage_set=StorageSetKey.METRICS,
table_name=self.table_name,
index_name="bf_tags_key_hash",
index_expression="tags.key",
index_type="bloom_filter()",
granularity=1,
),
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.METRICS,
table_name=self.dist_table_name,
columns=self.aggregated_cols,
engine=table_engines.Distributed(
local_table_name=self.table_name, sharding_key=None),
),
operations.AddColumn(
storage_set=StorageSetKey.METRICS,
table_name=self.dist_table_name,
column=Column(
"_tags_hash",
Array(UInt(64),
Modifiers(materialized=INT_TAGS_HASH_MAP_COLUMN)),
),
after="tags.value",
),
]
def forwards_dist(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="spans_experimental_dist",
columns=columns,
engine=table_engines.Distributed(
local_table_name="spans_experimental_local",
sharding_key="cityHash64(transaction_span_id)",
),
),
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="spans_experimental_dist",
column=Column(
"_tags_hash_map",
Array(UInt(64), Modifiers(materialized=TAGS_HASH_MAP_COLUMN)),
),
after="tags.value",
),
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.EVENTS,
table_name="errors_dist_new",
columns=columns,
engine=table_engines.Distributed(
local_table_name="errors_local",
sharding_key=sample_expr,
),
),
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="errors_dist_new",
column=Column(
"_tags_hash_map",
Array(UInt(64),
Modifiers(materialized=TAGS_HASH_MAP_COLUMN)),
),
after="tags",
),
operations.DropTable(
storage_set=StorageSetKey.EVENTS,
table_name="errors_dist",
),
operations.RenameTable(
storage_set=StorageSetKey.EVENTS,
old_table_name="errors_dist_new",
new_table_name="errors_dist",
),
operations.CreateTable(
storage_set=StorageSetKey.EVENTS_RO,
table_name="errors_dist_ro",
columns=columns,
engine=table_engines.Distributed(
local_table_name="errors_local",
sharding_key=sample_expr,
),
),
]
def forwards_local(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.EVENTS,
table_name="errors_local_new",
columns=columns,
engine=table_engines.ReplacingMergeTree(
storage_set=StorageSetKey.EVENTS,
version_column="deleted",
order_by=
"(project_id, toStartOfDay(timestamp), primary_hash, %s)" %
sample_expr,
partition_by="(retention_days, toMonday(timestamp))",
sample_by=sample_expr,
ttl="timestamp + toIntervalDay(retention_days)",
settings={"index_granularity": "8192"},
),
),
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="errors_local_new",
column=Column(
"_tags_hash_map",
Array(UInt(64),
Modifiers(materialized=TAGS_HASH_MAP_COLUMN)),
),
after="tags",
),
operations.DropTable(
storage_set=StorageSetKey.EVENTS,
table_name="errors_local",
),
operations.RenameTable(
storage_set=StorageSetKey.EVENTS,
old_table_name="errors_local_new",
new_table_name="errors_local",
),
]
def get_forward_migrations_dist(
dist_table_name: str,
local_table_name: str,
aggregation_col_schema: Sequence[Column[Modifiers]],
) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.METRICS,
table_name=dist_table_name,
columns=[*COMMON_AGGR_COLUMNS, *aggregation_col_schema],
engine=table_engines.Distributed(local_table_name=local_table_name,
sharding_key=None),
),
operations.AddColumn(
storage_set=StorageSetKey.METRICS,
table_name=dist_table_name,
column=Column(
"_tags_hash",
Array(UInt(64),
Modifiers(materialized=INT_TAGS_HASH_MAP_COLUMN)),
),
after="tags.value",
),
]
from typing import Sequence
from snuba.clickhouse.columns import Column, DateTime, UInt
from snuba.clusters.storage_sets import StorageSetKey
from snuba.migrations import migration, operations, table_engines
from snuba.migrations.columns import MigrationModifiers as Modifiers
columns: Sequence[Column[Modifiers]] = [
# Kafka topic offset
Column("offset", UInt(64)),
Column("record_deleted", UInt(8)),
# PG columns
Column("project_id", UInt(64)),
Column("group_id", UInt(64)),
Column("date_added", DateTime(Modifiers(nullable=True))),
Column("user_id", UInt(64, Modifiers(nullable=True))),
Column("team_id", UInt(64, Modifiers(nullable=True))),
]
class Migration(migration.ClickhouseNodeMigration):
blocking = False
def forwards_local(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.CDC,
table_name="groupassignee_local",
columns=columns,
engine=table_engines.ReplacingMergeTree(
storage_set=StorageSetKey.CDC,
from typing import Sequence
from snuba.clickhouse.columns import Column, DateTime, Enum, String, UInt
from snuba.clusters.storage_sets import StorageSetKey
from snuba.migrations import migration, operations
from snuba.migrations.columns import MigrationModifiers as Modifiers
from snuba.migrations.context import Context
from snuba.migrations.status import Status
from snuba.migrations.table_engines import Distributed, ReplacingMergeTree
columns: Sequence[Column[Modifiers]] = [
Column("group", String()),
Column("migration_id", String()),
Column("timestamp", DateTime()),
Column("status", Enum([("completed", 0), ("in_progress", 1), ("not_started", 2)]),),
Column("version", UInt(64, Modifiers(default="1"))),
]
class Migration(migration.Migration):
"""
This migration extends Migration instead of MultiStepMigration since it is
responsible for bootstrapping the migration system itself. It skips setting
the in progress status in the forwards method and the not started status in
the backwards method. Since the migration table doesn't exist yet, we can't
write any statuses until this migration is completed.
"""
blocking = False
def __forwards_local(self) -> Sequence[operations.Operation]:
Column,
DateTime,
String,
UInt,
)
from snuba.clusters.storage_sets import StorageSetKey
from snuba.migrations import migration, operations
from snuba.migrations.columns import MigrationModifiers as Modifiers
from snuba.processor import MAX_UINT32, NIL_UUID
from .matview import create_matview_v1
aggregate_columns: Sequence[Column[Modifiers]] = [
Column("org_id", UInt(64)),
Column("project_id", UInt(64)),
Column("started", DateTime()),
Column("release", String(Modifiers(low_cardinality=True))),
Column("environment", String(Modifiers(low_cardinality=True))),
Column("user_agent", String(Modifiers(low_cardinality=True))),
Column("os", String(Modifiers(low_cardinality=True))),
# durations
Column(
"duration_quantiles",
AggregateFunction("quantilesIf(0.5, 0.9)",
[UInt(32), UInt(8)]),
),
Column("duration_avg", AggregateFunction("avgIf",
[UInt(32), UInt(8)])),
# sessions:
Column("sessions", AggregateFunction("countIf", [UUID(), UInt(8)])),
Column("sessions_preaggr", AggregateFunction("sumIf",
[UInt(32), UInt(8)])),
from typing import Sequence
from snuba.clickhouse.columns import UUID, Column, DateTime, String, UInt
from snuba.clusters.storage_sets import StorageSetKey
from snuba.migrations import migration, operations, table_engines
from snuba.migrations.columns import MigrationModifiers as Modifiers
raw_columns = [
Column("org_id", UInt(64)),
Column("project_id", UInt(64)),
Column("key_id", UInt(64, Modifiers(nullable=True))),
Column("timestamp", DateTime()),
Column("outcome", UInt(8)),
Column("reason", String(Modifiers(nullable=True, low_cardinality=True))),
Column("event_id", UUID(Modifiers(nullable=True))),
]
hourly_columns = [
Column("org_id", UInt(64)),
Column("project_id", UInt(64)),
Column("key_id", UInt(64)),
Column("timestamp", DateTime()),
Column("outcome", UInt(8)),
Column("reason", String(Modifiers(low_cardinality=True))),
Column("times_seen", UInt(64)),
]
materialized_view_columns = [
Column("org_id", UInt(64)),
Column("project_id", UInt(64)),
Column("key_id", UInt(64)),
String,
UInt,
)
from snuba.clusters.storage_sets import StorageSetKey
from snuba.migrations import migration, operations, table_engines
from snuba.migrations.columns import Materialized
from snuba.migrations.columns import MigrationModifiers as Modifiers
UNKNOWN_SPAN_STATUS = 2
columns: List[Column[Modifiers]] = [
Column("project_id", UInt(64)),
Column("event_id", UUID()),
Column("trace_id", UUID()),
Column("span_id", UInt(64)),
Column("transaction_name", String(Modifiers(low_cardinality=True))),
Column(
"transaction_hash",
UInt(64, Modifiers(materialized="cityHash64(transaction_name)")),
),
Column("transaction_op", String(Modifiers(low_cardinality=True))),
Column("transaction_status",
UInt(8, Modifiers(default=str(UNKNOWN_SPAN_STATUS)))),
Column("start_ts", DateTime()),
Column("start_ms", UInt(16)),
Column("finish_ts", DateTime()),
Column("finish_ms", UInt(16)),
Column("duration", UInt(32)),
Column("platform", String(Modifiers(low_cardinality=True))),
Column("environment", String(Modifiers(nullable=True,
low_cardinality=True))),