def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.OUTCOMES,
table_name="outcomes_raw_dist",
column=Column("quantity", UInt(32)),
after=None,
),
operations.AddColumn(
storage_set=StorageSetKey.OUTCOMES,
table_name="outcomes_raw_dist",
column=Column("category", UInt(8)),
after=None,
),
operations.AddColumn(
storage_set=StorageSetKey.OUTCOMES,
table_name="outcomes_hourly_dist",
column=Column("quantity", UInt(64)),
after=None,
),
operations.AddColumn(
storage_set=StorageSetKey.OUTCOMES,
table_name="outcomes_hourly_dist",
column=Column("category", UInt(8)),
after=None,
),
]
def forwards_local(self) -> Sequence[operations.SqlOperation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.OUTCOMES,
table_name="outcomes_raw_local",
column=Column("quantity", UInt(32)),
after="reason",
),
operations.AddColumn(
storage_set=StorageSetKey.OUTCOMES,
table_name="outcomes_raw_local",
column=Column("category", UInt(8)),
after="timestamp",
),
operations.AddColumn(
storage_set=StorageSetKey.OUTCOMES,
table_name="outcomes_hourly_local",
column=Column("quantity", UInt(64)),
after="reason",
),
operations.RunSql(
storage_set=StorageSetKey.OUTCOMES,
statement="""
ALTER TABLE outcomes_hourly_local ADD COLUMN IF NOT EXISTS category UInt8 AFTER timestamp,
MODIFY ORDER BY (org_id, project_id, key_id, outcome, reason, timestamp, category);
""",
),
]
def backwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_dist",
column=Column("_contexts_flattened", String()),
),
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_dist",
column=Column("_tags_flattened", String()),
),
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="errors_dist",
column=Column("hierarchical_hashes", Array(UUID())),
after="primary_hash",
),
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="sentry_dist",
column=Column("hierarchical_hashes", Array(FixedString(32)),),
after="primary_hash",
),
]
def forwards_local(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="spans_experimental_local",
columns=columns,
engine=table_engines.ReplacingMergeTree(
storage_set=StorageSetKey.TRANSACTIONS,
version_column="deleted",
order_by=
("(project_id, toStartOfDay(finish_ts), transaction_name, "
"cityHash64(transaction_span_id), op, cityHash64(trace_id), "
"cityHash64(span_id))"),
partition_by="(toMonday(finish_ts))",
sample_by="cityHash64(span_id)",
ttl="finish_ts + toIntervalDay(retention_days)",
settings={"index_granularity": "8192"},
),
),
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="spans_experimental_local",
column=Column(
"_tags_hash_map",
Materialized(Array(UInt(64)), TAGS_HASH_MAP_COLUMN),
),
after="tags.value",
),
]
def __forward_migrations(
self, table_name: str) -> Sequence[operations.SqlOperation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.DISCOVER,
table_name=table_name,
column=Column("_tags_hash_map", Array(UInt(64))),
after="tags",
),
operations.AddColumn(
storage_set=StorageSetKey.DISCOVER,
table_name=table_name,
column=Column("deleted", UInt(8)),
after="contexts",
),
]
def forwards_local(self) -> Sequence[operations.Operation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="errors_local",
column=Column("http_method",
LowCardinality(Nullable(String()))),
after="sdk_version",
),
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="errors_local",
column=Column("http_referer", Nullable(String())),
after="http_method",
),
]
def forwards_dist(self) -> Sequence[operations.Operation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_dist",
column=Column("http_method",
LowCardinality(Nullable(String()))),
after="sdk_version",
),
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_dist",
column=Column("http_referer", Nullable(String())),
after="http_method",
),
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.SESSIONS,
table_name="sessions_raw_dist",
column=column,
after=after,
) for [column, after] in new_raw_columns
] + [
operations.AddColumn(
storage_set=StorageSetKey.SESSIONS,
table_name="sessions_hourly_dist",
column=column,
after=after,
) for [column, after] in new_dest_columns
]
def backwards_local(self) -> Sequence[operations.SqlOperation]:
sample_expr = "cityHash64(toString(event_id))"
return [
operations.CreateTable(
storage_set=StorageSetKey.EVENTS,
table_name="sentry_local",
columns=columns,
engine=table_engines.ReplacingMergeTree(
storage_set=StorageSetKey.EVENTS,
version_column="deleted",
order_by="(project_id, toStartOfDay(timestamp), %s)" % sample_expr,
partition_by="(toMonday(timestamp), if(equals(retention_days, 30), 30, 90))",
sample_by=sample_expr,
),
),
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="sentry_local",
column=Column(
"_tags_hash_map",
Array(UInt(64), Modifiers(materialized=TAGS_HASH_MAP_COLUMN)),
),
after="_tags_flattened",
),
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.GENERIC_METRICS_SETS,
table_name=self.table_name,
column=self.new_column,
)
]
def __forward_migrations(
self, table_name: str
) -> Sequence[operations.SqlOperation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name=table_name,
column=Column(
"type",
String(
Modifiers(low_cardinality=True, materialized="'transaction'")
),
),
after="deleted",
),
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name=table_name,
column=Column(
"message",
String(
Modifiers(low_cardinality=True, materialized="transaction_name")
),
),
after="type",
),
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name=table_name,
column=Column(
"title",
String(
Modifiers(low_cardinality=True, materialized="transaction_name")
),
),
after="message",
),
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name=table_name,
column=Column(
"timestamp", DateTime(Modifiers(materialized="finish_ts"))
),
after="title",
),
]
def get_forward_migrations_local(
source_table_name: str,
table_name: str,
mv_name: str,
aggregation_col_schema: Sequence[Column[Modifiers]],
aggregation_states: str,
granularity: int,
) -> Sequence[operations.SqlOperation]:
aggregated_cols = [*COMMON_AGGR_COLUMNS, *aggregation_col_schema]
return [
operations.CreateTable(
storage_set=StorageSetKey.METRICS,
table_name=table_name,
columns=aggregated_cols,
engine=table_engines.AggregatingMergeTree(
storage_set=StorageSetKey.METRICS,
order_by=
"(org_id, project_id, metric_id, granularity, timestamp, tags.key, tags.value)",
partition_by="(retention_days, toMonday(timestamp))",
settings={"index_granularity": "256"},
),
),
operations.AddColumn(
storage_set=StorageSetKey.METRICS,
table_name=table_name,
column=Column(
"_tags_hash",
Array(UInt(64),
Modifiers(materialized=INT_TAGS_HASH_MAP_COLUMN)),
),
after="tags.value",
),
operations.AddIndex(
storage_set=StorageSetKey.METRICS,
table_name=table_name,
index_name="bf_tags_hash",
index_expression="_tags_hash",
index_type="bloom_filter()",
granularity=1,
),
operations.AddIndex(
storage_set=StorageSetKey.METRICS,
table_name=table_name,
index_name="bf_tags_key_hash",
index_expression="tags.key",
index_type="bloom_filter()",
granularity=1,
),
] + [
get_forward_view_migration_local(
source_table_name,
table_name,
mv_name,
aggregation_col_schema,
aggregation_states,
granularity,
)
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_dist",
column=Column("spans.exclusive_time_32", Array(Float(32))),
after="spans.group",
),
]
def forwards_local(self) -> Sequence[operations.SqlOperation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="errors_local",
column=Column("message_timestamp", DateTime()),
after="offset",
)
]
def forwards_dist(self) -> Sequence[operations.Operation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="errors_dist",
column=Column(
"http_method",
String(Modifiers(nullable=True, low_cardinality=True)),
),
after="sdk_version",
),
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="errors_dist",
column=Column("http_referer", String(Modifiers(nullable=True))),
after="http_method",
),
]
def forwards_dist(self) -> Sequence[operations.Operation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="errors_dist",
column=Column("_tags_hash_map", Array(UInt(64)),),
after="_tags_flattened",
),
]
def fix_order_by() -> None:
cluster = get_cluster(StorageSetKey.EVENTS)
if not cluster.is_single_node():
return
clickhouse = cluster.get_query_connection(ClickhouseClientSettings.MIGRATE)
database = cluster.get_database()
new_primary_key = "project_id, id"
old_primary_key = "id"
((curr_primary_key,),) = clickhouse.execute(
f"SELECT primary_key FROM system.tables WHERE name = '{TABLE_NAME}' AND database = '{database}'"
)
assert curr_primary_key in [
new_primary_key,
old_primary_key,
], "Groupmessage table has invalid primary key"
if curr_primary_key != old_primary_key:
return
# Add the project_id column
add_column_sql = operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name=TABLE_NAME,
column=Column("project_id", UInt(64)),
after="record_deleted",
).format_sql()
clickhouse.execute(add_column_sql)
# There shouldn't be any data in the table yet
assert (
clickhouse.execute(f"SELECT COUNT() FROM {TABLE_NAME} FINAL;")[0][0] == 0
), f"{TABLE_NAME} is not empty"
new_order_by = f"ORDER BY ({new_primary_key})"
old_order_by = f"ORDER BY {old_primary_key}"
((curr_create_table_statement,),) = clickhouse.execute(
f"SHOW CREATE TABLE {database}.{TABLE_NAME}"
)
new_create_table_statement = curr_create_table_statement.replace(
TABLE_NAME, TABLE_NAME_NEW
).replace(old_order_by, new_order_by)
clickhouse.execute(new_create_table_statement)
clickhouse.execute(f"RENAME TABLE {TABLE_NAME} TO {TABLE_NAME_OLD};")
clickhouse.execute(f"RENAME TABLE {TABLE_NAME_NEW} TO {TABLE_NAME};")
clickhouse.execute(f"DROP TABLE {TABLE_NAME_OLD};")
def __forward_migrations(
self, table_name: str) -> Sequence[operations.SqlOperation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.DISCOVER,
table_name=table_name,
column=Column("trace_id", UUID(Modifiers(nullable=True))),
after="contexts",
),
]
def forwards_local(self) -> Sequence[operations.Operation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
column=Column(
"http_method",
String(Modifiers(nullable=True, low_cardinality=True)),
),
after="sdk_version",
),
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
column=Column("http_referer",
String(Modifiers(nullable=True))),
after="http_method",
),
]
def forwards_local(self) -> Sequence[operations.Operation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
column=Column(
"_tags_hash_map",
Materialized(Array(UInt(64)), TAGS_HASH_MAP_COLUMN),
),
after="_tags_flattened",
),
]
def forwards_local(self) -> Sequence[operations.Operation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="errors_local",
column=Column(
"_tags_hash_map",
Array(UInt(64), Modifiers(materialized=TAGS_HASH_MAP_COLUMN)),
),
after="_tags_flattened",
),
]
def forwards_dist(self) -> Sequence[operations.Operation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_dist",
column=Column(
"measurements",
Nested([("key", LowCardinality(String())), ("value", Float(64))]),
),
after="_contexts_flattened",
),
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_dist",
column=Column(
"_tags_hash_map",
Array(UInt(64)),
),
after="_tags_flattened",
),
]
def __forward_migrations(
self, table_name: str) -> Sequence[operations.SqlOperation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.METRICS,
table_name=table_name,
column=Column(
"histogram_buckets",
AggregateFunction("histogram(250)", [Float(64)]),
),
after="count",
)
]
def forwards_local(self) -> Sequence[operations.SqlOperation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
column=Column(
"measurements",
Nested([
("key", String(Modifiers(low_cardinality=True))),
("value", Float(64)),
]),
),
after="_contexts_flattened",
),
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_dist",
column=Column(
"span_op_breakdowns",
Nested([
("key", String(Modifiers(low_cardinality=True))),
("value", Float(64)),
]),
),
after="measurements.value",
),
]
def forwards_local(self) -> Sequence[operations.SqlOperation]:
return [
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
column=Column("spans.exclusive_time_32", Array(Float(32))),
after="spans.group",
),
operations.ModifyColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
column=Column("spans.exclusive_time_32", Array(Float(32))),
ttl_month=("finish_ts", 1),
),
]
def forwards_local(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.METRICS,
table_name=self.table_name,
columns=self.aggregated_cols,
engine=table_engines.AggregatingMergeTree(
storage_set=StorageSetKey.METRICS,
order_by=
"(use_case_id, org_id, project_id, metric_id, granularity, timestamp, tags.key, tags.value, retention_days)",
primary_key=
"(use_case_id, org_id, project_id, metric_id, granularity, timestamp)",
partition_by="(retention_days, toMonday(timestamp))",
settings={"index_granularity": self.granularity},
ttl="timestamp + toIntervalDay(retention_days)",
),
),
operations.AddColumn(
storage_set=StorageSetKey.METRICS,
table_name=self.table_name,
column=Column(
"_tags_hash",
Array(UInt(64),
Modifiers(materialized=INT_TAGS_HASH_MAP_COLUMN)),
),
after="tags.value",
),
operations.AddIndex(
storage_set=StorageSetKey.METRICS,
table_name=self.table_name,
index_name="bf_tags_hash",
index_expression="_tags_hash",
index_type="bloom_filter()",
granularity=1,
),
operations.AddIndex(
storage_set=StorageSetKey.METRICS,
table_name=self.table_name,
index_name="bf_tags_key_hash",
index_expression="tags.key",
index_type="bloom_filter()",
granularity=1,
),
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.METRICS,
table_name=self.dist_table_name,
columns=self.aggregated_cols,
engine=table_engines.Distributed(
local_table_name=self.table_name, sharding_key=None),
),
operations.AddColumn(
storage_set=StorageSetKey.METRICS,
table_name=self.dist_table_name,
column=Column(
"_tags_hash",
Array(UInt(64),
Modifiers(materialized=INT_TAGS_HASH_MAP_COLUMN)),
),
after="tags.value",
),
]
