def forwards_local(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.SESSIONS,
table_name="sessions_raw_local",
columns=raw_columns,
engine=table_engines.MergeTree(
storage_set=StorageSetKey.SESSIONS,
order_by=
"(org_id, project_id, release, environment, started)",
partition_by="(toMonday(started))",
settings={"index_granularity": "16384"},
),
),
operations.CreateTable(
storage_set=StorageSetKey.SESSIONS,
table_name="sessions_hourly_local",
columns=aggregate_columns_v1,
engine=table_engines.AggregatingMergeTree(
storage_set=StorageSetKey.SESSIONS,
order_by=
"(org_id, project_id, release, environment, started)",
partition_by="(toMonday(started))",
settings={"index_granularity": "256"},
),
),
create_matview_v1,
]
def forwards_local(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.SESSIONS,
table_name="sessions_raw_local",
columns=raw_columns,
engine=table_engines.MergeTree(
storage_set=StorageSetKey.SESSIONS,
order_by=
"(org_id, project_id, release, environment, started)",
partition_by="(toMonday(started))",
settings={"index_granularity": "16384"},
),
),
operations.CreateTable(
storage_set=StorageSetKey.SESSIONS,
table_name="sessions_hourly_local",
columns=aggregate_columns,
engine=table_engines.AggregatingMergeTree(
storage_set=StorageSetKey.SESSIONS,
order_by=
"(org_id, project_id, release, environment, started)",
partition_by="(toMonday(started))",
settings={"index_granularity": "256"},
),
),
operations.CreateMaterializedView(
storage_set=StorageSetKey.SESSIONS,
view_name="sessions_hourly_mv_local",
destination_table_name="sessions_hourly_local",
columns=aggregate_columns,
query=f"""
SELECT
org_id,
project_id,
toStartOfHour(started) as started,
release,
environment,
quantilesIfState(0.5, 0.9)(
duration,
duration <> {MAX_UINT32} AND status == 1
) as duration_quantiles,
countIfState(session_id, seq == 0) as sessions,
uniqIfState(distinct_id, distinct_id != '{NIL_UUID}') as users,
countIfState(session_id, status == 2) as sessions_crashed,
countIfState(session_id, status == 3) as sessions_abnormal,
uniqIfState(session_id, errors > 0) as sessions_errored,
uniqIfState(distinct_id, status == 2) as users_crashed,
uniqIfState(distinct_id, status == 3) as users_abnormal,
uniqIfState(distinct_id, errors > 0) as users_errored
FROM
sessions_raw_local
GROUP BY
org_id, project_id, started, release, environment
""",
),
]
def backwards_local(self) -> Sequence[operations.SqlOperation]:
sample_expr = "cityHash64(toString(event_id))"
return [
operations.CreateTable(
storage_set=StorageSetKey.EVENTS,
table_name="sentry_local",
columns=columns,
engine=table_engines.ReplacingMergeTree(
storage_set=StorageSetKey.EVENTS,
version_column="deleted",
order_by="(project_id, toStartOfDay(timestamp), %s)" % sample_expr,
partition_by="(toMonday(timestamp), if(equals(retention_days, 30), 30, 90))",
sample_by=sample_expr,
),
),
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="sentry_local",
column=Column(
"_tags_hash_map",
Array(UInt(64), Modifiers(materialized=TAGS_HASH_MAP_COLUMN)),
),
after="_tags_flattened",
),
]
def forwards_local(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="spans_experimental_local",
columns=columns,
engine=table_engines.ReplacingMergeTree(
storage_set=StorageSetKey.TRANSACTIONS,
version_column="deleted",
order_by=
("(project_id, toStartOfDay(finish_ts), transaction_name, "
"cityHash64(transaction_span_id), op, cityHash64(trace_id), "
"cityHash64(span_id))"),
partition_by="(toMonday(finish_ts))",
sample_by="cityHash64(span_id)",
ttl="finish_ts + toIntervalDay(retention_days)",
settings={"index_granularity": "8192"},
),
),
operations.AddColumn(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="spans_experimental_local",
column=Column(
"_tags_hash_map",
Materialized(Array(UInt(64)), TAGS_HASH_MAP_COLUMN),
),
after="tags.value",
),
]
def get_forward_migrations_local(
source_table_name: str,
table_name: str,
mv_name: str,
aggregation_col_schema: Sequence[Column[Modifiers]],
aggregation_states: str,
granularity: int,
) -> Sequence[operations.SqlOperation]:
aggregated_cols = [*COMMON_AGGR_COLUMNS, *aggregation_col_schema]
return [
operations.CreateTable(
storage_set=StorageSetKey.METRICS,
table_name=table_name,
columns=aggregated_cols,
engine=table_engines.AggregatingMergeTree(
storage_set=StorageSetKey.METRICS,
order_by=
"(org_id, project_id, metric_id, granularity, timestamp, tags.key, tags.value)",
partition_by="(retention_days, toMonday(timestamp))",
settings={"index_granularity": "256"},
),
),
operations.AddColumn(
storage_set=StorageSetKey.METRICS,
table_name=table_name,
column=Column(
"_tags_hash",
Array(UInt(64),
Modifiers(materialized=INT_TAGS_HASH_MAP_COLUMN)),
),
after="tags.value",
),
operations.AddIndex(
storage_set=StorageSetKey.METRICS,
table_name=table_name,
index_name="bf_tags_hash",
index_expression="_tags_hash",
index_type="bloom_filter()",
granularity=1,
),
operations.AddIndex(
storage_set=StorageSetKey.METRICS,
table_name=table_name,
index_name="bf_tags_key_hash",
index_expression="tags.key",
index_type="bloom_filter()",
granularity=1,
),
] + [
get_forward_view_migration_local(
source_table_name,
table_name,
mv_name,
aggregation_col_schema,
aggregation_states,
granularity,
)
]
def forwards_local(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.OUTCOMES,
table_name="outcomes_raw_local",
columns=raw_columns,
engine=table_engines.MergeTree(
storage_set=StorageSetKey.OUTCOMES,
order_by="(org_id, project_id, timestamp)",
partition_by="(toMonday(timestamp))",
settings={"index_granularity": "16384"},
),
),
operations.CreateTable(
storage_set=StorageSetKey.OUTCOMES,
table_name="outcomes_hourly_local",
columns=hourly_columns,
engine=table_engines.SummingMergeTree(
storage_set=StorageSetKey.OUTCOMES,
order_by=
"(org_id, project_id, key_id, outcome, reason, timestamp)",
partition_by="(toMonday(timestamp))",
settings={"index_granularity": "256"},
),
),
operations.CreateMaterializedView(
storage_set=StorageSetKey.OUTCOMES,
view_name="outcomes_mv_hourly_local",
destination_table_name="outcomes_hourly_local",
columns=materialized_view_columns,
query="""
SELECT
org_id,
project_id,
ifNull(key_id, 0) AS key_id,
toStartOfHour(timestamp) AS timestamp,
outcome,
ifNull(reason, 'none') AS reason,
count() AS times_seen
FROM outcomes_raw_local
GROUP BY org_id, project_id, key_id, timestamp, outcome, reason
""",
),
]
def forwards_local(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.DISCOVER,
table_name="discover_local",
columns=columns,
engine=table_engines.Merge(
table_name_regex="^errors_local$|^transactions_local$"),
),
]
def forwards_dist(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.SESSIONS,
table_name="sessions_raw_dist",
columns=raw_columns,
engine=table_engines.Distributed(
local_table_name="sessions_raw_local", sharding_key="org_id",
),
),
operations.CreateTable(
storage_set=StorageSetKey.SESSIONS,
table_name="sessions_hourly_dist",
columns=aggregate_columns,
engine=table_engines.Distributed(
local_table_name="sessions_hourly_local", sharding_key="org_id",
),
),
]
def __forwards_dist(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.MIGRATIONS,
table_name="migrations_dist",
columns=columns,
engine=Distributed(local_table_name="migrations_local",
sharding_key=None),
)
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.GENERIC_METRICS_SETS,
table_name="generic_metric_sets_aggregated_dist",
engine=table_engines.Distributed(
local_table_name=self.local_table_name, sharding_key=None
),
columns=self.columns,
)
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.CDC,
table_name="groupassignee_dist",
columns=columns,
engine=table_engines.Distributed(
local_table_name="groupassignee_local", sharding_key=None,
),
)
]
def forwards_dist(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.EVENTS,
table_name="errors_dist",
columns=columns,
engine=table_engines.Distributed(
local_table_name="errors_local", sharding_key="event_hash",
),
)
]
def forwards_dist(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.QUERYLOG,
table_name="querylog_dist",
columns=columns,
engine=table_engines.Distributed(
local_table_name="querylog_local", sharding_key=None,
),
)
]
def backwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.EVENTS,
table_name="sentry_dist",
columns=columns,
engine=table_engines.Distributed(
local_table_name="sentry_local",
sharding_key="cityHash64(toString(event_id))",
),
),
operations.CreateTable(
storage_set=StorageSetKey.EVENTS_RO,
table_name="sentry_dist_ro",
columns=columns,
engine=table_engines.Distributed(
local_table_name="sentry_local",
sharding_key="cityHash64(toString(event_id))",
),
),
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.REPLAYS,
table_name="replays_dist",
columns=raw_columns,
engine=table_engines.Distributed(
local_table_name="replays_local",
sharding_key="cityHash64(toString(replay_id))",
),
),
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.OUTCOMES,
table_name="outcomes_raw_dist",
columns=raw_columns,
engine=table_engines.Distributed(
local_table_name="outcomes_raw_local",
sharding_key="org_id",
),
),
operations.CreateTable(
storage_set=StorageSetKey.OUTCOMES,
table_name="outcomes_hourly_dist",
columns=hourly_columns,
engine=table_engines.Distributed(
local_table_name="outcomes_hourly_local",
sharding_key="org_id",
),
),
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.METRICS,
table_name=self.dist_table_name,
columns=self.column_list,
engine=table_engines.Distributed(
local_table_name=self.local_table_name,
sharding_key=None,
),
)
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.PROFILES,
table_name="profiles_dist",
columns=columns,
engine=table_engines.Distributed(
local_table_name="profiles_local",
sharding_key="cityHash64(profile_id)",
),
)
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_dist",
columns=columns,
engine=table_engines.Distributed(
local_table_name="transactions_local",
sharding_key="cityHash64(span_id)",
),
)
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.GENERIC_METRICS_SETS,
table_name=self.dist_table_name,
engine=table_engines.Distributed(
local_table_name=self.local_table_name,
sharding_key="cityHash64(timeseries_id)",
),
columns=self.columns,
)
]
def forwards_dist(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.EVENTS,
table_name="errors_dist_new",
columns=columns,
engine=table_engines.Distributed(
local_table_name="errors_local",
sharding_key=sample_expr,
),
),
operations.AddColumn(
storage_set=StorageSetKey.EVENTS,
table_name="errors_dist_new",
column=Column(
"_tags_hash_map",
Array(UInt(64),
Modifiers(materialized=TAGS_HASH_MAP_COLUMN)),
),
after="tags",
),
operations.DropTable(
storage_set=StorageSetKey.EVENTS,
table_name="errors_dist",
),
operations.RenameTable(
storage_set=StorageSetKey.EVENTS,
old_table_name="errors_dist_new",
new_table_name="errors_dist",
),
operations.CreateTable(
storage_set=StorageSetKey.EVENTS_RO,
table_name="errors_dist_ro",
columns=columns,
engine=table_engines.Distributed(
local_table_name="errors_local",
sharding_key=sample_expr,
),
),
]
def __forwards_local(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.MIGRATIONS,
table_name="migrations_local",
columns=columns,
engine=ReplacingMergeTree(
storage_set=StorageSetKey.MIGRATIONS,
version_column="version",
order_by="(group, migration_id)",
),
),
]
def forwards_local(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.CDC,
table_name="groupassignee_local",
columns=columns,
engine=table_engines.ReplacingMergeTree(
storage_set=StorageSetKey.CDC,
version_column="offset",
order_by="(project_id, group_id)",
unsharded=True,
),
)
]
def forwards_local(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.QUERYLOG,
table_name="querylog_local",
columns=columns,
engine=table_engines.MergeTree(
storage_set=StorageSetKey.QUERYLOG,
order_by="(toStartOfDay(timestamp), request_id)",
partition_by="(toMonday(timestamp))",
sample_by="request_id",
),
)
]
def forwards_local(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.EVENTS,
table_name="groupedmessage_local",
columns=columns,
engine=table_engines.ReplacingMergeTree(
storage_set=StorageSetKey.EVENTS,
version_column="offset",
order_by="(project_id, id)",
sample_by="id",
unsharded=True,
),
)
]
def forwards_local(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.METRICS,
table_name=self.local_table_name,
columns=self.column_list,
engine=table_engines.MergeTree(
storage_set=StorageSetKey.METRICS,
order_by=
"(use_case_id, metric_type, org_id, project_id, metric_id, timestamp)",
partition_by="(toStartOfDay(timestamp))",
ttl="timestamp + toIntervalDay(7)",
),
)
]
def forwards_local(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.EVENTS,
table_name="errors_local",
columns=columns,
engine=table_engines.ReplacingMergeTree(
storage_set=StorageSetKey.EVENTS,
version_column="deleted",
order_by="(org_id, project_id, toStartOfDay(timestamp), primary_hash_hex, event_hash)",
partition_by="(toMonday(timestamp), if(retention_days = 30, 30, 90))",
sample_by="event_hash",
ttl="timestamp + toIntervalDay(retention_days)",
settings={"index_granularity": "8192"},
),
)
]
def forwards_local(self) -> Sequence[operations.Operation]:
sample_expr = "cityHash64(toString(event_id))"
return [
operations.CreateTable(
storage_set=StorageSetKey.EVENTS,
table_name="sentry_local",
columns=columns,
engine=table_engines.ReplacingMergeTree(
storage_set=StorageSetKey.EVENTS,
version_column="deleted",
order_by="(project_id, toStartOfDay(timestamp), %s)" % sample_expr,
partition_by="(toMonday(timestamp), if(equals(retention_days, 30), 30, 90))",
sample_by=sample_expr,
),
),
]
def forwards_local(self) -> Sequence[operations.Operation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.TRANSACTIONS,
table_name="transactions_local",
columns=columns,
engine=table_engines.ReplacingMergeTree(
storage_set=StorageSetKey.TRANSACTIONS,
version_column="deleted",
order_by="(project_id, toStartOfDay(finish_ts), transaction_name, cityHash64(span_id))",
partition_by="(retention_days, toMonday(finish_ts))",
sample_by="cityHash64(span_id)",
ttl="finish_ts + toIntervalDay(retention_days)",
settings={"index_granularity": "8192"},
),
)
]
def forwards_local(self) -> Sequence[operations.SqlOperation]:
return [
operations.CreateTable(
storage_set=StorageSetKey.PROFILES,
table_name="profiles_local",
columns=columns,
engine=table_engines.ReplacingMergeTree(
order_by=
"(organization_id, project_id, toStartOfDay(received), cityHash64(profile_id))",
partition_by="(retention_days, toMonday(received))",
sample_by="cityHash64(profile_id)",
settings={"index_granularity": "8192"},
storage_set=StorageSetKey.PROFILES,
ttl="received + toIntervalDay(retention_days)",
),
)
]