def forwards_local(self) -> Sequence[operations.SqlOperation]: return [ operations.CreateTable( storage_set=StorageSetKey.SESSIONS, table_name="sessions_raw_local", columns=raw_columns, engine=table_engines.MergeTree( storage_set=StorageSetKey.SESSIONS, order_by= "(org_id, project_id, release, environment, started)", partition_by="(toMonday(started))", settings={"index_granularity": "16384"}, ), ), operations.CreateTable( storage_set=StorageSetKey.SESSIONS, table_name="sessions_hourly_local", columns=aggregate_columns_v1, engine=table_engines.AggregatingMergeTree( storage_set=StorageSetKey.SESSIONS, order_by= "(org_id, project_id, release, environment, started)", partition_by="(toMonday(started))", settings={"index_granularity": "256"}, ), ), create_matview_v1, ]
def forwards_local(self) -> Sequence[operations.Operation]: return [ operations.CreateTable( storage_set=StorageSetKey.SESSIONS, table_name="sessions_raw_local", columns=raw_columns, engine=table_engines.MergeTree( storage_set=StorageSetKey.SESSIONS, order_by= "(org_id, project_id, release, environment, started)", partition_by="(toMonday(started))", settings={"index_granularity": "16384"}, ), ), operations.CreateTable( storage_set=StorageSetKey.SESSIONS, table_name="sessions_hourly_local", columns=aggregate_columns, engine=table_engines.AggregatingMergeTree( storage_set=StorageSetKey.SESSIONS, order_by= "(org_id, project_id, release, environment, started)", partition_by="(toMonday(started))", settings={"index_granularity": "256"}, ), ), operations.CreateMaterializedView( storage_set=StorageSetKey.SESSIONS, view_name="sessions_hourly_mv_local", destination_table_name="sessions_hourly_local", columns=aggregate_columns, query=f""" SELECT org_id, project_id, toStartOfHour(started) as started, release, environment, quantilesIfState(0.5, 0.9)( duration, duration <> {MAX_UINT32} AND status == 1 ) as duration_quantiles, countIfState(session_id, seq == 0) as sessions, uniqIfState(distinct_id, distinct_id != '{NIL_UUID}') as users, countIfState(session_id, status == 2) as sessions_crashed, countIfState(session_id, status == 3) as sessions_abnormal, uniqIfState(session_id, errors > 0) as sessions_errored, uniqIfState(distinct_id, status == 2) as users_crashed, uniqIfState(distinct_id, status == 3) as users_abnormal, uniqIfState(distinct_id, errors > 0) as users_errored FROM sessions_raw_local GROUP BY org_id, project_id, started, release, environment """, ), ]
def backwards_local(self) -> Sequence[operations.SqlOperation]: sample_expr = "cityHash64(toString(event_id))" return [ operations.CreateTable( storage_set=StorageSetKey.EVENTS, table_name="sentry_local", columns=columns, engine=table_engines.ReplacingMergeTree( storage_set=StorageSetKey.EVENTS, version_column="deleted", order_by="(project_id, toStartOfDay(timestamp), %s)" % sample_expr, partition_by="(toMonday(timestamp), if(equals(retention_days, 30), 30, 90))", sample_by=sample_expr, ), ), operations.AddColumn( storage_set=StorageSetKey.EVENTS, table_name="sentry_local", column=Column( "_tags_hash_map", Array(UInt(64), Modifiers(materialized=TAGS_HASH_MAP_COLUMN)), ), after="_tags_flattened", ), ]
def forwards_local(self) -> Sequence[operations.Operation]: return [ operations.CreateTable( storage_set=StorageSetKey.TRANSACTIONS, table_name="spans_experimental_local", columns=columns, engine=table_engines.ReplacingMergeTree( storage_set=StorageSetKey.TRANSACTIONS, version_column="deleted", order_by= ("(project_id, toStartOfDay(finish_ts), transaction_name, " "cityHash64(transaction_span_id), op, cityHash64(trace_id), " "cityHash64(span_id))"), partition_by="(toMonday(finish_ts))", sample_by="cityHash64(span_id)", ttl="finish_ts + toIntervalDay(retention_days)", settings={"index_granularity": "8192"}, ), ), operations.AddColumn( storage_set=StorageSetKey.TRANSACTIONS, table_name="spans_experimental_local", column=Column( "_tags_hash_map", Materialized(Array(UInt(64)), TAGS_HASH_MAP_COLUMN), ), after="tags.value", ), ]
def get_forward_migrations_local( source_table_name: str, table_name: str, mv_name: str, aggregation_col_schema: Sequence[Column[Modifiers]], aggregation_states: str, granularity: int, ) -> Sequence[operations.SqlOperation]: aggregated_cols = [*COMMON_AGGR_COLUMNS, *aggregation_col_schema] return [ operations.CreateTable( storage_set=StorageSetKey.METRICS, table_name=table_name, columns=aggregated_cols, engine=table_engines.AggregatingMergeTree( storage_set=StorageSetKey.METRICS, order_by= "(org_id, project_id, metric_id, granularity, timestamp, tags.key, tags.value)", partition_by="(retention_days, toMonday(timestamp))", settings={"index_granularity": "256"}, ), ), operations.AddColumn( storage_set=StorageSetKey.METRICS, table_name=table_name, column=Column( "_tags_hash", Array(UInt(64), Modifiers(materialized=INT_TAGS_HASH_MAP_COLUMN)), ), after="tags.value", ), operations.AddIndex( storage_set=StorageSetKey.METRICS, table_name=table_name, index_name="bf_tags_hash", index_expression="_tags_hash", index_type="bloom_filter()", granularity=1, ), operations.AddIndex( storage_set=StorageSetKey.METRICS, table_name=table_name, index_name="bf_tags_key_hash", index_expression="tags.key", index_type="bloom_filter()", granularity=1, ), ] + [ get_forward_view_migration_local( source_table_name, table_name, mv_name, aggregation_col_schema, aggregation_states, granularity, ) ]
def forwards_local(self) -> Sequence[operations.Operation]: return [ operations.CreateTable( storage_set=StorageSetKey.OUTCOMES, table_name="outcomes_raw_local", columns=raw_columns, engine=table_engines.MergeTree( storage_set=StorageSetKey.OUTCOMES, order_by="(org_id, project_id, timestamp)", partition_by="(toMonday(timestamp))", settings={"index_granularity": "16384"}, ), ), operations.CreateTable( storage_set=StorageSetKey.OUTCOMES, table_name="outcomes_hourly_local", columns=hourly_columns, engine=table_engines.SummingMergeTree( storage_set=StorageSetKey.OUTCOMES, order_by= "(org_id, project_id, key_id, outcome, reason, timestamp)", partition_by="(toMonday(timestamp))", settings={"index_granularity": "256"}, ), ), operations.CreateMaterializedView( storage_set=StorageSetKey.OUTCOMES, view_name="outcomes_mv_hourly_local", destination_table_name="outcomes_hourly_local", columns=materialized_view_columns, query=""" SELECT org_id, project_id, ifNull(key_id, 0) AS key_id, toStartOfHour(timestamp) AS timestamp, outcome, ifNull(reason, 'none') AS reason, count() AS times_seen FROM outcomes_raw_local GROUP BY org_id, project_id, key_id, timestamp, outcome, reason """, ), ]
def forwards_local(self) -> Sequence[operations.SqlOperation]: return [ operations.CreateTable( storage_set=StorageSetKey.DISCOVER, table_name="discover_local", columns=columns, engine=table_engines.Merge( table_name_regex="^errors_local$|^transactions_local$"), ), ]
def forwards_dist(self) -> Sequence[operations.Operation]: return [ operations.CreateTable( storage_set=StorageSetKey.SESSIONS, table_name="sessions_raw_dist", columns=raw_columns, engine=table_engines.Distributed( local_table_name="sessions_raw_local", sharding_key="org_id", ), ), operations.CreateTable( storage_set=StorageSetKey.SESSIONS, table_name="sessions_hourly_dist", columns=aggregate_columns, engine=table_engines.Distributed( local_table_name="sessions_hourly_local", sharding_key="org_id", ), ), ]
def __forwards_dist(self) -> Sequence[operations.Operation]: return [ operations.CreateTable( storage_set=StorageSetKey.MIGRATIONS, table_name="migrations_dist", columns=columns, engine=Distributed(local_table_name="migrations_local", sharding_key=None), ) ]
def forwards_dist(self) -> Sequence[operations.SqlOperation]: return [ operations.CreateTable( storage_set=StorageSetKey.GENERIC_METRICS_SETS, table_name="generic_metric_sets_aggregated_dist", engine=table_engines.Distributed( local_table_name=self.local_table_name, sharding_key=None ), columns=self.columns, ) ]
def forwards_dist(self) -> Sequence[operations.SqlOperation]: return [ operations.CreateTable( storage_set=StorageSetKey.CDC, table_name="groupassignee_dist", columns=columns, engine=table_engines.Distributed( local_table_name="groupassignee_local", sharding_key=None, ), ) ]
def forwards_dist(self) -> Sequence[operations.Operation]: return [ operations.CreateTable( storage_set=StorageSetKey.EVENTS, table_name="errors_dist", columns=columns, engine=table_engines.Distributed( local_table_name="errors_local", sharding_key="event_hash", ), ) ]
def forwards_dist(self) -> Sequence[operations.Operation]: return [ operations.CreateTable( storage_set=StorageSetKey.QUERYLOG, table_name="querylog_dist", columns=columns, engine=table_engines.Distributed( local_table_name="querylog_local", sharding_key=None, ), ) ]
def backwards_dist(self) -> Sequence[operations.SqlOperation]: return [ operations.CreateTable( storage_set=StorageSetKey.EVENTS, table_name="sentry_dist", columns=columns, engine=table_engines.Distributed( local_table_name="sentry_local", sharding_key="cityHash64(toString(event_id))", ), ), operations.CreateTable( storage_set=StorageSetKey.EVENTS_RO, table_name="sentry_dist_ro", columns=columns, engine=table_engines.Distributed( local_table_name="sentry_local", sharding_key="cityHash64(toString(event_id))", ), ), ]
def forwards_dist(self) -> Sequence[operations.SqlOperation]: return [ operations.CreateTable( storage_set=StorageSetKey.REPLAYS, table_name="replays_dist", columns=raw_columns, engine=table_engines.Distributed( local_table_name="replays_local", sharding_key="cityHash64(toString(replay_id))", ), ), ]
def forwards_dist(self) -> Sequence[operations.SqlOperation]: return [ operations.CreateTable( storage_set=StorageSetKey.OUTCOMES, table_name="outcomes_raw_dist", columns=raw_columns, engine=table_engines.Distributed( local_table_name="outcomes_raw_local", sharding_key="org_id", ), ), operations.CreateTable( storage_set=StorageSetKey.OUTCOMES, table_name="outcomes_hourly_dist", columns=hourly_columns, engine=table_engines.Distributed( local_table_name="outcomes_hourly_local", sharding_key="org_id", ), ), ]
def forwards_dist(self) -> Sequence[operations.SqlOperation]: return [ operations.CreateTable( storage_set=StorageSetKey.METRICS, table_name=self.dist_table_name, columns=self.column_list, engine=table_engines.Distributed( local_table_name=self.local_table_name, sharding_key=None, ), ) ]
def forwards_dist(self) -> Sequence[operations.SqlOperation]: return [ operations.CreateTable( storage_set=StorageSetKey.PROFILES, table_name="profiles_dist", columns=columns, engine=table_engines.Distributed( local_table_name="profiles_local", sharding_key="cityHash64(profile_id)", ), ) ]
def forwards_dist(self) -> Sequence[operations.SqlOperation]: return [ operations.CreateTable( storage_set=StorageSetKey.TRANSACTIONS, table_name="transactions_dist", columns=columns, engine=table_engines.Distributed( local_table_name="transactions_local", sharding_key="cityHash64(span_id)", ), ) ]
def forwards_dist(self) -> Sequence[operations.SqlOperation]: return [ operations.CreateTable( storage_set=StorageSetKey.GENERIC_METRICS_SETS, table_name=self.dist_table_name, engine=table_engines.Distributed( local_table_name=self.local_table_name, sharding_key="cityHash64(timeseries_id)", ), columns=self.columns, ) ]
def forwards_dist(self) -> Sequence[operations.SqlOperation]: return [ operations.CreateTable( storage_set=StorageSetKey.EVENTS, table_name="errors_dist_new", columns=columns, engine=table_engines.Distributed( local_table_name="errors_local", sharding_key=sample_expr, ), ), operations.AddColumn( storage_set=StorageSetKey.EVENTS, table_name="errors_dist_new", column=Column( "_tags_hash_map", Array(UInt(64), Modifiers(materialized=TAGS_HASH_MAP_COLUMN)), ), after="tags", ), operations.DropTable( storage_set=StorageSetKey.EVENTS, table_name="errors_dist", ), operations.RenameTable( storage_set=StorageSetKey.EVENTS, old_table_name="errors_dist_new", new_table_name="errors_dist", ), operations.CreateTable( storage_set=StorageSetKey.EVENTS_RO, table_name="errors_dist_ro", columns=columns, engine=table_engines.Distributed( local_table_name="errors_local", sharding_key=sample_expr, ), ), ]
def __forwards_local(self) -> Sequence[operations.Operation]: return [ operations.CreateTable( storage_set=StorageSetKey.MIGRATIONS, table_name="migrations_local", columns=columns, engine=ReplacingMergeTree( storage_set=StorageSetKey.MIGRATIONS, version_column="version", order_by="(group, migration_id)", ), ), ]
def forwards_local(self) -> Sequence[operations.SqlOperation]: return [ operations.CreateTable( storage_set=StorageSetKey.CDC, table_name="groupassignee_local", columns=columns, engine=table_engines.ReplacingMergeTree( storage_set=StorageSetKey.CDC, version_column="offset", order_by="(project_id, group_id)", unsharded=True, ), ) ]
def forwards_local(self) -> Sequence[operations.Operation]: return [ operations.CreateTable( storage_set=StorageSetKey.QUERYLOG, table_name="querylog_local", columns=columns, engine=table_engines.MergeTree( storage_set=StorageSetKey.QUERYLOG, order_by="(toStartOfDay(timestamp), request_id)", partition_by="(toMonday(timestamp))", sample_by="request_id", ), ) ]
def forwards_local(self) -> Sequence[operations.Operation]: return [ operations.CreateTable( storage_set=StorageSetKey.EVENTS, table_name="groupedmessage_local", columns=columns, engine=table_engines.ReplacingMergeTree( storage_set=StorageSetKey.EVENTS, version_column="offset", order_by="(project_id, id)", sample_by="id", unsharded=True, ), ) ]
def forwards_local(self) -> Sequence[operations.SqlOperation]: return [ operations.CreateTable( storage_set=StorageSetKey.METRICS, table_name=self.local_table_name, columns=self.column_list, engine=table_engines.MergeTree( storage_set=StorageSetKey.METRICS, order_by= "(use_case_id, metric_type, org_id, project_id, metric_id, timestamp)", partition_by="(toStartOfDay(timestamp))", ttl="timestamp + toIntervalDay(7)", ), ) ]
def forwards_local(self) -> Sequence[operations.Operation]: return [ operations.CreateTable( storage_set=StorageSetKey.EVENTS, table_name="errors_local", columns=columns, engine=table_engines.ReplacingMergeTree( storage_set=StorageSetKey.EVENTS, version_column="deleted", order_by="(org_id, project_id, toStartOfDay(timestamp), primary_hash_hex, event_hash)", partition_by="(toMonday(timestamp), if(retention_days = 30, 30, 90))", sample_by="event_hash", ttl="timestamp + toIntervalDay(retention_days)", settings={"index_granularity": "8192"}, ), ) ]
def forwards_local(self) -> Sequence[operations.Operation]: sample_expr = "cityHash64(toString(event_id))" return [ operations.CreateTable( storage_set=StorageSetKey.EVENTS, table_name="sentry_local", columns=columns, engine=table_engines.ReplacingMergeTree( storage_set=StorageSetKey.EVENTS, version_column="deleted", order_by="(project_id, toStartOfDay(timestamp), %s)" % sample_expr, partition_by="(toMonday(timestamp), if(equals(retention_days, 30), 30, 90))", sample_by=sample_expr, ), ), ]
def forwards_local(self) -> Sequence[operations.Operation]: return [ operations.CreateTable( storage_set=StorageSetKey.TRANSACTIONS, table_name="transactions_local", columns=columns, engine=table_engines.ReplacingMergeTree( storage_set=StorageSetKey.TRANSACTIONS, version_column="deleted", order_by="(project_id, toStartOfDay(finish_ts), transaction_name, cityHash64(span_id))", partition_by="(retention_days, toMonday(finish_ts))", sample_by="cityHash64(span_id)", ttl="finish_ts + toIntervalDay(retention_days)", settings={"index_granularity": "8192"}, ), ) ]
def forwards_local(self) -> Sequence[operations.SqlOperation]: return [ operations.CreateTable( storage_set=StorageSetKey.PROFILES, table_name="profiles_local", columns=columns, engine=table_engines.ReplacingMergeTree( order_by= "(organization_id, project_id, toStartOfDay(received), cityHash64(profile_id))", partition_by="(retention_days, toMonday(received))", sample_by="cityHash64(profile_id)", settings={"index_granularity": "8192"}, storage_set=StorageSetKey.PROFILES, ttl="received + toIntervalDay(retention_days)", ), ) ]