("http_method", String(Modifiers(nullable=True))),
("http_referer", String(Modifiers(nullable=True))),
("tags", Nested([("key", String()), ("value", String())])),
("_tags_hash_map", Array(UInt(64))),
("contexts", Nested([("key", String()), ("value", String())])),
("deleted", UInt(8)),
])
schema = TableSchema(
columns=columns,
local_table_name="discover_local",
dist_table_name="discover_dist",
storage_set_key=StorageSetKey.DISCOVER,
mandatory_conditions=mandatory_conditions,
prewhere_candidates=[
"event_id",
"release",
"message",
"transaction_name",
"environment",
"project_id",
],
)
storage = ReadableTableStorage(
storage_key=StorageKey.DISCOVER,
storage_set_key=StorageSetKey.DISCOVER,
schema=schema,
query_processors=[
MappingOptimizer("tags", "_tags_hash_map", "tags_hash_map_enabled"),
EventIdColumnProcessor(),
("sdk_name", String(Modifiers(nullable=True))),
("sdk_version", String(Modifiers(nullable=True))),
("http_method", String(Modifiers(nullable=True))),
("http_referer", String(Modifiers(nullable=True))),
("tags", Nested([("key", String()), ("value", String())])),
("_tags_hash_map", Array(UInt(64))),
("contexts", Nested([("key", String()), ("value", String())])),
("trace_id", UUID(Modifiers(nullable=True))),
("deleted", UInt(8)),
]
)
schema = TableSchema(
columns=columns,
local_table_name="discover_local",
dist_table_name="discover_dist",
storage_set_key=StorageSetKey.DISCOVER,
mandatory_conditions=mandatory_conditions,
)
storage = ReadableTableStorage(
storage_key=StorageKey.DISCOVER,
storage_set_key=StorageSetKey.DISCOVER,
schema=schema,
query_processors=[
MappingColumnPromoter(
mapping_specs={
"tags": {
"environment": "environment",
"sentry:release": "release",
"sentry:dist": "dist",
Column("granularity", UInt(8)),
Column("timestamp", DateTime()),
Column("retention_days", UInt(16)),
Column(
"tags",
Nested([("key", UInt(64)), ("indexed_value", UInt(64)),
("raw_value", String())]),
),
Column("_raw_tags_hash", Array(UInt(64), SchemaModifiers(readonly=True))),
Column("_indexed_tags_hash", Array(UInt(64),
SchemaModifiers(readonly=True))),
]
sets_storage = ReadableTableStorage(
storage_key=StorageKey.GENERIC_METRICS_SETS,
storage_set_key=StorageSetKey.GENERIC_METRICS_SETS,
schema=TableSchema(
local_table_name="generic_metrics_sets_local",
dist_table_name="generic_metrics_sets_dist",
storage_set_key=StorageSetKey.GENERIC_METRICS_SETS,
columns=ColumnSet([
*aggregated_columns,
Column("value", AggregateFunction("uniqCombined64", [UInt(64)])),
]),
),
query_processors=[
ArrayJoinKeyValueOptimizer("tags"),
TableRateLimit(),
],
)
("sessions_preaggr", AggregateFunction("sumIf", [UInt(32), UInt(8)])),
("sessions_crashed", AggregateFunction("countIf", [UUID(), UInt(8)])),
("sessions_crashed_preaggr", AggregateFunction("sumIf", [UInt(32), UInt(8)])),
("sessions_abnormal", AggregateFunction("countIf", [UUID(), UInt(8)])),
("sessions_abnormal_preaggr", AggregateFunction("sumIf", [UInt(32), UInt(8)])),
("sessions_errored", AggregateFunction("uniqIf", [UUID(), UInt(8)])),
("sessions_errored_preaggr", AggregateFunction("sumIf", [UInt(32), UInt(8)])),
("users", AggregateFunction("uniqIf", [UUID(), UInt(8)])),
("users_crashed", AggregateFunction("uniqIf", [UUID(), UInt(8)])),
("users_abnormal", AggregateFunction("uniqIf", [UUID(), UInt(8)])),
("users_errored", AggregateFunction("uniqIf", [UUID(), UInt(8)])),
]
)
read_schema = TableSchema(
columns=read_columns,
local_table_name=READ_LOCAL_TABLE_NAME,
dist_table_name=READ_DIST_TABLE_NAME,
storage_set_key=StorageSetKey.SESSIONS,
)
materialized_view_schema = TableSchema(
local_table_name=READ_LOCAL_MV_NAME,
dist_table_name=READ_DIST_MV_NAME,
storage_set_key=StorageSetKey.SESSIONS,
columns=read_columns,
)
# The raw table we write onto, and that potentially we could
# query.
raw_storage = WritableTableStorage(
storage_key=StorageKey.SESSIONS_RAW,
storage_set_key=StorageSetKey.SESSIONS,
schema=raw_schema,
storage_set_key=StorageSetKey.OUTCOMES,
)
read_columns = ColumnSet([
("org_id", UInt(64)),
("project_id", UInt(64)),
("key_id", UInt(64)),
("timestamp", DateTime()),
("outcome", UInt(8)),
("reason", String()),
("times_seen", UInt(64)),
])
read_schema = TableSchema(
columns=read_columns,
local_table_name=READ_LOCAL_TABLE_NAME,
dist_table_name=READ_DIST_TABLE_NAME,
storage_set_key=StorageSetKey.OUTCOMES,
)
materialized_view_columns = ColumnSet([
("org_id", UInt(64)),
("project_id", UInt(64)),
("key_id", UInt(64)),
("timestamp", DateTime()),
("outcome", UInt(8)),
("reason", String()),
("times_seen", UInt(64)),
])
materialized_view_schema = TableSchema(
local_table_name="outcomes_mv_hourly_local",
"sessions_crashed",
AggregateFunction("countIf", UUID(), UInt(8)),
),
(
"sessions_abnormal",
AggregateFunction("countIf", UUID(), UInt(8)),
),
("sessions_errored", AggregateFunction("uniqIf", UUID(), UInt(8))),
("users_crashed", AggregateFunction("uniqIf", UUID(), UInt(8))),
("users_abnormal", AggregateFunction("uniqIf", UUID(), UInt(8))),
("users_errored", AggregateFunction("uniqIf", UUID(), UInt(8))),
])
read_schema = TableSchema(
columns=read_columns,
local_table_name=READ_LOCAL_TABLE_NAME,
dist_table_name=READ_DIST_TABLE_NAME,
storage_set_key=StorageSetKey.SESSIONS,
prewhere_candidates=["project_id", "org_id"],
)
materialized_view_schema = TableSchema(
local_table_name=READ_LOCAL_MV_NAME,
dist_table_name=READ_DIST_MV_NAME,
storage_set_key=StorageSetKey.SESSIONS,
prewhere_candidates=["project_id", "org_id"],
columns=read_columns,
)
# The raw table we write onto, and that potentially we could
# query.
raw_storage = WritableTableStorage(
storage_key=StorageKey.SESSIONS_RAW,
)
from snuba.clusters.storage_sets import StorageSetKey
from snuba.datasets.schemas.tables import TableSchema
from snuba.datasets.schemas.join import (
JoinConditionExpression,
JoinCondition,
JoinClause,
JoinType,
TableJoinNode,
)
table1 = TableSchema(
columns=ColumnSet([
("t1c1", UInt(64)),
("t1c2", String()),
("t1c3", Nested([("t11c4", UInt(64))])),
]),
local_table_name="table1",
dist_table_name="table1",
storage_set_key=StorageSetKey.EVENTS,
).get_data_source()
table2 = TableSchema(
columns=ColumnSet([
("t2c1", UInt(64)),
("t2c2", String()),
("t2c3", Nested([("t21c4", UInt(64))])),
]),
local_table_name="table2",
dist_table_name="table2",
storage_set_key=StorageSetKey.EVENTS,
).get_data_source()
from snuba.clusters.storage_sets import StorageSetKey
from snuba.datasets.storage import ReadableTableStorage
from snuba.datasets.schemas.tables import TableSchema
from snuba.datasets.storages import StorageKey
from snuba.datasets.storages.events_common import (
all_columns,
mandatory_conditions,
query_processors,
query_splitters,
)
schema = TableSchema(
columns=all_columns,
local_table_name="sentry_local",
dist_table_name="sentry_dist_ro",
storage_set_key=StorageSetKey.EVENTS_RO,
mandatory_conditions=mandatory_conditions,
)
storage = ReadableTableStorage(
storage_key=StorageKey.EVENTS_RO,
storage_set_key=StorageSetKey.EVENTS_RO,
schema=schema,
query_processors=query_processors,
query_splitters=query_splitters,
)
from snuba.clusters.storage_sets import StorageSetKey
from snuba.datasets.storage import ReadableTableStorage
from snuba.datasets.schemas.tables import TableSchema
from snuba.datasets.storages import StorageKey
from snuba.datasets.storages.errors_common import (
all_columns,
mandatory_conditions,
prewhere_candidates,
query_processors,
query_splitters,
)
schema = TableSchema(
columns=all_columns,
local_table_name="errors_local",
dist_table_name="errors_dist_ro",
storage_set_key=StorageSetKey.EVENTS_RO,
mandatory_conditions=mandatory_conditions,
prewhere_candidates=prewhere_candidates,
)
storage = ReadableTableStorage(
storage_key=StorageKey.ERRORS_RO,
storage_set_key=StorageSetKey.EVENTS_RO,
schema=schema,
query_processors=query_processors,
query_splitters=query_splitters,
)
from snuba.clusters.storage_sets import StorageSetKey
from snuba.datasets.schemas.tables import TableSchema
from snuba.datasets.storage import ReadableTableStorage
from snuba.datasets.storages import StorageKey
from snuba.datasets.storages.transactions_common import (
columns,
mandatory_condition_checkers,
query_processors,
query_splitters,
)
schema = TableSchema(
columns=columns,
local_table_name="transactions_local",
dist_table_name="transactions_dist",
storage_set_key=StorageSetKey.TRANSACTIONS_RO,
mandatory_conditions=[],
)
storage = ReadableTableStorage(
storage_key=StorageKey.TRANSACTIONS_RO,
storage_set_key=StorageSetKey.TRANSACTIONS_RO,
schema=schema,
query_processors=query_processors,
query_splitters=query_splitters,
mandatory_condition_checkers=mandatory_condition_checkers,
)
Column("metric_id", UInt(64)),
Column("granularity", UInt(32)),
Column("timestamp", DateTime()),
Column("retention_days", UInt(16)),
Column("tags", Nested([("key", UInt(64)), ("value", UInt(64))])),
Column("_tags_hash", Array(UInt(64), SchemaModifiers(readonly=True))),
]
sets_storage = ReadableTableStorage(
storage_key=StorageKey.METRICS_SETS,
storage_set_key=StorageSetKey.METRICS,
schema=TableSchema(
local_table_name="metrics_sets_local",
dist_table_name="metrics_sets_dist",
storage_set_key=StorageSetKey.METRICS,
columns=ColumnSet([
*aggregated_columns,
Column("value", AggregateFunction("uniqCombined64", [UInt(64)])),
]),
),
query_processors=[ArrayJoinKeyValueOptimizer("tags")],
)
counters_storage = ReadableTableStorage(
storage_key=StorageKey.METRICS_COUNTERS,
storage_set_key=StorageSetKey.METRICS,
schema=TableSchema(
local_table_name="metrics_counters_local",
dist_table_name="metrics_counters_dist",
storage_set_key=StorageSetKey.METRICS,
columns=ColumnSet([
from snuba.clusters.storage_sets import StorageSetKey
from snuba.datasets.schemas.tables import TableSchema
from snuba.datasets.storage import ReadableTableStorage
from snuba.datasets.storages import StorageKey
from snuba.datasets.storages.errors_common import (
all_columns,
mandatory_conditions,
query_splitters,
)
from snuba.datasets.storages.errors_v2 import query_processors
schema = TableSchema(
columns=all_columns,
local_table_name="errors_local",
dist_table_name="errors_dist_ro",
storage_set_key=StorageSetKey.ERRORS_V2_RO,
mandatory_conditions=mandatory_conditions,
)
storage = ReadableTableStorage(
storage_key=StorageKey.ERRORS_V2_RO,
storage_set_key=StorageSetKey.ERRORS_V2_RO,
schema=schema,
query_processors=query_processors,
query_splitters=query_splitters,
)
CounterAggregateProcessor(),
default_topic=Topic.METRICS,
dead_letter_queue_policy_creator=produce_policy_creator,
),
write_format=WriteFormat.VALUES,
)
org_counters_storage = ReadableTableStorage(
storage_key=StorageKey.ORG_METRICS_COUNTERS,
storage_set_key=StorageSetKey.METRICS,
schema=TableSchema(
local_table_name="metrics_counters_v2_local",
dist_table_name="metrics_counters_v2_dist",
storage_set_key=StorageSetKey.METRICS,
columns=ColumnSet([
Column("org_id", UInt(64)),
Column("project_id", UInt(64)),
Column("metric_id", UInt(64)),
Column("granularity", UInt(32)),
Column("timestamp", DateTime()),
]),
),
query_processors=[TableRateLimit()],
)
distributions_storage = WritableTableStorage(
storage_key=StorageKey.METRICS_DISTRIBUTIONS,
storage_set_key=StorageSetKey.METRICS,
schema=WritableTableSchema(
local_table_name="metrics_distributions_v2_local",
dist_table_name="metrics_distributions_v2_dist",
storage_set_key=StorageSetKey.METRICS,