def test_column_curried_function_translation() -> None:
assert ColumnToCurriedFunction(
None,
"duration_quantiles",
FunctionCall(
None,
"quantilesIfMerge",
(
Literal(None, 0.5),
Literal(None, 0.9),
),
),
(Column(None, None, "duration_quantiles"),),
).attempt_map(
Column("duration_quantiles", None, "duration_quantiles"),
SnubaClickhouseMappingTranslator(TranslationMappers()),
) == CurriedFunctionCall(
"duration_quantiles",
FunctionCall(
None,
"quantilesIfMerge",
(
Literal(None, 0.5),
Literal(None, 0.9),
),
),
(Column(None, None, "duration_quantiles"),),
)
def __init__(
self,
writable_storage_key: StorageKey,
readable_storage_key: StorageKey,
value_schema: Sequence[Column[SchemaModifiers]],
mappers: TranslationMappers,
) -> None:
writable_storage = get_writable_storage(writable_storage_key)
readable_storage = get_storage(readable_storage_key)
super().__init__(
storages=[writable_storage, readable_storage],
query_pipeline_builder=SimplePipelineBuilder(
query_plan_builder=SingleStorageQueryPlanBuilder(
readable_storage,
mappers=TranslationMappers(subscriptables=[
SubscriptableMapper(None, "tags", None, "tags"),
], ).concat(mappers),
)),
abstract_column_set=ColumnSet([
Column("org_id", UInt(64)),
Column("project_id", UInt(64)),
Column("metric_id", UInt(64)),
Column("timestamp", DateTime()),
Column("tags", Nested([("key", UInt(64)),
("value", UInt(64))])),
*value_schema,
]),
join_relationships={},
writable_storage=writable_storage,
validators=[
EntityRequiredColumnValidator({"org_id", "project_id"})
],
required_time_column="timestamp",
)
def test_col_nullable_measurements_translation() -> None:
translated = ColumnToMapping(
None, "lcp", None, "measurements", "lcp", nullable=True
).attempt_map(
Column("lcp", None, "lcp"),
SnubaClickhouseMappingTranslator(TranslationMappers()),
)
assert translated == _get_nullable_expr("lcp")
def test_default_translation(expression: Expression) -> None:
"""
Ensures that a translation that relies on the default translation rules
produces a deep copy of the original expression.
"""
translated = expression.accept(
SnubaClickhouseMappingTranslator(TranslationMappers()))
assert translated == expression
def __init__(self) -> None:
super().__init__(
writable_storage_key=StorageKey.METRICS_RAW,
readable_storage_key=StorageKey.METRICS_DISTRIBUTIONS,
value_schema=[
Column(
"percentiles",
AggregateFunction("quantiles(0.5, 0.75, 0.9, 0.95, 0.99)",
[Float(64)]),
),
Column("min", AggregateFunction("min", [Float(64)])),
Column("max", AggregateFunction("max", [Float(64)])),
Column("avg", AggregateFunction("avg", [Float(64)])),
Column("sum", AggregateFunction("sum", [Float(64)])),
Column("count", AggregateFunction("count", [Float(64)])),
Column(
"histogram_buckets",
AggregateFunction("histogram(250)", [Float(64)]),
),
],
mappers=TranslationMappers(
functions=[
AggregateFunctionMapper("value", "min", "minMerge", "min"),
AggregateFunctionMapper("value", "minIf", "minMergeIf",
"min"),
AggregateFunctionMapper("value", "max", "maxMerge", "max"),
AggregateFunctionMapper("value", "maxIf", "maxMergeIf",
"max"),
AggregateFunctionMapper("value", "avg", "avgMerge", "avg"),
AggregateFunctionMapper("value", "avgIf", "avgMergeIf",
"avg"),
AggregateFunctionMapper("value", "sum", "sumMerge", "sum"),
AggregateFunctionMapper("value", "sumIf", "sumMergeIf",
"sum"),
AggregateFunctionMapper("value", "count", "countMerge",
"count"),
AggregateFunctionMapper("value", "countIf", "countMergeIf",
"count"),
],
curried_functions=[
AggregateCurriedFunctionMapper("value", "quantiles",
"quantilesMerge",
"percentiles"),
AggregateCurriedFunctionMapper("value", "quantilesIf",
"quantilesMergeIf",
"percentiles"),
AggregateCurriedFunctionMapper("value", "histogram",
"histogramMerge",
"histogram_buckets"),
AggregateCurriedFunctionMapper("value", "histogramIf",
"histogramMergeIf",
"histogram_buckets"),
],
),
)
def __init__(self) -> None:
storage = get_writable_storage(StorageKey.SPANS)
super().__init__(
storages=[storage],
query_pipeline_builder=SimplePipelineBuilder(
query_plan_builder=SingleStorageQueryPlanBuilder(
storage=storage,
mappers=TranslationMappers(
subscriptables=[
SubscriptableMapper(None, "tags", None, "tags")
],
),
),
),
abstract_column_set=ColumnSet(
[
("project_id", UInt(64)),
("transaction_id", UUID()),
("trace_id", UUID()),
("transaction_span_id", UInt(64)),
("span_id", UInt(64)),
("parent_span_id", UInt(64, Modifiers(nullable=True))),
("transaction_name", String()),
("op", String()),
("status", UInt(8)),
("start_ts", DateTime()),
("start_ns", UInt(32)),
("finish_ts", DateTime()),
("finish_ns", UInt(32)),
("duration_ms", UInt(32)),
("tags", Nested([("key", String()), ("value", String())])),
]
),
join_relationships={
"contained": JoinRelationship(
rhs_entity=EntityKey.TRANSACTIONS,
columns=[
("project_id", "project_id"),
("transaction_span_id", "span_id"),
],
join_type=JoinType.INNER,
equivalences=[
ColumnEquivalence("transaction_id", "event_id"),
ColumnEquivalence("transaction_name", "transaction_name"),
ColumnEquivalence("trace_id", "trace_id"),
],
)
},
writable_storage=storage,
validators=[EntityRequiredColumnValidator({"project_id"})],
required_time_column=None,
)
def __init__(self) -> None:
super().__init__(
writable_storage_key=StorageKey.METRICS_RAW,
readable_storage_key=StorageKey.METRICS_COUNTERS,
value_schema=[
Column("value", AggregateFunction("sum", [Float(64)]))
],
mappers=TranslationMappers(functions=[
FunctionNameMapper("sum", "sumMerge"),
FunctionNameMapper("sumIf", "sumMergeIf"),
], ),
)
def __init__(self) -> None:
super().__init__(
writable_storage_key=StorageKey.METRICS_RAW,
readable_storage_key=StorageKey.METRICS_SETS,
value_schema=[
Column("value", AggregateFunction("uniqCombined64",
[UInt(64)])),
],
mappers=TranslationMappers(functions=[
FunctionNameMapper("uniq", "uniqCombined64Merge"),
FunctionNameMapper("uniqIf", "uniqCombined64MergeIf"),
], ),
)
def test_nullable_nested_translation() -> None:
translated = SubscriptableMapper(
None, "measurements", None, "measurements", nullable=True
).attempt_map(
SubscriptableReference(
"measurements[lcp]",
Column(None, None, "measurements"),
Literal(None, "lcp"),
),
SnubaClickhouseMappingTranslator(TranslationMappers()),
)
assert translated == _get_nullable_expr("measurements[lcp]")
def __init__(
self,
events_table: ReadableStorage,
events_ro_table: ReadableStorage,
abstract_events_columns: ColumnSet,
transactions_table: ReadableStorage,
abstract_transactions_columns: ColumnSet,
) -> None:
self.__events_table = events_table
self.__events_ro_table = events_ro_table
# Columns from the abstract model that map to storage columns present only
# in the Events table
self.__abstract_events_columns = abstract_events_columns
self.__transactions_table = transactions_table
# Columns from the abstract model that map to storage columns present only
# in the Transactions table
self.__abstract_transactions_columns = abstract_transactions_columns
self.__event_translator = event_translator.concat(
TranslationMappers(
columns=[
ColumnToMapping(None, "release", None, "tags", "sentry:release"),
ColumnToMapping(None, "dist", None, "tags", "sentry:dist"),
ColumnToMapping(None, "user", None, "tags", "sentry:user"),
DefaultNoneColumnMapper(self.__abstract_transactions_columns),
],
subscriptables=[DefaultNoneSubscriptMapper({"measurements"})],
)
)
self.__transaction_translator = transaction_translator.concat(
TranslationMappers(
columns=[
ColumnToLiteral(None, "group_id", 0),
DefaultNoneColumnMapper(self.__abstract_events_columns),
]
)
)
def test_column_function_translation() -> None:
assert ColumnToFunction(
None,
"ip_address",
"coalesce",
(Column(None, None, "ip_address_v4"), Column(None, None, "ip_address_v6")),
).attempt_map(
Column("ip_address", None, "ip_address"),
SnubaClickhouseMappingTranslator(TranslationMappers()),
) == FunctionCall(
"ip_address",
"coalesce",
(Column(None, None, "ip_address_v4"), Column(None, None, "ip_address_v6")),
)
def __init__(
self,
writable_storage_key: Optional[StorageKey],
readable_storage_key: StorageKey,
value_schema: Sequence[Column[SchemaModifiers]],
mappers: TranslationMappers,
abstract_column_set: Optional[ColumnSet] = None,
validators: Optional[Sequence[QueryValidator]] = None,
) -> None:
writable_storage = (get_writable_storage(writable_storage_key)
if writable_storage_key else None)
readable_storage = get_storage(readable_storage_key)
storages = [readable_storage]
if writable_storage:
storages.append(writable_storage)
if abstract_column_set is None:
abstract_column_set = ColumnSet([
Column("org_id", UInt(64)),
Column("project_id", UInt(64)),
Column("metric_id", UInt(64)),
Column("timestamp", DateTime()),
Column("bucketed_time", DateTime()),
Column("tags", Nested([("key", UInt(64)),
("value", UInt(64))])),
*value_schema,
])
if validators is None:
validators = [
EntityRequiredColumnValidator({"org_id", "project_id"}),
GranularityValidator(minimum=10),
]
super().__init__(
storages=storages,
query_pipeline_builder=SimplePipelineBuilder(
query_plan_builder=SingleStorageQueryPlanBuilder(
readable_storage,
mappers=TranslationMappers(subscriptables=[
SubscriptableMapper(None, "tags", None, "tags"),
], ).concat(mappers),
)),
abstract_column_set=abstract_column_set,
join_relationships={},
writable_storage=writable_storage,
validators=validators,
required_time_column="timestamp",
)
def __init__(self) -> None:
super().__init__(
writable_storage_key=None,
readable_storage_key=StorageKey.ORG_METRICS_COUNTERS,
value_schema=[],
mappers=TranslationMappers(),
abstract_column_set=ColumnSet([
Column("org_id", UInt(64)),
Column("project_id", UInt(64)),
Column("metric_id", UInt(64)),
Column("timestamp", DateTime()),
Column("bucketed_time", DateTime()),
]),
validators=[GranularityValidator(minimum=3600)],
)
def __init__(self) -> None:
super().__init__(
writable_storage_key=StorageKey.METRICS_COUNTERS_BUCKETS,
readable_storage_key=StorageKey.METRICS_COUNTERS,
value_schema=[
Column("value", AggregateFunction("sum", [Float(64)]))
],
mappers=TranslationMappers(columns=[
ColumnToFunction(
None,
"value",
"sumMerge",
(ColumnExpr(None, None, "value"), ),
),
], ),
)
def __init__(self) -> None:
super().__init__(
writable_storage_key=StorageKey.METRICS_BUCKETS,
readable_storage_key=StorageKey.METRICS_SETS,
value_schema=[
Column("value", AggregateFunction("uniqCombined64",
[UInt(64)])),
],
mappers=TranslationMappers(columns=[
ColumnToFunction(
None,
"value",
"uniqCombined64Merge",
(ColumnExpr(None, None, "value"), ),
),
], ),
)
def __init__(
self,
storage: ReadableStorage,
mappers: Optional[TranslationMappers] = None,
post_processors: Optional[Sequence[QueryProcessor]] = None,
) -> None:
# The storage the query is based on
self.__storage = storage
# The translation mappers to be used when translating the logical query
# into the clickhouse query.
self.__mappers = mappers if mappers is not None else TranslationMappers()
# This is a set of query processors that have to be executed on the
# query after the storage selection but that are defined by the dataset.
# Query processors defined by a Storage must be executable independently
# from the context the Storage is used (whether the storage is used by
# itself or whether it is joined with another storage).
# In a joined query we would have processors defined by multiple storages.
# that would have to be executed only once (like Prewhere). That is a
# candidate to be added here as post process.
self.__post_processors = post_processors or []
def test_tag_translation() -> None:
translated = SubscriptableMapper(None, "tags", None, "tags").attempt_map(
SubscriptableReference(
"tags[release]", Column(None, None, "tags"), Literal(None, "release")
),
SnubaClickhouseMappingTranslator(TranslationMappers()),
)
assert translated == FunctionCall(
"tags[release]",
"arrayElement",
(
Column(None, None, "tags.value"),
FunctionCall(
None,
"indexOf",
(Column(None, None, "tags.key"), Literal(None, "release")),
),
),
)
def test_col_tag_translation() -> None:
translated = ColumnToMapping(
None, "geo_country_code", None, "contexts", "geo.country_code"
).attempt_map(
Column("geo_country_code", None, "geo_country_code"),
SnubaClickhouseMappingTranslator(TranslationMappers()),
)
assert translated == FunctionCall(
"geo_country_code",
"arrayElement",
(
Column(None, None, "contexts.value"),
FunctionCall(
None,
"indexOf",
(Column(None, None, "contexts.key"), Literal(None, "geo.country_code")),
),
),
)
def __init__(self) -> None:
writable_storage = get_writable_storage(StorageKey.SESSIONS_RAW)
materialized_storage = get_storage(StorageKey.SESSIONS_HOURLY)
read_schema = materialized_storage.get_schema()
self.__time_group_columns = {"bucketed_started": "started"}
self.__time_parse_columns = ("started", "received")
super().__init__(
storages=[writable_storage, materialized_storage],
# TODO: Once we are ready to expose the raw data model and select whether to use
# materialized storage or the raw one here, replace this with a custom storage
# selector that decides when to use the materialized data.
query_plan_builder=SingleStorageQueryPlanBuilder(
storage=materialized_storage,
mappers=TranslationMappers(columns=[
ColumnToCurriedFunction(
None,
"duration_quantiles",
FunctionCall(
None,
"quantilesIfMerge",
(Literal(None, 0.5), Literal(None, 0.9)),
),
(Column(None, None, "duration_quantiles"), ),
),
function_rule("sessions", "countIfMerge"),
function_rule("sessions_crashed", "countIfMerge"),
function_rule("sessions_abnormal", "countIfMerge"),
function_rule("users", "uniqIfMerge"),
function_rule("sessions_errored", "uniqIfMerge"),
function_rule("users_crashed", "uniqIfMerge"),
function_rule("users_abnormal", "uniqIfMerge"),
function_rule("users_errored", "uniqIfMerge"),
]),
),
abstract_column_set=read_schema.get_columns(),
writable_storage=writable_storage,
)
def __init__(self) -> None:
super().__init__(
writable_storage_key=StorageKey.METRICS_DISTRIBUTIONS_BUCKETS,
readable_storage_key=StorageKey.METRICS_DISTRIBUTIONS,
value_schema=[
Column(
"percentiles",
AggregateFunction("quantiles(0.5, 0.75, 0.9, 0.95, 0.99)",
[Float(64)]),
),
Column("min", AggregateFunction("min", [Float(64)])),
Column("max", AggregateFunction("max", [Float(64)])),
Column("avg", AggregateFunction("avg", [Float(64)])),
Column("sum", AggregateFunction("sum", [Float(64)])),
Column("count", AggregateFunction("count", [Float(64)])),
],
mappers=TranslationMappers(columns=[
ColumnToCurriedFunction(
None,
"percentiles",
FunctionCall(
None,
"quantilesMerge",
tuple(
Literal(None, quant)
for quant in [0.5, 0.75, 0.9, 0.95, 0.99]),
),
(ColumnExpr(None, None, "percentiles"), ),
),
merge_mapper("min"),
merge_mapper("max"),
merge_mapper("avg"),
merge_mapper("sum"),
merge_mapper("count"),
], ),
)
def __init__(self) -> None:
self.__common_columns = ColumnSet([
("event_id", FixedString(32)),
("project_id", UInt(64)),
("type", String(Modifiers(nullable=True))),
("timestamp", DateTime()),
("platform", String(Modifiers(nullable=True))),
("environment", String(Modifiers(nullable=True))),
("release", String(Modifiers(nullable=True))),
("dist", String(Modifiers(nullable=True))),
("user", String(Modifiers(nullable=True))),
("transaction", String(Modifiers(nullable=True))),
("message", String(Modifiers(nullable=True))),
("title", String(Modifiers(nullable=True))),
# User
("user_id", String(Modifiers(nullable=True))),
("username", String(Modifiers(nullable=True))),
("email", String(Modifiers(nullable=True))),
("ip_address", String(Modifiers(nullable=True))),
# SDK
("sdk_name", String(Modifiers(nullable=True))),
("sdk_version", String(Modifiers(nullable=True))),
# geo location context
("geo_country_code", String(Modifiers(nullable=True))),
("geo_region", String(Modifiers(nullable=True))),
("geo_city", String(Modifiers(nullable=True))),
("http_method", String(Modifiers(nullable=True))),
("http_referer", String(Modifiers(nullable=True))),
# Other tags and context
("tags", Nested([("key", String()), ("value", String())])),
("contexts", Nested([("key", String()), ("value", String())])),
])
self.__events_columns = EVENTS_COLUMNS
self.__transactions_columns = TRANSACTIONS_COLUMNS
events_storage = get_storage(StorageKey.EVENTS)
events_pipeline_builder = SimplePipelineBuilder(
query_plan_builder=SelectedStorageQueryPlanBuilder(
selector=EventsQueryStorageSelector(
mappers=events_translation_mappers.
concat(transaction_translation_mappers).concat(
null_function_translation_mappers).concat(
TranslationMappers(
# XXX: Remove once we are using errors
columns=[
ColumnToMapping(None, "release", None,
"tags", "sentry:release"),
ColumnToMapping(None, "dist", None, "tags",
"sentry:dist"),
ColumnToMapping(None, "user", None, "tags",
"sentry:user"),
],
subscriptables=[
SubscriptableMapper(
None, "tags", None, "tags"),
SubscriptableMapper(
None, "contexts", None, "contexts"),
],
)))), )
discover_storage = get_storage(StorageKey.DISCOVER)
discover_pipeline_builder = SimplePipelineBuilder(
query_plan_builder=SingleStorageQueryPlanBuilder(
storage=discover_storage,
mappers=events_translation_mappers.concat(
transaction_translation_mappers).
concat(null_function_translation_mappers).concat(
TranslationMappers(columns=[
ColumnToFunction(
None,
"ip_address",
"coalesce",
(
FunctionCall(
None,
"IPv4NumToString",
(Column(None, None, "ip_address_v4"), ),
),
FunctionCall(
None,
"IPv6NumToString",
(Column(None, None, "ip_address_v6"), ),
),
),
),
ColumnToColumn(None, "transaction", None,
"transaction_name"),
ColumnToColumn(None, "username", None, "user_name"),
ColumnToColumn(None, "email", None, "user_email"),
ColumnToMapping(
None,
"geo_country_code",
None,
"contexts",
"geo.country_code",
),
ColumnToMapping(None, "geo_region", None, "contexts",
"geo.region"),
ColumnToMapping(None, "geo_city", None, "contexts",
"geo.city"),
ColumnToFunction(
None,
"user",
"nullIf",
(Column(None, None, "user"), Literal(None, "")),
),
])).concat(
TranslationMappers(subscriptables=[
SubscriptableMapper(None, "tags", None, "tags"),
SubscriptableMapper(None, "contexts", None,
"contexts"),
], )),
))
def selector_func(_query: Query) -> Tuple[str, List[str]]:
if random.random() < float(
state.get_config("discover_query_percentage", 0)):
return "events", ["discover"]
return "events", []
super().__init__(
storages=[events_storage, discover_storage],
query_pipeline_builder=PipelineDelegator(
query_pipeline_builders={
"events": events_pipeline_builder,
"discover": discover_pipeline_builder,
},
selector_func=selector_func,
callback_func=partial(callback_func, "discover"),
),
abstract_column_set=(self.__common_columns +
self.__events_columns +
self.__transactions_columns),
join_relationships={},
writable_storage=None,
)
sessions_hourly_translators = TranslationMappers(columns=[
ColumnToCurriedFunction(
None,
"duration_quantiles",
FunctionCall(None, "quantilesIfMerge", quantiles),
(Column(None, None, "duration_quantiles"), ),
),
function_column("duration_avg", "avgIfMerge"),
plus_columns(
"sessions",
function_call("sessions", "countIfMerge"),
function_call("sessions_preaggr", "sumIfMerge"),
),
plus_columns(
"sessions_crashed",
function_call("sessions_crashed", "countIfMerge"),
function_call("sessions_crashed_preaggr", "sumIfMerge"),
),
plus_columns(
"sessions_abnormal",
function_call("sessions_abnormal", "countIfMerge"),
function_call("sessions_abnormal_preaggr", "sumIfMerge"),
),
plus_columns(
"sessions_errored",
function_call("sessions_errored", "uniqIfMerge"),
function_call("sessions_errored_preaggr", "sumIfMerge"),
),
function_column("users", "uniqIfMerge"),
function_column("users_crashed", "uniqIfMerge"),
function_column("users_abnormal", "uniqIfMerge"),
function_column("users_errored", "uniqIfMerge"),
])
sessions_translators = TranslationMappers(columns=[
ColumnToCurriedFunction(
None,
"duration_quantiles",
FunctionCall(
None,
"quantilesIfMerge",
tuple(
Literal(None, quant)
for quant in [0.5, 0.75, 0.9, 0.95, 0.99, 1]),
),
(Column(None, None, "duration_quantiles"), ),
),
function_column("duration_avg", "avgIfMerge"),
plus_columns(
"sessions",
function_call("sessions", "countIfMerge"),
function_call("sessions_preaggr", "sumIfMerge"),
),
plus_columns(
"sessions_crashed",
function_call("sessions_crashed", "countIfMerge"),
function_call("sessions_crashed_preaggr", "sumIfMerge"),
),
plus_columns(
"sessions_abnormal",
function_call("sessions_abnormal", "countIfMerge"),
function_call("sessions_abnormal_preaggr", "sumIfMerge"),
),
plus_columns(
"sessions_errored",
function_call("sessions_errored", "uniqIfMerge"),
function_call("sessions_errored_preaggr", "sumIfMerge"),
),
function_column("users", "uniqIfMerge"),
function_column("users_crashed", "uniqIfMerge"),
function_column("users_abnormal", "uniqIfMerge"),
function_column("users_errored", "uniqIfMerge"),
])
("span_op_breakdowns", Nested([("key", String()), ("value", Float(64))])),
(
"spans",
Nested([
("op", String()),
("group", UInt(64)),
("exclusive_time", Float(64)),
("exclusive_time_32", Float(32)),
]),
),
])
events_translation_mappers = TranslationMappers(
columns=[DefaultNoneColumnMapper(TRANSACTIONS_COLUMNS)],
functions=[DefaultNoneFunctionMapper({"apdex", "failure_rate"})],
subscriptables=[
DefaultNoneSubscriptMapper({"measurements", "span_op_breakdowns"})
],
)
transaction_translation_mappers = TranslationMappers(
columns=[
ColumnToLiteral(None, "group_id", 0),
DefaultNoneColumnMapper(EVENTS_COLUMNS),
],
functions=[DefaultNoneFunctionMapper({"isHandled", "notHandled"})],
)
null_function_translation_mappers = TranslationMappers(
curried_functions=[DefaultIfNullCurriedFunctionMapper()],
functions=[DefaultIfNullFunctionMapper()],
from snuba.datasets.entities import EntityKey
from snuba.datasets.plans.translator.query import QueryTranslator
from snuba.query import SelectedExpression
from snuba.query.data_source.simple import Entity as QueryEntity
from snuba.query.data_source.simple import Table
from snuba.query.expressions import (
Column,
FunctionCall,
Literal,
SubscriptableReference,
)
from snuba.query.logical import Query as SnubaQuery
test_cases = [
pytest.param(
TranslationMappers(),
SnubaQuery(
from_clause=QueryEntity(EntityKey.EVENTS, ColumnSet([])),
selected_columns=[
SelectedExpression("alias", Column("alias", "table", "column")),
SelectedExpression(
"alias2",
FunctionCall(
"alias2",
"f1",
(Column(None, None, "column2"), Column(None, None, "column3")),
),
),
SelectedExpression(
name=None,
expression=SubscriptableReference(
def __init__(self) -> None:
self.__common_columns = ColumnSet([
("event_id", FixedString(32)),
("project_id", UInt(64)),
("type", String(Modifiers(nullable=True))),
("timestamp", DateTime()),
("platform", String(Modifiers(nullable=True))),
("environment", String(Modifiers(nullable=True))),
("release", String(Modifiers(nullable=True))),
("dist", String(Modifiers(nullable=True))),
("user", String(Modifiers(nullable=True))),
("transaction", String(Modifiers(nullable=True))),
("message", String(Modifiers(nullable=True))),
("title", String(Modifiers(nullable=True))),
# User
("user_id", String(Modifiers(nullable=True))),
("username", String(Modifiers(nullable=True))),
("email", String(Modifiers(nullable=True))),
("ip_address", String(Modifiers(nullable=True))),
# SDK
("sdk_name", String(Modifiers(nullable=True))),
("sdk_version", String(Modifiers(nullable=True))),
# geo location context
("geo_country_code", String(Modifiers(nullable=True))),
("geo_region", String(Modifiers(nullable=True))),
("geo_city", String(Modifiers(nullable=True))),
("http_method", String(Modifiers(nullable=True))),
("http_referer", String(Modifiers(nullable=True))),
# Other tags and context
("tags", Nested([("key", String()), ("value", String())])),
("contexts", Nested([("key", String()), ("value", String())])),
("trace_id", String(Modifiers(nullable=True))),
("span_id", UInt(64, Modifiers(nullable=True))),
])
self.__events_columns = EVENTS_COLUMNS
self.__transactions_columns = TRANSACTIONS_COLUMNS
discover_storage = get_storage(StorageKey.DISCOVER)
discover_storage_plan_builder = SingleStorageQueryPlanBuilder(
storage=discover_storage,
mappers=events_translation_mappers.
concat(transaction_translation_mappers).concat(
null_function_translation_mappers).concat(
TranslationMappers(columns=[
ColumnToFunction(
None,
"ip_address",
"coalesce",
(
FunctionCall(
None,
"IPv4NumToString",
(Column(None, None, "ip_address_v4"), ),
),
FunctionCall(
None,
"IPv6NumToString",
(Column(None, None, "ip_address_v6"), ),
),
),
),
ColumnToColumn(None, "transaction", None,
"transaction_name"),
ColumnToColumn(None, "username", None, "user_name"),
ColumnToColumn(None, "email", None, "user_email"),
ColumnToMapping(
None,
"geo_country_code",
None,
"contexts",
"geo.country_code",
nullable=True,
),
ColumnToMapping(
None,
"geo_region",
None,
"contexts",
"geo.region",
nullable=True,
),
ColumnToMapping(
None,
"geo_city",
None,
"contexts",
"geo.city",
nullable=True,
),
ColumnToFunction(
None,
"user",
"nullIf",
(Column(None, None, "user"), Literal(None, "")),
),
])).concat(
TranslationMappers(subscriptables=[
SubscriptableMapper(None, "tags", None, "tags"),
SubscriptableMapper(None, "contexts", None,
"contexts"),
], )),
)
discover_pipeline_builder = SimplePipelineBuilder(
query_plan_builder=discover_storage_plan_builder)
super().__init__(
storages=[discover_storage],
query_pipeline_builder=discover_pipeline_builder,
abstract_column_set=(self.__common_columns +
self.__events_columns +
self.__transactions_columns),
join_relationships={},
writable_storage=None,
validators=[EntityRequiredColumnValidator({"project_id"})],
required_time_column="timestamp",
)
from snuba.query.parsing import ParsingContext
from snuba.query.processors import QueryProcessor
from snuba.query.processors.basic_functions import BasicFunctionsProcessor
from snuba.query.processors.handled_functions import HandledFunctionsProcessor
from snuba.query.processors.tags_expander import TagsExpanderProcessor
from snuba.query.processors.timeseries_column_processor import TimeSeriesColumnProcessor
from snuba.query.project_extension import ProjectExtension
from snuba.query.timeseries_extension import TimeSeriesExtension
from snuba.request.request_settings import RequestSettings
from snuba.util import parse_datetime, qualified_column
# TODO: This will be a property of the relationship between entity and
# storage. Now we do not have entities so it is between dataset and
# storage.
event_translator = TranslationMappers(subscriptables=[
SubscriptableMapper(None, "tags", None, "tags"),
SubscriptableMapper(None, "contexts", None, "contexts"),
], )
class EventsQueryStorageSelector(QueryStorageSelector):
def __init__(
self,
events_table: ReadableStorage,
events_ro_table: ReadableStorage,
) -> None:
self.__events_table = events_table
self.__events_ro_table = events_ro_table
def select_storage(self, query: Query,
request_settings: RequestSettings) -> StorageAndMappers:
use_readonly_storage = (state.get_config(
transaction_translator = TranslationMappers(
columns=[
ColumnToFunction(
None,
"ip_address",
"coalesce",
(
FunctionCall(
None, "IPv4NumToString", (Column(None, None, "ip_address_v4"),),
),
FunctionCall(
None, "IPv6NumToString", (Column(None, None, "ip_address_v6"),),
),
),
),
ColumnToFunction(
None, "user", "nullIf", (Column(None, None, "user"), Literal(None, ""))
),
# These column aliases originally existed in the ``discover`` dataset,
# but now live here to maintain compatibility between the composite
# ``discover`` dataset and the standalone ``transaction`` dataset. In
# the future, these aliases should be defined on the Transaction entity
# instead of the dataset.
ColumnToLiteral(None, "type", "transaction"),
ColumnToColumn(None, "timestamp", None, "finish_ts"),
ColumnToColumn(None, "username", None, "user_name"),
ColumnToColumn(None, "email", None, "user_email"),
ColumnToColumn(None, "transaction", None, "transaction_name"),
ColumnToColumn(None, "message", None, "transaction_name"),
ColumnToColumn(None, "title", None, "transaction_name"),
ColumnToMapping(None, "geo_country_code", None, "contexts", "geo.country_code"),
ColumnToMapping(None, "geo_region", None, "contexts", "geo.region"),
ColumnToMapping(None, "geo_city", None, "contexts", "geo.city"),
],
subscriptables=[
SubscriptableMapper(None, "tags", None, "tags"),
SubscriptableMapper(None, "contexts", None, "contexts"),
SubscriptableMapper(None, "measurements", None, "measurements", nullable=True),
],
)
from snuba.query.logical import Query
from snuba.query.parsing import ParsingContext
from snuba.query.processors import QueryProcessor
from snuba.query.processors.basic_functions import BasicFunctionsProcessor
from snuba.query.processors.handled_functions import HandledFunctionsProcessor
from snuba.query.processors.tags_expander import TagsExpanderProcessor
from snuba.query.processors.timeseries_column_processor import TimeSeriesColumnProcessor
from snuba.query.project_extension import ProjectExtension
from snuba.query.timeseries_extension import TimeSeriesExtension
from snuba.util import parse_datetime
errors_translators = TranslationMappers(
columns=[
ColumnToFunction(
None, "user", "nullIf", (Column(None, None, "user"), Literal(None, ""))
),
]
)
class ErrorsEntity(Entity):
"""
Represents the collections of all event types that are not transactions.
This is meant to replace Events. They will both exist during the migration.
"""
def __init__(self) -> None:
storage = get_writable_storage(StorageKey.ERRORS)
schema = storage.get_table_writer().get_schema()
from snuba.query.processors.project_rate_limiter import ProjectRateLimiterProcessor
from snuba.query.processors.tags_expander import TagsExpanderProcessor
from snuba.query.processors.timeseries_processor import TimeSeriesProcessor
from snuba.query.project_extension import ProjectExtension
from snuba.query.timeseries_extension import TimeSeriesExtension
from snuba.request.request_settings import RequestSettings
from snuba.utils.metrics.wrapper import MetricsWrapper
from snuba.utils.threaded_function_delegator import Result
from snuba.web import QueryResult
event_translator = TranslationMappers(
columns=[
ColumnToMapping(None, "release", None, "tags", "sentry:release"),
ColumnToMapping(None, "dist", None, "tags", "sentry:dist"),
ColumnToMapping(None, "user", None, "tags", "sentry:user"),
],
subscriptables=[
SubscriptableMapper(None, "tags", None, "tags"),
SubscriptableMapper(None, "contexts", None, "contexts"),
],
)
errors_translators = TranslationMappers(
columns=[
ColumnToMapping(None, "release", None, "tags", "sentry:release"),
ColumnToMapping(None, "dist", None, "tags", "sentry:dist"),
ColumnToMapping(None, "user", None, "tags", "sentry:user"),
ColumnToFunction(
None,
"ip_address",
"coalesce",
