def test_column_curried_function_translation() -> None: assert ColumnToCurriedFunction( None, "duration_quantiles", FunctionCall( None, "quantilesIfMerge", ( Literal(None, 0.5), Literal(None, 0.9), ), ), (Column(None, None, "duration_quantiles"),), ).attempt_map( Column("duration_quantiles", None, "duration_quantiles"), SnubaClickhouseMappingTranslator(TranslationMappers()), ) == CurriedFunctionCall( "duration_quantiles", FunctionCall( None, "quantilesIfMerge", ( Literal(None, 0.5), Literal(None, 0.9), ), ), (Column(None, None, "duration_quantiles"),), )
def __init__( self, writable_storage_key: StorageKey, readable_storage_key: StorageKey, value_schema: Sequence[Column[SchemaModifiers]], mappers: TranslationMappers, ) -> None: writable_storage = get_writable_storage(writable_storage_key) readable_storage = get_storage(readable_storage_key) super().__init__( storages=[writable_storage, readable_storage], query_pipeline_builder=SimplePipelineBuilder( query_plan_builder=SingleStorageQueryPlanBuilder( readable_storage, mappers=TranslationMappers(subscriptables=[ SubscriptableMapper(None, "tags", None, "tags"), ], ).concat(mappers), )), abstract_column_set=ColumnSet([ Column("org_id", UInt(64)), Column("project_id", UInt(64)), Column("metric_id", UInt(64)), Column("timestamp", DateTime()), Column("tags", Nested([("key", UInt(64)), ("value", UInt(64))])), *value_schema, ]), join_relationships={}, writable_storage=writable_storage, validators=[ EntityRequiredColumnValidator({"org_id", "project_id"}) ], required_time_column="timestamp", )
def test_col_nullable_measurements_translation() -> None: translated = ColumnToMapping( None, "lcp", None, "measurements", "lcp", nullable=True ).attempt_map( Column("lcp", None, "lcp"), SnubaClickhouseMappingTranslator(TranslationMappers()), ) assert translated == _get_nullable_expr("lcp")
def test_default_translation(expression: Expression) -> None: """ Ensures that a translation that relies on the default translation rules produces a deep copy of the original expression. """ translated = expression.accept( SnubaClickhouseMappingTranslator(TranslationMappers())) assert translated == expression
def __init__(self) -> None: super().__init__( writable_storage_key=StorageKey.METRICS_RAW, readable_storage_key=StorageKey.METRICS_DISTRIBUTIONS, value_schema=[ Column( "percentiles", AggregateFunction("quantiles(0.5, 0.75, 0.9, 0.95, 0.99)", [Float(64)]), ), Column("min", AggregateFunction("min", [Float(64)])), Column("max", AggregateFunction("max", [Float(64)])), Column("avg", AggregateFunction("avg", [Float(64)])), Column("sum", AggregateFunction("sum", [Float(64)])), Column("count", AggregateFunction("count", [Float(64)])), Column( "histogram_buckets", AggregateFunction("histogram(250)", [Float(64)]), ), ], mappers=TranslationMappers( functions=[ AggregateFunctionMapper("value", "min", "minMerge", "min"), AggregateFunctionMapper("value", "minIf", "minMergeIf", "min"), AggregateFunctionMapper("value", "max", "maxMerge", "max"), AggregateFunctionMapper("value", "maxIf", "maxMergeIf", "max"), AggregateFunctionMapper("value", "avg", "avgMerge", "avg"), AggregateFunctionMapper("value", "avgIf", "avgMergeIf", "avg"), AggregateFunctionMapper("value", "sum", "sumMerge", "sum"), AggregateFunctionMapper("value", "sumIf", "sumMergeIf", "sum"), AggregateFunctionMapper("value", "count", "countMerge", "count"), AggregateFunctionMapper("value", "countIf", "countMergeIf", "count"), ], curried_functions=[ AggregateCurriedFunctionMapper("value", "quantiles", "quantilesMerge", "percentiles"), AggregateCurriedFunctionMapper("value", "quantilesIf", "quantilesMergeIf", "percentiles"), AggregateCurriedFunctionMapper("value", "histogram", "histogramMerge", "histogram_buckets"), AggregateCurriedFunctionMapper("value", "histogramIf", "histogramMergeIf", "histogram_buckets"), ], ), )
def __init__(self) -> None: storage = get_writable_storage(StorageKey.SPANS) super().__init__( storages=[storage], query_pipeline_builder=SimplePipelineBuilder( query_plan_builder=SingleStorageQueryPlanBuilder( storage=storage, mappers=TranslationMappers( subscriptables=[ SubscriptableMapper(None, "tags", None, "tags") ], ), ), ), abstract_column_set=ColumnSet( [ ("project_id", UInt(64)), ("transaction_id", UUID()), ("trace_id", UUID()), ("transaction_span_id", UInt(64)), ("span_id", UInt(64)), ("parent_span_id", UInt(64, Modifiers(nullable=True))), ("transaction_name", String()), ("op", String()), ("status", UInt(8)), ("start_ts", DateTime()), ("start_ns", UInt(32)), ("finish_ts", DateTime()), ("finish_ns", UInt(32)), ("duration_ms", UInt(32)), ("tags", Nested([("key", String()), ("value", String())])), ] ), join_relationships={ "contained": JoinRelationship( rhs_entity=EntityKey.TRANSACTIONS, columns=[ ("project_id", "project_id"), ("transaction_span_id", "span_id"), ], join_type=JoinType.INNER, equivalences=[ ColumnEquivalence("transaction_id", "event_id"), ColumnEquivalence("transaction_name", "transaction_name"), ColumnEquivalence("trace_id", "trace_id"), ], ) }, writable_storage=storage, validators=[EntityRequiredColumnValidator({"project_id"})], required_time_column=None, )
def __init__(self) -> None: super().__init__( writable_storage_key=StorageKey.METRICS_RAW, readable_storage_key=StorageKey.METRICS_COUNTERS, value_schema=[ Column("value", AggregateFunction("sum", [Float(64)])) ], mappers=TranslationMappers(functions=[ FunctionNameMapper("sum", "sumMerge"), FunctionNameMapper("sumIf", "sumMergeIf"), ], ), )
def __init__(self) -> None: super().__init__( writable_storage_key=StorageKey.METRICS_RAW, readable_storage_key=StorageKey.METRICS_SETS, value_schema=[ Column("value", AggregateFunction("uniqCombined64", [UInt(64)])), ], mappers=TranslationMappers(functions=[ FunctionNameMapper("uniq", "uniqCombined64Merge"), FunctionNameMapper("uniqIf", "uniqCombined64MergeIf"), ], ), )
def test_nullable_nested_translation() -> None: translated = SubscriptableMapper( None, "measurements", None, "measurements", nullable=True ).attempt_map( SubscriptableReference( "measurements[lcp]", Column(None, None, "measurements"), Literal(None, "lcp"), ), SnubaClickhouseMappingTranslator(TranslationMappers()), ) assert translated == _get_nullable_expr("measurements[lcp]")
def __init__( self, events_table: ReadableStorage, events_ro_table: ReadableStorage, abstract_events_columns: ColumnSet, transactions_table: ReadableStorage, abstract_transactions_columns: ColumnSet, ) -> None: self.__events_table = events_table self.__events_ro_table = events_ro_table # Columns from the abstract model that map to storage columns present only # in the Events table self.__abstract_events_columns = abstract_events_columns self.__transactions_table = transactions_table # Columns from the abstract model that map to storage columns present only # in the Transactions table self.__abstract_transactions_columns = abstract_transactions_columns self.__event_translator = event_translator.concat( TranslationMappers( columns=[ ColumnToMapping(None, "release", None, "tags", "sentry:release"), ColumnToMapping(None, "dist", None, "tags", "sentry:dist"), ColumnToMapping(None, "user", None, "tags", "sentry:user"), DefaultNoneColumnMapper(self.__abstract_transactions_columns), ], subscriptables=[DefaultNoneSubscriptMapper({"measurements"})], ) ) self.__transaction_translator = transaction_translator.concat( TranslationMappers( columns=[ ColumnToLiteral(None, "group_id", 0), DefaultNoneColumnMapper(self.__abstract_events_columns), ] ) )
def test_column_function_translation() -> None: assert ColumnToFunction( None, "ip_address", "coalesce", (Column(None, None, "ip_address_v4"), Column(None, None, "ip_address_v6")), ).attempt_map( Column("ip_address", None, "ip_address"), SnubaClickhouseMappingTranslator(TranslationMappers()), ) == FunctionCall( "ip_address", "coalesce", (Column(None, None, "ip_address_v4"), Column(None, None, "ip_address_v6")), )
def __init__( self, writable_storage_key: Optional[StorageKey], readable_storage_key: StorageKey, value_schema: Sequence[Column[SchemaModifiers]], mappers: TranslationMappers, abstract_column_set: Optional[ColumnSet] = None, validators: Optional[Sequence[QueryValidator]] = None, ) -> None: writable_storage = (get_writable_storage(writable_storage_key) if writable_storage_key else None) readable_storage = get_storage(readable_storage_key) storages = [readable_storage] if writable_storage: storages.append(writable_storage) if abstract_column_set is None: abstract_column_set = ColumnSet([ Column("org_id", UInt(64)), Column("project_id", UInt(64)), Column("metric_id", UInt(64)), Column("timestamp", DateTime()), Column("bucketed_time", DateTime()), Column("tags", Nested([("key", UInt(64)), ("value", UInt(64))])), *value_schema, ]) if validators is None: validators = [ EntityRequiredColumnValidator({"org_id", "project_id"}), GranularityValidator(minimum=10), ] super().__init__( storages=storages, query_pipeline_builder=SimplePipelineBuilder( query_plan_builder=SingleStorageQueryPlanBuilder( readable_storage, mappers=TranslationMappers(subscriptables=[ SubscriptableMapper(None, "tags", None, "tags"), ], ).concat(mappers), )), abstract_column_set=abstract_column_set, join_relationships={}, writable_storage=writable_storage, validators=validators, required_time_column="timestamp", )
def __init__(self) -> None: super().__init__( writable_storage_key=None, readable_storage_key=StorageKey.ORG_METRICS_COUNTERS, value_schema=[], mappers=TranslationMappers(), abstract_column_set=ColumnSet([ Column("org_id", UInt(64)), Column("project_id", UInt(64)), Column("metric_id", UInt(64)), Column("timestamp", DateTime()), Column("bucketed_time", DateTime()), ]), validators=[GranularityValidator(minimum=3600)], )
def __init__(self) -> None: super().__init__( writable_storage_key=StorageKey.METRICS_COUNTERS_BUCKETS, readable_storage_key=StorageKey.METRICS_COUNTERS, value_schema=[ Column("value", AggregateFunction("sum", [Float(64)])) ], mappers=TranslationMappers(columns=[ ColumnToFunction( None, "value", "sumMerge", (ColumnExpr(None, None, "value"), ), ), ], ), )
def __init__(self) -> None: super().__init__( writable_storage_key=StorageKey.METRICS_BUCKETS, readable_storage_key=StorageKey.METRICS_SETS, value_schema=[ Column("value", AggregateFunction("uniqCombined64", [UInt(64)])), ], mappers=TranslationMappers(columns=[ ColumnToFunction( None, "value", "uniqCombined64Merge", (ColumnExpr(None, None, "value"), ), ), ], ), )
def __init__( self, storage: ReadableStorage, mappers: Optional[TranslationMappers] = None, post_processors: Optional[Sequence[QueryProcessor]] = None, ) -> None: # The storage the query is based on self.__storage = storage # The translation mappers to be used when translating the logical query # into the clickhouse query. self.__mappers = mappers if mappers is not None else TranslationMappers() # This is a set of query processors that have to be executed on the # query after the storage selection but that are defined by the dataset. # Query processors defined by a Storage must be executable independently # from the context the Storage is used (whether the storage is used by # itself or whether it is joined with another storage). # In a joined query we would have processors defined by multiple storages. # that would have to be executed only once (like Prewhere). That is a # candidate to be added here as post process. self.__post_processors = post_processors or []
def test_tag_translation() -> None: translated = SubscriptableMapper(None, "tags", None, "tags").attempt_map( SubscriptableReference( "tags[release]", Column(None, None, "tags"), Literal(None, "release") ), SnubaClickhouseMappingTranslator(TranslationMappers()), ) assert translated == FunctionCall( "tags[release]", "arrayElement", ( Column(None, None, "tags.value"), FunctionCall( None, "indexOf", (Column(None, None, "tags.key"), Literal(None, "release")), ), ), )
def test_col_tag_translation() -> None: translated = ColumnToMapping( None, "geo_country_code", None, "contexts", "geo.country_code" ).attempt_map( Column("geo_country_code", None, "geo_country_code"), SnubaClickhouseMappingTranslator(TranslationMappers()), ) assert translated == FunctionCall( "geo_country_code", "arrayElement", ( Column(None, None, "contexts.value"), FunctionCall( None, "indexOf", (Column(None, None, "contexts.key"), Literal(None, "geo.country_code")), ), ), )
def __init__(self) -> None: writable_storage = get_writable_storage(StorageKey.SESSIONS_RAW) materialized_storage = get_storage(StorageKey.SESSIONS_HOURLY) read_schema = materialized_storage.get_schema() self.__time_group_columns = {"bucketed_started": "started"} self.__time_parse_columns = ("started", "received") super().__init__( storages=[writable_storage, materialized_storage], # TODO: Once we are ready to expose the raw data model and select whether to use # materialized storage or the raw one here, replace this with a custom storage # selector that decides when to use the materialized data. query_plan_builder=SingleStorageQueryPlanBuilder( storage=materialized_storage, mappers=TranslationMappers(columns=[ ColumnToCurriedFunction( None, "duration_quantiles", FunctionCall( None, "quantilesIfMerge", (Literal(None, 0.5), Literal(None, 0.9)), ), (Column(None, None, "duration_quantiles"), ), ), function_rule("sessions", "countIfMerge"), function_rule("sessions_crashed", "countIfMerge"), function_rule("sessions_abnormal", "countIfMerge"), function_rule("users", "uniqIfMerge"), function_rule("sessions_errored", "uniqIfMerge"), function_rule("users_crashed", "uniqIfMerge"), function_rule("users_abnormal", "uniqIfMerge"), function_rule("users_errored", "uniqIfMerge"), ]), ), abstract_column_set=read_schema.get_columns(), writable_storage=writable_storage, )
def __init__(self) -> None: super().__init__( writable_storage_key=StorageKey.METRICS_DISTRIBUTIONS_BUCKETS, readable_storage_key=StorageKey.METRICS_DISTRIBUTIONS, value_schema=[ Column( "percentiles", AggregateFunction("quantiles(0.5, 0.75, 0.9, 0.95, 0.99)", [Float(64)]), ), Column("min", AggregateFunction("min", [Float(64)])), Column("max", AggregateFunction("max", [Float(64)])), Column("avg", AggregateFunction("avg", [Float(64)])), Column("sum", AggregateFunction("sum", [Float(64)])), Column("count", AggregateFunction("count", [Float(64)])), ], mappers=TranslationMappers(columns=[ ColumnToCurriedFunction( None, "percentiles", FunctionCall( None, "quantilesMerge", tuple( Literal(None, quant) for quant in [0.5, 0.75, 0.9, 0.95, 0.99]), ), (ColumnExpr(None, None, "percentiles"), ), ), merge_mapper("min"), merge_mapper("max"), merge_mapper("avg"), merge_mapper("sum"), merge_mapper("count"), ], ), )
def __init__(self) -> None: self.__common_columns = ColumnSet([ ("event_id", FixedString(32)), ("project_id", UInt(64)), ("type", String(Modifiers(nullable=True))), ("timestamp", DateTime()), ("platform", String(Modifiers(nullable=True))), ("environment", String(Modifiers(nullable=True))), ("release", String(Modifiers(nullable=True))), ("dist", String(Modifiers(nullable=True))), ("user", String(Modifiers(nullable=True))), ("transaction", String(Modifiers(nullable=True))), ("message", String(Modifiers(nullable=True))), ("title", String(Modifiers(nullable=True))), # User ("user_id", String(Modifiers(nullable=True))), ("username", String(Modifiers(nullable=True))), ("email", String(Modifiers(nullable=True))), ("ip_address", String(Modifiers(nullable=True))), # SDK ("sdk_name", String(Modifiers(nullable=True))), ("sdk_version", String(Modifiers(nullable=True))), # geo location context ("geo_country_code", String(Modifiers(nullable=True))), ("geo_region", String(Modifiers(nullable=True))), ("geo_city", String(Modifiers(nullable=True))), ("http_method", String(Modifiers(nullable=True))), ("http_referer", String(Modifiers(nullable=True))), # Other tags and context ("tags", Nested([("key", String()), ("value", String())])), ("contexts", Nested([("key", String()), ("value", String())])), ]) self.__events_columns = EVENTS_COLUMNS self.__transactions_columns = TRANSACTIONS_COLUMNS events_storage = get_storage(StorageKey.EVENTS) events_pipeline_builder = SimplePipelineBuilder( query_plan_builder=SelectedStorageQueryPlanBuilder( selector=EventsQueryStorageSelector( mappers=events_translation_mappers. concat(transaction_translation_mappers).concat( null_function_translation_mappers).concat( TranslationMappers( # XXX: Remove once we are using errors columns=[ ColumnToMapping(None, "release", None, "tags", "sentry:release"), ColumnToMapping(None, "dist", None, "tags", "sentry:dist"), ColumnToMapping(None, "user", None, "tags", "sentry:user"), ], subscriptables=[ SubscriptableMapper( None, "tags", None, "tags"), SubscriptableMapper( None, "contexts", None, "contexts"), ], )))), ) discover_storage = get_storage(StorageKey.DISCOVER) discover_pipeline_builder = SimplePipelineBuilder( query_plan_builder=SingleStorageQueryPlanBuilder( storage=discover_storage, mappers=events_translation_mappers.concat( transaction_translation_mappers). concat(null_function_translation_mappers).concat( TranslationMappers(columns=[ ColumnToFunction( None, "ip_address", "coalesce", ( FunctionCall( None, "IPv4NumToString", (Column(None, None, "ip_address_v4"), ), ), FunctionCall( None, "IPv6NumToString", (Column(None, None, "ip_address_v6"), ), ), ), ), ColumnToColumn(None, "transaction", None, "transaction_name"), ColumnToColumn(None, "username", None, "user_name"), ColumnToColumn(None, "email", None, "user_email"), ColumnToMapping( None, "geo_country_code", None, "contexts", "geo.country_code", ), ColumnToMapping(None, "geo_region", None, "contexts", "geo.region"), ColumnToMapping(None, "geo_city", None, "contexts", "geo.city"), ColumnToFunction( None, "user", "nullIf", (Column(None, None, "user"), Literal(None, "")), ), ])).concat( TranslationMappers(subscriptables=[ SubscriptableMapper(None, "tags", None, "tags"), SubscriptableMapper(None, "contexts", None, "contexts"), ], )), )) def selector_func(_query: Query) -> Tuple[str, List[str]]: if random.random() < float( state.get_config("discover_query_percentage", 0)): return "events", ["discover"] return "events", [] super().__init__( storages=[events_storage, discover_storage], query_pipeline_builder=PipelineDelegator( query_pipeline_builders={ "events": events_pipeline_builder, "discover": discover_pipeline_builder, }, selector_func=selector_func, callback_func=partial(callback_func, "discover"), ), abstract_column_set=(self.__common_columns + self.__events_columns + self.__transactions_columns), join_relationships={}, writable_storage=None, )
sessions_hourly_translators = TranslationMappers(columns=[ ColumnToCurriedFunction( None, "duration_quantiles", FunctionCall(None, "quantilesIfMerge", quantiles), (Column(None, None, "duration_quantiles"), ), ), function_column("duration_avg", "avgIfMerge"), plus_columns( "sessions", function_call("sessions", "countIfMerge"), function_call("sessions_preaggr", "sumIfMerge"), ), plus_columns( "sessions_crashed", function_call("sessions_crashed", "countIfMerge"), function_call("sessions_crashed_preaggr", "sumIfMerge"), ), plus_columns( "sessions_abnormal", function_call("sessions_abnormal", "countIfMerge"), function_call("sessions_abnormal_preaggr", "sumIfMerge"), ), plus_columns( "sessions_errored", function_call("sessions_errored", "uniqIfMerge"), function_call("sessions_errored_preaggr", "sumIfMerge"), ), function_column("users", "uniqIfMerge"), function_column("users_crashed", "uniqIfMerge"), function_column("users_abnormal", "uniqIfMerge"), function_column("users_errored", "uniqIfMerge"), ])
sessions_translators = TranslationMappers(columns=[ ColumnToCurriedFunction( None, "duration_quantiles", FunctionCall( None, "quantilesIfMerge", tuple( Literal(None, quant) for quant in [0.5, 0.75, 0.9, 0.95, 0.99, 1]), ), (Column(None, None, "duration_quantiles"), ), ), function_column("duration_avg", "avgIfMerge"), plus_columns( "sessions", function_call("sessions", "countIfMerge"), function_call("sessions_preaggr", "sumIfMerge"), ), plus_columns( "sessions_crashed", function_call("sessions_crashed", "countIfMerge"), function_call("sessions_crashed_preaggr", "sumIfMerge"), ), plus_columns( "sessions_abnormal", function_call("sessions_abnormal", "countIfMerge"), function_call("sessions_abnormal_preaggr", "sumIfMerge"), ), plus_columns( "sessions_errored", function_call("sessions_errored", "uniqIfMerge"), function_call("sessions_errored_preaggr", "sumIfMerge"), ), function_column("users", "uniqIfMerge"), function_column("users_crashed", "uniqIfMerge"), function_column("users_abnormal", "uniqIfMerge"), function_column("users_errored", "uniqIfMerge"), ])
("span_op_breakdowns", Nested([("key", String()), ("value", Float(64))])), ( "spans", Nested([ ("op", String()), ("group", UInt(64)), ("exclusive_time", Float(64)), ("exclusive_time_32", Float(32)), ]), ), ]) events_translation_mappers = TranslationMappers( columns=[DefaultNoneColumnMapper(TRANSACTIONS_COLUMNS)], functions=[DefaultNoneFunctionMapper({"apdex", "failure_rate"})], subscriptables=[ DefaultNoneSubscriptMapper({"measurements", "span_op_breakdowns"}) ], ) transaction_translation_mappers = TranslationMappers( columns=[ ColumnToLiteral(None, "group_id", 0), DefaultNoneColumnMapper(EVENTS_COLUMNS), ], functions=[DefaultNoneFunctionMapper({"isHandled", "notHandled"})], ) null_function_translation_mappers = TranslationMappers( curried_functions=[DefaultIfNullCurriedFunctionMapper()], functions=[DefaultIfNullFunctionMapper()],
from snuba.datasets.entities import EntityKey from snuba.datasets.plans.translator.query import QueryTranslator from snuba.query import SelectedExpression from snuba.query.data_source.simple import Entity as QueryEntity from snuba.query.data_source.simple import Table from snuba.query.expressions import ( Column, FunctionCall, Literal, SubscriptableReference, ) from snuba.query.logical import Query as SnubaQuery test_cases = [ pytest.param( TranslationMappers(), SnubaQuery( from_clause=QueryEntity(EntityKey.EVENTS, ColumnSet([])), selected_columns=[ SelectedExpression("alias", Column("alias", "table", "column")), SelectedExpression( "alias2", FunctionCall( "alias2", "f1", (Column(None, None, "column2"), Column(None, None, "column3")), ), ), SelectedExpression( name=None, expression=SubscriptableReference(
def __init__(self) -> None: self.__common_columns = ColumnSet([ ("event_id", FixedString(32)), ("project_id", UInt(64)), ("type", String(Modifiers(nullable=True))), ("timestamp", DateTime()), ("platform", String(Modifiers(nullable=True))), ("environment", String(Modifiers(nullable=True))), ("release", String(Modifiers(nullable=True))), ("dist", String(Modifiers(nullable=True))), ("user", String(Modifiers(nullable=True))), ("transaction", String(Modifiers(nullable=True))), ("message", String(Modifiers(nullable=True))), ("title", String(Modifiers(nullable=True))), # User ("user_id", String(Modifiers(nullable=True))), ("username", String(Modifiers(nullable=True))), ("email", String(Modifiers(nullable=True))), ("ip_address", String(Modifiers(nullable=True))), # SDK ("sdk_name", String(Modifiers(nullable=True))), ("sdk_version", String(Modifiers(nullable=True))), # geo location context ("geo_country_code", String(Modifiers(nullable=True))), ("geo_region", String(Modifiers(nullable=True))), ("geo_city", String(Modifiers(nullable=True))), ("http_method", String(Modifiers(nullable=True))), ("http_referer", String(Modifiers(nullable=True))), # Other tags and context ("tags", Nested([("key", String()), ("value", String())])), ("contexts", Nested([("key", String()), ("value", String())])), ("trace_id", String(Modifiers(nullable=True))), ("span_id", UInt(64, Modifiers(nullable=True))), ]) self.__events_columns = EVENTS_COLUMNS self.__transactions_columns = TRANSACTIONS_COLUMNS discover_storage = get_storage(StorageKey.DISCOVER) discover_storage_plan_builder = SingleStorageQueryPlanBuilder( storage=discover_storage, mappers=events_translation_mappers. concat(transaction_translation_mappers).concat( null_function_translation_mappers).concat( TranslationMappers(columns=[ ColumnToFunction( None, "ip_address", "coalesce", ( FunctionCall( None, "IPv4NumToString", (Column(None, None, "ip_address_v4"), ), ), FunctionCall( None, "IPv6NumToString", (Column(None, None, "ip_address_v6"), ), ), ), ), ColumnToColumn(None, "transaction", None, "transaction_name"), ColumnToColumn(None, "username", None, "user_name"), ColumnToColumn(None, "email", None, "user_email"), ColumnToMapping( None, "geo_country_code", None, "contexts", "geo.country_code", nullable=True, ), ColumnToMapping( None, "geo_region", None, "contexts", "geo.region", nullable=True, ), ColumnToMapping( None, "geo_city", None, "contexts", "geo.city", nullable=True, ), ColumnToFunction( None, "user", "nullIf", (Column(None, None, "user"), Literal(None, "")), ), ])).concat( TranslationMappers(subscriptables=[ SubscriptableMapper(None, "tags", None, "tags"), SubscriptableMapper(None, "contexts", None, "contexts"), ], )), ) discover_pipeline_builder = SimplePipelineBuilder( query_plan_builder=discover_storage_plan_builder) super().__init__( storages=[discover_storage], query_pipeline_builder=discover_pipeline_builder, abstract_column_set=(self.__common_columns + self.__events_columns + self.__transactions_columns), join_relationships={}, writable_storage=None, validators=[EntityRequiredColumnValidator({"project_id"})], required_time_column="timestamp", )
from snuba.query.parsing import ParsingContext from snuba.query.processors import QueryProcessor from snuba.query.processors.basic_functions import BasicFunctionsProcessor from snuba.query.processors.handled_functions import HandledFunctionsProcessor from snuba.query.processors.tags_expander import TagsExpanderProcessor from snuba.query.processors.timeseries_column_processor import TimeSeriesColumnProcessor from snuba.query.project_extension import ProjectExtension from snuba.query.timeseries_extension import TimeSeriesExtension from snuba.request.request_settings import RequestSettings from snuba.util import parse_datetime, qualified_column # TODO: This will be a property of the relationship between entity and # storage. Now we do not have entities so it is between dataset and # storage. event_translator = TranslationMappers(subscriptables=[ SubscriptableMapper(None, "tags", None, "tags"), SubscriptableMapper(None, "contexts", None, "contexts"), ], ) class EventsQueryStorageSelector(QueryStorageSelector): def __init__( self, events_table: ReadableStorage, events_ro_table: ReadableStorage, ) -> None: self.__events_table = events_table self.__events_ro_table = events_ro_table def select_storage(self, query: Query, request_settings: RequestSettings) -> StorageAndMappers: use_readonly_storage = (state.get_config(
transaction_translator = TranslationMappers( columns=[ ColumnToFunction( None, "ip_address", "coalesce", ( FunctionCall( None, "IPv4NumToString", (Column(None, None, "ip_address_v4"),), ), FunctionCall( None, "IPv6NumToString", (Column(None, None, "ip_address_v6"),), ), ), ), ColumnToFunction( None, "user", "nullIf", (Column(None, None, "user"), Literal(None, "")) ), # These column aliases originally existed in the ``discover`` dataset, # but now live here to maintain compatibility between the composite # ``discover`` dataset and the standalone ``transaction`` dataset. In # the future, these aliases should be defined on the Transaction entity # instead of the dataset. ColumnToLiteral(None, "type", "transaction"), ColumnToColumn(None, "timestamp", None, "finish_ts"), ColumnToColumn(None, "username", None, "user_name"), ColumnToColumn(None, "email", None, "user_email"), ColumnToColumn(None, "transaction", None, "transaction_name"), ColumnToColumn(None, "message", None, "transaction_name"), ColumnToColumn(None, "title", None, "transaction_name"), ColumnToMapping(None, "geo_country_code", None, "contexts", "geo.country_code"), ColumnToMapping(None, "geo_region", None, "contexts", "geo.region"), ColumnToMapping(None, "geo_city", None, "contexts", "geo.city"), ], subscriptables=[ SubscriptableMapper(None, "tags", None, "tags"), SubscriptableMapper(None, "contexts", None, "contexts"), SubscriptableMapper(None, "measurements", None, "measurements", nullable=True), ], )
from snuba.query.logical import Query from snuba.query.parsing import ParsingContext from snuba.query.processors import QueryProcessor from snuba.query.processors.basic_functions import BasicFunctionsProcessor from snuba.query.processors.handled_functions import HandledFunctionsProcessor from snuba.query.processors.tags_expander import TagsExpanderProcessor from snuba.query.processors.timeseries_column_processor import TimeSeriesColumnProcessor from snuba.query.project_extension import ProjectExtension from snuba.query.timeseries_extension import TimeSeriesExtension from snuba.util import parse_datetime errors_translators = TranslationMappers( columns=[ ColumnToFunction( None, "user", "nullIf", (Column(None, None, "user"), Literal(None, "")) ), ] ) class ErrorsEntity(Entity): """ Represents the collections of all event types that are not transactions. This is meant to replace Events. They will both exist during the migration. """ def __init__(self) -> None: storage = get_writable_storage(StorageKey.ERRORS) schema = storage.get_table_writer().get_schema()
from snuba.query.processors.project_rate_limiter import ProjectRateLimiterProcessor from snuba.query.processors.tags_expander import TagsExpanderProcessor from snuba.query.processors.timeseries_processor import TimeSeriesProcessor from snuba.query.project_extension import ProjectExtension from snuba.query.timeseries_extension import TimeSeriesExtension from snuba.request.request_settings import RequestSettings from snuba.utils.metrics.wrapper import MetricsWrapper from snuba.utils.threaded_function_delegator import Result from snuba.web import QueryResult event_translator = TranslationMappers( columns=[ ColumnToMapping(None, "release", None, "tags", "sentry:release"), ColumnToMapping(None, "dist", None, "tags", "sentry:dist"), ColumnToMapping(None, "user", None, "tags", "sentry:user"), ], subscriptables=[ SubscriptableMapper(None, "tags", None, "tags"), SubscriptableMapper(None, "contexts", None, "contexts"), ], ) errors_translators = TranslationMappers( columns=[ ColumnToMapping(None, "release", None, "tags", "sentry:release"), ColumnToMapping(None, "dist", None, "tags", "sentry:dist"), ColumnToMapping(None, "user", None, "tags", "sentry:user"), ColumnToFunction( None, "ip_address", "coalesce",