# the future, these aliases should be defined on the Transaction entity
# instead of the dataset.
ColumnToLiteral(None, "type", "transaction"),
ColumnToColumn(None, "timestamp", None, "finish_ts"),
ColumnToColumn(None, "username", None, "user_name"),
ColumnToColumn(None, "email", None, "user_email"),
ColumnToColumn(None, "transaction", None, "transaction_name"),
ColumnToColumn(None, "message", None, "transaction_name"),
ColumnToColumn(None, "title", None, "transaction_name"),
ColumnToMapping(None, "geo_country_code", None, "contexts",
"geo.country_code"),
ColumnToMapping(None, "geo_region", None, "contexts", "geo.region"),
ColumnToMapping(None, "geo_city", None, "contexts", "geo.city"),
],
subscriptables=[
SubscriptableMapper(None, "tags", None, "tags"),
SubscriptableMapper(None, "contexts", None, "contexts"),
SubscriptableMapper(None,
"measurements",
None,
"measurements",
nullable=True),
],
)
class BaseTransactionsEntity(Entity, ABC):
def __init__(self,
custom_mappers: Optional[TranslationMappers] = None) -> None:
storage = get_writable_storage(StorageKey.TRANSACTIONS)
schema = storage.get_table_writer().get_schema()
def __init__(self) -> None:
self.__common_columns = ColumnSet([
("event_id", FixedString(32)),
("project_id", UInt(64)),
("type", String(Modifiers(nullable=True))),
("timestamp", DateTime()),
("platform", String(Modifiers(nullable=True))),
("environment", String(Modifiers(nullable=True))),
("release", String(Modifiers(nullable=True))),
("dist", String(Modifiers(nullable=True))),
("user", String(Modifiers(nullable=True))),
("transaction", String(Modifiers(nullable=True))),
("message", String(Modifiers(nullable=True))),
("title", String(Modifiers(nullable=True))),
# User
("user_id", String(Modifiers(nullable=True))),
("username", String(Modifiers(nullable=True))),
("email", String(Modifiers(nullable=True))),
("ip_address", String(Modifiers(nullable=True))),
# SDK
("sdk_name", String(Modifiers(nullable=True))),
("sdk_version", String(Modifiers(nullable=True))),
# geo location context
("geo_country_code", String(Modifiers(nullable=True))),
("geo_region", String(Modifiers(nullable=True))),
("geo_city", String(Modifiers(nullable=True))),
("http_method", String(Modifiers(nullable=True))),
("http_referer", String(Modifiers(nullable=True))),
# Other tags and context
("tags", Nested([("key", String()), ("value", String())])),
("contexts", Nested([("key", String()), ("value", String())])),
])
self.__events_columns = EVENTS_COLUMNS
self.__transactions_columns = TRANSACTIONS_COLUMNS
events_storage = get_storage(StorageKey.EVENTS)
super().__init__(
storages=[events_storage],
query_plan_builder=SelectedStorageQueryPlanBuilder(
selector=EventsQueryStorageSelector(
mappers=events_translation_mappers.
concat(transaction_translation_mappers).concat(
null_function_translation_mappers).concat(
TranslationMappers(
# XXX: Remove once we are using errors
columns=[
ColumnToMapping(None, "release", None,
"tags", "sentry:release"),
ColumnToMapping(None, "dist", None, "tags",
"sentry:dist"),
ColumnToMapping(None, "user", None, "tags",
"sentry:user"),
],
subscriptables=[
SubscriptableMapper(
None, "tags", None, "tags"),
SubscriptableMapper(
None, "contexts", None, "contexts"),
],
)))),
abstract_column_set=(self.__common_columns +
self.__events_columns +
self.__transactions_columns),
writable_storage=None,
)
def __init__(self) -> None:
self.__common_columns = ColumnSet([
("event_id", FixedString(32)),
("project_id", UInt(64)),
("type", String(Modifiers(nullable=True))),
("timestamp", DateTime()),
("platform", String(Modifiers(nullable=True))),
("environment", String(Modifiers(nullable=True))),
("release", String(Modifiers(nullable=True))),
("dist", String(Modifiers(nullable=True))),
("user", String(Modifiers(nullable=True))),
("transaction", String(Modifiers(nullable=True))),
("message", String(Modifiers(nullable=True))),
("title", String(Modifiers(nullable=True))),
# User
("user_id", String(Modifiers(nullable=True))),
("username", String(Modifiers(nullable=True))),
("email", String(Modifiers(nullable=True))),
("ip_address", String(Modifiers(nullable=True))),
# SDK
("sdk_name", String(Modifiers(nullable=True))),
("sdk_version", String(Modifiers(nullable=True))),
# geo location context
("geo_country_code", String(Modifiers(nullable=True))),
("geo_region", String(Modifiers(nullable=True))),
("geo_city", String(Modifiers(nullable=True))),
("http_method", String(Modifiers(nullable=True))),
("http_referer", String(Modifiers(nullable=True))),
# Other tags and context
("tags", Nested([("key", String()), ("value", String())])),
("contexts", Nested([("key", String()), ("value", String())])),
])
self.__events_columns = EVENTS_COLUMNS
self.__transactions_columns = TRANSACTIONS_COLUMNS
events_storage = get_storage(StorageKey.EVENTS)
events_pipeline_builder = SimplePipelineBuilder(
query_plan_builder=SelectedStorageQueryPlanBuilder(
selector=EventsQueryStorageSelector(
mappers=events_translation_mappers.
concat(transaction_translation_mappers).concat(
null_function_translation_mappers).concat(
TranslationMappers(
# XXX: Remove once we are using errors
columns=[
ColumnToMapping(None, "release", None,
"tags", "sentry:release"),
ColumnToMapping(None, "dist", None, "tags",
"sentry:dist"),
ColumnToMapping(None, "user", None, "tags",
"sentry:user"),
],
subscriptables=[
SubscriptableMapper(
None, "tags", None, "tags"),
SubscriptableMapper(
None, "contexts", None, "contexts"),
],
)))), )
discover_storage = get_storage(StorageKey.DISCOVER)
discover_pipeline_builder = SimplePipelineBuilder(
query_plan_builder=SingleStorageQueryPlanBuilder(
storage=discover_storage,
mappers=events_translation_mappers.concat(
transaction_translation_mappers).
concat(null_function_translation_mappers).concat(
TranslationMappers(columns=[
ColumnToFunction(
None,
"ip_address",
"coalesce",
(
FunctionCall(
None,
"IPv4NumToString",
(Column(None, None, "ip_address_v4"), ),
),
FunctionCall(
None,
"IPv6NumToString",
(Column(None, None, "ip_address_v6"), ),
),
),
),
ColumnToColumn(None, "transaction", None,
"transaction_name"),
ColumnToColumn(None, "username", None, "user_name"),
ColumnToColumn(None, "email", None, "user_email"),
ColumnToMapping(
None,
"geo_country_code",
None,
"contexts",
"geo.country_code",
),
ColumnToMapping(None, "geo_region", None, "contexts",
"geo.region"),
ColumnToMapping(None, "geo_city", None, "contexts",
"geo.city"),
ColumnToFunction(
None,
"user",
"nullIf",
(Column(None, None, "user"), Literal(None, "")),
),
])).concat(
TranslationMappers(subscriptables=[
SubscriptableMapper(None, "tags", None, "tags"),
SubscriptableMapper(None, "contexts", None,
"contexts"),
], )),
))
def selector_func(_query: Query) -> Tuple[str, List[str]]:
if random.random() < float(
state.get_config("discover_query_percentage", 0)):
return "events", ["discover"]
return "events", []
super().__init__(
storages=[events_storage, discover_storage],
query_pipeline_builder=PipelineDelegator(
query_pipeline_builders={
"events": events_pipeline_builder,
"discover": discover_pipeline_builder,
},
selector_func=selector_func,
callback_func=partial(callback_func, "discover"),
),
abstract_column_set=(self.__common_columns +
self.__events_columns +
self.__transactions_columns),
join_relationships={},
writable_storage=None,
)
test_data = [
pytest.param(
TranslationMappers(columns=[ColumnToColumn(None, "col", None, "col2")]),
Column(None, None, "col3"),
Column(None, None, "col3"),
id="default rule",
),
pytest.param(
TranslationMappers(columns=[ColumnToColumn(None, "col", None, "col2")]),
Column("col", None, "col"),
Column("col", None, "col2"),
id="simple column",
),
pytest.param(
TranslationMappers(
subscriptables=[SubscriptableMapper(None, "tags", "table", "tags")]
),
SubscriptableReference(
"tags[release]", Column(None, None, "tags"), Literal(None, "release")
),
FunctionCall(
"tags[release]",
"arrayElement",
(
Column(None, "table", "tags.value"),
FunctionCall(
None,
"indexOf",
(Column(None, "table", "tags.key"), Literal(None, "release")),
),
),
from snuba.query.extensions import QueryExtension
from snuba.query.logical import Query
from snuba.query.processors import QueryProcessor
from snuba.query.processors.basic_functions import BasicFunctionsProcessor
from snuba.query.processors.handled_functions import HandledFunctionsProcessor
from snuba.query.processors.tags_expander import TagsExpanderProcessor
from snuba.query.processors.timeseries_processor import TimeSeriesProcessor
from snuba.query.project_extension import ProjectExtension
from snuba.query.timeseries_extension import TimeSeriesExtension
from snuba.request.request_settings import RequestSettings
# TODO: This will be a property of the relationship between entity and
# storage. Now we do not have entities so it is between dataset and
# storage.
event_translator = TranslationMappers(subscriptables=[
SubscriptableMapper(None, "tags", None, "tags"),
SubscriptableMapper(None, "contexts", None, "contexts"),
], )
class EventsQueryStorageSelector(QueryStorageSelector):
def __init__(
self,
events_table: ReadableStorage,
events_ro_table: ReadableStorage,
) -> None:
self.__events_table = events_table
self.__events_ro_table = events_ro_table
def select_storage(self, query: Query,
request_settings: RequestSettings) -> StorageAndMappers:
),
ColumnToMapping(None,
"geo_region",
None,
"contexts",
"geo.region",
nullable=True),
ColumnToMapping(None,
"geo_city",
None,
"contexts",
"geo.city",
nullable=True),
],
subscriptables=[
SubscriptableMapper(None, "tags", None, "tags"),
SubscriptableMapper(None, "contexts", None, "contexts"),
SubscriptableMapper(None,
"measurements",
None,
"measurements",
nullable=True),
SubscriptableMapper(None,
"span_op_breakdowns",
None,
"span_op_breakdowns",
nullable=True),
],
)
def __init__(self) -> None:
self.__common_columns = ColumnSet([
("event_id", FixedString(32)),
("project_id", UInt(64)),
("type", String(Modifiers(nullable=True))),
("timestamp", DateTime()),
("platform", String(Modifiers(nullable=True))),
("environment", String(Modifiers(nullable=True))),
("release", String(Modifiers(nullable=True))),
("dist", String(Modifiers(nullable=True))),
("user", String(Modifiers(nullable=True))),
("transaction", String(Modifiers(nullable=True))),
("message", String(Modifiers(nullable=True))),
("title", String(Modifiers(nullable=True))),
# User
("user_id", String(Modifiers(nullable=True))),
("username", String(Modifiers(nullable=True))),
("email", String(Modifiers(nullable=True))),
("ip_address", String(Modifiers(nullable=True))),
# SDK
("sdk_name", String(Modifiers(nullable=True))),
("sdk_version", String(Modifiers(nullable=True))),
# geo location context
("geo_country_code", String(Modifiers(nullable=True))),
("geo_region", String(Modifiers(nullable=True))),
("geo_city", String(Modifiers(nullable=True))),
("http_method", String(Modifiers(nullable=True))),
("http_referer", String(Modifiers(nullable=True))),
# Other tags and context
("tags", Nested([("key", String()), ("value", String())])),
("contexts", Nested([("key", String()), ("value", String())])),
("trace_id", String(Modifiers(nullable=True))),
])
self.__events_columns = EVENTS_COLUMNS
self.__transactions_columns = TRANSACTIONS_COLUMNS
events_storage = get_storage(StorageKey.EVENTS)
events_pipeline_builder = SimplePipelineBuilder(
query_plan_builder=SelectedStorageQueryPlanBuilder(
selector=EventsQueryStorageSelector(
mappers=events_translation_mappers.
concat(transaction_translation_mappers).concat(
null_function_translation_mappers).concat(
TranslationMappers(
# XXX: Remove once we are using errors
columns=[
ColumnToMapping(None, "release", None,
"tags", "sentry:release"),
ColumnToMapping(None, "dist", None, "tags",
"sentry:dist"),
ColumnToMapping(None, "user", None, "tags",
"sentry:user"),
],
subscriptables=[
SubscriptableMapper(
None, "tags", None, "tags"),
SubscriptableMapper(
None, "contexts", None, "contexts"),
],
)))), )
discover_storage = get_storage(StorageKey.DISCOVER)
discover_storage_plan_builder = SingleStorageQueryPlanBuilder(
storage=discover_storage,
mappers=events_translation_mappers.
concat(transaction_translation_mappers).concat(
null_function_translation_mappers).concat(
TranslationMappers(columns=[
ColumnToFunction(
None,
"ip_address",
"coalesce",
(
FunctionCall(
None,
"IPv4NumToString",
(Column(None, None, "ip_address_v4"), ),
),
FunctionCall(
None,
"IPv6NumToString",
(Column(None, None, "ip_address_v6"), ),
),
),
),
ColumnToColumn(None, "transaction", None,
"transaction_name"),
ColumnToColumn(None, "username", None, "user_name"),
ColumnToColumn(None, "email", None, "user_email"),
ColumnToMapping(
None,
"geo_country_code",
None,
"contexts",
"geo.country_code",
nullable=True,
),
ColumnToMapping(
None,
"geo_region",
None,
"contexts",
"geo.region",
nullable=True,
),
ColumnToMapping(
None,
"geo_city",
None,
"contexts",
"geo.city",
nullable=True,
),
ColumnToFunction(
None,
"user",
"nullIf",
(Column(None, None, "user"), Literal(None, "")),
),
])).concat(
TranslationMappers(subscriptables=[
SubscriptableMapper(None, "tags", None, "tags"),
SubscriptableMapper(None, "contexts", None,
"contexts"),
], )),
)
discover_pipeline_builder = SimplePipelineBuilder(
query_plan_builder=discover_storage_plan_builder)
pipeline_builder: Union[PipelineDelegator, SimplePipelineBuilder]
if settings.ERRORS_ROLLOUT_ALL:
storage = discover_storage
sampled_pipeline_builder = SampledSimplePipelineBuilder(
query_plan_builder=discover_storage_plan_builder)
pipeline_builder = PipelineDelegator(
query_pipeline_builders={
"primary": discover_pipeline_builder,
"sampler": sampled_pipeline_builder,
},
selector_func=sampling_selector_func,
callback_func=sampling_callback_func,
)
else:
storage = events_storage
pipeline_builder = events_pipeline_builder
super().__init__(
storages=[storage],
query_pipeline_builder=pipeline_builder,
abstract_column_set=(self.__common_columns +
self.__events_columns +
self.__transactions_columns),
join_relationships={},
writable_storage=None,
validators=[EntityRequiredColumnValidator({"project_id"})],
required_time_column="timestamp",
)
def __init__(self) -> None:
self.__common_columns = ColumnSet([
("event_id", FixedString(32)),
("project_id", UInt(64)),
("type", String(Modifiers(nullable=True))),
("timestamp", DateTime()),
("platform", String(Modifiers(nullable=True))),
("environment", String(Modifiers(nullable=True))),
("release", String(Modifiers(nullable=True))),
("dist", String(Modifiers(nullable=True))),
("user", String(Modifiers(nullable=True))),
("transaction", String(Modifiers(nullable=True))),
("message", String(Modifiers(nullable=True))),
("title", String(Modifiers(nullable=True))),
# User
("user_id", String(Modifiers(nullable=True))),
("username", String(Modifiers(nullable=True))),
("email", String(Modifiers(nullable=True))),
("ip_address", String(Modifiers(nullable=True))),
# SDK
("sdk_name", String(Modifiers(nullable=True))),
("sdk_version", String(Modifiers(nullable=True))),
# geo location context
("geo_country_code", String(Modifiers(nullable=True))),
("geo_region", String(Modifiers(nullable=True))),
("geo_city", String(Modifiers(nullable=True))),
("http_method", String(Modifiers(nullable=True))),
("http_referer", String(Modifiers(nullable=True))),
# Other tags and context
("tags", Nested([("key", String()), ("value", String())])),
("contexts", Nested([("key", String()), ("value", String())])),
])
self.__events_columns = EVENTS_COLUMNS
self.__transactions_columns = TRANSACTIONS_COLUMNS
events_storage = get_storage(StorageKey.EVENTS)
events_pipeline_builder = SimplePipelineBuilder(
query_plan_builder=SelectedStorageQueryPlanBuilder(
selector=EventsQueryStorageSelector(
mappers=events_translation_mappers.
concat(transaction_translation_mappers).concat(
null_function_translation_mappers).concat(
TranslationMappers(
# XXX: Remove once we are using errors
columns=[
ColumnToMapping(None, "release", None,
"tags", "sentry:release"),
ColumnToMapping(None, "dist", None, "tags",
"sentry:dist"),
ColumnToMapping(None, "user", None, "tags",
"sentry:user"),
],
subscriptables=[
SubscriptableMapper(
None, "tags", None, "tags"),
SubscriptableMapper(
None, "contexts", None, "contexts"),
],
)))), )
discover_storage = get_storage(StorageKey.DISCOVER)
discover_pipeline_builder = SimplePipelineBuilder(
query_plan_builder=SingleStorageQueryPlanBuilder(
storage=discover_storage,
mappers=events_translation_mappers.concat(
transaction_translation_mappers).
concat(null_function_translation_mappers).concat(
TranslationMappers(columns=[
ColumnToFunction(
None,
"ip_address",
"coalesce",
(
FunctionCall(
None,
"IPv4NumToString",
(Column(None, None, "ip_address_v4"), ),
),
FunctionCall(
None,
"IPv6NumToString",
(Column(None, None, "ip_address_v6"), ),
),
),
),
ColumnToColumn(None, "transaction", None,
"transaction_name"),
ColumnToColumn(None, "username", None, "user_name"),
ColumnToColumn(None, "email", None, "user_email"),
ColumnToMapping(
None,
"geo_country_code",
None,
"contexts",
"geo.country_code",
nullable=True,
),
ColumnToMapping(
None,
"geo_region",
None,
"contexts",
"geo.region",
nullable=True,
),
ColumnToMapping(
None,
"geo_city",
None,
"contexts",
"geo.city",
nullable=True,
),
ColumnToFunction(
None,
"user",
"nullIf",
(Column(None, None, "user"), Literal(None, "")),
),
])).concat(
TranslationMappers(subscriptables=[
SubscriptableMapper(None, "tags", None, "tags"),
SubscriptableMapper(None, "contexts", None,
"contexts"),
], )),
))
def selector_func(_query: Query,
referrer: str) -> Tuple[str, List[str]]:
# In case something goes wrong, set this to 1 to revert to the events storage.
kill_rollout = state.get_config("errors_rollout_killswitch", 0)
assert isinstance(kill_rollout, (int, str))
if int(kill_rollout):
return "events", []
if referrer in settings.ERRORS_ROLLOUT_BY_REFERRER:
return "discover", []
if settings.ERRORS_ROLLOUT_ALL:
return "discover", []
default_threshold = state.get_config("discover_query_percentage",
0)
assert isinstance(default_threshold, (float, int, str))
threshold = settings.ERRORS_QUERY_PERCENTAGE_BY_REFERRER.get(
referrer, default_threshold)
if random.random() < float(threshold):
return "events", ["discover"]
return "events", []
super().__init__(
storages=[events_storage, discover_storage],
query_pipeline_builder=PipelineDelegator(
query_pipeline_builders={
"events": events_pipeline_builder,
"discover": discover_pipeline_builder,
},
selector_func=selector_func,
callback_func=partial(callback_func, "discover"),
),
abstract_column_set=(self.__common_columns +
self.__events_columns +
self.__transactions_columns),
join_relationships={},
writable_storage=None,
required_filter_columns=["project_id"],
required_time_column="timestamp",
)
