class EventsEntity(Entity):
"""
Represents the collection of classic sentry "error" type events
and the particular quirks of storing and querying them.
"""
def __init__(self) -> None:
storage = get_writable_storage(StorageKey.EVENTS)
schema = storage.get_table_writer().get_schema()
columns = schema.get_columns()
ro_storage = get_storage(StorageKey.EVENTS_RO)
self.__time_group_columns = {"time": "timestamp", "rtime": "received"}
self.__time_parse_columns = ("timestamp", "received")
super().__init__(
storages=[storage],
query_plan_builder=SelectedStorageQueryPlanBuilder(
selector=EventsQueryStorageSelector(
events_table=storage,
events_ro_table=ro_storage,
)),
abstract_column_set=columns,
writable_storage=storage,
)
self.__tags_processor = TagColumnProcessor(
columns=columns,
promoted_columns=get_promoted_columns(),
column_tag_map=get_column_tag_map(),
)
def column_expr(
self,
column_name: str,
query: Query,
parsing_context: ParsingContext,
table_alias: str = "",
) -> Union[None, Any]:
processed_column = self.__tags_processor.process_column_expression(
column_name, query, parsing_context, table_alias)
if processed_column:
# If processed_column is None, this was not a tag/context expression
# This conversion must not be ported to the errors dataset. We should
# not support promoting tags/contexts with boolean values. There is
# no way to convert them back consistently to the value provided by
# the client when the event is ingested, in all ways to access
# tags/contexts. Once the errors dataset is in use, we will not have
# boolean promoted tags/contexts so this constraint will be easy to enforce.
boolean_contexts = {
"contexts[device.simulator]",
"contexts[device.online]",
"contexts[device.charging]",
}
boolean_context_template = (
"multiIf(equals(%(processed_column)s, ''), '', "
"in(%(processed_column)s, tuple('1', 'True')), 'True', 'False')"
)
if column_name in boolean_contexts:
return boolean_context_template % ({
"processed_column":
processed_column
})
return processed_column
elif column_name == "group_id":
return f"nullIf({qualified_column('group_id', table_alias)}, 0)"
elif column_name == "message":
# Because of the rename from message->search_message without backfill,
# records will have one or the other of these fields.
# TODO this can be removed once all data has search_message filled in.
search_message = qualified_column("search_message", table_alias)
message = qualified_column("message", table_alias)
return f"coalesce({search_message}, {message})"
else:
return super().column_expr(column_name, query, parsing_context,
table_alias)
def get_extensions(self) -> Mapping[str, QueryExtension]:
return {
"project":
ProjectExtension(project_column="project_id"),
"timeseries":
TimeSeriesExtension(
default_granularity=3600,
default_window=timedelta(days=5),
timestamp_column="timestamp",
),
}
def get_query_processors(self) -> Sequence[QueryProcessor]:
return [
TagsExpanderProcessor(),
BasicFunctionsProcessor(),
TimeSeriesColumnProcessor(self.__time_group_columns),
HandledFunctionsProcessor("exception_stacks.mechanism_handled",
self.get_data_model()),
]
# TODO: This needs to burned with fire, for so many reasons.
# It's here now to reduce the scope of the initial entity changes
# but can be moved to a processor if not removed entirely.
def process_condition(
self, condition: Tuple[str, str, Any]) -> Tuple[str, str, Any]:
lhs, op, lit = condition
if (lhs in self.__time_parse_columns
and op in (">", "<", ">=", "<=", "=", "!=")
and isinstance(lit, str)):
lit = parse_datetime(lit)
return lhs, op, lit
class ErrorsEntity(Entity):
"""
Represents the collections of all event types that are not transactions.
This is meant to replace Events. They will both exist during the migration.
"""
def __init__(self) -> None:
storage = get_writable_storage(StorageKey.ERRORS)
schema = storage.get_table_writer().get_schema()
columns = schema.get_columns()
self.__time_group_columns = {"time": "timestamp", "rtime": "received"}
self.__time_parse_columns = ("timestamp", "received")
super().__init__(
storages=[storage],
query_plan_builder=SingleStorageQueryPlanBuilder(
storage=storage, mappers=errors_translators
),
abstract_column_set=columns,
writable_storage=storage,
)
self.__tags_processor = TagColumnProcessor(
columns=columns,
promoted_columns=self._get_promoted_columns(),
column_tag_map=self._get_column_tag_map(),
)
def column_expr(
self,
column_name: str,
query: Query,
parsing_context: ParsingContext,
table_alias: str = "",
) -> Union[None, Any]:
processed_column = self.__tags_processor.process_column_expression(
column_name, query, parsing_context, table_alias
)
return processed_column or super().column_expr(
column_name, query, parsing_context, table_alias
)
def _get_promoted_columns(self) -> Mapping[str, FrozenSet[str]]:
return {
"tags": frozenset(promoted_tag_columns.values()),
"contexts": frozenset(),
}
def _get_column_tag_map(self) -> Mapping[str, Mapping[str, str]]:
return {
"tags": {col: tag for tag, col in promoted_tag_columns.items()},
"contexts": {},
}
def get_extensions(self) -> Mapping[str, QueryExtension]:
return {
"project": ProjectExtension(project_column="project_id"),
"timeseries": TimeSeriesExtension(
default_granularity=3600,
default_window=timedelta(days=5),
timestamp_column="timestamp",
),
}
def get_query_processors(self) -> Sequence[QueryProcessor]:
return [
TagsExpanderProcessor(),
BasicFunctionsProcessor(),
TimeSeriesColumnProcessor(self.__time_group_columns),
HandledFunctionsProcessor(
"exception_stacks.mechanism_handled", self.get_data_model()
),
]
# TODO: This needs to burned with fire, for so many reasons.
# It's here now to reduce the scope of the initial entity changes
# but can be moved to a processor if not removed entirely.
def process_condition(
self, condition: Tuple[str, str, Any]
) -> Tuple[str, str, Any]:
lhs, op, lit = condition
if (
lhs in self.__time_parse_columns
and op in (">", "<", ">=", "<=", "=", "!=")
and isinstance(lit, str)
):
lit = parse_datetime(lit)
return lhs, op, lit
class TransactionsDataset(TimeSeriesDataset):
def __init__(self) -> None:
columns = ColumnSet(
[
("project_id", UInt(64)),
("event_id", UUID()),
("trace_id", UUID()),
("span_id", UInt(64)),
("transaction_name", LowCardinality(String())),
(
"transaction_hash",
Materialized(UInt(64), "cityHash64(transaction_name)",),
),
("transaction_op", LowCardinality(String())),
("transaction_status", WithDefault(UInt(8), UNKNOWN_SPAN_STATUS)),
("start_ts", DateTime()),
("start_ms", UInt(16)),
("_start_date", Materialized(Date(), "toDate(start_ts)"),),
("finish_ts", DateTime()),
("finish_ms", UInt(16)),
("_finish_date", Materialized(Date(), "toDate(finish_ts)"),),
("duration", UInt(32)),
("platform", LowCardinality(String())),
("environment", LowCardinality(Nullable(String()))),
("release", LowCardinality(Nullable(String()))),
("dist", LowCardinality(Nullable(String()))),
("ip_address_v4", Nullable(IPv4())),
("ip_address_v6", Nullable(IPv6())),
("user", WithDefault(String(), "''",)),
("user_hash", Materialized(UInt(64), "cityHash64(user)"),),
("user_id", Nullable(String())),
("user_name", Nullable(String())),
("user_email", Nullable(String())),
("sdk_name", WithDefault(LowCardinality(String()), "''")),
("sdk_version", WithDefault(LowCardinality(String()), "''")),
("tags", Nested([("key", String()), ("value", String())])),
("_tags_flattened", String()),
("contexts", Nested([("key", String()), ("value", String())])),
("_contexts_flattened", String()),
("partition", UInt(16)),
("offset", UInt(64)),
("retention_days", UInt(16)),
("deleted", UInt(8)),
]
)
schema = ReplacingMergeTreeSchema(
columns=columns,
local_table_name="transactions_local",
dist_table_name="transactions_dist",
mandatory_conditions=[],
prewhere_candidates=["event_id", "project_id"],
order_by="(project_id, _finish_date, transaction_name, cityHash64(span_id))",
partition_by="(retention_days, toMonday(_finish_date))",
version_column="deleted",
sample_expr=None,
migration_function=transactions_migrations,
)
dataset_schemas = DatasetSchemas(read_schema=schema, write_schema=schema,)
self.__tags_processor = TagColumnProcessor(
columns=columns,
promoted_columns=self._get_promoted_columns(),
column_tag_map=self._get_column_tag_map(),
)
super().__init__(
dataset_schemas=dataset_schemas,
table_writer=TransactionsTableWriter(
write_schema=schema,
stream_loader=KafkaStreamLoader(
processor=TransactionsMessageProcessor(), default_topic="events",
),
),
time_group_columns={
"bucketed_start": "start_ts",
"bucketed_end": "finish_ts",
},
time_parse_columns=("start_ts", "finish_ts"),
)
def _get_promoted_columns(self):
# TODO: Support promoted tags
return {
"tags": frozenset(),
"contexts": frozenset(),
}
def _get_column_tag_map(self):
# TODO: Support promoted tags
return {
"tags": {},
"contexts": {},
}
def get_extensions(self) -> Mapping[str, QueryExtension]:
return {
"project": ProjectExtension(
processor=ProjectExtensionProcessor(project_column="project_id")
),
"timeseries": TimeSeriesExtension(
default_granularity=3600,
default_window=timedelta(days=5),
timestamp_column="start_ts",
),
}
def column_expr(
self,
column_name,
query: Query,
parsing_context: ParsingContext,
table_alias: str = "",
):
# TODO remove these casts when clickhouse-driver is >= 0.0.19
if column_name == "ip_address_v4":
return "IPv4NumToString(ip_address_v4)"
if column_name == "ip_address_v6":
return "IPv6NumToString(ip_address_v6)"
if column_name == "ip_address":
return f"coalesce(IPv4NumToString(ip_address_v4), IPv6NumToString(ip_address_v6))"
if column_name == "event_id":
return "replaceAll(toString(event_id), '-', '')"
processed_column = self.__tags_processor.process_column_expression(
column_name, query, parsing_context, table_alias
)
if processed_column:
# If processed_column is None, this was not a tag/context expression
return processed_column
return super().column_expr(column_name, query, parsing_context)
def get_split_query_spec(self) -> Union[None, ColumnSplitSpec]:
return ColumnSplitSpec(
id_column="event_id",
project_column="project_id",
timestamp_column="start_ts",
)
def get_query_processors(self) -> Sequence[QueryProcessor]:
return [
BasicFunctionsProcessor(),
ApdexProcessor(),
ImpactProcessor(),
PrewhereProcessor(),
NestedFieldConditionOptimizer(
"tags", "_tags_flattened", {"start_ts", "finish_ts"}, BEGINNING_OF_TIME
),
NestedFieldConditionOptimizer(
"contexts",
"_contexts_flattened",
{"start_ts", "finish_ts"},
BEGINNING_OF_TIME,
),
]
class TransactionsDataset(TimeSeriesDataset):
def __init__(self) -> None:
storage = get_writable_storage("transactions")
schema = storage.get_table_writer().get_schema()
columns = schema.get_columns()
self.__tags_processor = TagColumnProcessor(
columns=columns,
promoted_columns=self._get_promoted_columns(),
column_tag_map=self._get_column_tag_map(),
)
self.__time_group_columns = {
"bucketed_start": "start_ts",
"bucketed_end": "finish_ts",
}
super().__init__(
storages=[storage],
query_plan_builder=SingleStorageQueryPlanBuilder(storage=storage),
abstract_column_set=schema.get_columns(),
writable_storage=storage,
time_group_columns=self.__time_group_columns,
time_parse_columns=("start_ts", "finish_ts"),
)
def _get_promoted_columns(self):
# TODO: Support promoted tags
return {
"tags": frozenset(),
"contexts": frozenset(),
}
def _get_column_tag_map(self):
# TODO: Support promoted tags
return {
"tags": {},
"contexts": {},
}
def get_extensions(self) -> Mapping[str, QueryExtension]:
return {
"project":
ProjectExtension(processor=ProjectExtensionProcessor(
project_column="project_id")),
"timeseries":
TimeSeriesExtension(
default_granularity=3600,
default_window=timedelta(days=5),
timestamp_column="start_ts",
),
}
def column_expr(
self,
column_name,
query: Query,
parsing_context: ParsingContext,
table_alias: str = "",
):
# TODO remove these casts when clickhouse-driver is >= 0.0.19
if column_name == "ip_address_v4":
return "IPv4NumToString(ip_address_v4)"
if column_name == "ip_address_v6":
return "IPv6NumToString(ip_address_v6)"
if column_name == "ip_address":
return f"coalesce(IPv4NumToString(ip_address_v4), IPv6NumToString(ip_address_v6))"
if column_name == "event_id":
return "replaceAll(toString(event_id), '-', '')"
processed_column = self.__tags_processor.process_column_expression(
column_name, query, parsing_context, table_alias)
if processed_column:
# If processed_column is None, this was not a tag/context expression
return processed_column
return super().column_expr(column_name, query, parsing_context)
def get_split_query_spec(self) -> Union[None, ColumnSplitSpec]:
return ColumnSplitSpec(
id_column="event_id",
project_column="project_id",
timestamp_column="start_ts",
)
def get_query_processors(self) -> Sequence[QueryProcessor]:
return [
BasicFunctionsProcessor(),
ApdexProcessor(),
ImpactProcessor(),
TimeSeriesColumnProcessor(self.__time_group_columns),
]
class TransactionsDataset(TimeSeriesDataset):
def __init__(self):
columns = ColumnSet([
('project_id', UInt(64)),
('event_id', UUID()),
('trace_id', UUID()),
('span_id', UInt(64)),
('transaction_name', String()),
('transaction_hash',
Materialized(
UInt(64),
'cityHash64(transaction_name)',
)),
('transaction_op', LowCardinality(String())),
('start_ts', DateTime()),
('start_ms', UInt(16)),
('finish_ts', DateTime()),
('finish_ms', UInt(16)),
('duration', UInt(32)),
('platform', LowCardinality(String())),
('environment', Nullable(String())),
('release', Nullable(String())),
('dist', Nullable(String())),
('ip_address_v4', Nullable(IPv4())),
('ip_address_v6', Nullable(IPv6())),
('user', WithDefault(
String(),
"''",
)),
('user_id', Nullable(String())),
('user_name', Nullable(String())),
('user_email', Nullable(String())),
('tags', Nested([
('key', String()),
('value', String()),
])),
('contexts', Nested([
('key', String()),
('value', String()),
])),
('partition', UInt(16)),
('offset', UInt(64)),
('retention_days', UInt(16)),
('deleted', UInt(8)),
])
schema = ReplacingMergeTreeSchema(
columns=columns,
local_table_name='transactions_local',
dist_table_name='transactions_dist',
mandatory_conditions=[],
order_by=
'(project_id, toStartOfDay(start_ts), transaction_hash, start_ts, start_ms, trace_id, span_id)',
partition_by='(retention_days, toMonday(start_ts))',
version_column='deleted',
sample_expr=None,
migration_function=transactions_migrations,
)
dataset_schemas = DatasetSchemas(
read_schema=schema,
write_schema=schema,
)
self.__tags_processor = TagColumnProcessor(
columns=columns,
promoted_columns=self._get_promoted_columns(),
column_tag_map=self._get_column_tag_map(),
)
super().__init__(dataset_schemas=dataset_schemas,
table_writer=TransactionsTableWriter(
write_schema=schema,
stream_loader=KafkaStreamLoader(
processor=TransactionsMessageProcessor(),
default_topic="events",
),
),
time_group_columns={
'bucketed_start': 'start_ts',
'bucketed_end': 'finish_ts',
},
time_parse_columns=('start_ts', 'finish_ts'))
def _get_promoted_columns(self):
# TODO: Support promoted tags
return {
'tags': frozenset(),
'contexts': frozenset(),
}
def _get_column_tag_map(self):
# TODO: Support promoted tags
return {
'tags': {},
'contexts': {},
}
def get_extensions(self) -> Mapping[str, QueryExtension]:
return {
'project':
ProjectExtension(processor=ProjectExtensionProcessor(
project_column="project_id")),
'timeseries':
TimeSeriesExtension(
default_granularity=3600,
default_window=timedelta(days=5),
timestamp_column='start_ts',
),
}
def column_expr(self,
column_name,
query: Query,
parsing_context: ParsingContext,
table_alias: str = ""):
# TODO remove these casts when clickhouse-driver is >= 0.0.19
if column_name == 'ip_address_v4':
return 'IPv4NumToString(ip_address_v4)'
if column_name == 'ip_address_v6':
return 'IPv6NumToString(ip_address_v6)'
if column_name == 'event_id':
return 'replaceAll(toString(event_id), \'-\', \'\')'
processed_column = self.__tags_processor.process_column_expression(
column_name, query, parsing_context, table_alias)
if processed_column:
# If processed_column is None, this was not a tag/context expression
return processed_column
return super().column_expr(column_name, query, parsing_context)
def get_split_query_spec(self) -> Union[None, ColumnSplitSpec]:
return ColumnSplitSpec(
id_column="event_id",
project_column="project_id",
timestamp_column="start_ts",
)
def get_prewhere_keys(self) -> Sequence[str]:
return ['event_id', 'project_id']
class EventsDataset(TimeSeriesDataset):
"""
Represents the collection of classic sentry "error" type events
and the particular quirks of storing and querying them.
"""
def __init__(self):
metadata_columns = ColumnSet([
# optional stream related data
('offset', Nullable(UInt(64))),
('partition', Nullable(UInt(16))),
])
promoted_tag_columns = ColumnSet([
# These are the classic tags, they are saved in Snuba exactly as they
# appear in the event body.
('level', Nullable(String())),
('logger', Nullable(String())),
('server_name', Nullable(String())), # future name: device_id?
('transaction', Nullable(String())),
('environment', Nullable(String())),
('sentry:release', Nullable(String())),
('sentry:dist', Nullable(String())),
('sentry:user', Nullable(String())),
('site', Nullable(String())),
('url', Nullable(String())),
])
promoted_context_tag_columns = ColumnSet([
# These are promoted tags that come in in `tags`, but are more closely
# related to contexts. To avoid naming confusion with Clickhouse nested
# columns, they are stored in the database with s/./_/
# promoted tags
('app_device', Nullable(String())),
('device', Nullable(String())),
('device_family', Nullable(String())),
('runtime', Nullable(String())),
('runtime_name', Nullable(String())),
('browser', Nullable(String())),
('browser_name', Nullable(String())),
('os', Nullable(String())),
('os_name', Nullable(String())),
('os_rooted', Nullable(UInt(8))),
])
promoted_context_columns = ColumnSet([
('os_build', Nullable(String())),
('os_kernel_version', Nullable(String())),
('device_name', Nullable(String())),
('device_brand', Nullable(String())),
('device_locale', Nullable(String())),
('device_uuid', Nullable(String())),
('device_model_id', Nullable(String())),
('device_arch', Nullable(String())),
('device_battery_level', Nullable(Float(32))),
('device_orientation', Nullable(String())),
('device_simulator', Nullable(UInt(8))),
('device_online', Nullable(UInt(8))),
('device_charging', Nullable(UInt(8))),
])
required_columns = ColumnSet([
('event_id', FixedString(32)),
('project_id', UInt(64)),
('group_id', UInt(64)),
('timestamp', DateTime()),
('deleted', UInt(8)),
('retention_days', UInt(16)),
])
all_columns = required_columns + [
# required for non-deleted
('platform', Nullable(String())),
('message', Nullable(String())),
('primary_hash', Nullable(FixedString(32))),
('received', Nullable(DateTime())),
('search_message', Nullable(String())),
('title', Nullable(String())),
('location', Nullable(String())),
# optional user
('user_id', Nullable(String())),
('username', Nullable(String())),
('email', Nullable(String())),
('ip_address', Nullable(String())),
# optional geo
('geo_country_code', Nullable(String())),
('geo_region', Nullable(String())),
('geo_city', Nullable(String())),
('sdk_name', Nullable(String())),
('sdk_version', Nullable(String())),
('type', Nullable(String())),
('version', Nullable(String())),
] + metadata_columns \
+ promoted_context_columns \
+ promoted_tag_columns \
+ promoted_context_tag_columns \
+ [
# other tags
('tags', Nested([
('key', String()),
('value', String()),
])),
# other context
('contexts', Nested([
('key', String()),
('value', String()),
])),
# http interface
('http_method', Nullable(String())),
('http_referer', Nullable(String())),
# exception interface
('exception_stacks', Nested([
('type', Nullable(String())),
('value', Nullable(String())),
('mechanism_type', Nullable(String())),
('mechanism_handled', Nullable(UInt(8))),
])),
('exception_frames', Nested([
('abs_path', Nullable(String())),
('filename', Nullable(String())),
('package', Nullable(String())),
('module', Nullable(String())),
('function', Nullable(String())),
('in_app', Nullable(UInt(8))),
('colno', Nullable(UInt(32))),
('lineno', Nullable(UInt(32))),
('stack_level', UInt(16)),
])),
# These are columns we added later in the life of the (current) production
# database. They don't necessarily belong here in a logical/readability sense
# but they are here to match the order of columns in production becase
# `insert_distributed_sync` is very sensitive to column existence and ordering.
('culprit', Nullable(String())),
('sdk_integrations', Array(String())),
('modules', Nested([
('name', String()),
('version', String()),
])),
]
sample_expr = 'cityHash64(toString(event_id))'
schema = ReplacingMergeTreeSchema(
columns=all_columns,
local_table_name='sentry_local',
dist_table_name='sentry_dist',
mandatory_conditions=[('deleted', '=', 0)],
order_by='(project_id, toStartOfDay(timestamp), %s)' % sample_expr,
partition_by='(toMonday(timestamp), if(equals(retention_days, 30), 30, 90))',
version_column='deleted',
sample_expr=sample_expr,
migration_function=events_migrations)
dataset_schemas = DatasetSchemas(
read_schema=schema,
write_schema=schema,
)
table_writer = TableWriter(
write_schema=schema,
stream_loader=KafkaStreamLoader(
processor=EventsProcessor(promoted_tag_columns),
default_topic="events",
replacement_topic="event-replacements",
commit_log_topic="snuba-commit-log",
)
)
super(EventsDataset, self).__init__(
dataset_schemas=dataset_schemas,
table_writer=table_writer,
time_group_columns={
'time': 'timestamp',
'rtime': 'received'
},
time_parse_columns=('timestamp', 'received')
)
self.__metadata_columns = metadata_columns
self.__promoted_tag_columns = promoted_tag_columns
self.__promoted_context_tag_columns = promoted_context_tag_columns
self.__promoted_context_columns = promoted_context_columns
self.__required_columns = required_columns
self.__tags_processor = TagColumnProcessor(
columns=all_columns,
promoted_columns=self._get_promoted_columns(),
column_tag_map=self._get_column_tag_map(),
)
def get_split_query_spec(self) -> Union[None, ColumnSplitSpec]:
return ColumnSplitSpec(
id_column="event_id",
project_column="project_id",
timestamp_column="timestamp",
)
def column_expr(self, column_name, query: Query, parsing_context: ParsingContext, table_alias: str=""):
processed_column = self.__tags_processor.process_column_expression(column_name, query, parsing_context, table_alias)
if processed_column:
# If processed_column is None, this was not a tag/context expression
return processed_column
elif column_name == 'issue' or column_name == 'group_id':
return f"nullIf({qualified_column('group_id', table_alias)}, 0)"
elif column_name == 'message':
# Because of the rename from message->search_message without backfill,
# records will have one or the other of these fields.
# TODO this can be removed once all data has search_message filled in.
search_message = qualified_column('search_message', table_alias)
message = qualified_column('message', table_alias)
return f"coalesce({search_message}, {message})"
else:
return super().column_expr(column_name, query, parsing_context, table_alias)
def get_promoted_tag_columns(self):
return self.__promoted_tag_columns
def _get_promoted_context_tag_columns(self):
return self.__promoted_context_tag_columns
def _get_promoted_context_columns(self):
return self.__promoted_context_columns
def get_required_columns(self):
return self.__required_columns
def _get_promoted_columns(self):
# The set of columns, and associated keys that have been promoted
# to the top level table namespace.
return {
'tags': frozenset(col.flattened for col in (self.get_promoted_tag_columns() + self._get_promoted_context_tag_columns())),
'contexts': frozenset(col.flattened for col in self._get_promoted_context_columns()),
}
def _get_column_tag_map(self):
# For every applicable promoted column, a map of translations from the column
# name we save in the database to the tag we receive in the query.
promoted_context_tag_columns = self._get_promoted_context_tag_columns()
return {
'tags': {col.flattened: col.flattened.replace('_', '.') for col in promoted_context_tag_columns},
'contexts': {},
}
def get_tag_column_map(self):
# And a reverse map from the tags the client expects to the database columns
return {
col: dict(map(reversed, trans.items())) for col, trans in self._get_column_tag_map().items()
}
def get_promoted_tags(self):
# The canonical list of foo.bar strings that you can send as a `tags[foo.bar]` query
# and they can/will use a promoted column.
return {
col: [self._get_column_tag_map()[col].get(x, x) for x in self._get_promoted_columns()[col]]
for col in self._get_promoted_columns()
}
def get_extensions(self) -> Mapping[str, QueryExtension]:
return {
'project': ProjectExtension(
processor=ProjectWithGroupsProcessor(project_column="project_id")
),
'timeseries': TimeSeriesExtension(
default_granularity=3600,
default_window=timedelta(days=5),
timestamp_column='timestamp',
),
}
def get_prewhere_keys(self) -> Sequence[str]:
return ['event_id', 'issue', 'tags[sentry:release]', 'message', 'environment', 'project_id']
class ErrorsDataset(TimeSeriesDataset):
"""
Represents the collections of all event types that are not transactions.
This is meant to replace Events. They will both exist during the migration.
"""
def __init__(self) -> None:
storage = get_writable_storage("errors")
schema = storage.get_table_writer().get_schema()
columns = schema.get_columns()
self.__time_group_columns = {"time": "timestamp", "rtime": "received"}
super().__init__(
storages=[storage],
query_plan_builder=SingleStorageQueryPlanBuilder(storage=storage),
abstract_column_set=columns,
writable_storage=storage,
time_group_columns=self.__time_group_columns,
time_parse_columns=("timestamp", "received"),
)
self.__tags_processor = TagColumnProcessor(
columns=columns,
promoted_columns=self._get_promoted_columns(),
column_tag_map=self._get_column_tag_map(),
)
def get_split_query_spec(self) -> Union[None, ColumnSplitSpec]:
return ColumnSplitSpec(
id_column="event_id",
project_column="project_id",
timestamp_column="timestamp",
)
def column_expr(
self,
column_name,
query: Query,
parsing_context: ParsingContext,
table_alias: str = "",
):
processed_column = self.__tags_processor.process_column_expression(
column_name, query, parsing_context, table_alias
)
return processed_column or super().column_expr(
column_name, query, parsing_context, table_alias
)
def _get_promoted_columns(self) -> Mapping[str, FrozenSet[str]]:
return {
"tags": frozenset(promoted_tag_columns.values()),
"contexts": frozenset(),
}
def _get_column_tag_map(self) -> Mapping[str, Mapping[str, str]]:
return {
"tags": {col: tag for tag, col in promoted_tag_columns.items()},
"contexts": {},
}
def get_extensions(self) -> Mapping[str, QueryExtension]:
return {
"project": ProjectExtension(
processor=ProjectWithGroupsProcessor(
project_column="project_id",
replacer_state_name=ReplacerState.ERRORS,
)
),
"timeseries": TimeSeriesExtension(
default_granularity=3600,
default_window=timedelta(days=5),
timestamp_column="timestamp",
),
}
def get_query_processors(self) -> Sequence[QueryProcessor]:
return [
BasicFunctionsProcessor(),
TimeSeriesColumnProcessor(self.__time_group_columns),
]
class EventsDataset(TimeSeriesDataset):
"""
Represents the collection of classic sentry "error" type events
and the particular quirks of storing and querying them.
"""
def __init__(self) -> None:
metadata_columns = ColumnSet([
# optional stream related data
("offset", Nullable(UInt(64))),
("partition", Nullable(UInt(16))),
])
promoted_tag_columns = ColumnSet([
# These are the classic tags, they are saved in Snuba exactly as they
# appear in the event body.
("level", Nullable(String())),
("logger", Nullable(String())),
("server_name", Nullable(String())), # future name: device_id?
("transaction", Nullable(String())),
("environment", Nullable(String())),
("sentry:release", Nullable(String())),
("sentry:dist", Nullable(String())),
("sentry:user", Nullable(String())),
("site", Nullable(String())),
("url", Nullable(String())),
])
promoted_context_tag_columns = ColumnSet([
# These are promoted tags that come in in `tags`, but are more closely
# related to contexts. To avoid naming confusion with Clickhouse nested
# columns, they are stored in the database with s/./_/
# promoted tags
("app_device", Nullable(String())),
("device", Nullable(String())),
("device_family", Nullable(String())),
("runtime", Nullable(String())),
("runtime_name", Nullable(String())),
("browser", Nullable(String())),
("browser_name", Nullable(String())),
("os", Nullable(String())),
("os_name", Nullable(String())),
("os_rooted", Nullable(UInt(8))),
])
promoted_context_columns = ColumnSet([
("os_build", Nullable(String())),
("os_kernel_version", Nullable(String())),
("device_name", Nullable(String())),
("device_brand", Nullable(String())),
("device_locale", Nullable(String())),
("device_uuid", Nullable(String())),
("device_model_id", Nullable(String())),
("device_arch", Nullable(String())),
("device_battery_level", Nullable(Float(32))),
("device_orientation", Nullable(String())),
("device_simulator", Nullable(UInt(8))),
("device_online", Nullable(UInt(8))),
("device_charging", Nullable(UInt(8))),
])
required_columns = ColumnSet([
("event_id", FixedString(32)),
("project_id", UInt(64)),
("group_id", UInt(64)),
("timestamp", DateTime()),
("deleted", UInt(8)),
("retention_days", UInt(16)),
])
all_columns = (
required_columns + [
# required for non-deleted
("platform", Nullable(String())),
("message", Nullable(String())),
("primary_hash", Nullable(FixedString(32))),
("received", Nullable(DateTime())),
("search_message", Nullable(String())),
("title", Nullable(String())),
("location", Nullable(String())),
# optional user
("user_id", Nullable(String())),
("username", Nullable(String())),
("email", Nullable(String())),
("ip_address", Nullable(String())),
# optional geo
("geo_country_code", Nullable(String())),
("geo_region", Nullable(String())),
("geo_city", Nullable(String())),
("sdk_name", Nullable(String())),
("sdk_version", Nullable(String())),
("type", Nullable(String())),
("version", Nullable(String())),
] + metadata_columns + promoted_context_columns +
promoted_tag_columns + promoted_context_tag_columns + [
# other tags
("tags", Nested([("key", String()), ("value", String())])),
("_tags_flattened", String()),
# other context
("contexts", Nested([("key", String()), ("value", String())])),
# http interface
("http_method", Nullable(String())),
("http_referer", Nullable(String())),
# exception interface
(
"exception_stacks",
Nested([
("type", Nullable(String())),
("value", Nullable(String())),
("mechanism_type", Nullable(String())),
("mechanism_handled", Nullable(UInt(8))),
]),
),
(
"exception_frames",
Nested([
("abs_path", Nullable(String())),
("filename", Nullable(String())),
("package", Nullable(String())),
("module", Nullable(String())),
("function", Nullable(String())),
("in_app", Nullable(UInt(8))),
("colno", Nullable(UInt(32))),
("lineno", Nullable(UInt(32))),
("stack_level", UInt(16)),
]),
),
# These are columns we added later in the life of the (current) production
# database. They don't necessarily belong here in a logical/readability sense
# but they are here to match the order of columns in production becase
# `insert_distributed_sync` is very sensitive to column existence and ordering.
("culprit", Nullable(String())),
("sdk_integrations", Array(String())),
("modules", Nested([("name", String()),
("version", String())])),
])
sample_expr = "cityHash64(toString(event_id))"
schema = ReplacingMergeTreeSchema(
columns=all_columns,
local_table_name="sentry_local",
dist_table_name="sentry_dist",
mandatory_conditions=[("deleted", "=", 0)],
prewhere_candidates=[
"event_id",
"group_id",
"tags[sentry:release]",
"message",
"environment",
"project_id",
],
order_by="(project_id, toStartOfDay(timestamp), %s)" % sample_expr,
partition_by=
"(toMonday(timestamp), if(equals(retention_days, 30), 30, 90))",
version_column="deleted",
sample_expr=sample_expr,
migration_function=events_migrations,
)
dataset_schemas = DatasetSchemas(
read_schema=schema,
write_schema=schema,
)
table_writer = TableWriter(
write_schema=schema,
stream_loader=KafkaStreamLoader(
processor=EventsProcessor(promoted_tag_columns),
default_topic="events",
replacement_topic="event-replacements",
commit_log_topic="snuba-commit-log",
),
)
super(EventsDataset, self).__init__(
dataset_schemas=dataset_schemas,
table_writer=table_writer,
time_group_columns={
"time": "timestamp",
"rtime": "received"
},
time_parse_columns=("timestamp", "received"),
)
self.__metadata_columns = metadata_columns
self.__promoted_tag_columns = promoted_tag_columns
self.__promoted_context_tag_columns = promoted_context_tag_columns
self.__promoted_context_columns = promoted_context_columns
self.__required_columns = required_columns
self.__tags_processor = TagColumnProcessor(
columns=all_columns,
promoted_columns=self._get_promoted_columns(),
column_tag_map=self._get_column_tag_map(),
)
def get_split_query_spec(self) -> Union[None, ColumnSplitSpec]:
return ColumnSplitSpec(
id_column="event_id",
project_column="project_id",
timestamp_column="timestamp",
)
def column_expr(
self,
column_name,
query: Query,
parsing_context: ParsingContext,
table_alias: str = "",
):
processed_column = self.__tags_processor.process_column_expression(
column_name, query, parsing_context, table_alias)
if processed_column:
# If processed_column is None, this was not a tag/context expression
return processed_column
elif column_name == "group_id":
return f"nullIf({qualified_column('group_id', table_alias)}, 0)"
elif column_name == "message":
# Because of the rename from message->search_message without backfill,
# records will have one or the other of these fields.
# TODO this can be removed once all data has search_message filled in.
search_message = qualified_column("search_message", table_alias)
message = qualified_column("message", table_alias)
return f"coalesce({search_message}, {message})"
else:
return super().column_expr(column_name, query, parsing_context,
table_alias)
def get_promoted_tag_columns(self):
return self.__promoted_tag_columns
def _get_promoted_context_tag_columns(self):
return self.__promoted_context_tag_columns
def _get_promoted_context_columns(self):
return self.__promoted_context_columns
def get_required_columns(self):
return self.__required_columns
def _get_promoted_columns(self):
# The set of columns, and associated keys that have been promoted
# to the top level table namespace.
return {
"tags":
frozenset(col.flattened
for col in (self.get_promoted_tag_columns() +
self._get_promoted_context_tag_columns())),
"contexts":
frozenset(col.flattened
for col in self._get_promoted_context_columns()),
}
def _get_column_tag_map(self):
# For every applicable promoted column, a map of translations from the column
# name we save in the database to the tag we receive in the query.
promoted_context_tag_columns = self._get_promoted_context_tag_columns()
return {
"tags": {
col.flattened: col.flattened.replace("_", ".")
for col in promoted_context_tag_columns
},
"contexts": {},
}
def get_tag_column_map(self):
# And a reverse map from the tags the client expects to the database columns
return {
col: dict(map(reversed, trans.items()))
for col, trans in self._get_column_tag_map().items()
}
def get_promoted_tags(self):
# The canonical list of foo.bar strings that you can send as a `tags[foo.bar]` query
# and they can/will use a promoted column.
return {
col: [
self._get_column_tag_map()[col].get(x, x)
for x in self._get_promoted_columns()[col]
]
for col in self._get_promoted_columns()
}
def get_extensions(self) -> Mapping[str, QueryExtension]:
return {
"project":
ProjectExtension(processor=ProjectWithGroupsProcessor(
project_column="project_id")),
"timeseries":
TimeSeriesExtension(
default_granularity=3600,
default_window=timedelta(days=5),
timestamp_column="timestamp",
),
}
def get_query_processors(self) -> Sequence[QueryProcessor]:
return [BasicFunctionsProcessor(), PrewhereProcessor()]
class ErrorsDataset(TimeSeriesDataset):
"""
Represents the collections of all event types that are not transactions.
This is meant to replace Events. They will both exist during the migration.
"""
def __init__(self) -> None:
all_columns = ColumnSet([
("org_id", UInt(64)),
("project_id", UInt(64)),
("timestamp", DateTime()),
("event_id", WithCodecs(UUID(), ["NONE"])),
(
"event_hash",
WithCodecs(
Materialized(
UInt(64),
"cityHash64(toString(event_id))",
),
["NONE"],
),
),
("platform", LowCardinality(String())),
("environment", LowCardinality(Nullable(String()))),
("release", LowCardinality(Nullable(String()))),
("dist", LowCardinality(Nullable(String()))),
("ip_address_v4", Nullable(IPv4())),
("ip_address_v6", Nullable(IPv6())),
("user", WithDefault(String(), "''")),
(
"user_hash",
Materialized(UInt(64), "cityHash64(user)"),
),
("user_id", Nullable(String())),
("user_name", Nullable(String())),
("user_email", Nullable(String())),
("sdk_name", LowCardinality(Nullable(String()))),
("sdk_version", LowCardinality(Nullable(String()))),
("tags", Nested([("key", String()), ("value", String())])),
("_tags_flattened", String()),
("contexts", Nested([("key", String()), ("value", String())])),
("_contexts_flattened", String()),
("transaction_name", WithDefault(LowCardinality(String()), "''")),
(
"transaction_hash",
Materialized(UInt(64), "cityHash64(transaction_name)"),
),
("span_id", Nullable(UInt(64))),
("trace_id", Nullable(UUID())),
("partition", UInt(16)),
("offset", WithCodecs(UInt(64), ["DoubleDelta", "LZ4"])),
("retention_days", UInt(16)),
("deleted", UInt(8)),
("group_id", UInt(64)),
("primary_hash", FixedString(32)),
("primary_hash_hex", Materialized(UInt(64), "hex(primary_hash)")),
("event_string", WithCodecs(String(), ["NONE"])),
("received", DateTime()),
("message", String()),
("title", String()),
("culprit", String()),
("level", LowCardinality(String())),
("location", Nullable(String())),
("version", LowCardinality(Nullable(String()))),
("type", LowCardinality(String())),
(
"exception_stacks",
Nested([
("type", Nullable(String())),
("value", Nullable(String())),
("mechanism_type", Nullable(String())),
("mechanism_handled", Nullable(UInt(8))),
]),
),
(
"exception_frames",
Nested([
("abs_path", Nullable(String())),
("colno", Nullable(UInt(32))),
("filename", Nullable(String())),
("function", Nullable(String())),
("lineno", Nullable(UInt(32))),
("in_app", Nullable(UInt(8))),
("package", Nullable(String())),
("module", Nullable(String())),
("stack_level", Nullable(UInt(16))),
]),
),
("sdk_integrations", Array(String())),
("modules", Nested([("name", String()), ("version", String())])),
])
self.__promoted_tag_columns = {
"environment": "environment",
"sentry:release": "release",
"sentry:dist": "dist",
"sentry:user": "******",
"transaction": "transaction_name",
"level": "level",
}
schema = ReplacingMergeTreeSchema(
columns=all_columns,
local_table_name="errors_local",
dist_table_name="errors_dist",
mandatory_conditions=[("deleted", "=", 0)],
prewhere_candidates=[
"event_id",
"group_id",
"tags[sentry:release]",
"message",
"environment",
"project_id",
],
order_by=
"(org_id, project_id, toStartOfDay(timestamp), primary_hash_hex, event_hash)",
partition_by=
"(toMonday(timestamp), if(retention_days = 30, 30, 90))",
version_column="deleted",
sample_expr="event_hash",
ttl_expr="timestamp + toIntervalDay(retention_days)",
settings={"index_granularity": "8192"},
)
dataset_schemas = DatasetSchemas(
read_schema=schema,
write_schema=schema,
)
table_writer = TableWriter(
write_schema=schema,
stream_loader=KafkaStreamLoader(
processor=ErrorsProcessor(self.__promoted_tag_columns),
default_topic="events",
),
)
super().__init__(
dataset_schemas=dataset_schemas,
table_writer=table_writer,
time_group_columns={
"time": "timestamp",
"rtime": "received"
},
time_parse_columns=("timestamp", "received"),
)
self.__tags_processor = TagColumnProcessor(
columns=all_columns,
promoted_columns=self._get_promoted_columns(),
column_tag_map=self._get_column_tag_map(),
)
def get_split_query_spec(self) -> Union[None, ColumnSplitSpec]:
return ColumnSplitSpec(
id_column="event_id",
project_column="project_id",
timestamp_column="timestamp",
)
def column_expr(
self,
column_name,
query: Query,
parsing_context: ParsingContext,
table_alias: str = "",
):
processed_column = self.__tags_processor.process_column_expression(
column_name, query, parsing_context, table_alias)
return processed_column or super().column_expr(
column_name, query, parsing_context, table_alias)
def _get_promoted_columns(self) -> Mapping[str, FrozenSet[str]]:
return {
"tags": frozenset(self.__promoted_tag_columns.values()),
"contexts": frozenset(),
}
def _get_column_tag_map(self) -> Mapping[str, Mapping[str, str]]:
return {
"tags":
{col: tag
for tag, col in self.__promoted_tag_columns.items()},
"contexts": {},
}
def get_extensions(self) -> Mapping[str, QueryExtension]:
return {
"project":
ProjectExtension(processor=ProjectWithGroupsProcessor(
project_column="project_id")),
"timeseries":
TimeSeriesExtension(
default_granularity=3600,
default_window=timedelta(days=5),
timestamp_column="timestamp",
),
}
def get_query_processors(self) -> Sequence[QueryProcessor]:
return [BasicFunctionsProcessor(), PrewhereProcessor()]