def expected_query(match, select, extra_groupby, metric_name):
function, column, alias = select
return Query(
dataset="metrics",
match=Entity(match),
select=[
Function(
OP_TO_SNUBA_FUNCTION[match][alias],
[
Column("value"),
Function(
"equals",
[Column("metric_id"),
resolve_weak(metric_name)]),
],
alias=f"{alias}({metric_name})",
)
],
groupby=[Column("tags[8]"), Column("tags[2]")] + extra_groupby,
where=[
Condition(Column("org_id"), Op.EQ, 1),
Condition(Column("project_id"), Op.IN, [1]),
Condition(Column("timestamp"), Op.GTE,
datetime(2021, 5, 28, 0, tzinfo=pytz.utc)),
Condition(Column("timestamp"), Op.LT,
datetime(2021, 8, 26, 0, tzinfo=pytz.utc)),
Condition(Column("tags[6]"), Op.IN, [10]),
Condition(Column("metric_id"), Op.IN,
[resolve_weak(metric_name)]),
],
limit=Limit(MAX_POINTS),
offset=Offset(0),
granularity=Granularity(query_definition.rollup),
)
def test_counter_sum_aggregation_on_session_status(self):
for status, func in [
("init", init_sessions),
("crashed", crashed_sessions),
("errored_preaggr", errored_preaggr_sessions),
]:
assert func(self.metric_ids, alias=status) == Function(
"sumIf",
[
Column("value"),
Function(
"and",
[
Function(
"equals",
[
Column(
f"tags[{resolve_weak('session.status')}]"
),
resolve_weak(status),
],
),
Function(
"in",
[Column("metric_id"),
list(self.metric_ids)]),
],
),
],
status,
)
def resolve_tags(input_: Any) -> Any:
"""Translate tags in snuba condition
This assumes that all strings are either tag names or tag values, so do not
pass Column("metric_id") or Column("project_id") into this function.
"""
if isinstance(input_, list):
return [resolve_tags(item) for item in input_]
if isinstance(input_, Function):
if input_.function == "ifNull":
# This was wrapped automatically by QueryBuilder, remove wrapper
return resolve_tags(input_.parameters[0])
return Function(
function=input_.function,
parameters=input_.parameters and [resolve_tags(item) for item in input_.parameters],
)
if isinstance(input_, Condition):
return Condition(lhs=resolve_tags(input_.lhs), op=input_.op, rhs=resolve_tags(input_.rhs))
if isinstance(input_, BooleanCondition):
return input_.__class__(conditions=[resolve_tags(item) for item in input_.conditions])
if isinstance(input_, Column):
# HACK: Some tags already take the form "tags[...]" in discover, take that into account:
if input_.subscriptable == "tags":
name = input_.key
else:
name = input_.name
return Column(name=resolve_tag_key(name))
if isinstance(input_, str):
return resolve_weak(input_)
return input_
def _snql_on_session_status_factory(session_status,
metric_ids,
alias=None):
return Function(
aggregate,
[
Column("value"),
Function(
"and",
[
Function(
"equals",
[
Column(
f"tags[{resolve_weak('session.status')}]"),
resolve_weak(session_status),
],
),
Function("in", [Column("metric_id"),
list(metric_ids)]),
],
),
],
alias,
)
def test_set_uniq_aggregation_on_session_status(self):
for status, func in [
("init", all_users),
("crashed", crashed_users),
]:
assert func(self.metric_ids, alias=status) == Function(
"uniqIf",
[
Column("value"),
Function(
"and",
[
Function(
"equals",
[
Column(
f"tags[{resolve_weak('session.status')}]"
),
resolve_weak(status),
],
),
Function(
"in",
[Column("metric_id"),
list(self.metric_ids)]),
],
),
],
status,
)
def test_query_and_environment_users_metrics(self):
env = self.create_environment(self.project, name="development")
org_id = self.organization.id
for tag in [
SessionMetricKey.USER.value,
"session.status",
"environment",
"development",
"init",
"crashed",
"release",
"[email protected]",
]:
indexer.record(org_id, tag)
entity_subscription = get_entity_subscription_for_dataset(
dataset=QueryDatasets.METRICS,
time_window=3600,
aggregate=
"percentage(users_crashed, users) AS _crash_rate_alert_aggregate",
extra_fields={"org_id": org_id},
)
snuba_filter = build_snuba_filter(
entity_subscription,
query="release:[email protected]",
environment=env,
)
assert snuba_filter
assert snuba_filter.aggregations == [["uniq(value)", None, "value"]]
assert snuba_filter.groupby == [resolve_tag_key("session.status")]
assert snuba_filter.conditions == [
["metric_id", "=",
resolve(SessionMetricKey.USER.value)],
[
resolve_tag_key("session.status"),
"IN",
resolve_many_weak(["crashed", "init"]),
],
[resolve_tag_key("environment"), "=",
resolve_weak("development")],
[resolve_tag_key("release"), "=",
resolve_weak("[email protected]")],
]
def build_snuba_filter(
self,
query: str,
environment: Optional[Environment],
params: Optional[Mapping[str, Any]] = None,
) -> Filter:
snuba_filter = get_filter(query, params=params)
conditions = copy(snuba_filter.conditions)
session_status_tag_values = resolve_many_weak(["crashed", "init"])
snuba_filter.update_with({
"aggregations":
[[f"{self.aggregation_func}(value)", None, "value"]],
"conditions": [
["metric_id", "=",
resolve(self.metric_key.value)],
[self.session_status, "IN", session_status_tag_values],
],
"groupby":
self.get_query_groupby(),
"rollup":
self.get_granularity(),
})
if environment:
snuba_filter.conditions.append([
resolve_tag_key("environment"), "=",
resolve_weak(environment.name)
])
if query and len(conditions) > 0:
release_conditions = [
condition for condition in conditions
if condition[0] == "release"
]
for release_condition in release_conditions:
snuba_filter.conditions.append([
resolve_tag_key(release_condition[0]),
release_condition[1],
resolve_weak(release_condition[2]),
])
return snuba_filter
def __build_conditional_aggregate_for_metric(
self, entity: MetricEntity) -> Function:
snuba_function = OP_TO_SNUBA_FUNCTION[entity][self.op]
return Function(
snuba_function,
[
Column("value"),
Function("equals",
[Column("metric_id"),
resolve_weak(self.metric_name)]),
],
alias=f"{self.op}({self.metric_name})",
)
def __recursively_generate_metric_ids(
cls, derived_metric_name: str) -> Set[int]:
"""
Method that traverses a derived metric dependency tree to return a set of the metric ids
of its constituent metrics
"""
if derived_metric_name not in DERIVED_METRICS:
return set()
derived_metric = DERIVED_METRICS[derived_metric_name]
ids = set()
for metric_name in derived_metric.metrics:
if metric_name not in DERIVED_METRICS:
ids.add(resolve_weak(metric_name))
else:
ids |= cls.__recursively_generate_metric_ids(metric_name)
return ids
def generate_metric_ids(self) -> Set[int]:
return {resolve_weak(self.metric_name)}
def test_build_snuba_query_orderby(mock_now, mock_now2, monkeypatch):
monkeypatch.setattr("sentry.sentry_metrics.indexer.resolve",
MockIndexer().resolve)
query_params = MultiValueDict({
"query":
["release:staging"
], # weird release but we need a string exising in mock indexer
"groupBy": ["session.status", "environment"],
"field": [
"sum(sentry.sessions.session)",
],
"orderBy": ["-sum(sentry.sessions.session)"],
})
query_definition = QueryDefinition(query_params,
paginator_kwargs={"limit": 3})
snuba_queries, _ = SnubaQueryBuilder([PseudoProject(1, 1)],
query_definition).get_snuba_queries()
counter_queries = snuba_queries.pop("metrics_counters")
assert not snuba_queries
op = "sum"
metric_name = "sentry.sessions.session"
select = Function(
OP_TO_SNUBA_FUNCTION["metrics_counters"]["sum"],
[
Column("value"),
Function("equals",
[Column("metric_id"),
resolve_weak(metric_name)])
],
alias=f"{op}({metric_name})",
)
assert counter_queries["totals"] == Query(
dataset="metrics",
match=Entity("metrics_counters"),
select=[select],
groupby=[
Column("tags[8]"),
Column("tags[2]"),
],
where=[
Condition(Column("org_id"), Op.EQ, 1),
Condition(Column("project_id"), Op.IN, [1]),
Condition(Column("timestamp"), Op.GTE,
datetime(2021, 5, 28, 0, tzinfo=pytz.utc)),
Condition(Column("timestamp"), Op.LT,
datetime(2021, 8, 26, 0, tzinfo=pytz.utc)),
Condition(Column("tags[6]", entity=None), Op.IN, [10]),
Condition(Column("metric_id"), Op.IN, [9]),
],
orderby=[OrderBy(select, Direction.DESC)],
limit=Limit(3),
offset=Offset(0),
granularity=Granularity(query_definition.rollup),
)
assert counter_queries["series"] == Query(
dataset="metrics",
match=Entity("metrics_counters"),
select=[select],
groupby=[
Column("tags[8]"),
Column("tags[2]"),
Column("bucketed_time"),
],
where=[
Condition(Column("org_id"), Op.EQ, 1),
Condition(Column("project_id"), Op.IN, [1]),
Condition(Column("timestamp"), Op.GTE,
datetime(2021, 5, 28, 0, tzinfo=pytz.utc)),
Condition(Column("timestamp"), Op.LT,
datetime(2021, 8, 26, 0, tzinfo=pytz.utc)),
Condition(Column("tags[6]", entity=None), Op.IN, [10]),
Condition(Column("metric_id"), Op.IN, [9]),
],
orderby=[OrderBy(select, Direction.DESC)],
limit=Limit(6480),
offset=Offset(0),
granularity=Granularity(query_definition.rollup),
)
def test_build_snuba_query_derived_metrics(mock_now, mock_now2, monkeypatch):
monkeypatch.setattr("sentry.sentry_metrics.indexer.resolve",
MockIndexer().resolve)
# Your typical release health query querying everything
query_params = MultiValueDict({
"groupBy": [],
"field": [
"session.errored",
"session.crash_free_rate",
"session.all",
],
"interval": ["1d"],
"statsPeriod": ["2d"],
})
query_definition = QueryDefinition(query_params)
query_builder = SnubaQueryBuilder([PseudoProject(1, 1)], query_definition)
snuba_queries, fields_in_entities = query_builder.get_snuba_queries()
assert fields_in_entities == {
"metrics_counters": [
(None, "session.errored_preaggregated"),
(None, "session.crash_free_rate"),
(None, "session.all"),
],
"metrics_sets": [
(None, "session.errored_set"),
],
}
for key in ("totals", "series"):
groupby = [] if key == "totals" else [Column("bucketed_time")]
assert snuba_queries["metrics_counters"][key] == (Query(
dataset="metrics",
match=Entity("metrics_counters"),
select=[
errored_preaggr_sessions(
metric_ids=[resolve_weak("sentry.sessions.session")],
alias="session.errored_preaggregated",
),
percentage(
crashed_sessions(
metric_ids=[resolve_weak("sentry.sessions.session")],
alias="session.crashed",
),
all_sessions(
metric_ids=[resolve_weak("sentry.sessions.session")],
alias="session.all",
),
alias="session.crash_free_rate",
),
all_sessions(
metric_ids=[resolve_weak("sentry.sessions.session")],
alias="session.all"),
],
groupby=groupby,
where=[
Condition(Column("org_id"), Op.EQ, 1),
Condition(Column("project_id"), Op.IN, [1]),
Condition(Column("timestamp"), Op.GTE,
datetime(2021, 8, 24, 0, tzinfo=pytz.utc)),
Condition(Column("timestamp"), Op.LT,
datetime(2021, 8, 26, 0, tzinfo=pytz.utc)),
Condition(Column("metric_id"), Op.IN,
[resolve_weak("sentry.sessions.session")]),
],
limit=Limit(MAX_POINTS),
offset=Offset(0),
granularity=Granularity(query_definition.rollup),
))
assert snuba_queries["metrics_sets"][key] == (Query(
dataset="metrics",
match=Entity("metrics_sets"),
select=[
sessions_errored_set(
metric_ids=[resolve_weak("sentry.sessions.session.error")],
alias="session.errored_set",
),
],
groupby=groupby,
where=[
Condition(Column("org_id"), Op.EQ, 1),
Condition(Column("project_id"), Op.IN, [1]),
Condition(Column("timestamp"), Op.GTE,
datetime(2021, 8, 24, 0, tzinfo=pytz.utc)),
Condition(Column("timestamp"), Op.LT,
datetime(2021, 8, 26, 0, tzinfo=pytz.utc)),
Condition(Column("metric_id"), Op.IN,
[resolve_weak("sentry.sessions.session.error")]),
],
limit=Limit(MAX_POINTS),
offset=Offset(0),
granularity=Granularity(query_definition.rollup),
))
