def expected_query(match, select, extra_groupby, metric_name): function, column, alias = select return Query( dataset="metrics", match=Entity(match), select=[ Function( OP_TO_SNUBA_FUNCTION[match][alias], [ Column("value"), Function( "equals", [Column("metric_id"), resolve_weak(metric_name)]), ], alias=f"{alias}({metric_name})", ) ], groupby=[Column("tags[8]"), Column("tags[2]")] + extra_groupby, where=[ Condition(Column("org_id"), Op.EQ, 1), Condition(Column("project_id"), Op.IN, [1]), Condition(Column("timestamp"), Op.GTE, datetime(2021, 5, 28, 0, tzinfo=pytz.utc)), Condition(Column("timestamp"), Op.LT, datetime(2021, 8, 26, 0, tzinfo=pytz.utc)), Condition(Column("tags[6]"), Op.IN, [10]), Condition(Column("metric_id"), Op.IN, [resolve_weak(metric_name)]), ], limit=Limit(MAX_POINTS), offset=Offset(0), granularity=Granularity(query_definition.rollup), )
def test_counter_sum_aggregation_on_session_status(self): for status, func in [ ("init", init_sessions), ("crashed", crashed_sessions), ("errored_preaggr", errored_preaggr_sessions), ]: assert func(self.metric_ids, alias=status) == Function( "sumIf", [ Column("value"), Function( "and", [ Function( "equals", [ Column( f"tags[{resolve_weak('session.status')}]" ), resolve_weak(status), ], ), Function( "in", [Column("metric_id"), list(self.metric_ids)]), ], ), ], status, )
def resolve_tags(input_: Any) -> Any: """Translate tags in snuba condition This assumes that all strings are either tag names or tag values, so do not pass Column("metric_id") or Column("project_id") into this function. """ if isinstance(input_, list): return [resolve_tags(item) for item in input_] if isinstance(input_, Function): if input_.function == "ifNull": # This was wrapped automatically by QueryBuilder, remove wrapper return resolve_tags(input_.parameters[0]) return Function( function=input_.function, parameters=input_.parameters and [resolve_tags(item) for item in input_.parameters], ) if isinstance(input_, Condition): return Condition(lhs=resolve_tags(input_.lhs), op=input_.op, rhs=resolve_tags(input_.rhs)) if isinstance(input_, BooleanCondition): return input_.__class__(conditions=[resolve_tags(item) for item in input_.conditions]) if isinstance(input_, Column): # HACK: Some tags already take the form "tags[...]" in discover, take that into account: if input_.subscriptable == "tags": name = input_.key else: name = input_.name return Column(name=resolve_tag_key(name)) if isinstance(input_, str): return resolve_weak(input_) return input_
def _snql_on_session_status_factory(session_status, metric_ids, alias=None): return Function( aggregate, [ Column("value"), Function( "and", [ Function( "equals", [ Column( f"tags[{resolve_weak('session.status')}]"), resolve_weak(session_status), ], ), Function("in", [Column("metric_id"), list(metric_ids)]), ], ), ], alias, )
def test_set_uniq_aggregation_on_session_status(self): for status, func in [ ("init", all_users), ("crashed", crashed_users), ]: assert func(self.metric_ids, alias=status) == Function( "uniqIf", [ Column("value"), Function( "and", [ Function( "equals", [ Column( f"tags[{resolve_weak('session.status')}]" ), resolve_weak(status), ], ), Function( "in", [Column("metric_id"), list(self.metric_ids)]), ], ), ], status, )
def test_query_and_environment_users_metrics(self): env = self.create_environment(self.project, name="development") org_id = self.organization.id for tag in [ SessionMetricKey.USER.value, "session.status", "environment", "development", "init", "crashed", "release", "[email protected]", ]: indexer.record(org_id, tag) entity_subscription = get_entity_subscription_for_dataset( dataset=QueryDatasets.METRICS, time_window=3600, aggregate= "percentage(users_crashed, users) AS _crash_rate_alert_aggregate", extra_fields={"org_id": org_id}, ) snuba_filter = build_snuba_filter( entity_subscription, query="release:[email protected]", environment=env, ) assert snuba_filter assert snuba_filter.aggregations == [["uniq(value)", None, "value"]] assert snuba_filter.groupby == [resolve_tag_key("session.status")] assert snuba_filter.conditions == [ ["metric_id", "=", resolve(SessionMetricKey.USER.value)], [ resolve_tag_key("session.status"), "IN", resolve_many_weak(["crashed", "init"]), ], [resolve_tag_key("environment"), "=", resolve_weak("development")], [resolve_tag_key("release"), "=", resolve_weak("[email protected]")], ]
def build_snuba_filter( self, query: str, environment: Optional[Environment], params: Optional[Mapping[str, Any]] = None, ) -> Filter: snuba_filter = get_filter(query, params=params) conditions = copy(snuba_filter.conditions) session_status_tag_values = resolve_many_weak(["crashed", "init"]) snuba_filter.update_with({ "aggregations": [[f"{self.aggregation_func}(value)", None, "value"]], "conditions": [ ["metric_id", "=", resolve(self.metric_key.value)], [self.session_status, "IN", session_status_tag_values], ], "groupby": self.get_query_groupby(), "rollup": self.get_granularity(), }) if environment: snuba_filter.conditions.append([ resolve_tag_key("environment"), "=", resolve_weak(environment.name) ]) if query and len(conditions) > 0: release_conditions = [ condition for condition in conditions if condition[0] == "release" ] for release_condition in release_conditions: snuba_filter.conditions.append([ resolve_tag_key(release_condition[0]), release_condition[1], resolve_weak(release_condition[2]), ]) return snuba_filter
def __build_conditional_aggregate_for_metric( self, entity: MetricEntity) -> Function: snuba_function = OP_TO_SNUBA_FUNCTION[entity][self.op] return Function( snuba_function, [ Column("value"), Function("equals", [Column("metric_id"), resolve_weak(self.metric_name)]), ], alias=f"{self.op}({self.metric_name})", )
def __recursively_generate_metric_ids( cls, derived_metric_name: str) -> Set[int]: """ Method that traverses a derived metric dependency tree to return a set of the metric ids of its constituent metrics """ if derived_metric_name not in DERIVED_METRICS: return set() derived_metric = DERIVED_METRICS[derived_metric_name] ids = set() for metric_name in derived_metric.metrics: if metric_name not in DERIVED_METRICS: ids.add(resolve_weak(metric_name)) else: ids |= cls.__recursively_generate_metric_ids(metric_name) return ids
def generate_metric_ids(self) -> Set[int]: return {resolve_weak(self.metric_name)}
def test_build_snuba_query_orderby(mock_now, mock_now2, monkeypatch): monkeypatch.setattr("sentry.sentry_metrics.indexer.resolve", MockIndexer().resolve) query_params = MultiValueDict({ "query": ["release:staging" ], # weird release but we need a string exising in mock indexer "groupBy": ["session.status", "environment"], "field": [ "sum(sentry.sessions.session)", ], "orderBy": ["-sum(sentry.sessions.session)"], }) query_definition = QueryDefinition(query_params, paginator_kwargs={"limit": 3}) snuba_queries, _ = SnubaQueryBuilder([PseudoProject(1, 1)], query_definition).get_snuba_queries() counter_queries = snuba_queries.pop("metrics_counters") assert not snuba_queries op = "sum" metric_name = "sentry.sessions.session" select = Function( OP_TO_SNUBA_FUNCTION["metrics_counters"]["sum"], [ Column("value"), Function("equals", [Column("metric_id"), resolve_weak(metric_name)]) ], alias=f"{op}({metric_name})", ) assert counter_queries["totals"] == Query( dataset="metrics", match=Entity("metrics_counters"), select=[select], groupby=[ Column("tags[8]"), Column("tags[2]"), ], where=[ Condition(Column("org_id"), Op.EQ, 1), Condition(Column("project_id"), Op.IN, [1]), Condition(Column("timestamp"), Op.GTE, datetime(2021, 5, 28, 0, tzinfo=pytz.utc)), Condition(Column("timestamp"), Op.LT, datetime(2021, 8, 26, 0, tzinfo=pytz.utc)), Condition(Column("tags[6]", entity=None), Op.IN, [10]), Condition(Column("metric_id"), Op.IN, [9]), ], orderby=[OrderBy(select, Direction.DESC)], limit=Limit(3), offset=Offset(0), granularity=Granularity(query_definition.rollup), ) assert counter_queries["series"] == Query( dataset="metrics", match=Entity("metrics_counters"), select=[select], groupby=[ Column("tags[8]"), Column("tags[2]"), Column("bucketed_time"), ], where=[ Condition(Column("org_id"), Op.EQ, 1), Condition(Column("project_id"), Op.IN, [1]), Condition(Column("timestamp"), Op.GTE, datetime(2021, 5, 28, 0, tzinfo=pytz.utc)), Condition(Column("timestamp"), Op.LT, datetime(2021, 8, 26, 0, tzinfo=pytz.utc)), Condition(Column("tags[6]", entity=None), Op.IN, [10]), Condition(Column("metric_id"), Op.IN, [9]), ], orderby=[OrderBy(select, Direction.DESC)], limit=Limit(6480), offset=Offset(0), granularity=Granularity(query_definition.rollup), )
def test_build_snuba_query_derived_metrics(mock_now, mock_now2, monkeypatch): monkeypatch.setattr("sentry.sentry_metrics.indexer.resolve", MockIndexer().resolve) # Your typical release health query querying everything query_params = MultiValueDict({ "groupBy": [], "field": [ "session.errored", "session.crash_free_rate", "session.all", ], "interval": ["1d"], "statsPeriod": ["2d"], }) query_definition = QueryDefinition(query_params) query_builder = SnubaQueryBuilder([PseudoProject(1, 1)], query_definition) snuba_queries, fields_in_entities = query_builder.get_snuba_queries() assert fields_in_entities == { "metrics_counters": [ (None, "session.errored_preaggregated"), (None, "session.crash_free_rate"), (None, "session.all"), ], "metrics_sets": [ (None, "session.errored_set"), ], } for key in ("totals", "series"): groupby = [] if key == "totals" else [Column("bucketed_time")] assert snuba_queries["metrics_counters"][key] == (Query( dataset="metrics", match=Entity("metrics_counters"), select=[ errored_preaggr_sessions( metric_ids=[resolve_weak("sentry.sessions.session")], alias="session.errored_preaggregated", ), percentage( crashed_sessions( metric_ids=[resolve_weak("sentry.sessions.session")], alias="session.crashed", ), all_sessions( metric_ids=[resolve_weak("sentry.sessions.session")], alias="session.all", ), alias="session.crash_free_rate", ), all_sessions( metric_ids=[resolve_weak("sentry.sessions.session")], alias="session.all"), ], groupby=groupby, where=[ Condition(Column("org_id"), Op.EQ, 1), Condition(Column("project_id"), Op.IN, [1]), Condition(Column("timestamp"), Op.GTE, datetime(2021, 8, 24, 0, tzinfo=pytz.utc)), Condition(Column("timestamp"), Op.LT, datetime(2021, 8, 26, 0, tzinfo=pytz.utc)), Condition(Column("metric_id"), Op.IN, [resolve_weak("sentry.sessions.session")]), ], limit=Limit(MAX_POINTS), offset=Offset(0), granularity=Granularity(query_definition.rollup), )) assert snuba_queries["metrics_sets"][key] == (Query( dataset="metrics", match=Entity("metrics_sets"), select=[ sessions_errored_set( metric_ids=[resolve_weak("sentry.sessions.session.error")], alias="session.errored_set", ), ], groupby=groupby, where=[ Condition(Column("org_id"), Op.EQ, 1), Condition(Column("project_id"), Op.IN, [1]), Condition(Column("timestamp"), Op.GTE, datetime(2021, 8, 24, 0, tzinfo=pytz.utc)), Condition(Column("timestamp"), Op.LT, datetime(2021, 8, 26, 0, tzinfo=pytz.utc)), Condition(Column("metric_id"), Op.IN, [resolve_weak("sentry.sessions.session.error")]), ], limit=Limit(MAX_POINTS), offset=Offset(0), granularity=Granularity(query_definition.rollup), ))