def expected_query(match, select, extra_groupby):
function, column, alias = select
return Query(
dataset="metrics",
match=Entity(match),
select=[Function(function, [Column(column)], alias)],
groupby=[
Column("metric_id"),
Column("tags[8]"),
Column("tags[2]")
] + extra_groupby,
where=[
Condition(Column("org_id"), Op.EQ, 1),
Condition(Column("project_id"), Op.IN, [1]),
Condition(Column("metric_id"), Op.IN, [9, 11, 7]),
Condition(Column("timestamp"), Op.GTE,
datetime(2021, 5, 28, 0, tzinfo=pytz.utc)),
Condition(Column("timestamp"), Op.LT,
datetime(2021, 8, 26, 0, tzinfo=pytz.utc)),
Condition(Column("tags[6]"), Op.IN, [10]),
],
limit=Limit(MAX_POINTS),
offset=Offset(0),
granularity=Granularity(query_definition.rollup),
)
def run_metrics_query(
*,
entity_key: EntityKey,
select: List[Column],
where: List[Condition],
groupby: List[Column],
projects: Sequence[Project],
org_id: int,
referrer: str,
) -> Mapping[str, Any]:
# Round timestamp to minute to get cache efficiency:
now = datetime.now().replace(second=0, microsecond=0)
query = Query(
dataset=Dataset.Metrics.value,
match=Entity(entity_key.value),
select=select,
groupby=groupby,
where=[
Condition(Column("org_id"), Op.EQ, org_id),
Condition(Column("project_id"), Op.IN, [p.id for p in projects]),
Condition(Column(TS_COL_QUERY), Op.GTE, now - timedelta(hours=24)),
Condition(Column(TS_COL_QUERY), Op.LT, now),
] + where,
granularity=Granularity(GRANULARITY),
)
result = raw_snql_query(query, referrer, use_cache=True)
return result["data"]
def _build_totals_and_series_queries(
entity, select, where, groupby, orderby, limit, offset, rollup, intervals_len
):
totals_query = Query(
dataset=Dataset.Metrics.value,
match=Entity(entity),
groupby=groupby,
select=select,
where=where,
limit=Limit(limit or MAX_POINTS),
offset=Offset(offset or 0),
granularity=Granularity(rollup),
orderby=orderby,
)
series_query = totals_query.set_groupby(
(totals_query.groupby or []) + [Column(TS_COL_GROUP)]
)
# In a series query, we also need to factor in the len of the intervals array
series_limit = MAX_POINTS
if limit:
series_limit = limit * intervals_len
series_query = series_query.set_limit(series_limit)
return {"totals": totals_query, "series": series_query}
def expected_query(match, select, extra_groupby, metric_name):
function, column, alias = select
return Query(
dataset="metrics",
match=Entity(match),
select=[
Function(
OP_TO_SNUBA_FUNCTION[match][alias],
[
Column("value"),
Function(
"equals",
[Column("metric_id"),
resolve_weak(metric_name)]),
],
alias=f"{alias}({metric_name})",
)
],
groupby=[Column("tags[8]"), Column("tags[2]")] + extra_groupby,
where=[
Condition(Column("org_id"), Op.EQ, 1),
Condition(Column("project_id"), Op.IN, [1]),
Condition(Column("timestamp"), Op.GTE,
datetime(2021, 5, 28, 0, tzinfo=pytz.utc)),
Condition(Column("timestamp"), Op.LT,
datetime(2021, 8, 26, 0, tzinfo=pytz.utc)),
Condition(Column("tags[6]"), Op.IN, [10]),
Condition(Column("metric_id"), Op.IN,
[resolve_weak(metric_name)]),
],
limit=Limit(MAX_POINTS),
offset=Offset(0),
granularity=Granularity(query_definition.rollup),
)
def monitor_release_adoption(**kwargs):
metrics.incr("sentry.tasks.monitor_release_adoption.start", sample_rate=1.0)
# 1. Query snuba for all project ids that have sessions.
with metrics.timer(
"sentry.tasks.monitor_release_adoption.aggregate_projects.loop", sample_rate=1.0
):
aggregated_projects = defaultdict(list)
start_time = time.time()
offset = 0
while (time.time() - start_time) < MAX_SECONDS:
query = (
Query(
dataset="sessions",
match=Entity("org_sessions"),
select=[
Column("org_id"),
Column("project_id"),
],
groupby=[Column("org_id"), Column("project_id")],
where=[
Condition(
Column("started"), Op.GTE, datetime.utcnow() - timedelta(hours=6)
),
Condition(Column("started"), Op.LT, datetime.utcnow()),
],
granularity=Granularity(3600),
orderby=[
OrderBy(Column("org_id"), Direction.ASC),
OrderBy(Column("project_id"), Direction.ASC),
],
)
.set_limit(CHUNK_SIZE + 1)
.set_offset(offset)
)
data = snuba.raw_snql_query(query, referrer="tasks.monitor_release_adoption")["data"]
count = len(data)
more_results = count > CHUNK_SIZE
offset += CHUNK_SIZE
if more_results:
data = data[:-1]
for row in data:
aggregated_projects[row["org_id"]].append(row["project_id"])
if not more_results:
break
else:
logger.info(
"monitor_release_adoption.loop_timeout",
sample_rate=1.0,
extra={"offset": offset},
)
with metrics.timer(
"sentry.tasks.monitor_release_adoption.process_projects_with_sessions", sample_rate=1.0
):
for org_id in aggregated_projects:
process_projects_with_sessions.delay(org_id, aggregated_projects[org_id])
def test_build_snuba_query_orderby(mock_now, mock_now2, mock_indexer):
mock_indexer.resolve = MockIndexer().resolve
query_params = MultiValueDict({
"query":
["release:staging"
], # weird release but we need a string exising in mock indexer
"groupBy": ["session.status", "environment"],
"field": [
"sum(sentry.sessions.session)",
],
"orderBy": ["-sum(sentry.sessions.session)"],
"limit": [3],
})
query_definition = QueryDefinition(query_params)
snuba_queries = SnubaQueryBuilder([PseudoProject(1, 1)],
query_definition).get_snuba_queries()
counter_queries = snuba_queries.pop("metrics_counters")
assert not snuba_queries
assert counter_queries["series"] is None # No series because of orderBy
assert counter_queries["totals"] == Query(
dataset="metrics",
match=Entity("metrics_counters"),
select=[Function("sum", [Column("value")], "value")],
groupby=[
Column("metric_id"),
Column("tags[8]"),
Column("tags[2]"),
],
where=[
Condition(Column("org_id"), Op.EQ, 1),
Condition(Column("project_id"), Op.IN, [1]),
Condition(Column("metric_id"), Op.IN, [9]),
Condition(Column("timestamp"), Op.GTE,
datetime(2021, 5, 28, 0, tzinfo=pytz.utc)),
Condition(Column("timestamp"), Op.LT,
datetime(2021, 8, 26, 0, tzinfo=pytz.utc)),
Condition(Column("tags[6]", entity=None), Op.IN, [10]),
],
orderby=[OrderBy(Column("value"), Direction.DESC)],
limit=Limit(3),
offset=Offset(0),
granularity=Granularity(query_definition.rollup),
)
def _build_queries_for_entity(self, query_definition, entity, fields,
where, groupby):
totals_query = Query(
dataset="metrics",
match=Entity(entity),
groupby=groupby,
select=list(
map(
Column,
{_OP_TO_FIELD[entity][op]
for op, _ in fields},
)),
where=where,
limit=Limit(MAX_POINTS),
offset=Offset(0),
granularity=Granularity(query_definition.rollup),
)
series_query = totals_query.set_groupby((totals_query.groupby or []) +
[Column(TS_COL_GROUP)])
return {
"totals": totals_query,
"series": series_query,
}
def _build_queries_for_entity(self, query_definition, entity, fields, where, groupby):
totals_query = Query(
dataset=Dataset.Metrics.value,
match=Entity(entity),
groupby=groupby,
select=list(self._build_select(entity, fields)),
where=where,
limit=Limit(query_definition.limit or MAX_POINTS),
offset=Offset(0),
granularity=Granularity(query_definition.rollup),
orderby=self._build_orderby(query_definition, entity),
)
if totals_query.orderby is None:
series_query = totals_query.set_groupby(
(totals_query.groupby or []) + [Column(TS_COL_GROUP)]
)
else:
series_query = None
return {
"totals": totals_query,
"series": series_query,
}
def _get_snuba_query(
org_id: int,
query: QueryDefinition,
entity_key: EntityKey,
metric_id: int,
columns: List[SelectableExpression],
series: bool,
limit_state: _LimitState,
extra_conditions: List[Condition],
) -> Optional[Query]:
"""Build the snuba query
Return None if the results from the initial totals query was empty.
"""
conditions = [
Condition(Column("org_id"), Op.EQ, org_id),
Condition(Column("project_id"), Op.IN,
query.filter_keys["project_id"]),
Condition(Column("metric_id"), Op.EQ, metric_id),
Condition(Column(TS_COL_QUERY), Op.GTE, query.start),
Condition(Column(TS_COL_QUERY), Op.LT, query.end),
]
conditions += _get_filter_conditions(org_id, query.conditions)
conditions += extra_conditions
groupby = {}
for field in query.raw_groupby:
if field == "session.status":
# This will be handled by conditional aggregates
continue
if field == "project":
groupby["project"] = Column("project_id")
continue
try:
groupby[field] = Column(resolve_tag_key(field))
except MetricIndexNotFound:
# exclude unresolved keys from groupby
pass
full_groupby = list(set(groupby.values()))
if series:
full_groupby.append(Column(TS_COL_GROUP))
query_args = dict(
dataset=Dataset.Metrics.value,
match=Entity(entity_key.value),
select=columns,
groupby=full_groupby,
where=conditions,
granularity=Granularity(query.rollup),
)
# In case of group by, either set a limit or use the groups from the
# first query to limit the results:
if query.raw_groupby:
if not limit_state.initialized:
# Set limit and order by to be consistent with sessions_v2
max_groups = SNUBA_LIMIT // len(get_timestamps(query))
query_args["limit"] = Limit(max_groups)
query_args["orderby"] = [OrderBy(columns[0], Direction.DESC)]
else:
if limit_state.limiting_conditions is None:
# Initial query returned no results, no need to run any more queries
return None
query_args["where"] += limit_state.limiting_conditions
query_args["limit"] = Limit(SNUBA_LIMIT)
return Query(**query_args)
def sum_sessions_and_releases(org_id, project_ids):
# Takes a single org id and a list of project ids
# returns counts of releases and sessions across all environments and passed project_ids for the last 6 hours
start_time = time.time()
offset = 0
totals = defaultdict(dict)
with metrics.timer(
"sentry.tasks.monitor_release_adoption.process_projects_with_sessions.loop"
):
while (time.time() - start_time) < MAX_SECONDS:
with metrics.timer(
"sentry.tasks.monitor_release_adoption.process_projects_with_sessions.query"
):
query = (Query(
dataset="sessions",
match=Entity("sessions"),
select=[
Column("sessions"),
],
groupby=[
Column("org_id"),
Column("project_id"),
Column("release"),
Column("environment"),
],
where=[
Condition(Column("started"), Op.GTE,
datetime.utcnow() - timedelta(hours=6)),
Condition(Column("started"), Op.LT, datetime.utcnow()),
Condition(Column("org_id"), Op.EQ, org_id),
Condition(Column("project_id"), Op.IN, project_ids),
],
granularity=Granularity(21600),
orderby=[
OrderBy(Column("org_id"), Direction.ASC),
OrderBy(Column("project_id"), Direction.ASC),
],
).set_limit(CHUNK_SIZE + 1).set_offset(offset))
data = snuba.raw_snql_query(
query,
referrer="tasks.process_projects_with_sessions.session_count"
)["data"]
count = len(data)
more_results = count > CHUNK_SIZE
offset += CHUNK_SIZE
if more_results:
data = data[:-1]
for row in data:
row_totals = totals[row["project_id"]].setdefault(
row["environment"], {
"total_sessions": 0,
"releases": defaultdict(int)
})
row_totals["total_sessions"] += row["sessions"]
row_totals["releases"][row["release"]] += row["sessions"]
if not more_results:
break
else:
logger.info(
"process_projects_with_sessions.loop_timeout",
extra={
"org_id": org_id,
"project_ids": project_ids
},
)
return totals
)
from snuba_sdk.query_validation import InvalidMatchError
from snuba_sdk.query_visitors import InvalidQueryError
NOW = datetime(2021, 1, 2, 3, 4, 5, 6, timezone.utc)
tests = [
pytest.param(
Query(
dataset="discover",
match=Entity("events"),
select=[Column("event_id")],
groupby=None,
where=[Condition(Column("timestamp"), Op.GT, NOW)],
limit=Limit(10),
offset=Offset(1),
granularity=Granularity(3600),
),
id="basic query",
),
pytest.param(
Query(
dataset="discover",
match=Entity("events", "ev", 0.2),
select=[
Column("title"),
Column("tags[release:1]"),
Function("uniq", [Column("event_id")], "uniq_events"),
],
groupby=[Column("title"), Column("tags[release:1]")],
where=[
Condition(Column("timestamp"), Op.GT, NOW),
def test_build_snuba_query_orderby(mock_now, mock_now2, monkeypatch):
monkeypatch.setattr("sentry.sentry_metrics.indexer.resolve",
MockIndexer().resolve)
query_params = MultiValueDict({
"query":
["release:staging"
], # weird release but we need a string exising in mock indexer
"groupBy": ["session.status", "environment"],
"field": [
"sum(sentry.sessions.session)",
],
"orderBy": ["-sum(sentry.sessions.session)"],
})
query_definition = QueryDefinition(query_params,
paginator_kwargs={"limit": 3})
snuba_queries, _ = SnubaQueryBuilder([PseudoProject(1, 1)],
query_definition).get_snuba_queries()
counter_queries = snuba_queries.pop("metrics_counters")
assert not snuba_queries
op = "sum"
metric_name = "sentry.sessions.session"
select = Function(
OP_TO_SNUBA_FUNCTION["metrics_counters"]["sum"],
[
Column("value"),
Function("equals",
[Column("metric_id"),
resolve_weak(metric_name)])
],
alias=f"{op}({metric_name})",
)
assert counter_queries["totals"] == Query(
dataset="metrics",
match=Entity("metrics_counters"),
select=[select],
groupby=[
Column("tags[8]"),
Column("tags[2]"),
],
where=[
Condition(Column("org_id"), Op.EQ, 1),
Condition(Column("project_id"), Op.IN, [1]),
Condition(Column("timestamp"), Op.GTE,
datetime(2021, 5, 28, 0, tzinfo=pytz.utc)),
Condition(Column("timestamp"), Op.LT,
datetime(2021, 8, 26, 0, tzinfo=pytz.utc)),
Condition(Column("tags[6]", entity=None), Op.IN, [10]),
Condition(Column("metric_id"), Op.IN, [9]),
],
orderby=[OrderBy(select, Direction.DESC)],
limit=Limit(3),
offset=Offset(0),
granularity=Granularity(query_definition.rollup),
)
assert counter_queries["series"] == Query(
dataset="metrics",
match=Entity("metrics_counters"),
select=[select],
groupby=[
Column("tags[8]"),
Column("tags[2]"),
Column("bucketed_time"),
],
where=[
Condition(Column("org_id"), Op.EQ, 1),
Condition(Column("project_id"), Op.IN, [1]),
Condition(Column("timestamp"), Op.GTE,
datetime(2021, 5, 28, 0, tzinfo=pytz.utc)),
Condition(Column("timestamp"), Op.LT,
datetime(2021, 8, 26, 0, tzinfo=pytz.utc)),
Condition(Column("tags[6]", entity=None), Op.IN, [10]),
Condition(Column("metric_id"), Op.IN, [9]),
],
orderby=[OrderBy(select, Direction.DESC)],
limit=Limit(6480),
offset=Offset(0),
granularity=Granularity(query_definition.rollup),
)
def test_build_snuba_query_derived_metrics(mock_now, mock_now2, monkeypatch):
monkeypatch.setattr("sentry.sentry_metrics.indexer.resolve",
MockIndexer().resolve)
# Your typical release health query querying everything
query_params = MultiValueDict({
"groupBy": [],
"field": [
"session.errored",
"session.crash_free_rate",
"session.all",
],
"interval": ["1d"],
"statsPeriod": ["2d"],
})
query_definition = QueryDefinition(query_params)
query_builder = SnubaQueryBuilder([PseudoProject(1, 1)], query_definition)
snuba_queries, fields_in_entities = query_builder.get_snuba_queries()
assert fields_in_entities == {
"metrics_counters": [
(None, "session.errored_preaggregated"),
(None, "session.crash_free_rate"),
(None, "session.all"),
],
"metrics_sets": [
(None, "session.errored_set"),
],
}
for key in ("totals", "series"):
groupby = [] if key == "totals" else [Column("bucketed_time")]
assert snuba_queries["metrics_counters"][key] == (Query(
dataset="metrics",
match=Entity("metrics_counters"),
select=[
errored_preaggr_sessions(
metric_ids=[resolve_weak("sentry.sessions.session")],
alias="session.errored_preaggregated",
),
percentage(
crashed_sessions(
metric_ids=[resolve_weak("sentry.sessions.session")],
alias="session.crashed",
),
all_sessions(
metric_ids=[resolve_weak("sentry.sessions.session")],
alias="session.all",
),
alias="session.crash_free_rate",
),
all_sessions(
metric_ids=[resolve_weak("sentry.sessions.session")],
alias="session.all"),
],
groupby=groupby,
where=[
Condition(Column("org_id"), Op.EQ, 1),
Condition(Column("project_id"), Op.IN, [1]),
Condition(Column("timestamp"), Op.GTE,
datetime(2021, 8, 24, 0, tzinfo=pytz.utc)),
Condition(Column("timestamp"), Op.LT,
datetime(2021, 8, 26, 0, tzinfo=pytz.utc)),
Condition(Column("metric_id"), Op.IN,
[resolve_weak("sentry.sessions.session")]),
],
limit=Limit(MAX_POINTS),
offset=Offset(0),
granularity=Granularity(query_definition.rollup),
))
assert snuba_queries["metrics_sets"][key] == (Query(
dataset="metrics",
match=Entity("metrics_sets"),
select=[
sessions_errored_set(
metric_ids=[resolve_weak("sentry.sessions.session.error")],
alias="session.errored_set",
),
],
groupby=groupby,
where=[
Condition(Column("org_id"), Op.EQ, 1),
Condition(Column("project_id"), Op.IN, [1]),
Condition(Column("timestamp"), Op.GTE,
datetime(2021, 8, 24, 0, tzinfo=pytz.utc)),
Condition(Column("timestamp"), Op.LT,
datetime(2021, 8, 26, 0, tzinfo=pytz.utc)),
Condition(Column("metric_id"), Op.IN,
[resolve_weak("sentry.sessions.session.error")]),
],
limit=Limit(MAX_POINTS),
offset=Offset(0),
granularity=Granularity(query_definition.rollup),
))
