def test_field_aliasing_in_aggregate_functions_and_groupby(self):
result = discover.query(
selected_columns=["project.id", "count_unique(user.email)"],
query="",
params={"project_id": [self.project.id]},
auto_fields=True,
)
data = result["data"]
assert len(data) == 1
assert data[0]["project.id"] == self.project.id
assert data[0]["count_unique_user_email"] == 1
def data_fn(offset, limit):
return discover.query(
selected_columns=fields,
query=query,
params=params,
offset=offset,
limit=limit,
referrer="data_export.tasks.discover",
auto_fields=True,
use_aggregate_conditions=True,
)
def test_field_aliasing_in_conditions(self):
result = discover.query(
selected_columns=["project.id", "user.email"],
query="user.email:[email protected]",
params={"project_id": [self.project.id]},
auto_fields=True,
)
data = result["data"]
assert len(data) == 1
assert data[0]["project.id"] == self.project.id
assert data[0]["user.email"] == "*****@*****.**"
def test_query_no_fields(self, mock_query):
mock_query.return_value = {
"meta": [{
"name": "transaction"
}, {
"name": "duration"
}],
"data": [{
"transaction": "api.do_things",
"duration": 200
}],
}
with pytest.raises(InvalidSearchQuery) as err:
discover.query(
selected_columns=[],
query="event.type:transaction",
params={"project_id": [self.project.id]},
)
assert "No fields" in six.text_type(err)
assert mock_query.call_count == 0
def data_fn(offset, limit):
return discover.query(
selected_columns=["geo.country_code", maybe_aggregate],
query=request.GET.get("query"),
params=params,
offset=offset,
limit=limit,
referrer=request.GET.get("referrer",
"api.organization-events-geo"),
use_aggregate_conditions=True,
)
def query_tag_data(
params: Mapping[str, str],
referrer: str,
use_snql: bool,
filter_query: Optional[str] = None,
aggregate_column: Optional[str] = None,
) -> Optional[Dict]:
"""
Fetch general data about all the transactions with this transaction name to feed into the facet query
:return: Returns the row with aggregate and count if the query was successful
Returns None if query was not successful which causes the endpoint to return early
"""
with sentry_sdk.start_span(
op="discover.discover", description="facets.filter_transform"
) as span:
span.set_data("query", filter_query)
snuba_filter = get_filter(filter_query, params)
# Resolve the public aliases into the discover dataset names.
snuba_filter, translated_columns = discover.resolve_discover_aliases(snuba_filter)
translated_aggregate_column = discover.resolve_discover_column(aggregate_column)
with sentry_sdk.start_span(op="discover.discover", description="facets.frequent_tags"):
# Get the average and count to use to filter the next request to facets
tag_data = discover.query(
selected_columns=[
"count()",
f"avg({aggregate_column}) as aggregate",
f"max({aggregate_column}) as max",
f"min({aggregate_column}) as min",
],
conditions=[
[translated_aggregate_column, "IS NOT NULL", None],
],
query=filter_query,
params=params,
referrer=f"{referrer}.all_transactions",
use_snql=use_snql,
limit=1,
)
if len(tag_data["data"]) != 1:
return None
counts = [r["count"] for r in tag_data["data"]]
aggregates = [r["aggregate"] for r in tag_data["data"]]
# Return early to avoid doing more queries with 0 count transactions or aggregates for columns that don't exist
if counts[0] == 0 or aggregates[0] is None:
return None
if not tag_data["data"][0]:
return None
return tag_data["data"][0]
def get(self, request, organization):
""" Get the Key Transactions for a user """
if not self.has_feature(request, organization):
return self.response(status=404)
params = self.get_filter_params(request, organization)
fields = request.GET.getlist("field")[:]
orderby = self.get_orderby(request)
queryset = KeyTransaction.objects.filter(organization=organization,
owner=request.user)
results = query(
fields,
None,
params,
orderby=orderby,
referrer="discover.key_transactions",
# The snuba query for transactions is of the form
# (transaction="1" AND project=1) OR (transaction="2" and project=2) ...
# which the schema intentionally doesn't support so we cannot do an AND in OR
# so here the "and" operator is being instead to do an AND in OR query
conditions=[[
# First layer is Ands
[
# Second layer is Ors
[
"and",
[
[
"equals",
# Without the outer ' here, the transaction will be treated as another column
# instead of a string. This isn't an injection risk since snuba is smart enough to
# handle escaping for us.
[
"transaction",
u"'{}'".format(transaction.transaction)
],
],
["equals", ["project_id", transaction.project.id]],
],
],
"=",
1,
] for transaction in queryset
]],
)
return Response(
self.handle_results_with_meta(request, organization,
params["project_id"], results),
status=200,
)
def _get_one_transaction_name(project: Project):
result = discover.query(
query="event.type:transaction",
selected_columns=["transaction"],
limit=1,
params={
"organization_id": project.organization_id,
"project_id": [project.id],
},
referrer="sandbox.demo_start._get_one_transaction_name",
)
return result["data"][0]["transaction"]
def data_fn(offset, limit):
return discover.query(
selected_columns=request.GET.getlist("field")[:],
query=request.GET.get("query"),
params=params,
orderby=self.get_orderby(request),
offset=offset,
limit=limit,
referrer=request.GET.get("referrer", "api.organization-events-v2"),
auto_fields=True,
use_aggregate_conditions=True,
)
def test_auto_fields_aggregates(self):
result = discover.query(
selected_columns=["count_unique(user.email)"],
query="",
params={"project_id": [self.project.id]},
auto_fields=True,
)
data = result["data"]
assert len(data) == 1
assert data[0]["projectid"] == self.project.id
assert data[0]["latest_event"] == self.event.event_id
assert data[0]["count_unique_user_email"] == 1
def data_fn(offset, limit):
return discover.query(
selected_columns=request.GET.getlist("field")[:],
query=request.GET.get("query"),
params=params,
reference_event=self.reference_event(request, organization),
orderby=self.get_orderby(request),
offset=offset,
limit=limit,
referrer="api.organization-events-v2",
auto_fields=True,
)
def test_selected_columns_aliasing_in_function(self, mock_query):
mock_query.return_value = {
"meta": [{
"name": "transaction"
}, {
"name": "duration"
}],
"data": [{
"transaction": "api.do_things",
"duration": 200
}],
}
discover.query(
selected_columns=[
"transaction",
"transaction.duration",
"count_unique(transaction.duration)",
],
query="",
params={"project_id": [self.project.id]},
auto_fields=True,
)
mock_query.assert_called_with(
selected_columns=["transaction", "duration"],
aggregations=[
["uniq", "duration", "count_unique_transaction_duration"],
["argMax", ["event_id", "timestamp"], "latest_event"],
["argMax", ["project_id", "timestamp"], "projectid"],
],
filter_keys={"project_id": [self.project.id]},
dataset=Dataset.Discover,
end=None,
start=None,
conditions=[],
groupby=["transaction", "duration"],
orderby=None,
limit=50,
offset=None,
referrer=None,
)
def test_conditions_order_and_groupby_aliasing(self, mock_query):
mock_query.return_value = {
"meta": [{
"name": "transaction"
}, {
"name": "duration"
}],
"data": [{
"transaction": "api.do_things",
"duration": 200
}],
}
discover.query(
selected_columns=[
"timestamp", "transaction", "transaction.duration", "count()"
],
query=
"transaction.duration:200 sdk.name:python tags[projectid]:123",
params={"project_id": [self.project.id]},
orderby=["-timestamp", "-count"],
)
mock_query.assert_called_with(
selected_columns=["timestamp", "transaction", "duration"],
aggregations=[["count", None, "count"]],
conditions=[
["duration", "=", 200],
["sdk_name", "=", "python"],
[["ifNull", ["tags[projectid]", "''"]], "=", "123"],
],
filter_keys={"project_id": [self.project.id]},
groupby=["timestamp", "transaction", "duration"],
orderby=["-timestamp", "-count"],
dataset=Dataset.Discover,
end=None,
start=None,
limit=50,
offset=None,
referrer=None,
)
def test_condition_transform(self, mock_query):
mock_query.return_value = {
"meta": [{"name": "transaction"}, {"name": "duration"}],
"data": [{"transaction": "api.do_things", "duration": 200}],
}
discover.query(
selected_columns=["transaction", "transaction.duration"],
query="http.method:GET",
params={"project_id": [self.project.id]},
)
mock_query.assert_called_with(
selected_columns=["transaction", "duration", "event_id", "project_id"],
conditions=[["http_method", "=", "GET"]],
filter_keys={"project_id": [self.project.id]},
groupby=[],
dataset=Dataset.Discover,
aggregations=[],
orderby=None,
end=None,
start=None,
referrer=None,
)
def data_fn(offset, limit):
return discover.query(
selected_columns=selected_columns + trend_columns,
query=query,
params=params,
orderby=orderby,
offset=offset,
limit=limit,
referrer="api.trends.get-percentage-change",
auto_fields=True,
auto_aggregations=True,
use_aggregate_conditions=True,
)
def data_fn(offset, limit):
return discover.query(
selected_columns=self.get_field_list(organization, request),
query=request.GET.get("query"),
params=params,
equations=self.get_equation_list(organization, request),
orderby=self.get_orderby(request),
offset=offset,
limit=limit,
referrer=referrer,
auto_fields=True,
auto_aggregations=True,
use_aggregate_conditions=True,
)
def data_fn(offset, limit):
return discover.query(
selected_columns=["geo.country_code", maybe_aggregate],
query=f"{request.GET.get('query', '')} has:geo.country_code",
params=params,
offset=offset,
limit=limit,
referrer=referrer,
use_aggregate_conditions=True,
orderby=self.get_orderby(request) or maybe_aggregate,
use_snql=features.has("organizations:discover-use-snql",
organization,
actor=request.user),
)
def test_orderby(self, mock_query):
mock_query.return_value = {
"meta": [{"name": "title"}, {"name": "project.id"}],
"data": [{"project.id": "tester", "title": "test title"}],
}
discover.query(
selected_columns=["project.id", "title"],
query="",
params={"project_id": [self.project.id]},
orderby=["project.id"],
)
mock_query.assert_called_with(
selected_columns=["project_id", "title", "event_id"],
filter_keys={"project_id": [self.project.id]},
dataset=Dataset.Discover,
orderby=["project_id"],
aggregations=[],
end=None,
start=None,
conditions=[],
groupby=[],
referrer=None,
)
def test_params_forward(self, mock_query):
mock_query.return_value = {
"meta": [{
"name": "transaction"
}, {
"name": "duration"
}],
"data": [{
"transaction": "api.do_things",
"duration": 200
}],
}
start_time = before_now(minutes=10)
end_time = before_now(seconds=1)
discover.query(
selected_columns=["transaction", "transaction.duration"],
query="http.method:GET",
params={
"project_id": [self.project.id],
"start": start_time,
"end": end_time
},
)
mock_query.assert_called_with(
selected_columns=["transaction", "duration"],
conditions=[["http_method", "=", "GET"]],
filter_keys={"project_id": [self.project.id]},
groupby=[],
dataset=Dataset.Discover,
aggregations=[],
end=end_time,
start=start_time,
orderby=None,
limit=50,
offset=None,
referrer=None,
)
def get_errors(self, organization, trace_id, params, *args):
""" Ignores current_event since we get all errors """
with sentry_sdk.start_span(op="discover", description="getting trace errors"):
with self.handle_query_errors():
# This can't be combined with the transaction query since we need dataset specific fields
error_results = discover.query(
selected_columns=ERROR_COLUMNS,
orderby=["-timestamp", "id"],
params=params,
query=f"!event.type:transaction trace:{trace_id}",
limit=MAX_TRACE_SIZE,
auto_fields=False,
referrer="api.trace-view.get-errors",
)
return error_results["data"]
def get(self, request, organization):
try:
params = self.get_snuba_params(request, organization)
except NoProjects:
return Response({"count": 0})
with self.handle_query_errors():
result = discover.query(
selected_columns=["count()"],
params=params,
query=request.query_params.get("query"),
referrer="api.organization-events-meta",
)
return Response({"count": result["data"][0]["count"]})
def get(self, request, organization):
try:
params = self.get_filter_params(request, organization)
except OrganizationEventsError as exc:
return Response({"detail": exc.message}, status=400)
except NoProjects:
return Response({"count": 0})
result = discover.query(
selected_columns=["count()"],
params=params,
query=request.query_params.get("query"),
referrer="api.organization-events-meta",
)
return Response({"count": result["data"][0]["count"]})
def get(self, request, organization):
try:
params = self.get_filter_params(request, organization)
except NoProjects:
return Response({"count": 0})
try:
result = discover.query(
selected_columns=["count()"],
params=params,
query=request.query_params.get("query"),
referrer="api.organization-events-meta",
)
except (discover.InvalidSearchQuery, snuba.QueryOutsideRetentionError) as error:
raise ParseError(detail=six.text_type(error))
return Response({"count": result["data"][0]["count"]})
def get(self, request: Request, organization) -> Response:
try:
params = self.get_snuba_params(request, organization)
except NoProjects:
return Response({"count": 0})
with self.handle_query_errors():
result = discover.query(
selected_columns=["count()"],
params=params,
query=request.query_params.get("query"),
referrer="api.organization-events-meta",
use_snql=features.has("organizations:discover-use-snql",
organization,
actor=request.user),
)
return Response({"count": result["data"][0]["count"]})
def test_field_aliasing_in_selected_columns(self):
result = discover.query(
selected_columns=["project.id", "user.email", "release"],
query="",
params={"project_id": [self.project.id]},
)
data = result["data"]
assert len(data) == 1
assert data[0]["id"] == self.event.event_id
assert data[0]["project.id"] == self.project.id
assert data[0]["user.email"] == "*****@*****.**"
assert data[0]["release"] == "first-release"
assert len(result["meta"]) == 4
assert result["meta"][0] == {"name": "project.id", "type": "UInt64"}
assert result["meta"][1] == {"name": "user.email", "type": "Nullable(String)"}
assert result["meta"][2] == {"name": "release", "type": "Nullable(String)"}
assert result["meta"][3] == {"name": "id", "type": "FixedString(32)"}
def get(self, request, organization):
with sentry_sdk.start_span(op="discover.endpoint", description="filter_params") as span:
span.set_data("organization", organization)
try:
params = self.get_filter_params(request, organization)
except NoProjects:
return Response({"count": 0})
params = self.quantize_date_params(request, params)
with self.handle_query_errors():
result = discover.query(
selected_columns=["count()"],
params=params,
query=request.query_params.get("query"),
referrer="api.organization-events-meta",
)
return Response({"count": result["data"][0]["count"]})
def get_one_transaction(org: Organization, project_slug: Optional[str]):
project = _get_project(org, project_slug)
# find the most recent transaction
result = discover.query(
query="event.type:transaction",
orderby="-timestamp",
selected_columns=["id", "timestamp"],
limit=1,
params={
"organization_id": org.id,
"project_id": [project.id],
},
referrer="sandbox.demo_start.get_one_transaction",
)
transaction_id = result["data"][0]["id"]
return f"/organizations/{org.slug}/discover/{project.slug}:{transaction_id}/"
def get(self, request, organization):
project_ids = self.get_requested_project_ids(request)
projects = self.get_projects(request, organization, project_ids)
with self.handle_query_errors():
result = discover.query(
query="has:sdk.version",
selected_columns=["project", "sdk.name", "sdk.version", "last_seen()"],
orderby="-project",
params={
"start": timezone.now() - timedelta(days=1),
"end": timezone.now(),
"organization_id": organization.id,
"project_id": [p.id for p in projects],
},
referrer="api.organization-sdk-updates",
)
return Response(serialize(result["data"], projects))
def _get(self, request: Request, organization):
projects = self.get_projects(request, organization)
if len(projects) == 0:
return None
browser_name_list = request.GET.getlist("userAgents")
query = " OR ".join(
map(lambda x: f"browser.name:{x}", browser_name_list))
with self.handle_query_errors():
# TODO: update discover query for null browser and client_os_name is android or ios
result = discover.query(
query=query,
orderby="-timestamp",
selected_columns=[
"browser.name", "client_os.name", "timestamp"
],
limit=1,
params={
"start": timezone.now() - timedelta(days=1),
"end": timezone.now(),
"organization_id": organization.id,
"project_id": [p.id for p in projects],
},
referrer="api.organization-has-mobile-app-events",
use_snql=features.has("organizations:performance-use-snql",
organization,
actor=request.user),
)
data = result["data"]
if not data:
return None
one_result = data[0]
# only send back browserName and clientOsName for now
return {
"browserName": one_result["browser.name"],
"clientOsName": one_result["client_os.name"],
}
def data_fn(offset, limit):
return discover.query(
selected_columns=selected_columns
+ [
trend_column["format"].format(*columns, start=start, end=middle, index="1"),
trend_column["format"].format(*columns, start=middle, end=end, index="2"),
percentage_column["format"].format(alias=trend_column["alias"]),
"minus({alias}2,{alias}1)".format(alias=trend_column["alias"]),
count_column["format"].format(start=start, end=middle, index="1"),
count_column["format"].format(start=middle, end=end, index="2"),
percentage_column["format"].format(alias=count_column["alias"]),
],
query=query,
params=params,
orderby=orderby,
offset=offset,
limit=limit,
referrer="api.trends.get-percentage-change",
auto_fields=True,
auto_aggregations=True,
use_aggregate_conditions=True,
)
