def test_find_projects(
query_body: MutableMapping[str, Any], expected_projects: Optional[Set[int]]
) -> None:
events = get_dataset("events")
if expected_projects is None:
with pytest.raises(ParsingException):
snql_query = json_to_snql(query_body, "events")
query, _ = parse_snql_query(str(snql_query), events)
identity_translate(query)
else:
snql_query = json_to_snql(query_body, "events")
query, _ = parse_snql_query(str(snql_query), events)
query = identity_translate(query)
project_ids_ast = get_object_ids_in_query_ast(query, "project_id")
assert project_ids_ast == expected_projects
def _get_filter_conditions(org_id: int, conditions: Sequence[Condition]) -> Any:
"""Translate given conditions to snql"""
dummy_entity = EntityKey.MetricsSets.value
filter_conditions = json_to_snql(
{"selected_columns": ["value"], "conditions": conditions}, entity=dummy_entity
).where
return _translate_conditions(org_id, filter_conditions)
def test_select_storage(query_body: MutableMapping[str, Any],
is_subscription: bool, expected_table: str) -> None:
sessions = get_dataset("sessions")
snql_query = json_to_snql(query_body, "sessions")
query, snql_anonymized = parse_snql_query(str(snql_query), sessions)
query_body = json.loads(snql_query.snuba())
subscription_settings = (SubscriptionQuerySettings
if is_subscription else HTTPQuerySettings)
request = Request(
id="a",
original_body=query_body,
query=query,
snql_anonymized=snql_anonymized,
query_settings=subscription_settings(referrer=""),
attribution_info=AttributionInfo(get_app_id("default"), "blah", None,
None, None),
)
def query_runner(query: Query, settings: QuerySettings,
reader: Reader) -> QueryResult:
assert query.get_from_clause().table_name == expected_table
return QueryResult({}, {})
sessions.get_default_entity().get_query_pipeline_builder(
).build_execution_pipeline(request, query_runner).execute()
def test_failures(
query_body: MutableMapping[str, Any],
expected_exception: Type[InvalidQueryException],
) -> None:
with pytest.raises(expected_exception):
events = get_dataset("events")
snql_query = json_to_snql(query_body, "events")
parse_snql_query(str(snql_query), events)
def _create_in_snuba(subscription):
snuba_query = subscription.snuba_query
snuba_filter = build_snuba_filter(
QueryDatasets(snuba_query.dataset),
snuba_query.query,
snuba_query.aggregate,
snuba_query.environment,
snuba_query.event_types,
)
body = {
"project_id": subscription.project_id,
"project": subscription.project_id, # for SnQL SDK
"dataset": snuba_query.dataset,
"conditions": snuba_filter.conditions,
"aggregations": snuba_filter.aggregations,
"time_window": snuba_query.time_window,
"resolution": snuba_query.resolution,
}
if Dataset(snuba_query.dataset) == Dataset.Sessions:
body.update({
"organization": subscription.project.organization_id,
})
try:
metrics.incr("snuba.snql.subscription.create",
tags={"dataset": snuba_query.dataset})
snql_query = json_to_snql(body, snuba_query.dataset)
snql_query.validate()
body["query"] = str(snql_query)
body["type"] = "delegate" # mark this as a combined subscription
except Exception as e:
logger.warning(
"snuba.snql.subscription.parsing.error",
extra={
"error": str(e),
"params": json.dumps(body),
"dataset": snuba_query.dataset
},
)
metrics.incr("snuba.snql.subscription.parsing.error",
tags={"dataset": snuba_query.dataset})
response = _snuba_pool.urlopen(
"POST",
f"/{snuba_query.dataset}/subscriptions",
body=json.dumps(body),
)
if response.status != 202:
metrics.incr("snuba.snql.subscription.http.error",
tags={"dataset": snuba_query.dataset})
raise SnubaError("HTTP %s response from Snuba!" % response.status)
return json.loads(response.data)["subscription_id"]
def test_alias_validation(query_body: MutableMapping[str, Any],
expected_result: bool) -> None:
events = get_dataset("events")
snql_query = json_to_snql(query_body, "events")
query, _ = parse_snql_query(str(snql_query), events)
settings = HTTPQuerySettings()
query_plan = (
events.get_default_entity().get_query_pipeline_builder().build_planner(
query, settings)).build_best_plan()
execute_all_clickhouse_processors(query_plan, settings)
assert query_plan.query.validate_aliases() == expected_result
def _create_in_snuba(subscription: QuerySubscription) -> str:
snuba_query = subscription.snuba_query
entity_subscription = get_entity_subscription_for_dataset(
dataset=QueryDatasets(snuba_query.dataset),
aggregate=snuba_query.aggregate,
time_window=snuba_query.time_window,
extra_fields={
"org_id": subscription.project.organization_id,
"event_types": snuba_query.event_types,
},
)
snuba_filter = build_snuba_filter(
entity_subscription,
snuba_query.query,
snuba_query.environment,
)
body = {
"project_id": subscription.project_id,
"project": subscription.project_id, # for SnQL SDK
"dataset": snuba_query.dataset,
"conditions": snuba_filter.conditions,
"aggregations": snuba_filter.aggregations,
"time_window": snuba_query.time_window,
"resolution": snuba_query.resolution,
**entity_subscription.get_entity_extra_params(),
}
try:
metrics.incr("snuba.snql.subscription.create", tags={"dataset": snuba_query.dataset})
snql_query = json_to_snql(body, entity_subscription.entity_key.value)
snql_query.validate()
body["query"] = str(snql_query)
body["type"] = "delegate" # mark this as a combined subscription
except Exception as e:
logger.warning(
"snuba.snql.subscription.parsing.error",
extra={"error": str(e), "params": json.dumps(body), "dataset": snuba_query.dataset},
)
metrics.incr("snuba.snql.subscription.parsing.error", tags={"dataset": snuba_query.dataset})
response = _snuba_pool.urlopen(
"POST",
f"/{snuba_query.dataset}/{entity_subscription.entity_key.value}/subscriptions",
body=json.dumps(body),
)
if response.status != 202:
metrics.incr("snuba.snql.subscription.http.error", tags={"dataset": snuba_query.dataset})
raise SnubaError("HTTP %s response from Snuba!" % response.status)
return json.loads(response.data)["subscription_id"]
def test_prewhere(
query_body: MutableMapping[str, Any],
keys: Sequence[str],
omit_if_final_keys: Sequence[str],
new_ast_condition: Optional[Expression],
new_prewhere_ast_condition: Optional[Expression],
final: bool,
) -> None:
settings.MAX_PREWHERE_CONDITIONS = 2
events = get_dataset("events")
# HACK until we migrate these tests to SnQL
query_body["selected_columns"] = ["project_id"]
query_body["conditions"] += [
["timestamp", ">=", "2021-01-01T00:00:00"],
["timestamp", "<", "2021-01-02T00:00:00"],
["project_id", "=", 1],
]
snql_query = json_to_snql(query_body, "events")
query, _ = parse_snql_query(str(snql_query), events)
query = identity_translate(query)
query.set_from_clause(Table("my_table", all_columns, final=final))
query_settings = HTTPQuerySettings()
processor = PrewhereProcessor(keys, omit_if_final=omit_if_final_keys)
processor.process_query(query, query_settings)
# HACK until we migrate these tests to SnQL
def verify_expressions(top_level: Expression,
expected: Expression) -> bool:
actual_conds = get_first_level_and_conditions(top_level)
expected_conds = get_first_level_and_conditions(expected)
for cond in expected_conds:
if cond not in actual_conds:
return False
return True
if new_ast_condition:
condition = query.get_condition()
assert condition is not None
assert verify_expressions(condition, new_ast_condition)
if new_prewhere_ast_condition:
prewhere = query.get_prewhere_ast()
assert prewhere is not None
assert verify_expressions(prewhere, new_prewhere_ast_condition)
def test_get_all_columns_legacy() -> None:
query_body = {
"selected_columns": [
["f1", ["column1", "column2"], "f1_alias"],
["f2", [], "f2_alias"],
["formatDateTime", ["timestamp", "'%Y-%m-%d'"], "formatted_time"],
],
"aggregations": [
["count", "platform", "platforms"],
["uniq", "platform", "uniq_platforms"],
["testF", ["platform", ["anotherF", ["field2"]]], "top_platforms"],
],
"conditions": [
["tags[sentry:dist]", "IN", ["dist1", "dist2"]],
["timestamp", ">=", "2020-01-01T12:00:00"],
["timestamp", "<", "2020-01-02T12:00:00"],
["project_id", "=", 1],
],
"having": [["times_seen", ">", 1]],
"groupby": [["format_eventid", ["event_id"]]],
}
events = get_dataset("events")
snql_query = json_to_snql(query_body, "events")
query, _ = parse_snql_query(str(snql_query), events)
assert query.get_all_ast_referenced_columns() == {
Column("_snuba_column1", None, "column1"),
Column("_snuba_column2", None, "column2"),
Column("_snuba_platform", None, "platform"),
Column("_snuba_field2", None, "field2"),
Column("_snuba_tags", None, "tags"),
Column("_snuba_times_seen", None, "times_seen"),
Column("_snuba_event_id", None, "event_id"),
Column("_snuba_timestamp", None, "timestamp"),
Column("_snuba_project_id", None, "project_id"),
}
assert query.get_all_ast_referenced_subscripts() == {
SubscriptableReference(
"_snuba_tags[sentry:dist]",
Column("_snuba_tags", None, "tags"),
Literal(None, "sentry:dist"),
)
}
def test_data_source(
query_body: MutableMapping[str, Any],
expected_entity: EntityKey,
) -> None:
dataset = get_dataset("discover")
# HACK until these are converted to proper SnQL queries
if not query_body.get("conditions"):
query_body["conditions"] = []
query_body["conditions"] += [
["timestamp", ">=", "2020-01-01T12:00:00"],
["timestamp", "<", "2020-01-02T12:00:00"],
["project_id", "=", 1],
]
if not query_body.get("selected_columns"):
query_body["selected_columns"] = ["project_id"]
snql_query = json_to_snql(query_body, "discover")
query, _ = parse_snql_query(str(snql_query), dataset)
assert query.get_from_clause().key == expected_entity
def test_col_split_conditions(id_column: str, project_column: str,
timestamp_column: str, query,
expected_result) -> None:
dataset = get_dataset("events")
snql_query = json_to_snql(query, "events")
query, _ = parse_snql_query(str(snql_query), dataset)
splitter = ColumnSplitQueryStrategy(id_column, project_column,
timestamp_column)
def do_query(query: ClickhouseQuery,
query_settings: QuerySettings = None) -> QueryResult:
return QueryResult(
{
"data": [{
id_column: "asd123",
project_column: 123,
timestamp_column: "2019-10-01 22:33:42",
}]
},
{},
)
assert (splitter.execute(query, HTTPQuerySettings(), do_query)
is not None) == expected_result
def convert(data: str, entity: str) -> str:
legacy = json.loads(data)
sdk_output = json_to_snql(legacy, entity)
return sdk_output.snuba()
def test_json_to_snuba_for_sessions(json_body: Mapping[str, Any],
clauses: Sequence[str]) -> None:
expected = "\n".join(clauses)
query = json_to_snql(json_body, "sessions")
assert query.print() == expected
def validate(self, data):
"""
Performs validation on an alert rule's data.
This includes ensuring there is either 1 or 2 triggers, which each have
actions, and have proper thresholds set. The critical trigger should
both alert and resolve 'after' the warning trigger (whether that means
> or < the value depends on threshold type).
"""
data.setdefault("dataset", QueryDatasets.EVENTS)
project_id = data.get("projects")
if not project_id:
# We just need a valid project id from the org so that we can verify
# the query. We don't use the returned data anywhere, so it doesn't
# matter which.
project_id = list(
self.context["organization"].project_set.all()[:1])
try:
entity_subscription = get_entity_subscription_for_dataset(
dataset=QueryDatasets(data["dataset"]),
aggregate=data["aggregate"],
time_window=int(
timedelta(minutes=data["time_window"]).total_seconds()),
extra_fields={
"org_id": project_id[0].organization_id,
"event_types": data.get("event_types"),
},
)
except UnsupportedQuerySubscription as e:
raise serializers.ValidationError(f"{e}")
try:
snuba_filter = build_snuba_filter(
entity_subscription,
data["query"],
data.get("environment"),
params={
"project_id": [p.id for p in project_id],
"start": timezone.now() - timedelta(minutes=10),
"end": timezone.now(),
},
)
if any(cond[0] == "project_id"
for cond in snuba_filter.conditions):
raise serializers.ValidationError(
{"query": "Project is an invalid search term"})
except (InvalidSearchQuery, ValueError) as e:
raise serializers.ValidationError(f"Invalid Query or Metric: {e}")
else:
if not snuba_filter.aggregations:
raise serializers.ValidationError(
"Invalid Metric: Please pass a valid function for aggregation"
)
dataset = Dataset(data["dataset"].value)
self._validate_time_window(dataset, data.get("time_window"))
conditions = copy(snuba_filter.conditions)
time_col = entity_subscription.time_col
conditions += [
[time_col, ">=", snuba_filter.start],
[time_col, "<", snuba_filter.end],
]
body = {
"project": project_id[0].id,
"project_id": project_id[0].id,
"aggregations": snuba_filter.aggregations,
"conditions": conditions,
"filter_keys": snuba_filter.filter_keys,
"having": snuba_filter.having,
"dataset": dataset.value,
"limit": 1,
**entity_subscription.get_entity_extra_params(),
}
try:
snql_query = json_to_snql(body,
entity_subscription.entity_key.value)
snql_query.validate()
except Exception as e:
raise serializers.ValidationError(str(e),
params={
"params":
json.dumps(body),
"dataset":
data["dataset"].value
})
try:
raw_snql_query(snql_query,
referrer="alertruleserializer.test_query")
except Exception:
logger.exception(
"Error while validating snuba alert rule query")
raise serializers.ValidationError(
"Invalid Query or Metric: An error occurred while attempting "
"to run the query")
triggers = data.get("triggers", [])
if not triggers:
raise serializers.ValidationError(
"Must include at least one trigger")
if len(triggers) > 2:
raise serializers.ValidationError(
"Must send 1 or 2 triggers - A critical trigger, and an optional warning trigger"
)
event_types = data.get("event_types")
valid_event_types = dataset_valid_event_types.get(
data["dataset"], set())
if event_types and set(event_types) - valid_event_types:
raise serializers.ValidationError(
"Invalid event types for this dataset. Valid event types are %s"
% sorted(et.name.lower() for et in valid_event_types))
for i, (trigger, expected_label) in enumerate(
zip(triggers,
(CRITICAL_TRIGGER_LABEL, WARNING_TRIGGER_LABEL))):
if trigger.get("label", None) != expected_label:
raise serializers.ValidationError(
f'Trigger {i + 1} must be labeled "{expected_label}"')
threshold_type = data["threshold_type"]
self._translate_thresholds(threshold_type,
data.get("comparison_delta"), triggers,
data)
critical = triggers[0]
self._validate_trigger_thresholds(threshold_type, critical,
data.get("resolve_threshold"))
if len(triggers) == 2:
warning = triggers[1]
self._validate_trigger_thresholds(threshold_type, warning,
data.get("resolve_threshold"))
self._validate_critical_warning_triggers(threshold_type, critical,
warning)
return data
