def validate(self, value, dataset: Dataset, referrer: str) -> Request:
try:
value = validate_jsonschema(value, self.__composite_schema)
except jsonschema.ValidationError as error:
raise JsonSchemaValidationException(str(error)) from error
query_body = {
key: value.pop(key)
for key in self.__query_schema["properties"].keys() if key in value
}
settings = {
key: value.pop(key)
for key in self.__settings_schema["properties"].keys()
if key in value
}
extensions = {}
for extension_name, extension_schema in self.__extension_schemas.items(
):
extensions[extension_name] = {
key: value.pop(key)
for key in extension_schema["properties"].keys()
if key in value
}
query = parse_query(query_body, dataset)
request_id = uuid.uuid4().hex
return Request(request_id, query, self.__setting_class(**settings),
extensions, referrer)
def validate(self, value: MutableMapping[str, Any]) -> RequestParts:
try:
value = validate_jsonschema(value, self.__composite_schema)
except jsonschema.ValidationError as error:
raise JsonSchemaValidationException(str(error)) from error
query_body = {
key: value.get(key)
for key in self.__query_schema["properties"].keys()
if key in value
}
query_settings = {
key: value.get(key)
for key in self.__query_settings_schema["properties"].keys()
if key in value
}
attribution_info = {
key: value.get(key)
for key in self.__attribution_info_schema["properties"].keys()
if key in value
}
return RequestParts(
query=query_body,
query_settings=query_settings,
attribution_info=attribution_info,
)
def test_query_extension_processing(
raw_data: dict,
expected_conditions: Sequence[Condition],
expected_granularity: int,
):
state.set_config('max_days', 1)
extension = TimeSeriesExtension(
default_granularity=60,
default_window=datetime.timedelta(days=5),
timestamp_column='timestamp',
)
valid_data = validate_jsonschema(raw_data, extension.get_schema())
query = Query(
{"conditions": []},
TableSource("my_table", ColumnSet([])),
)
request_settings = RequestSettings(turbo=False,
consistent=False,
debug=False)
extension.get_processor().process_query(query, valid_data,
request_settings)
assert query.get_conditions() == expected_conditions
assert query.get_granularity() == expected_granularity
def validate(self, value: MutableMapping[str, Any]) -> RequestParts:
try:
value = validate_jsonschema(value, self.__composite_schema)
except jsonschema.ValidationError as error:
raise JsonSchemaValidationException(str(error)) from error
query_body = {
key: value.pop(key)
for key in self.__query_schema["properties"].keys() if key in value
}
settings = {
key: value.pop(key)
for key in self.__settings_schema["properties"].keys()
if key in value
}
extensions = {}
for extension_name, extension_schema in self.__extension_schemas.items(
):
extensions[extension_name] = {
key: value.pop(key)
for key in extension_schema["properties"].keys()
if key in value
}
return RequestParts(query=query_body,
settings=settings,
extensions=extensions)
def validate(self, value) -> Request:
value = validate_jsonschema(value, self.__composite_schema)
query_body = {
key: value.pop(key)
for key in self.__query_schema['properties'].keys() if key in value
}
settings = {
key: value.pop(key)
for key in self.__settings_schema['properties'].keys()
if key in value
}
extensions = {}
for extension_name, extension_schema in self.__extension_schemas.items(
):
extensions[extension_name] = {
key: value.pop(key)
for key in extension_schema['properties'].keys()
if key in value
}
return Request(
Query(query_body),
RequestSettings(settings['turbo'], settings['consistent'],
settings['debug']), extensions)
def validate(self, value, dataset: Dataset, referrer: str) -> Request:
value = validate_jsonschema(value, self.__composite_schema)
query_body = {
key: value.pop(key)
for key in self.__query_schema["properties"].keys() if key in value
}
settings = {
key: value.pop(key)
for key in self.__settings_schema["properties"].keys()
if key in value
}
extensions = {}
for extension_name, extension_schema in self.__extension_schemas.items(
):
extensions[extension_name] = {
key: value.pop(key)
for key in extension_schema["properties"].keys()
if key in value
}
query = parse_query(query_body, dataset)
return Request(query, self.__setting_class(**settings), extensions,
referrer)
def test_project_extension_query_adds_rate_limits():
extension = ProjectExtension(processor=ProjectExtensionProcessor(
project_column="project_id"))
raw_data = {'project': [2, 3]}
valid_data = validate_jsonschema(raw_data, extension.get_schema())
query = Query(
{"conditions": []},
TableSource("my_table", ColumnSet([])),
)
request_settings = RequestSettings(turbo=False,
consistent=False,
debug=False)
num_rate_limits_before_processing = len(
request_settings.get_rate_limit_params())
extension.get_processor().process_query(query, valid_data,
request_settings)
rate_limits = request_settings.get_rate_limit_params()
# make sure a rate limit was added by the processing
assert len(rate_limits) == num_rate_limits_before_processing + 1
most_recent_rate_limit = rate_limits[-1]
assert most_recent_rate_limit.bucket == '2'
assert most_recent_rate_limit.per_second_limit == 1000
assert most_recent_rate_limit.concurrent_limit == 1000
def setup_method(self, test_method):
super().setup_method(test_method)
raw_data = {"project": 2}
self.extension = ProjectExtension(
processor=ProjectWithGroupsProcessor(project_column="project_id")
)
self.valid_data = validate_jsonschema(raw_data, self.extension.get_schema())
self.query = Query({"conditions": []}, TableSource("my_table", ColumnSet([])),)
def setup_method(self, test_method):
super().setup_method(test_method)
raw_data = {'project': 2}
self.extension = ProjectExtension(
processor=ProjectWithGroupsProcessor()
)
self.valid_data = validate_jsonschema(raw_data, self.extension.get_schema())
self.query = Query({
"conditions": []
})
def test_project_extension_query_processing(
raw_data: Mapping[str, Any],
expected_conditions: Sequence[Condition],
expected_ast_conditions: Expression,
) -> None:
extension = ProjectExtension(project_column="project_id")
valid_data = validate_jsonschema(raw_data, extension.get_schema())
query = Query({"conditions": []}, QueryEntity(EntityKey.EVENTS, ColumnSet([])))
request_settings = HTTPRequestSettings()
extension.get_processor().process_query(query, valid_data, request_settings)
assert query.get_condition_from_ast() == expected_ast_conditions
def validate(
self, value: MutableMapping[str, Any], dataset: Dataset, referrer: str
) -> Request:
try:
value = validate_jsonschema(value, self.__composite_schema)
except jsonschema.ValidationError as error:
raise JsonSchemaValidationException(str(error)) from error
query_body = {
key: value.pop(key)
for key in self.__query_schema["properties"].keys()
if key in value
}
settings = {
key: value.pop(key)
for key in self.__settings_schema["properties"].keys()
if key in value
}
class_name = self.__setting_class
if isinstance(class_name, type(HTTPRequestSettings)):
settings_obj: Union[
HTTPRequestSettings, SubscriptionRequestSettings
] = class_name(**settings)
elif isinstance(class_name, type(SubscriptionRequestSettings)):
settings_obj = class_name()
extensions = {}
for extension_name, extension_schema in self.__extension_schemas.items():
extensions[extension_name] = {
key: value.pop(key)
for key in extension_schema["properties"].keys()
if key in value
}
if self.__language == Language.SNQL:
query = parse_snql_query(query_body["query"], dataset)
else:
query = parse_query(query_body, dataset)
apply_query_extensions(query, extensions, settings_obj)
request_id = uuid.uuid4().hex
return Request(
request_id,
# TODO: Replace this with the actual query raw body.
# this can have an impact on subscriptions so we need
# to be careful with the change.
ChainMap(query_body, *extensions.values()),
query,
settings_obj,
referrer,
)
def test_query_extension_processing(raw_data: dict,
expected_conditions: Sequence[Condition]):
state.set_config('max_days', 1)
extension = TimeSeriesExtension(
default_granularity=3600,
default_window=datetime.timedelta(days=5),
timestamp_column='timestamp',
)
valid_data = validate_jsonschema(raw_data, extension.get_schema())
query = Query({"conditions": []})
extension.get_processor().process_query(query, valid_data)
assert query.get_conditions() == expected_conditions
def test_project_extension_query_processing(raw_data: dict, expected_conditions: Sequence[Condition]):
extension = ProjectExtension(
processor=ProjectExtensionProcessor()
)
valid_data = validate_jsonschema(raw_data, extension.get_schema())
query = Query({
"conditions": []
})
request_settings = RequestSettings(turbo=False, consistent=False, debug=False)
extension.get_processor().process_query(query, valid_data, request_settings)
assert query.get_conditions() == expected_conditions
def test_organization_extension_query_processing_happy_path():
extension = OrganizationExtension()
raw_data = {"organization": 2}
valid_data = validate_jsonschema(raw_data, extension.get_schema())
query = Query({"conditions": []}, TableSource("my_table", ColumnSet([])))
request_settings = HTTPRequestSettings()
extension.get_processor().process_query(query, valid_data,
request_settings)
assert query.get_condition_from_ast() == binary_condition(
None, ConditionFunctions.EQ, Column(None, None, "org_id"),
Literal(None, 2))
def test_organization_extension_query_processing_happy_path() -> None:
extension = OrganizationExtension()
raw_data = {"organization": 2}
valid_data = validate_jsonschema(raw_data, extension.get_schema())
query = Query({"conditions": []},
QueryEntity(EntityKey.EVENTS, ColumnSet([])))
request_settings = HTTPRequestSettings()
extension.get_processor().process_query(query, valid_data,
request_settings)
assert query.get_condition_from_ast() == binary_condition(
ConditionFunctions.EQ, Column("_snuba_org_id", None, "org_id"),
Literal(None, 2))
def test_organization_extension_query_processing_happy_path():
extension = OrganizationExtension()
raw_data = {"organization": 2}
valid_data = validate_jsonschema(raw_data, extension.get_schema())
query = Query({"conditions": []})
request_settings = RequestSettings(turbo=False,
consistent=False,
debug=False)
extension.get_processor().process_query(query, valid_data,
request_settings)
assert query.get_conditions() == [("org_id", "=", 2)]
def test_project_extension_query_processing(
raw_data: dict,
expected_conditions: Sequence[Condition],
expected_ast_conditions: Expression,
):
extension = ProjectExtension(project_column="project_id")
valid_data = validate_jsonschema(raw_data, extension.get_schema())
query = Query({"conditions": []}, TableSource("my_table", ColumnSet([])),)
request_settings = HTTPRequestSettings()
extension.get_processor().process_query(query, valid_data, request_settings)
assert query.get_conditions() == expected_conditions
assert query.get_condition_from_ast() == expected_ast_conditions
def test_invalid_data_does_not_validate():
extension = OrganizationExtension()
with pytest.raises(ValidationError):
validate_jsonschema({"organization": "2"}, extension.get_schema())
with pytest.raises(ValidationError):
validate_jsonschema({"organization": 0}, extension.get_schema())
with pytest.raises(ValidationError):
validate_jsonschema({"organization": [2]}, extension.get_schema())
def test_project_extension_project_rate_limits_are_overridden() -> None:
extension = ProjectExtension(project_column="project_id")
raw_data = {"project": [3, 4]}
valid_data = validate_jsonschema(raw_data, extension.get_schema())
query = Query({"conditions": []}, QueryEntity(EntityKey.EVENTS, ColumnSet([])))
request_settings = HTTPRequestSettings()
state.set_config("project_per_second_limit_3", 5)
state.set_config("project_concurrent_limit_3", 10)
extension.get_processor().process_query(query, valid_data, request_settings)
rate_limits = request_settings.get_rate_limit_params()
most_recent_rate_limit = rate_limits[-1]
assert most_recent_rate_limit.bucket == "3"
assert most_recent_rate_limit.per_second_limit == 5
assert most_recent_rate_limit.concurrent_limit == 10
def test_invalid_data_does_not_validate() -> None:
extension = OrganizationExtension()
schema = cast(MutableMapping[str, Any], extension.get_schema())
with pytest.raises(ValidationError):
validate_jsonschema({"organization": "2"}, schema)
with pytest.raises(ValidationError):
validate_jsonschema({"organization": 0}, schema)
with pytest.raises(ValidationError):
validate_jsonschema({"organization": [2]}, schema)
def test_project_extension_query_adds_rate_limits() -> None:
extension = ProjectExtension(project_column="project_id")
raw_data = {"project": [1, 2]}
valid_data = validate_jsonschema(raw_data, extension.get_schema())
query = Query({"conditions": []}, QueryEntity(EntityKey.EVENTS, ColumnSet([])))
request_settings = HTTPRequestSettings()
num_rate_limits_before_processing = len(request_settings.get_rate_limit_params())
extension.get_processor().process_query(query, valid_data, request_settings)
rate_limits = request_settings.get_rate_limit_params()
# make sure a rate limit was added by the processing
assert len(rate_limits) == num_rate_limits_before_processing + 1
most_recent_rate_limit = rate_limits[-1]
assert most_recent_rate_limit.bucket == "1"
assert most_recent_rate_limit.per_second_limit == 1000
assert most_recent_rate_limit.concurrent_limit == 1000
def test_query_extension_processing(
raw_data: Mapping[str, Any],
expected_ast_condition: Expression,
expected_granularity: int,
) -> None:
state.set_config("max_days", 1)
extension = TimeSeriesExtension(
default_granularity=60,
default_window=timedelta(days=5),
timestamp_column="timestamp",
)
valid_data = validate_jsonschema(raw_data, extension.get_schema())
query = Query({"conditions": []}, QueryEntity(EntityKey.EVENTS, ColumnSet([])))
request_settings = HTTPRequestSettings()
extension.get_processor().process_query(query, valid_data, request_settings)
assert query.get_condition_from_ast() == expected_ast_condition
assert query.get_granularity() == expected_granularity
def test_project_extension_project_rate_limits_are_overridden():
extension = ProjectExtension(
processor=ProjectExtensionProcessor(project_column="project_id")
)
raw_data = {"project": [2, 3]}
valid_data = validate_jsonschema(raw_data, extension.get_schema())
query = Query({"conditions": []}, TableSource("my_table", ColumnSet([])),)
request_settings = HTTPRequestSettings()
state.set_config("project_per_second_limit_2", 5)
state.set_config("project_concurrent_limit_2", 10)
extension.get_processor().process_query(query, valid_data, request_settings)
rate_limits = request_settings.get_rate_limit_params()
most_recent_rate_limit = rate_limits[-1]
assert most_recent_rate_limit.bucket == "2"
assert most_recent_rate_limit.per_second_limit == 5
assert most_recent_rate_limit.concurrent_limit == 10
def test_query_extension_processing(
raw_data: dict,
expected_conditions: Sequence[Condition],
expected_ast_condition: Expression,
expected_granularity: int,
):
state.set_config("max_days", 1)
extension = TimeSeriesExtension(
default_granularity=60,
default_window=timedelta(days=5),
timestamp_column="timestamp",
)
valid_data = validate_jsonschema(raw_data, extension.get_schema())
query = Query({"conditions": []}, TableSource("my_table", ColumnSet([])),)
request_settings = HTTPRequestSettings()
extension.get_processor().process_query(query, valid_data, request_settings)
assert query.get_conditions() == expected_conditions
assert query.get_condition_from_ast() == expected_ast_condition
assert query.get_granularity() == expected_granularity
def test_project_extension_project_rate_limits_are_overridden():
extension = ProjectExtension(
processor=ProjectExtensionProcessor()
)
raw_data = {
'project': [2, 3]
}
valid_data = validate_jsonschema(raw_data, extension.get_schema())
query = Query({
'conditions': []
})
request_settings = RequestSettings(turbo=False, consistent=False, debug=False)
state.set_config('project_per_second_limit_2', 5)
state.set_config('project_concurrent_limit_2', 10)
extension.get_processor().process_query(query, valid_data, request_settings)
rate_limits = request_settings.get_rate_limit_params()
most_recent_rate_limit = rate_limits[-1]
assert most_recent_rate_limit.bucket == '2'
assert most_recent_rate_limit.per_second_limit == 5
assert most_recent_rate_limit.concurrent_limit == 10
