def validate(self, value, dataset: Dataset, referrer: str) -> Request: try: value = validate_jsonschema(value, self.__composite_schema) except jsonschema.ValidationError as error: raise JsonSchemaValidationException(str(error)) from error query_body = { key: value.pop(key) for key in self.__query_schema["properties"].keys() if key in value } settings = { key: value.pop(key) for key in self.__settings_schema["properties"].keys() if key in value } extensions = {} for extension_name, extension_schema in self.__extension_schemas.items( ): extensions[extension_name] = { key: value.pop(key) for key in extension_schema["properties"].keys() if key in value } query = parse_query(query_body, dataset) request_id = uuid.uuid4().hex return Request(request_id, query, self.__setting_class(**settings), extensions, referrer)
def validate(self, value: MutableMapping[str, Any]) -> RequestParts: try: value = validate_jsonschema(value, self.__composite_schema) except jsonschema.ValidationError as error: raise JsonSchemaValidationException(str(error)) from error query_body = { key: value.get(key) for key in self.__query_schema["properties"].keys() if key in value } query_settings = { key: value.get(key) for key in self.__query_settings_schema["properties"].keys() if key in value } attribution_info = { key: value.get(key) for key in self.__attribution_info_schema["properties"].keys() if key in value } return RequestParts( query=query_body, query_settings=query_settings, attribution_info=attribution_info, )
def test_query_extension_processing( raw_data: dict, expected_conditions: Sequence[Condition], expected_granularity: int, ): state.set_config('max_days', 1) extension = TimeSeriesExtension( default_granularity=60, default_window=datetime.timedelta(days=5), timestamp_column='timestamp', ) valid_data = validate_jsonschema(raw_data, extension.get_schema()) query = Query( {"conditions": []}, TableSource("my_table", ColumnSet([])), ) request_settings = RequestSettings(turbo=False, consistent=False, debug=False) extension.get_processor().process_query(query, valid_data, request_settings) assert query.get_conditions() == expected_conditions assert query.get_granularity() == expected_granularity
def validate(self, value: MutableMapping[str, Any]) -> RequestParts: try: value = validate_jsonschema(value, self.__composite_schema) except jsonschema.ValidationError as error: raise JsonSchemaValidationException(str(error)) from error query_body = { key: value.pop(key) for key in self.__query_schema["properties"].keys() if key in value } settings = { key: value.pop(key) for key in self.__settings_schema["properties"].keys() if key in value } extensions = {} for extension_name, extension_schema in self.__extension_schemas.items( ): extensions[extension_name] = { key: value.pop(key) for key in extension_schema["properties"].keys() if key in value } return RequestParts(query=query_body, settings=settings, extensions=extensions)
def validate(self, value) -> Request: value = validate_jsonschema(value, self.__composite_schema) query_body = { key: value.pop(key) for key in self.__query_schema['properties'].keys() if key in value } settings = { key: value.pop(key) for key in self.__settings_schema['properties'].keys() if key in value } extensions = {} for extension_name, extension_schema in self.__extension_schemas.items( ): extensions[extension_name] = { key: value.pop(key) for key in extension_schema['properties'].keys() if key in value } return Request( Query(query_body), RequestSettings(settings['turbo'], settings['consistent'], settings['debug']), extensions)
def validate(self, value, dataset: Dataset, referrer: str) -> Request: value = validate_jsonschema(value, self.__composite_schema) query_body = { key: value.pop(key) for key in self.__query_schema["properties"].keys() if key in value } settings = { key: value.pop(key) for key in self.__settings_schema["properties"].keys() if key in value } extensions = {} for extension_name, extension_schema in self.__extension_schemas.items( ): extensions[extension_name] = { key: value.pop(key) for key in extension_schema["properties"].keys() if key in value } query = parse_query(query_body, dataset) return Request(query, self.__setting_class(**settings), extensions, referrer)
def test_project_extension_query_adds_rate_limits(): extension = ProjectExtension(processor=ProjectExtensionProcessor( project_column="project_id")) raw_data = {'project': [2, 3]} valid_data = validate_jsonschema(raw_data, extension.get_schema()) query = Query( {"conditions": []}, TableSource("my_table", ColumnSet([])), ) request_settings = RequestSettings(turbo=False, consistent=False, debug=False) num_rate_limits_before_processing = len( request_settings.get_rate_limit_params()) extension.get_processor().process_query(query, valid_data, request_settings) rate_limits = request_settings.get_rate_limit_params() # make sure a rate limit was added by the processing assert len(rate_limits) == num_rate_limits_before_processing + 1 most_recent_rate_limit = rate_limits[-1] assert most_recent_rate_limit.bucket == '2' assert most_recent_rate_limit.per_second_limit == 1000 assert most_recent_rate_limit.concurrent_limit == 1000
def setup_method(self, test_method): super().setup_method(test_method) raw_data = {"project": 2} self.extension = ProjectExtension( processor=ProjectWithGroupsProcessor(project_column="project_id") ) self.valid_data = validate_jsonschema(raw_data, self.extension.get_schema()) self.query = Query({"conditions": []}, TableSource("my_table", ColumnSet([])),)
def setup_method(self, test_method): super().setup_method(test_method) raw_data = {'project': 2} self.extension = ProjectExtension( processor=ProjectWithGroupsProcessor() ) self.valid_data = validate_jsonschema(raw_data, self.extension.get_schema()) self.query = Query({ "conditions": [] })
def test_project_extension_query_processing( raw_data: Mapping[str, Any], expected_conditions: Sequence[Condition], expected_ast_conditions: Expression, ) -> None: extension = ProjectExtension(project_column="project_id") valid_data = validate_jsonschema(raw_data, extension.get_schema()) query = Query({"conditions": []}, QueryEntity(EntityKey.EVENTS, ColumnSet([]))) request_settings = HTTPRequestSettings() extension.get_processor().process_query(query, valid_data, request_settings) assert query.get_condition_from_ast() == expected_ast_conditions
def validate( self, value: MutableMapping[str, Any], dataset: Dataset, referrer: str ) -> Request: try: value = validate_jsonschema(value, self.__composite_schema) except jsonschema.ValidationError as error: raise JsonSchemaValidationException(str(error)) from error query_body = { key: value.pop(key) for key in self.__query_schema["properties"].keys() if key in value } settings = { key: value.pop(key) for key in self.__settings_schema["properties"].keys() if key in value } class_name = self.__setting_class if isinstance(class_name, type(HTTPRequestSettings)): settings_obj: Union[ HTTPRequestSettings, SubscriptionRequestSettings ] = class_name(**settings) elif isinstance(class_name, type(SubscriptionRequestSettings)): settings_obj = class_name() extensions = {} for extension_name, extension_schema in self.__extension_schemas.items(): extensions[extension_name] = { key: value.pop(key) for key in extension_schema["properties"].keys() if key in value } if self.__language == Language.SNQL: query = parse_snql_query(query_body["query"], dataset) else: query = parse_query(query_body, dataset) apply_query_extensions(query, extensions, settings_obj) request_id = uuid.uuid4().hex return Request( request_id, # TODO: Replace this with the actual query raw body. # this can have an impact on subscriptions so we need # to be careful with the change. ChainMap(query_body, *extensions.values()), query, settings_obj, referrer, )
def test_query_extension_processing(raw_data: dict, expected_conditions: Sequence[Condition]): state.set_config('max_days', 1) extension = TimeSeriesExtension( default_granularity=3600, default_window=datetime.timedelta(days=5), timestamp_column='timestamp', ) valid_data = validate_jsonschema(raw_data, extension.get_schema()) query = Query({"conditions": []}) extension.get_processor().process_query(query, valid_data) assert query.get_conditions() == expected_conditions
def test_project_extension_query_processing(raw_data: dict, expected_conditions: Sequence[Condition]): extension = ProjectExtension( processor=ProjectExtensionProcessor() ) valid_data = validate_jsonschema(raw_data, extension.get_schema()) query = Query({ "conditions": [] }) request_settings = RequestSettings(turbo=False, consistent=False, debug=False) extension.get_processor().process_query(query, valid_data, request_settings) assert query.get_conditions() == expected_conditions
def test_organization_extension_query_processing_happy_path(): extension = OrganizationExtension() raw_data = {"organization": 2} valid_data = validate_jsonschema(raw_data, extension.get_schema()) query = Query({"conditions": []}, TableSource("my_table", ColumnSet([]))) request_settings = HTTPRequestSettings() extension.get_processor().process_query(query, valid_data, request_settings) assert query.get_condition_from_ast() == binary_condition( None, ConditionFunctions.EQ, Column(None, None, "org_id"), Literal(None, 2))
def test_organization_extension_query_processing_happy_path() -> None: extension = OrganizationExtension() raw_data = {"organization": 2} valid_data = validate_jsonschema(raw_data, extension.get_schema()) query = Query({"conditions": []}, QueryEntity(EntityKey.EVENTS, ColumnSet([]))) request_settings = HTTPRequestSettings() extension.get_processor().process_query(query, valid_data, request_settings) assert query.get_condition_from_ast() == binary_condition( ConditionFunctions.EQ, Column("_snuba_org_id", None, "org_id"), Literal(None, 2))
def test_organization_extension_query_processing_happy_path(): extension = OrganizationExtension() raw_data = {"organization": 2} valid_data = validate_jsonschema(raw_data, extension.get_schema()) query = Query({"conditions": []}) request_settings = RequestSettings(turbo=False, consistent=False, debug=False) extension.get_processor().process_query(query, valid_data, request_settings) assert query.get_conditions() == [("org_id", "=", 2)]
def test_project_extension_query_processing( raw_data: dict, expected_conditions: Sequence[Condition], expected_ast_conditions: Expression, ): extension = ProjectExtension(project_column="project_id") valid_data = validate_jsonschema(raw_data, extension.get_schema()) query = Query({"conditions": []}, TableSource("my_table", ColumnSet([])),) request_settings = HTTPRequestSettings() extension.get_processor().process_query(query, valid_data, request_settings) assert query.get_conditions() == expected_conditions assert query.get_condition_from_ast() == expected_ast_conditions
def test_invalid_data_does_not_validate(): extension = OrganizationExtension() with pytest.raises(ValidationError): validate_jsonschema({"organization": "2"}, extension.get_schema()) with pytest.raises(ValidationError): validate_jsonschema({"organization": 0}, extension.get_schema()) with pytest.raises(ValidationError): validate_jsonschema({"organization": [2]}, extension.get_schema())
def test_project_extension_project_rate_limits_are_overridden() -> None: extension = ProjectExtension(project_column="project_id") raw_data = {"project": [3, 4]} valid_data = validate_jsonschema(raw_data, extension.get_schema()) query = Query({"conditions": []}, QueryEntity(EntityKey.EVENTS, ColumnSet([]))) request_settings = HTTPRequestSettings() state.set_config("project_per_second_limit_3", 5) state.set_config("project_concurrent_limit_3", 10) extension.get_processor().process_query(query, valid_data, request_settings) rate_limits = request_settings.get_rate_limit_params() most_recent_rate_limit = rate_limits[-1] assert most_recent_rate_limit.bucket == "3" assert most_recent_rate_limit.per_second_limit == 5 assert most_recent_rate_limit.concurrent_limit == 10
def test_invalid_data_does_not_validate() -> None: extension = OrganizationExtension() schema = cast(MutableMapping[str, Any], extension.get_schema()) with pytest.raises(ValidationError): validate_jsonschema({"organization": "2"}, schema) with pytest.raises(ValidationError): validate_jsonschema({"organization": 0}, schema) with pytest.raises(ValidationError): validate_jsonschema({"organization": [2]}, schema)
def test_project_extension_query_adds_rate_limits() -> None: extension = ProjectExtension(project_column="project_id") raw_data = {"project": [1, 2]} valid_data = validate_jsonschema(raw_data, extension.get_schema()) query = Query({"conditions": []}, QueryEntity(EntityKey.EVENTS, ColumnSet([]))) request_settings = HTTPRequestSettings() num_rate_limits_before_processing = len(request_settings.get_rate_limit_params()) extension.get_processor().process_query(query, valid_data, request_settings) rate_limits = request_settings.get_rate_limit_params() # make sure a rate limit was added by the processing assert len(rate_limits) == num_rate_limits_before_processing + 1 most_recent_rate_limit = rate_limits[-1] assert most_recent_rate_limit.bucket == "1" assert most_recent_rate_limit.per_second_limit == 1000 assert most_recent_rate_limit.concurrent_limit == 1000
def test_query_extension_processing( raw_data: Mapping[str, Any], expected_ast_condition: Expression, expected_granularity: int, ) -> None: state.set_config("max_days", 1) extension = TimeSeriesExtension( default_granularity=60, default_window=timedelta(days=5), timestamp_column="timestamp", ) valid_data = validate_jsonschema(raw_data, extension.get_schema()) query = Query({"conditions": []}, QueryEntity(EntityKey.EVENTS, ColumnSet([]))) request_settings = HTTPRequestSettings() extension.get_processor().process_query(query, valid_data, request_settings) assert query.get_condition_from_ast() == expected_ast_condition assert query.get_granularity() == expected_granularity
def test_project_extension_project_rate_limits_are_overridden(): extension = ProjectExtension( processor=ProjectExtensionProcessor(project_column="project_id") ) raw_data = {"project": [2, 3]} valid_data = validate_jsonschema(raw_data, extension.get_schema()) query = Query({"conditions": []}, TableSource("my_table", ColumnSet([])),) request_settings = HTTPRequestSettings() state.set_config("project_per_second_limit_2", 5) state.set_config("project_concurrent_limit_2", 10) extension.get_processor().process_query(query, valid_data, request_settings) rate_limits = request_settings.get_rate_limit_params() most_recent_rate_limit = rate_limits[-1] assert most_recent_rate_limit.bucket == "2" assert most_recent_rate_limit.per_second_limit == 5 assert most_recent_rate_limit.concurrent_limit == 10
def test_query_extension_processing( raw_data: dict, expected_conditions: Sequence[Condition], expected_ast_condition: Expression, expected_granularity: int, ): state.set_config("max_days", 1) extension = TimeSeriesExtension( default_granularity=60, default_window=timedelta(days=5), timestamp_column="timestamp", ) valid_data = validate_jsonschema(raw_data, extension.get_schema()) query = Query({"conditions": []}, TableSource("my_table", ColumnSet([])),) request_settings = HTTPRequestSettings() extension.get_processor().process_query(query, valid_data, request_settings) assert query.get_conditions() == expected_conditions assert query.get_condition_from_ast() == expected_ast_condition assert query.get_granularity() == expected_granularity
def test_project_extension_project_rate_limits_are_overridden(): extension = ProjectExtension( processor=ProjectExtensionProcessor() ) raw_data = { 'project': [2, 3] } valid_data = validate_jsonschema(raw_data, extension.get_schema()) query = Query({ 'conditions': [] }) request_settings = RequestSettings(turbo=False, consistent=False, debug=False) state.set_config('project_per_second_limit_2', 5) state.set_config('project_concurrent_limit_2', 10) extension.get_processor().process_query(query, valid_data, request_settings) rate_limits = request_settings.get_rate_limit_params() most_recent_rate_limit = rate_limits[-1] assert most_recent_rate_limit.bucket == '2' assert most_recent_rate_limit.per_second_limit == 5 assert most_recent_rate_limit.concurrent_limit == 10