def get_date_range_from_params(params, optional=False, validate_window=True): """ Gets a date range from standard date range params we pass to the api. If `statsPeriod` is passed then convert to a time delta and make sure it fits within our min/max period length. Values are in the format <number><period_type>, where period type is one of `s` (seconds), `m` (minutes), `h` (hours) or `d` (days). Similarly, `statsPeriodStart` and `statsPeriodEnd` allow for selecting a relative range, for example: 15 days ago through 8 days ago. This uses the same format as `statsPeriod` :param params: If `start` end `end` are passed, validate them, convert to `datetime` and returns them if valid. :param optional: When True, if no params passed then return `(None, None)`. :param validate_window: When True, validate against min / max time delta. :return: A length 2 tuple containing start/end or raises an `InvalidParams` exception """ now = timezone.now() end = now start = now - MAX_STATS_PERIOD stats_period = params.get('statsPeriod') stats_period_start = params.get('statsPeriodStart') stats_period_end = params.get('statsPeriodEnd') if stats_period is not None: start = get_datetime_from_stats_period(stats_period, now) elif stats_period_start or stats_period_end: if not all([stats_period_start, stats_period_end]): raise InvalidParams( 'statsPeriodStart and statsPeriodEnd are both required') start = get_datetime_from_stats_period(stats_period_start, now) end = get_datetime_from_stats_period(stats_period_end, now) elif params.get('start') or params.get('end'): if not all([params.get('start'), params.get('end')]): raise InvalidParams('start and end are both required') try: start = parse_datetime_string(params['start']) end = parse_datetime_string(params['end']) except InvalidQuery as exc: raise InvalidParams(exc.message) elif optional: return None, None if start > end: raise InvalidParams('start must be before end') if validate_window: delta = end - start if delta > MAX_STATS_PERIOD: raise InvalidParams(INVALID_PERIOD_ERROR) return start, end
def test_converts_stats_period_start_end(self): """ Ensures that statsPeriodStart and statsPeriodEnd is converted to start/end. """ payload = { "query_type": ExportQueryType.DISCOVER_STR, "query_info": { "env": "test", "statsPeriodStart": "1w", "statsPeriodEnd": "5d" }, } with self.feature({ "organizations:data-export": True, "organizations:discover-basic": True }): response = self.get_valid_response(self.organization.slug, status_code=201, **payload) data_export = ExportedData.objects.get(id=response.data["id"]) query_info = data_export.query_info assert parse_datetime_string( query_info["start"]) == parse_datetime_string( "2020-05-12T14:00:00") assert parse_datetime_string( query_info["end"]) == parse_datetime_string("2020-05-14T14:00:00") assert "statsPeriod" not in query_info assert "statsPeriodStart" not in query_info assert "statsPeriodSEnd" not in query_info
def test_preserves_start_end(self): """ Ensures that start/end is preserved """ payload = { "query_type": ExportQueryType.DISCOVER_STR, "query_info": { "env": "test", "start": "2020-05-18T14:00:00", "end": "2020-05-19T14:00:00", }, } with self.feature("organizations:discover-query"): response = self.get_valid_response(self.organization.slug, status_code=201, **payload) data_export = ExportedData.objects.get(id=response.data["id"]) query_info = data_export.query_info assert parse_datetime_string(query_info["start"]) == parse_datetime_string( "2020-05-18T14:00:00" ) assert parse_datetime_string(query_info["end"]) == parse_datetime_string( "2020-05-19T14:00:00" ) assert "statsPeriod" not in query_info assert "statsPeriodStart" not in query_info assert "statsPeriodSEnd" not in query_info
def get_date_range_from_params(params): # Returns (start, end) or raises an `InvalidParams` exception now = timezone.now() end = now start = now - MAX_STATS_PERIOD stats_period = params.get('statsPeriod') if stats_period is not None: stats_period = parse_stats_period(stats_period) if stats_period is None or stats_period < MIN_STATS_PERIOD or stats_period >= MAX_STATS_PERIOD: raise InvalidParams('Invalid statsPeriod') start = now - stats_period elif params.get('start') or params.get('end'): if not all([params.get('start'), params.get('end')]): raise InvalidParams('start and end are both required') try: start = parse_datetime_string(params['start']) end = parse_datetime_string(params['end']) except InvalidQuery as exc: raise InvalidParams(exc.message) if start > end: raise InvalidParams('start must be before end') return (start, end)
def get_date_range_from_params(params, optional=False): """ Gets a date range from standard date range params we pass to the api. If `statsPeriod` is passed then convert to a time delta and make sure it fits within our min/max period length. Values are in the format <number><period_type>, where period type is one of `s` (seconds), `m` (minutes), `h` (hours) or `d` (days). Similarly, `statsPeriodStart` and `statsPeriodEnd` allow for selecting a relative range, for example: 15 days ago through 8 days ago. This uses the same format as `statsPeriod` :param params: If `start` end `end` are passed, validate them, convert to `datetime` and returns them if valid. :param optional: When True, if no params passed then return `(None, None)`. :return: A length 2 tuple containing start/end or raises an `InvalidParams` exception """ now = timezone.now() end = now start = now - MAX_STATS_PERIOD stats_period = params.get('statsPeriod') stats_period_start = params.get('statsPeriodStart') stats_period_end = params.get('statsPeriodEnd') if stats_period is not None: start = get_datetime_from_stats_period(stats_period, now) elif stats_period_start or stats_period_end: if not all([stats_period_start, stats_period_end]): raise InvalidParams('statsPeriodStart and statsPeriodEnd are both required') start = get_datetime_from_stats_period(stats_period_start, now) end = get_datetime_from_stats_period(stats_period_end, now) elif params.get('start') or params.get('end'): if not all([params.get('start'), params.get('end')]): raise InvalidParams('start and end are both required') try: start = parse_datetime_string(params['start']) end = parse_datetime_string(params['end']) except InvalidQuery as exc: raise InvalidParams(exc.message) elif optional: return None, None if start > end: raise InvalidParams('start must be before end') delta = end - start if delta < MIN_STATS_PERIOD or delta > MAX_STATS_PERIOD: raise InvalidParams(INVALID_PERIOD_ERROR) return start, end
def get_date_range_from_params(params, optional=False): """ Gets a date range from standard date range params we pass to the api. If `statsPeriod` is passed then convert to a time delta and make sure it fits within our min/max period length. Values are in the format <number><period_type>, where period type is one of `s` (seconds), `m` (minutes), `h` (hours) or `d` (days). Similarly, `statsPeriodStart` and `statsPeriodEnd` allow for selecting a relative range, for example: 15 days ago through 8 days ago. This uses the same format as `statsPeriod` :param params: If `start` end `end` are passed, validate them, convert to `datetime` and returns them if valid. :param optional: When True, if no params passed then return `(None, None)`. :return: A length 2 tuple containing start/end or raises an `InvalidParams` exception """ now = timezone.now() start, end = default_start_end_dates(now) stats_period = params.get("statsPeriod") stats_period_start = params.get("statsPeriodStart") stats_period_end = params.get("statsPeriodEnd") if stats_period is not None: start = get_datetime_from_stats_period(stats_period, now) elif stats_period_start or stats_period_end: if not all([stats_period_start, stats_period_end]): raise InvalidParams( "statsPeriodStart and statsPeriodEnd are both required") start = get_datetime_from_stats_period(stats_period_start, now) end = get_datetime_from_stats_period(stats_period_end, now) elif params.get("start") or params.get("end"): if not all([params.get("start"), params.get("end")]): raise InvalidParams("start and end are both required") try: start = parse_datetime_string(params["start"]) end = parse_datetime_string(params["end"]) except InvalidQuery as e: raise InvalidParams(str(e)) elif optional: return None, None if start >= end: raise InvalidParams("start must be before end") return start, end
def test_converts_stats_period_start_end(self): """ Ensures that statsPeriodStart and statsPeriodEnd is converted to start/end. """ payload = self.make_payload("discover", {"statsPeriodStart": "1w", "statsPeriodEnd": "5d"}) with self.feature("organizations:discover-query"): response = self.get_valid_response(self.org.slug, status_code=201, **payload) data_export = ExportedData.objects.get(id=response.data["id"]) query_info = data_export.query_info assert parse_datetime_string(query_info["start"]) == parse_datetime_string( "2020-05-12T14:00:00" ) assert parse_datetime_string(query_info["end"]) == parse_datetime_string( "2020-05-14T14:00:00" ) assert "statsPeriod" not in query_info assert "statsPeriodStart" not in query_info assert "statsPeriodSEnd" not in query_info
def test_preserves_start_end(self): """ Ensures that start/end is preserved """ payload = self.discover_payload.copy() payload["query_info"].update({"start": "2020-05-18T14:00:00", "end": "2020-05-19T14:00:00"}) with self.feature("organizations:discover-query"): response = self.get_valid_response(self.org.slug, status_code=201, **payload) data_export = ExportedData.objects.get(id=response.data["id"]) query_info = data_export.query_info assert parse_datetime_string(query_info["start"]) == parse_datetime_string( "2020-05-18T14:00:00" ) assert parse_datetime_string(query_info["end"]) == parse_datetime_string( "2020-05-19T14:00:00" ) assert "statsPeriod" not in query_info assert "statsPeriodStart" not in query_info assert "statsPeriodSEnd" not in query_info
def visit_time_filter(self, node, children): (search_key, _, operator, search_value) = children if search_key.name in self.date_keys: try: search_value = parse_datetime_string(search_value) except InvalidQuery as exc: raise InvalidSearchQuery(exc.message) return SearchFilter(search_key, operator, SearchValue(search_value)) else: search_value = operator + search_value if operator != '=' else search_value return self._handle_basic_filter(search_key, '=', SearchValue(search_value))
def visit_time_filter(self, node, children): (search_key, _, operator, search_value) = children if search_key.name in self.date_keys: try: search_value = parse_datetime_string(search_value) except InvalidQuery as exc: raise InvalidSearchQuery(six.text_type(exc)) return SearchFilter(search_key, operator, SearchValue(search_value)) else: search_value = operator + search_value if operator != "=" else search_value return self._handle_basic_filter(search_key, "=", SearchValue(search_value))
def get_date_range_from_params(params, optional=False): """ Gets a date range from standard date range params we pass to the api. If `statsPeriod` is passed then convert to a time delta and make sure it fits within our min/max period length. Values are in the format <number><period_type>, where period type is one of `s` (seconds), `m` (minutes), `h` (hours) or `d` (days). :param params: If `start` end `end` are passed, validate them, convert to `datetime` and returns them if valid. :param optional: When True, if no params passed then return `(None, None)`. :return: A length 2 tuple containing start/end or raises an `InvalidParams` exception """ now = timezone.now() end = now start = now - MAX_STATS_PERIOD stats_period = params.get('statsPeriod') if stats_period is not None: stats_period = parse_stats_period(stats_period) if stats_period is None or stats_period < MIN_STATS_PERIOD or stats_period >= MAX_STATS_PERIOD: raise InvalidParams('Invalid statsPeriod') start = now - stats_period elif params.get('start') or params.get('end'): if not all([params.get('start'), params.get('end')]): raise InvalidParams('start and end are both required') try: start = parse_datetime_string(params['start']) end = parse_datetime_string(params['end']) except InvalidQuery as exc: raise InvalidParams(exc.message) if start > end: raise InvalidParams('start must be before end') elif optional: return None, None return start, end
def visit_aggregate_date_filter(self, node, children): (negation, search_key, _, operator, search_value) = children search_value = search_value[0] operator = self.handle_negation(negation, operator) is_date_aggregate = any(key in search_key.name for key in self.date_keys) if is_date_aggregate: try: search_value = parse_datetime_string(search_value) except InvalidQuery as exc: raise InvalidSearchQuery(str(exc)) return AggregateFilter(search_key, operator, SearchValue(search_value)) else: search_value = operator + search_value if operator != "=" else search_value return AggregateFilter(search_key, "=", SearchValue(search_value))
def visit_aggregate_date_filter(self, node, children): (search_key, _, operator, search_value) = children search_value = search_value[0] operator = operator[0] if not isinstance(operator, Node) else "=" is_date_aggregate = any(key in search_key.name for key in self.date_keys) if is_date_aggregate: try: search_value = parse_datetime_string(search_value) except InvalidQuery as exc: raise InvalidSearchQuery(six.text_type(exc)) return AggregateFilter(search_key, operator, SearchValue(search_value)) else: search_value = operator + search_value if operator != "=" else search_value return AggregateFilter(search_key, "=", SearchValue(search_value))
def visit_time_filter(self, node, children): search_key_node, operator, search_value = children search_key = search_key_node.text try: search_value = parse_datetime_string(search_value) except InvalidQuery as exc: raise InvalidSearchQuery(exc.message) try: return SearchFilter( SearchKey(search_key), operator, SearchValue(search_value, FIELD_LOOKUP[search_key]['type']), ) except KeyError: raise InvalidSearchQuery('Unsupported search term: %s' % (search_key,))
def visit_time_filter(self, node, children): search_key, _, operator, search_value = children try: search_value = parse_datetime_string(search_value) except InvalidQuery as exc: raise InvalidSearchQuery(exc.message) try: return SearchFilter( search_key, operator, SearchValue(search_value), ) except KeyError: raise InvalidSearchQuery('Unsupported search term: %s' % (search_key, ))
def visit_time_filter(self, node, children): search_key, _, operator, search_value = children if search_key.name in self.date_keys: try: search_value = parse_datetime_string(search_value) except InvalidQuery as exc: raise InvalidSearchQuery(exc.message) else: search_value = operator + search_value if operator != '=' else search_value operator = '=' try: return SearchFilter( search_key, operator, SearchValue(search_value), ) except KeyError: raise InvalidSearchQuery('Unsupported search term: %s' % (search_key,))
def get(self, request: Request, organization) -> Response: if not self.has_feature(organization, request): return Response(status=404) use_snql = self.has_snql_feature(organization, request) sentry_sdk.set_tag("discover.use-snql", use_snql) try: params = self.get_snuba_params(request, organization) except NoProjects: return Response([]) with sentry_sdk.start_span(op="discover.endpoint", description="trend_dates"): middle_date = request.GET.get("middle") if middle_date: try: middle = parse_datetime_string(middle_date) except InvalidQuery: raise ParseError(detail=f"{middle_date} is not a valid date format") if middle <= params["start"] or middle >= params["end"]: raise ParseError( detail="The middle date should be within the duration of the query" ) else: middle = params["start"] + timedelta( seconds=(params["end"] - params["start"]).total_seconds() * 0.5 ) middle = datetime.strftime(middle, DateArg.date_format) trend_type = request.GET.get("trendType", REGRESSION) if trend_type not in TREND_TYPES: raise ParseError(detail=f"{trend_type} is not a supported trend type") trend_function = request.GET.get("trendFunction", "p50()") try: function, columns, _ = parse_function(trend_function) except InvalidSearchQuery as error: raise ParseError(detail=error) if len(columns) == 0: # Default to duration column = "transaction.duration" else: column = columns[0] selected_columns = self.get_field_list(organization, request) orderby = self.get_orderby(request) query = request.GET.get("query") if use_snql: with self.handle_query_errors(): trend_query = TrendQueryBuilder( dataset=Dataset.Discover, params=params, selected_columns=selected_columns, auto_fields=False, auto_aggregations=True, use_aggregate_conditions=True, ) snql_trend_columns = self.resolve_trend_columns( trend_query, function, column, middle ) trend_query.columns.extend(snql_trend_columns.values()) trend_query.aggregates.extend(snql_trend_columns.values()) trend_query.params["aliases"] = self.get_snql_function_aliases( snql_trend_columns, trend_type ) # Both orderby and conditions need to be resolved after the columns because of aliasing trend_query.orderby = trend_query.resolve_orderby(orderby) where, having = trend_query.resolve_conditions(query, use_aggregate_conditions=True) trend_query.where += where trend_query.having += having else: params["aliases"] = self.get_function_aliases(trend_type) trend_columns = self.get_trend_columns(function, column, middle) def data_fn(offset, limit): if use_snql: trend_query.offset = Offset(offset) trend_query.limit = Limit(limit) result = raw_snql_query( trend_query.get_snql_query(), referrer="api.trends.get-percentage-change.wip-snql", ) result = discover.transform_results( result, trend_query.function_alias_map, {}, None ) return result else: return discover.query( selected_columns=selected_columns + trend_columns, query=query, params=params, orderby=orderby, offset=offset, limit=limit, referrer="api.trends.get-percentage-change", auto_fields=True, auto_aggregations=True, use_aggregate_conditions=True, ) with self.handle_query_errors(): return self.paginate( request=request, paginator=GenericOffsetPaginator(data_fn=data_fn), on_results=self.build_result_handler( request, organization, params, trend_function, selected_columns, orderby, query, use_snql, ), default_per_page=5, max_per_page=5, )
try: search_value = SearchValue(int(search_value.text)) except ValueError: raise InvalidSearchQuery('Invalid numeric query: %s' % (search_key, )) return SearchFilter(search_key, operator, search_value) else: search_value = SearchValue( operator + search_value.text if operator != '=' else search_value.text, ) return self._handle_basic_filter(search_key, '=', search_value) def visit_time_filter(self, node, (search_key, _, operator, search_value)): if search_key.name in self.date_keys: try: search_value = parse_datetime_string(search_value) except InvalidQuery as exc: raise InvalidSearchQuery(exc.message) return SearchFilter(search_key, operator, SearchValue(search_value)) else: search_value = operator + search_value if operator != '=' else search_value return self._handle_basic_filter(search_key, '=', SearchValue(search_value)) def visit_rel_time_filter(self, node, (search_key, _, value)): if search_key.name in self.date_keys: try: from_val, to_val = parse_datetime_range(value.text) except InvalidQuery as exc: raise InvalidSearchQuery(exc.message)
def node_visitor(token): if token["type"] == "spaces": return None if token["type"] == "filter": # Filters with an invalid reason raises to signal to the test # runner that we should expect this exception if token.get("invalid"): raise InvalidSearchQuery(token["invalid"]["reason"]) # Transform the operator to match for list values if token["value"]["type"] in ["valueTextList", "valueNumberList"]: operator = "NOT IN" if token["negated"] else "IN" else: # Negate the operator if the filter is negated to match operator = token["operator"] or "=" operator = f"!{operator}" if token["negated"] else operator key = node_visitor(token["key"]) value = node_visitor(token["value"]) if token["filter"] == "boolean" and token["negated"]: operator = "=" value = SearchValue(raw_value=1 if value.raw_value == 0 else 0) return SearchFilter(key, operator, value) if token["type"] == "keySimple": return SearchKey(name=token["value"]) if token["type"] == "keyExplicitTag": return SearchKey(name=f"tags[{token['key']['value']}]") if token["type"] == "keyAggregate": name = node_visitor(token["name"]).name # Consistent join aggregate function parameters args = ", ".join(arg["value"]["value"] for arg in token["args"]["args"]) return AggregateKey(name=f"{name}({args})") if token["type"] == "valueText": # Noramlize values by removing the escaped quotes value = token["value"].replace('\\"', '"') return SearchValue(raw_value=value) if token["type"] == "valueNumber": return SearchValue( raw_value=parse_numeric_value(token["value"], token["unit"])) if token["type"] == "valueTextList": return SearchValue( raw_value=[item["value"]["value"] for item in token["items"]]) if token["type"] == "valueNumberList": return SearchValue(raw_value=[ item["value"]["rawValue"] for item in token["items"] ]) if token["type"] == "valueIso8601Date": return SearchValue(raw_value=parse_datetime_string(token["value"])) if token["type"] == "valueDuration": return SearchValue( raw_value=parse_duration(token["value"], token["unit"])) if token["type"] == "valueRelativeDate": return SearchValue( raw_value=parse_duration(token["value"], token["unit"])) if token["type"] == "valueBoolean": return SearchValue(raw_value=int(token["value"])) if token["type"] == "freeText": if token["quoted"]: # Normalize quotes value = token["value"].replace('\\"', '"') else: # Normalize spacing value = token["value"].strip(" ") if value == "": return None return SearchFilter( key=SearchKey(name="message"), operator="=", value=SearchValue(raw_value=value), )
def get(self, request, organization): if not self.has_feature(organization, request): return Response(status=404) try: params = self.get_snuba_params(request, organization) except NoProjects: return Response([]) with sentry_sdk.start_span(op="discover.endpoint", description="trend_dates"): middle_date = request.GET.get("middle") if middle_date: try: middle = parse_datetime_string(middle_date) except InvalidQuery: raise ParseError(detail="{} is not a valid date format".format(middle_date)) if middle <= params["start"] or middle >= params["end"]: raise ParseError( detail="The middle date should be within the duration of the query" ) else: middle = params["start"] + timedelta( seconds=(params["end"] - params["start"]).total_seconds() * 0.5 ) start, middle, end = ( datetime.strftime(params["start"], DateArg.date_format), datetime.strftime(middle, DateArg.date_format), datetime.strftime(params["end"], DateArg.date_format), ) trend_type = request.GET.get("trendType", REGRESSION) if trend_type not in TREND_TYPES: raise ParseError(detail=u"{} is not a supported trend type".format(trend_type)) params["aliases"] = self.get_function_aliases(trend_type) trend_function = request.GET.get("trendFunction", "p50()") function, columns = parse_function(trend_function) trend_columns = self.get_trend_columns(function, columns, start, middle, end) selected_columns = request.GET.getlist("field")[:] orderby = self.get_orderby(request) query = request.GET.get("query") def data_fn(offset, limit): return discover.query( selected_columns=selected_columns + trend_columns, query=query, params=params, orderby=orderby, offset=offset, limit=limit, referrer="api.trends.get-percentage-change", auto_fields=True, auto_aggregations=True, use_aggregate_conditions=True, ) with self.handle_query_errors(): return self.paginate( request=request, paginator=GenericOffsetPaginator(data_fn=data_fn), on_results=self.build_result_handler( request, organization, params, trend_function, selected_columns, orderby, query ), default_per_page=5, max_per_page=5, )