def build_snuba_filter(dataset, query, aggregate, environment, params=None): resolve_func = (resolve_column(Dataset.Events) if dataset == QueryDatasets.EVENTS else resolve_column( Dataset.Transactions)) query = apply_dataset_query_conditions(dataset, query) snuba_filter = get_filter(query, params=params) snuba_filter.update_with( resolve_field_list([aggregate], snuba_filter, auto_fields=False)) snuba_filter = resolve_snuba_aliases(snuba_filter, resolve_func)[0] if environment: snuba_filter.conditions.append(["environment", "=", environment.name]) return snuba_filter
def build_snuba_filter( self, query: str, environment: Optional[Environment], params: Optional[Mapping[str, Any]] = None, ) -> Filter: resolve_func = resolve_column(Dataset(self.dataset.value)) aggregations = [self.aggregate] # This aggregation is added to return the total number of sessions in crash # rate alerts that is used to identify if we are below a general minimum alert threshold count_col = re.search(r"(sessions|users)", self.aggregate) if not count_col: raise UnsupportedQuerySubscription( "Only crash free percentage queries are supported for subscriptions" "over the sessions dataset" ) count_col_matched = count_col.group() aggregations += [f"identity({count_col_matched}) AS {CRASH_RATE_ALERT_SESSION_COUNT_ALIAS}"] functions_acl = ["identity"] snuba_filter = get_filter(query, params=params) snuba_filter.update_with( resolve_field_list( aggregations, snuba_filter, auto_fields=False, functions_acl=functions_acl ) ) snuba_filter = resolve_snuba_aliases(snuba_filter, resolve_func)[0] if environment: snuba_filter.conditions.append(["environment", "=", environment.name]) return snuba_filter
def build_snuba_filter(dataset, query, aggregate, environment, params=None): snuba_filter = get_filter(query, params=params) snuba_filter.update_with(resolve_field_list([aggregate], snuba_filter, auto_fields=False)) snuba_filter = resolve_snuba_aliases(snuba_filter, resolve_column(Dataset.Discover))[0] if environment: snuba_filter.conditions.append(["environment", "=", environment.name]) snuba_filter.conditions = apply_dataset_conditions(dataset, snuba_filter.conditions) return snuba_filter
def build_snuba_filter(dataset, query, aggregate, environment, event_types, params=None): resolve_func = { QueryDatasets.EVENTS: resolve_column(Dataset.Events), QueryDatasets.SESSIONS: resolve_column(Dataset.Sessions), QueryDatasets.TRANSACTIONS: resolve_column(Dataset.Transactions), }[dataset] functions_acl = None aggregations = [aggregate] if dataset == QueryDatasets.SESSIONS: # This aggregation is added to return the total number of sessions in crash # rate alerts that is used to identify if we are below a general minimum alert threshold count_col = re.search(r"(sessions|users)", aggregate) count_col_matched = count_col.group() aggregations += [ f"identity({count_col_matched}) AS {CRASH_RATE_ALERT_SESSION_COUNT_ALIAS}" ] functions_acl = ["identity"] query = apply_dataset_query_conditions(dataset, query, event_types) snuba_filter = get_filter(query, params=params) snuba_filter.update_with( resolve_field_list(aggregations, snuba_filter, auto_fields=False, functions_acl=functions_acl)) snuba_filter = resolve_snuba_aliases(snuba_filter, resolve_func)[0] if snuba_filter.group_ids: snuba_filter.conditions.append( ["group_id", "IN", list(map(int, snuba_filter.group_ids))]) if environment: snuba_filter.conditions.append(["environment", "=", environment.name]) return snuba_filter
def __init__(self, dataset: Dataset, params: ParamsType): self.params = params self.dataset = dataset # Function is a subclass of CurriedFunction self.where: List[WhereType] = [] self.aggregates: List[CurriedFunction] = [] self.columns: List[SelectType] = [] self.orderby: List[OrderBy] = [] self.projects_to_filter: Set[int] = set() self.resolve_column_name = resolve_column(self.dataset)
def __init__( self, dataset: Dataset, params: ParamsType, orderby: Optional[List[str]] = None, ): # Function is a subclass of CurriedFunction self.aggregates: List[CurriedFunction] = [] self.columns: List[SelectType] = [] self.where: List[WhereType] = [] self.params = params self.dataset = dataset self.orderby_columns: List[str] = orderby if orderby else [] self.resolve_column_name = resolve_column(self.dataset)
def __init__(self, dataset: Dataset, params: ParamsType, functions_acl: Optional[List[str]] = None): self.dataset = dataset self.params = params self.functions_acl = set() if functions_acl is None else functions_acl # Function is a subclass of CurriedFunction self.where: List[WhereType] = [] self.aggregates: List[CurriedFunction] = [] self.columns: List[SelectType] = [] self.orderby: List[OrderBy] = [] self.projects_to_filter: Set[int] = set() self.resolve_column_name = resolve_column(self.dataset)
def build_snuba_filter( self, query: str, environment: Optional[Environment], params: Optional[Mapping[str, Any]] = None, ) -> Filter: resolve_func = resolve_column(Dataset(self.dataset.value)) query = apply_dataset_query_conditions(QueryDatasets(self.dataset), query, self.event_types) snuba_filter = get_filter(query, params=params) snuba_filter.update_with( resolve_field_list([self.aggregate], snuba_filter, auto_fields=False) ) snuba_filter = resolve_snuba_aliases(snuba_filter, resolve_func)[0] if snuba_filter.group_ids: snuba_filter.conditions.append( ["group_id", "IN", list(map(int, snuba_filter.group_ids))] ) if environment: snuba_filter.conditions.append(["environment", "=", environment.name]) return snuba_filter
def __init__( self, dataset: Dataset, params: ParamsType, auto_fields: bool = False, functions_acl: Optional[List[str]] = None, equation_config: Optional[Dict[str, bool]] = None, ): self.dataset = dataset self.params = params self.auto_fields = auto_fields self.functions_acl = set() if functions_acl is None else functions_acl self.equation_config = equation_config if equation_config is not None else {} # Function is a subclass of CurriedFunction self.where: List[WhereType] = [] # The list of aggregates to be selected self.aggregates: List[CurriedFunction] = [] self.columns: List[SelectType] = [] self.orderby: List[OrderBy] = [] self.projects_to_filter: Set[int] = set() self.resolve_column_name = resolve_column(self.dataset)
"get_facets", "transform_data", "zerofill", "histogram_query", "check_multihistogram_fields", ) logger = logging.getLogger(__name__) PaginationResult = namedtuple("PaginationResult", ["next", "previous", "oldest", "latest"]) FacetResult = namedtuple("FacetResult", ["key", "value", "count"]) PerformanceFacetResult = namedtuple("PerformanceFacetResult", ["key", "value", "performance", "count"]) resolve_discover_column = resolve_column(Dataset.Discover) def is_real_column(col): """ Return true if col corresponds to an actual column to be fetched (not an aggregate function or field alias) """ if is_function(col): return False if col in FIELD_ALIASES: return False return True
def get_tag_value_paginator_for_projects( self, projects, environments, key, start=None, end=None, query=None, order_by="-last_seen", include_transactions=False, ): from sentry.api.paginator import SequencePaginator if not order_by == "-last_seen": raise ValueError("Unsupported order_by: %s" % order_by) dataset = Dataset.Events snuba_key = snuba.get_snuba_column_name(key) if include_transactions and snuba_key.startswith("tags["): snuba_key = snuba.get_snuba_column_name(key, dataset=Dataset.Discover) if not snuba_key.startswith("tags["): dataset = Dataset.Discover # We cannot search the values of these columns like we do other columns because they are # a different type, and as such, LIKE and != do not work on them. Furthermore, because the # use case for these values in autosuggestion is minimal, so we choose to disable them here. # # event_id: This is a FixedString which disallows us to use LIKE on it when searching, # but does work with !=. However, for consistency sake we disallow it # entirely, furthermore, suggesting an event_id is not a very useful feature # as they are not human readable. # timestamp: This is a DateTime which disallows us to use both LIKE and != on it when # searching. Suggesting a timestamp can potentially be useful but as it does # work at all, we opt to disable it here. A potential solution can be to # generate a time range to bound where they are searching. e.g. if a user # enters 2020-07 we can generate the following conditions: # >= 2020-07-01T00:00:00 AND <= 2020-07-31T23:59:59 # time: This is a column computed from timestamp so it suffers the same issues if snuba_key in {"event_id", "timestamp", "time"}: return SequencePaginator([]) # These columns have fixed values and we don't need to emit queries to find out the # potential options. if key in {"error.handled", "error.unhandled"}: return SequencePaginator( [ ( 1, TagValue( key=key, value="true", times_seen=None, first_seen=None, last_seen=None ), ), ( 2, TagValue( key=key, value="false", times_seen=None, first_seen=None, last_seen=None ), ), ] ) conditions = [] # transaction status needs a special case so that the user interacts with the names and not codes transaction_status = snuba_key == "transaction_status" if include_transactions and transaction_status: # Here we want to use the status codes during filtering, # but want to do this with names that include our query status_codes = [ span_key for span_key, value in six.iteritems(SPAN_STATUS_CODE_TO_NAME) if (query and query in value) or (not query) ] if status_codes: conditions.append([snuba_key, "IN", status_codes]) else: return SequencePaginator([]) elif key in FUZZY_NUMERIC_KEYS: converted_query = int(query) if query is not None and query.isdigit() else None if converted_query is not None: conditions.append([snuba_key, ">=", converted_query - FUZZY_NUMERIC_DISTANCE]) conditions.append([snuba_key, "<=", converted_query + FUZZY_NUMERIC_DISTANCE]) elif include_transactions and key == PROJECT_ALIAS: project_filters = { "id__in": projects, } if query: project_filters["slug__icontains"] = query project_queryset = Project.objects.filter(**project_filters).values("id", "slug") if not project_queryset.exists(): return SequencePaginator([]) project_slugs = {project["id"]: project["slug"] for project in project_queryset} projects = [project["id"] for project in project_queryset] snuba_key = "project_id" dataset = Dataset.Discover else: snuba_name = snuba_key is_user_alias = include_transactions and key == USER_DISPLAY_ALIAS if is_user_alias: # user.alias is a pseudo column in discover. It is computed by coalescing # together multiple user attributes. Here we get the coalese function used, # and resolve it to the corresponding snuba query dataset = Dataset.Discover resolver = snuba.resolve_column(dataset) snuba_name = FIELD_ALIASES[USER_DISPLAY_ALIAS].get_field() snuba.resolve_complex_column(snuba_name, resolver) elif snuba_name in BLACKLISTED_COLUMNS: snuba_name = "tags[%s]" % (key,) if query: conditions.append([snuba_name, "LIKE", u"%{}%".format(query)]) else: conditions.append([snuba_name, "!=", ""]) filters = {"project_id": projects} if environments: filters["environment"] = environments results = snuba.query( dataset=dataset, start=start, end=end, groupby=[snuba_key], filter_keys=filters, aggregations=[ ["count()", "", "times_seen"], ["min", "timestamp", "first_seen"], ["max", "timestamp", "last_seen"], ], conditions=conditions, orderby=order_by, # TODO: This means they can't actually paginate all TagValues. limit=1000, arrayjoin=snuba.get_arrayjoin(snuba_key), referrer="tagstore.get_tag_value_paginator_for_projects", ) if include_transactions: # With transaction_status we need to map the ids back to their names if transaction_status: results = OrderedDict( [ (SPAN_STATUS_CODE_TO_NAME[result_key], data) for result_key, data in six.iteritems(results) ] ) # With project names we map the ids back to the project slugs elif key == PROJECT_ALIAS: results = OrderedDict( [ (project_slugs[value], data) for value, data in six.iteritems(results) if value in project_slugs ] ) tag_values = [ TagValue(key=key, value=six.text_type(value), **fix_tag_value_data(data)) for value, data in six.iteritems(results) ] desc = order_by.startswith("-") score_field = order_by.lstrip("-") return SequencePaginator( [(int(to_timestamp(getattr(tv, score_field)) * 1000), tv) for tv in tag_values], reverse=desc, )