def test_invalid_query_set() -> None: query = Query("discover", Entity("events")) tests: Mapping[str, Sequence[Any]] = { "match": (0, "0 must be a valid Entity"), "select": ( (0, [], [0]), "select clause must be a non-empty list of Column and/or Function", ), "groupby": ( [0, [0]], "groupby clause must be a list of Column and/or Function", ), "where": ([0, [0]], "where clause must be a list of Condition"), "having": ([0, [0]], "having clause must be a list of Condition"), "orderby": ([0, [0]], "orderby clause must be a list of OrderBy"), "limitby": ("a", "limitby clause must be a LimitBy"), "limit": (100000, "limit '100000' is capped at 10,000"), "offset": ("", "offset '' must be an integer"), "granularity": (-1, "granularity '-1' must be at least 1"), } match, err = tests["match"] with pytest.raises(InvalidQuery, match=re.escape(err)): query.set_match(match) for val in tests["select"][0]: with pytest.raises(InvalidQuery, match=re.escape(tests["select"][1])): query.set_select(val) for val in tests["groupby"][0]: with pytest.raises(InvalidQuery, match=re.escape(tests["groupby"][1])): query.set_groupby(val) for val in tests["where"][0]: with pytest.raises(InvalidQuery, match=re.escape(tests["where"][1])): query.set_where(val) for val in tests["having"][0]: with pytest.raises(InvalidQuery, match=re.escape(tests["having"][1])): query.set_having(val) for val in tests["orderby"][0]: with pytest.raises(InvalidQuery, match=re.escape(tests["orderby"][1])): query.set_orderby(val) with pytest.raises(InvalidQuery, match=re.escape(tests["limitby"][1])): query.set_limitby(tests["limitby"][0]) with pytest.raises(InvalidExpression, match=re.escape(tests["limit"][1])): query.set_limit(tests["limit"][0]) with pytest.raises(InvalidExpression, match=re.escape(tests["offset"][1])): query.set_offset(tests["offset"][0]) with pytest.raises(InvalidExpression, match=re.escape(tests["granularity"][1])): query.set_granularity(tests["granularity"][0])
def test_entity_validate_match(query: Query, exception: Optional[Exception]) -> None: query = query.set_where([ Condition(Column("required1"), Op.IN, [1, 2, 3]), Condition(Column("required2"), Op.EQ, 1), Condition(Column("time"), Op.GTE, BEFORE), Condition(Column("time"), Op.LT, AFTER), ], ) if exception is not None: with pytest.raises(type(exception), match=re.escape(str(exception))): validate_match(query, SEARCHER) else: validate_match(query, SEARCHER)
def test_entity_validate_match( conditions: ConditionGroup, entity: Entity, exception: Optional[Exception], ) -> None: query = Query(dataset="test", match=entity, select=[Column("test1"), Column("required1")]) query = query.set_where(conditions) if exception is not None: with pytest.raises(type(exception), match=re.escape(str(exception))): validate_required_columns(query) else: validate_required_columns(query)
def json_to_snql(body: Mapping[str, Any], entity: str) -> Query: """ This will output a Query object that matches the Legacy query body that was passed in. The entity is necessary since the SnQL API requires an explicit entity. This doesn't support subquery or joins. :param body: The legacy API body. :type body: Mapping[str, Any] :param entity: The name of the entity being queried. :type entity: str :raises InvalidExpressionError, InvalidQueryError: If the legacy body is invalid, the SDK will raise an exception. """ dataset = body.get("dataset") or entity sample = body.get("sample") if sample is not None: sample = float(sample) query = Query(dataset, Entity(entity, None, sample)) selected_columns = [] for a in body.get("aggregations", []): selected_columns.append(parse_exp(list(a))) selected = [] for s in body.get("selected_columns", []): if isinstance(s, tuple): selected.append(list(s)) else: selected.append(s) selected_columns.extend(list(map(parse_exp, selected))) arrayjoin = body.get("arrayjoin") if arrayjoin: query = query.set_array_join([Column(arrayjoin)]) query = query.set_select(selected_columns) groupby = body.get("groupby", []) if groupby and not isinstance(groupby, list): groupby = [groupby] parsed_groupby = [] for g in groupby: if isinstance(g, tuple): g = list(g) parsed_groupby.append(parse_exp(g)) query = query.set_groupby(parsed_groupby) conditions: list[Union[Or, Condition]] = [] if body.get("organization"): org_cond = parse_extension_condition("org_id", body["organization"]) if org_cond: conditions.append(org_cond) assert isinstance(query.match, Entity) time_column = get_required_time_column(query.match.name) if time_column: time_cols = (("from_date", Op.GTE), ("to_date", Op.LT)) for col, op in time_cols: date_val = body.get(col) if date_val: conditions.append( Condition(Column(time_column), op, parse_datetime(date_val))) if body.get("project"): proj_cond = parse_extension_condition("project_id", body["project"], True) if proj_cond: conditions.append(proj_cond) for cond in body.get("conditions", []): if not is_condition(cond): or_conditions = [] for or_cond in cond: or_conditions.append(parse_condition(or_cond)) if len(or_conditions) > 1: conditions.append(Or(or_conditions)) else: conditions.extend(or_conditions) else: conditions.append(parse_condition(cond)) query = query.set_where(conditions) having: list[Union[Or, Condition]] = [] for cond in body.get("having", []): if not is_condition(cond): or_conditions = [] for or_cond in cond: or_conditions.append(parse_condition(or_cond)) having.append(Or(or_conditions)) else: having.append(parse_condition(cond)) query = query.set_having(having) order_by = body.get("orderby") if order_by: if not isinstance(order_by, list): order_by = [order_by] order_bys = [] for o in order_by: direction = Direction.ASC if isinstance(o, list): first = o[0] if isinstance(first, str) and first.startswith("-"): o[0] = first.lstrip("-") direction = Direction.DESC part = parse_exp(o) elif isinstance(o, str): if o.startswith("-"): direction = Direction.DESC part = parse_exp(o.lstrip("-")) else: part = parse_exp(o) order_bys.append(OrderBy(part, direction)) query = query.set_orderby(order_bys) limitby = body.get("limitby") if limitby: limit, name = limitby query = query.set_limitby(LimitBy([Column(name)], int(limit))) extras = ( "limit", "offset", "granularity", "totals", "consistent", "turbo", "debug", "dry_run", "parent_api", ) for extra in extras: if body.get(extra) is not None: query = getattr(query, f"set_{extra}")(body.get(extra)) query.set_legacy(True) return query
def json_to_snql(body: Mapping[str, Any], entity: str) -> Query: dataset = body.get("dataset", "") sample = body.get("sample") query = Query(dataset, Entity(entity, sample)) selected_columns = list(map(parse_exp, body.get("selected_columns", []))) for a in body.get("aggregations", []): selected_columns.append(parse_exp(a)) arrayjoin = body.get("arrayjoin") if arrayjoin: selected_columns.append( Function("arrayJoin", [Column(arrayjoin)], arrayjoin)) query = query.set_select(selected_columns) groupby = body.get("groupby", []) if groupby and not isinstance(groupby, list): groupby = [groupby] query = query.set_groupby(list(map(parse_exp, groupby))) conditions = [] for cond in body.get("conditions", []): if len(cond) != 3 or not isinstance(cond[1], str): raise InvalidQuery("OR conditions not supported yet") conditions.append( Condition(parse_exp(cond[0]), Op(cond[1]), parse_scalar(cond[2]))) extra_conditions = [("project", "project_id"), ("organization", "org_id")] for cond, col in extra_conditions: column = Column(col) values = body.get(cond) if isinstance(values, int): conditions.append(Condition(column, Op.EQ, values)) elif isinstance(values, list): rhs: Sequence[Any] = list(map(parse_scalar, values)) conditions.append(Condition(column, Op.IN, rhs)) elif isinstance(values, tuple): rhs = tuple(map(parse_scalar, values)) conditions.append(Condition(column, Op.IN, rhs)) date_conds = [("from_date", Op.GT), ("to_date", Op.LTE)] for cond, op in date_conds: date_str = body.get(cond, "") if date_str: # HACK: This is to get sessions working quickly. # The time column should depend on the entity. conditions.append( Condition(Column("started"), op, parse_datetime(date_str))) query = query.set_where(conditions) having = [] for cond in body.get("having", []): if len(cond) != 3 or not isinstance(cond[1], str): raise InvalidQuery("OR conditions not supported yet") having.append( Condition(parse_exp(cond[0]), Op(cond[1]), parse_scalar(cond[2]))) query = query.set_having(having) order_by = body.get("orderby") if order_by: if not isinstance(order_by, list): order_by = [order_by] order_bys = [] for o in order_by: direction = Direction.ASC if isinstance(o, str) and o.startswith("-"): direction = Direction.DESC o = o.lstrip("-") order_bys.append(OrderBy(parse_exp(o), direction)) query = query.set_orderby(order_bys) limitby = body.get("limitby") if limitby: limit, name = limitby query = query.set_limitby(LimitBy(Column(name), int(limit))) extras = ( "limit", "offset", "granularity", "totals", "consistent", "turbo", "debug", ) for extra in extras: if body.get(extra) is not None: query = getattr(query, f"set_{extra}")(body.get(extra)) return query