def test_not_many_groups_to_exclude(query: ClickhouseQuery) -> None:
state.set_config("max_group_ids_exclude", 5)
set_project_exclude_groups(2, [100, 101, 102], ReplacerState.EVENTS)
PostReplacementConsistencyEnforcer(
"project_id", ReplacerState.EVENTS
).process_query(query, HTTPRequestSettings())
assert query.get_condition_from_ast() == FunctionCall(
None,
BooleanFunctions.AND,
(
FunctionCall(
None,
"notIn",
(
FunctionCall(
None, "assumeNotNull", (Column(None, None, "group_id"),)
),
FunctionCall(
None,
"tuple",
(Literal(None, 100), Literal(None, 101), Literal(None, 102),),
),
),
),
build_in("project_id", [2]),
),
)
assert not query.get_from_clause().final
def test_query_time_flags(self):
project_ids = [1, 2]
assert errors_replacer.get_projects_query_flags(
project_ids, ReplacerState.ERRORS
) == (False, [],)
errors_replacer.set_project_needs_final(100, ReplacerState.ERRORS)
assert errors_replacer.get_projects_query_flags(
project_ids, ReplacerState.ERRORS
) == (False, [],)
errors_replacer.set_project_needs_final(1, ReplacerState.ERRORS)
assert errors_replacer.get_projects_query_flags(
project_ids, ReplacerState.ERRORS
) == (True, [],)
assert errors_replacer.get_projects_query_flags(
project_ids, ReplacerState.EVENTS
) == (False, [],)
errors_replacer.set_project_needs_final(2, ReplacerState.ERRORS)
assert errors_replacer.get_projects_query_flags(
project_ids, ReplacerState.ERRORS
) == (True, [],)
errors_replacer.set_project_exclude_groups(1, [1, 2], ReplacerState.ERRORS)
errors_replacer.set_project_exclude_groups(2, [3, 4], ReplacerState.ERRORS)
assert errors_replacer.get_projects_query_flags(
project_ids, ReplacerState.ERRORS
) == (True, [1, 2, 3, 4],)
assert errors_replacer.get_projects_query_flags(
project_ids, ReplacerState.EVENTS
) == (False, [],)
def test_multiple_not_too_many_excludes(
query_with_multiple_group_ids: ClickhouseQuery, ) -> None:
"""
Query is looking for multiple groups and there are not too many groups to exclude, but
there are fewer groups queried for than replaced.
"""
enforcer = PostReplacementConsistencyEnforcer("project_id",
ReplacerState.ERRORS)
set_project_exclude_groups(
2,
[100, 101, 102],
ReplacerState.ERRORS,
ReplacementType.
EXCLUDE_GROUPS, # Arbitrary replacement type, no impact on tests
)
enforcer._set_query_final(query_with_multiple_group_ids, True)
state.set_config("max_group_ids_exclude", 5)
enforcer.process_query(query_with_multiple_group_ids, HTTPQuerySettings())
assert query_with_multiple_group_ids.get_condition() == build_and(
build_not_in("group_id", [101, 102]),
build_and(build_in("project_id", [2]),
build_in("group_id", [101, 102])),
)
assert not query_with_multiple_group_ids.get_from_clause().final
def test_single_too_many_exclude(
query_with_single_group_id: ClickhouseQuery) -> None:
"""
Query is looking for a group that has been replaced, and there are too many
groups to exclude.
"""
enforcer = PostReplacementConsistencyEnforcer("project_id",
ReplacerState.ERRORS)
set_project_exclude_groups(
2,
[100, 101, 102],
ReplacerState.ERRORS,
ReplacementType.
EXCLUDE_GROUPS, # Arbitrary replacement type, no impact on tests
)
enforcer._set_query_final(query_with_single_group_id, True)
state.set_config("max_group_ids_exclude", 2)
enforcer.process_query(query_with_single_group_id, HTTPQuerySettings())
assert query_with_single_group_id.get_condition() == build_and(
build_not_in("group_id", [101]),
build_and(build_in("project_id", [2]), build_in("group_id", [101])),
)
assert not query_with_single_group_id.get_from_clause().final
def test_not_many_groups_to_exclude(query: ClickhouseQuery) -> None:
state.set_config("max_group_ids_exclude", 5)
set_project_exclude_groups(
2,
[100, 101, 102],
ReplacerState.ERRORS,
ReplacementType.
EXCLUDE_GROUPS, # Arbitrary replacement type, no impact on tests
)
PostReplacementConsistencyEnforcer("project_id",
ReplacerState.ERRORS).process_query(
query, HTTPQuerySettings())
assert query.get_condition() == build_and(
FunctionCall(
None,
"notIn",
(
FunctionCall(None, "assumeNotNull",
(Column(None, None, "group_id"), )),
FunctionCall(
None,
"tuple",
(
Literal(None, 100),
Literal(None, 101),
Literal(None, 102),
),
),
),
),
build_in("project_id", [2]),
)
assert not query.get_from_clause().final
def test_query_overlaps_replacements_processor(
query: ClickhouseQuery,
query_with_timestamp: ClickhouseQuery,
query_with_future_timestamp: ClickhouseQuery,
) -> None:
enforcer = PostReplacementConsistencyEnforcer("project_id",
ReplacerState.ERRORS)
# replacement time unknown, default to "overlaps" but no groups to exclude so shouldn't be final
enforcer._set_query_final(query_with_timestamp, True)
enforcer.process_query(query_with_timestamp, HTTPQuerySettings())
assert not query_with_timestamp.get_from_clause().final
# overlaps replacement and should be final due to too many groups to exclude
state.set_config("max_group_ids_exclude", 2)
set_project_exclude_groups(
2,
[100, 101, 102],
ReplacerState.ERRORS,
ReplacementType.
EXCLUDE_GROUPS, # Arbitrary replacement type, no impact on tests
)
enforcer._set_query_final(query_with_timestamp, False)
enforcer.process_query(query_with_timestamp, HTTPQuerySettings())
assert query_with_timestamp.get_from_clause().final
# query time range unknown and should be final due to too many groups to exclude
enforcer._set_query_final(query, False)
enforcer.process_query(query, HTTPQuerySettings())
assert query.get_from_clause().final
# doesn't overlap replacements
enforcer._set_query_final(query_with_future_timestamp, True)
enforcer.process_query(query_with_future_timestamp, HTTPQuerySettings())
assert not query_with_future_timestamp.get_from_clause().final
def test_query_time_flags_project_and_groups(self) -> None:
"""
Tests errors_replacer.set_project_needs_final() and
errors_replacer.set_project_exclude_groups() work together as expected.
ReplacementType's are arbitrary, just need to show up in
getter appropriately once set.
"""
redis_client.flushdb()
project_ids = [7, 8, 9]
errors_replacer.set_project_needs_final(7, ReplacerState.ERRORS,
ReplacementType.EXCLUDE_GROUPS)
errors_replacer.set_project_exclude_groups(7, [1, 2],
ReplacerState.ERRORS,
ReplacementType.START_MERGE)
flags = ProjectsQueryFlags.load_from_redis(project_ids,
ReplacerState.ERRORS)
assert (
flags.needs_final,
flags.group_ids_to_exclude,
flags.replacement_types,
) == (
True,
{1, 2},
# exclude_groups from project setter, start_merge from group setter
{ReplacementType.EXCLUDE_GROUPS, ReplacementType.START_MERGE},
)
def test_too_many_groups_to_exclude(query: ClickhouseQuery) -> None:
state.set_config("max_group_ids_exclude", 2)
set_project_exclude_groups(2, [100, 101, 102], ReplacerState.EVENTS)
PostReplacementConsistencyEnforcer(
"project_id", ReplacerState.EVENTS
).process_query(query, HTTPRequestSettings())
assert query.get_condition_from_ast() == build_in("project_id", [2])
assert query.get_from_clause().final
def test_when_there_are_too_many_groups_to_exclude(self):
request_settings = HTTPRequestSettings()
state.set_config("max_group_ids_exclude", 2)
set_project_exclude_groups(2, [100, 101, 102], ReplacerState.EVENTS)
self.extension.get_processor().process_query(self.query,
self.valid_data,
request_settings)
assert self.query.get_conditions() == [("project_id", "IN", [2])]
assert self.query.get_condition_from_ast() == build_in(
"project_id", [2])
assert self.query.get_final()
def test_too_many_groups_to_exclude(query: ClickhouseQuery) -> None:
state.set_config("max_group_ids_exclude", 2)
set_project_exclude_groups(
2,
[100, 101, 102],
ReplacerState.ERRORS,
ReplacementType.
EXCLUDE_GROUPS, # Arbitrary replacement type, no impact on tests
)
PostReplacementConsistencyEnforcer("project_id",
ReplacerState.ERRORS).process_query(
query, HTTPQuerySettings())
assert query.get_condition() == build_in("project_id", [2])
assert query.get_from_clause().final
def test_no_groups_too_many_excludes(query: ClickhouseQuery) -> None:
"""
Query has no groups, and too many to exclude.
"""
enforcer = PostReplacementConsistencyEnforcer("project_id",
ReplacerState.ERRORS)
set_project_exclude_groups(
2,
[100, 101, 102],
ReplacerState.ERRORS,
ReplacementType.
EXCLUDE_GROUPS, # Arbitrary replacement type, no impact on tests
)
enforcer._set_query_final(query, True)
state.set_config("max_group_ids_exclude", 1)
enforcer.process_query(query, HTTPQuerySettings())
assert query.get_condition() == build_in("project_id", [2])
assert query.get_from_clause().final
def test_when_there_are_not_many_groups_to_exclude(self):
request_settings = HTTPRequestSettings()
state.set_config("max_group_ids_exclude", 5)
set_project_exclude_groups(2, [100, 101, 102], ReplacerState.EVENTS)
self.extension.get_processor().process_query(self.query,
self.valid_data,
request_settings)
expected = [
("project_id", "IN", [2]),
(["assumeNotNull", ["group_id"]], "NOT IN", [100, 101, 102]),
]
assert self.query.get_conditions() == expected
assert self.query.get_condition_from_ast() == FunctionCall(
None,
BooleanFunctions.AND,
(
FunctionCall(
None,
"notIn",
(
FunctionCall(None, "assumeNotNull",
(Column(None, "group_id", None), )),
FunctionCall(
None,
"tuple",
(
Literal(None, 100),
Literal(None, 101),
Literal(None, 102),
),
),
),
),
build_in("project_id", [2]),
),
)
assert not self.query.get_final()
def test_multiple_disjoint_replaced(
query_with_multiple_group_ids: ClickhouseQuery, ) -> None:
"""
Query is looking for multiple groups and there are replaced groups, but these
sets of group ids are disjoint. (No queried groups have been replaced)
"""
enforcer = PostReplacementConsistencyEnforcer("project_id",
ReplacerState.ERRORS)
set_project_exclude_groups(
2,
[110, 120, 130],
ReplacerState.ERRORS,
ReplacementType.
EXCLUDE_GROUPS, # Arbitrary replacement type, no impact on tests
)
enforcer._set_query_final(query_with_multiple_group_ids, True)
state.set_config("max_group_ids_exclude", 5)
enforcer.process_query(query_with_multiple_group_ids, HTTPQuerySettings())
assert query_with_multiple_group_ids.get_condition() == build_and(
build_in("project_id", [2]), build_in("group_id", [101, 102]))
assert not query_with_multiple_group_ids.get_from_clause().final
def test_query_time_flags_groups(self) -> None:
"""
Tests errors_replacer.set_project_exclude_groups()
ReplacementType's are arbitrary, just need to show up in
getter appropriately once set.
"""
redis_client.flushdb()
project_ids = [4, 5, 6]
errors_replacer.set_project_exclude_groups(
4, [1, 2], ReplacerState.ERRORS, ReplacementType.EXCLUDE_GROUPS)
errors_replacer.set_project_exclude_groups(5, [3, 4],
ReplacerState.ERRORS,
ReplacementType.START_MERGE)
flags = ProjectsQueryFlags.load_from_redis(project_ids,
ReplacerState.ERRORS)
assert (
flags.needs_final,
flags.group_ids_to_exclude,
flags.replacement_types,
) == (
False,
{1, 2, 3, 4},
{ReplacementType.EXCLUDE_GROUPS, ReplacementType.START_MERGE},
)
errors_replacer.set_project_exclude_groups(
4, [1, 2], ReplacerState.ERRORS, ReplacementType.EXCLUDE_GROUPS)
errors_replacer.set_project_exclude_groups(
5, [3, 4], ReplacerState.ERRORS, ReplacementType.EXCLUDE_GROUPS)
errors_replacer.set_project_exclude_groups(
6, [5, 6], ReplacerState.ERRORS, ReplacementType.START_UNMERGE)
flags = ProjectsQueryFlags.load_from_redis(project_ids,
ReplacerState.ERRORS)
assert (
flags.needs_final,
flags.group_ids_to_exclude,
flags.replacement_types,
) == (
False,
{1, 2, 3, 4, 5, 6},
{
ReplacementType.EXCLUDE_GROUPS,
# start_merge should show up from previous setter on project id 2
ReplacementType.START_MERGE,
ReplacementType.START_UNMERGE,
},
)
flags = ProjectsQueryFlags.load_from_redis([4, 5],
ReplacerState.ERRORS)
assert (
flags.needs_final,
flags.group_ids_to_exclude,
flags.replacement_types,
) == (
False,
{1, 2, 3, 4},
{ReplacementType.EXCLUDE_GROUPS, ReplacementType.START_MERGE},
)
flags = ProjectsQueryFlags.load_from_redis([4], ReplacerState.ERRORS)
assert (
flags.needs_final,
flags.group_ids_to_exclude,
flags.replacement_types,
) == (
False,
{1, 2},
{ReplacementType.EXCLUDE_GROUPS},
)
