def test_group_types_to_query(self): group_types_to_query = lambda filter: EnterpriseColumnOptimizer( filter, self.team.id).group_types_to_query self.assertEqual(group_types_to_query(BASE_FILTER), set()) self.assertEqual(group_types_to_query(FILTER_WITH_PROPERTIES), {2}) self.assertEqual(group_types_to_query(FILTER_WITH_GROUPS), {2})
def test_materialized_columns_checks(self): optimizer = lambda: EnterpriseColumnOptimizer(FILTER_WITH_PROPERTIES, self.team.id) optimizer_groups = lambda: EnterpriseColumnOptimizer(FILTER_WITH_GROUPS, self.team.id) self.assertEqual(optimizer().event_columns_to_query, {"properties"}) self.assertEqual(optimizer().person_columns_to_query, {"properties"}) self.assertEqual(optimizer_groups().event_columns_to_query, {"properties"}) self.assertEqual(optimizer_groups().person_columns_to_query, {"properties"}) materialize("events", "event_prop") materialize("person", "person_prop") self.assertEqual(optimizer().event_columns_to_query, {"mat_event_prop"}) self.assertEqual(optimizer().person_columns_to_query, {"pmat_person_prop"}) self.assertEqual(optimizer_groups().event_columns_to_query, {"mat_event_prop"}) self.assertEqual(optimizer_groups().person_columns_to_query, {"pmat_person_prop"})
def __init__( self, filter: Union[Filter, PathFilter, RetentionFilter, StickinessFilter, SessionRecordingsFilter], team: Team, round_interval=False, should_join_distinct_ids=False, should_join_persons=False, # Extra events/person table columns to fetch since parent query needs them extra_fields: List[ColumnName] = [], extra_event_properties: List[PropertyName] = [], extra_person_fields: List[ColumnName] = [], override_aggregate_users_by_distinct_id: Optional[bool] = None, using_person_on_events: bool = False, **kwargs, ) -> None: super().__init__( filter=filter, team=team, round_interval=round_interval, should_join_distinct_ids=should_join_distinct_ids, should_join_persons=should_join_persons, extra_fields=extra_fields, extra_event_properties=extra_event_properties, extra_person_fields=extra_person_fields, override_aggregate_users_by_distinct_id=override_aggregate_users_by_distinct_id, using_person_on_events=using_person_on_events, **kwargs, ) self._column_optimizer = EnterpriseColumnOptimizer(self._filter, self._team_id)
def __init__( self, entity: Entity, filter: Filter, team: Team, column_optimizer: Optional[EnterpriseColumnOptimizer] = None ): self.entity = entity self.filter = filter self.team = team self.team_id = team.pk self.params: Dict[str, Any] = {"team_id": team.pk} self.column_optimizer = column_optimizer or EnterpriseColumnOptimizer(self.filter, self.team_id)
def test_properties_used_in_filter_with_actions(self): action = Action.objects.create(team=self.team) ActionStep.objects.create( event="$autocapture", action=action, url="https://example.com/donate", url_matching=ActionStep.EXACT, ) ActionStep.objects.create( action=action, event="$autocapture", tag_name="button", text="Pay $10", properties=[{ "key": "$browser", "value": "Chrome", "type": "person" }], ) filter = Filter(data={"actions": [{"id": action.id, "math": "dau"}]}) self.assertEqual( EnterpriseColumnOptimizer(filter, self.team.id).properties_used_in_filter, { ("$current_url", "event", None): 1, ("$browser", "person", None): 1 }, ) filter = BASE_FILTER.with_data( {"exclusions": [{ "id": action.id, "type": "actions" }]}) self.assertEqual( EnterpriseColumnOptimizer(filter, self.team.id).properties_used_in_filter, { ("$current_url", "event", None): 1, ("$browser", "person", None): 1 }, )
def __init__( self, filter: Union[Filter, PathFilter, RetentionFilter, StickinessFilter], team_id: int, column_optimizer: Optional[EnterpriseColumnOptimizer] = None, join_key: Optional[str] = None, ) -> None: self._filter = filter self._team_id = team_id self._column_optimizer = column_optimizer or EnterpriseColumnOptimizer( self._filter, self._team_id) self._join_key = join_key
def test_should_query_element_chain_column_with_actions(self): action = Action.objects.create(team=self.team) ActionStep.objects.create( event="$autocapture", action=action, url="https://example.com/donate", url_matching=ActionStep.EXACT, ) filter = Filter(data={"actions": [{"id": action.id, "math": "dau"}]}) self.assertEqual( EnterpriseColumnOptimizer( filter, self.team.id).should_query_elements_chain_column, False, ) ActionStep.objects.create( action=action, event="$autocapture", tag_name="button", text="Pay $10", ) self.assertEqual( EnterpriseColumnOptimizer( filter, self.team.id).should_query_elements_chain_column, True, ) filter = BASE_FILTER.with_data( {"exclusions": [{ "id": action.id, "type": "actions" }]}) self.assertEqual( EnterpriseColumnOptimizer( filter, self.team.id).should_query_elements_chain_column, True, )
def test_should_query_element_chain_column(self): should_query_elements_chain_column = lambda filter: EnterpriseColumnOptimizer( filter, self.team.id ).should_query_elements_chain_column self.assertEqual(should_query_elements_chain_column(BASE_FILTER), False) self.assertEqual(should_query_elements_chain_column(FILTER_WITH_PROPERTIES), True) self.assertEqual(should_query_elements_chain_column(FILTER_WITH_GROUPS), True) filter = Filter( data={"events": [{"id": "$pageview", "type": "events", "order": 0, "properties": PROPERTIES_OF_ALL_TYPES,}]} ) self.assertEqual(should_query_elements_chain_column(filter), True)
def _get_aggregation_join_query(self): if self._filter.aggregation_group_type_index is None: person_query, person_query_params = PersonQuery( self._filter, self._team.pk, EnterpriseColumnOptimizer(self._filter, self._team.pk) ).get_query() return ( f""" JOIN ({person_query}) person ON person.id = funnel_actors.actor_id """, person_query_params, ) else: return GroupsJoinQuery(self._filter, self._team.pk, join_key="funnel_actors.actor_id").get_join_query()
def test_properties_used_in_filter(self): properties_used_in_filter = lambda filter: EnterpriseColumnOptimizer( filter, self.team.id).properties_used_in_filter self.assertEqual(properties_used_in_filter(BASE_FILTER), {}) self.assertEqual( properties_used_in_filter(FILTER_WITH_PROPERTIES), { ("event_prop", "event", None): 1, ("person_prop", "person", None): 1, ("id", "cohort", None): 1, ("tag_name", "element", None): 1, ("group_prop", "group", 2): 1, }, ) self.assertEqual( properties_used_in_filter(FILTER_WITH_GROUPS), { ("event_prop", "event", None): 1, ("person_prop", "person", None): 1, ("id", "cohort", None): 1, ("tag_name", "element", None): 1, ("group_prop", "group", 2): 1, }, ) # Breakdown cases filter = BASE_FILTER.with_data({ "breakdown": "some_prop", "breakdown_type": "person" }) self.assertEqual(properties_used_in_filter(filter), {("some_prop", "person", None): 1}) filter = BASE_FILTER.with_data({ "breakdown": "some_prop", "breakdown_type": "event" }) self.assertEqual(properties_used_in_filter(filter), {("some_prop", "event", None): 1}) filter = BASE_FILTER.with_data({ "breakdown": [11], "breakdown_type": "cohort" }) self.assertEqual(properties_used_in_filter(filter), {}) filter = BASE_FILTER.with_data({ "breakdown": "some_prop", "breakdown_type": "group", "breakdown_group_type_index": 1 }) self.assertEqual(properties_used_in_filter(filter), {("some_prop", "group", 1): 1}) # Funnel Correlation cases filter = BASE_FILTER.with_data({ "funnel_correlation_type": "events", "funnel_correlation_names": ["random_column"] }) self.assertEqual(properties_used_in_filter(filter), {}) filter = BASE_FILTER.with_data({ "funnel_correlation_type": "properties", "funnel_correlation_names": ["random_column", "$browser"] }) self.assertEqual(properties_used_in_filter(filter), { ("random_column", "person", None): 1, ("$browser", "person", None): 1 }) filter = BASE_FILTER.with_data({ "funnel_correlation_type": "properties", "funnel_correlation_names": ["random_column", "$browser"], "aggregation_group_type_index": 2, }) self.assertEqual(properties_used_in_filter(filter), { ("random_column", "group", 2): 1, ("$browser", "group", 2): 1 }) filter = BASE_FILTER.with_data( {"funnel_correlation_type": "properties"}) self.assertEqual(properties_used_in_filter(filter), {}) filter = Filter( data={ "events": [{ "id": "$pageview", "type": "events", "order": 0, "math": "sum", "math_property": "numeric_prop", "properties": PROPERTIES_OF_ALL_TYPES, }] }) self.assertEqual( properties_used_in_filter(filter), { ("numeric_prop", "event", None): 1, ("event_prop", "event", None): 1, ("person_prop", "person", None): 1, ("id", "cohort", None): 1, ("tag_name", "element", None): 1, ("group_prop", "group", 2): 1, }, ) filter = Filter( data={ "events": [{ "id": "$pageview", "type": "events", "order": 0, "math": "unique_group", "math_group_type_index": 1, }] }) self.assertEqual( properties_used_in_filter(filter), { ("$group_1", "event", None): 1, }, ) filter = Filter( data={ "events": [{ "id": "$pageview", "type": "events", "order": 0, "math": "unique_session", }] }) self.assertEqual( properties_used_in_filter(filter), { ("$session_id", "event", None): 1, }, )
def get_breakdown_prop_values( filter: Filter, entity: Entity, aggregate_operation: str, team_id: int, limit: int = BREAKDOWN_VALUES_LIMIT, extra_params={}, column_optimizer: Optional[EnterpriseColumnOptimizer] = None, ): """ Returns the top N breakdown prop values for event/person breakdown e.g. for Browser with limit 3 might return ['Chrome', 'Safari', 'Firefox', 'Other'] """ column_optimizer = column_optimizer or EnterpriseColumnOptimizer( filter, team_id) parsed_date_from, parsed_date_to, date_params = parse_timestamps( filter=filter, team_id=team_id) props_to_filter = filter.property_groups.combine_property_group( PropertyOperatorType.AND, entity.property_groups) outer_properties = column_optimizer.property_optimizer.parse_property_groups( props_to_filter).outer prop_filters, prop_filter_params = parse_prop_grouped_clauses( team_id=team_id, property_group=outer_properties, table_name="e", prepend="e_brkdwn", person_properties_mode=PersonPropertiesMode. USING_PERSON_PROPERTIES_COLUMN, allow_denormalized_props=True, ) entity_params, entity_format_params = get_entity_filtering_params( entity=entity, team_id=team_id, table_name="e") value_expression = _to_value_expression(filter.breakdown_type, filter.breakdown, filter.breakdown_group_type_index) person_join_clauses = "" person_join_params: Dict = {} person_query = PersonQuery(filter, team_id, column_optimizer=column_optimizer, entity=entity) if person_query.is_used: person_subquery, person_join_params = person_query.get_query() person_join_clauses = f""" INNER JOIN ({get_team_distinct_ids_query(team_id)}) AS pdi ON e.distinct_id = pdi.distinct_id INNER JOIN ({person_subquery}) person ON pdi.person_id = person.id """ groups_join_condition, groups_join_params = GroupsJoinQuery( filter, team_id, column_optimizer).get_join_query() elements_query = TOP_ELEMENTS_ARRAY_OF_KEY_SQL.format( value_expression=value_expression, parsed_date_from=parsed_date_from, parsed_date_to=parsed_date_to, prop_filters=prop_filters, aggregate_operation=aggregate_operation, person_join_clauses=person_join_clauses, groups_join_clauses=groups_join_condition, **entity_format_params, ) return sync_execute( elements_query, { "key": filter.breakdown, "limit": limit, "team_id": team_id, "offset": filter.offset, **prop_filter_params, **entity_params, **person_join_params, **groups_join_params, **extra_params, **date_params, }, )[0][0]