def _save_team(team: Team, event_names: Dict[str, Dict], event_properties: Dict[str, Dict]) -> None: def _sort(to_sort: List) -> List: return sorted(to_sort, key=lambda item: (item.get("usage_count", 0), item.get("volume", 0)), reverse=True) team.event_names_with_usage = _sort([val for _, val in event_names.items()]) team.event_properties_with_usage = _sort([val for _, val in event_properties.items()]) team.save()
def store_names_and_properties(team: Team, event: str, properties: Dict) -> None: # In _capture we only prefetch a couple of fields in Team to avoid fetching too much data save = False if not team.ingested_event: # First event for the team captured for user in Team.objects.get(pk=team.pk).users.all(): posthoganalytics.capture(user.distinct_id, "first team event ingested", {"team": str(team.uuid)}) team.ingested_event = True save = True if event not in team.event_names: save = True team.event_names.append(event) for key, value in properties.items(): if key not in team.event_properties: team.event_properties.append(key) save = True if isinstance(value, Number) and key not in team.event_properties_numerical: team.event_properties_numerical.append(key) save = True if save: team.save()
def _calculate_by_filter(filter: FilterType, key: str, team_id: int, cache_type: CacheType) -> List[Dict[str, Any]]: insight_class = CACHE_TYPE_TO_INSIGHT_CLASS[cache_type] if cache_type == CacheType.PATHS: result = insight_class(filter, Team(pk=team_id)).run(filter, Team(pk=team_id)) else: result = insight_class().run(filter, Team(pk=team_id)) return result
def _store_names_and_properties(team: Team, event: str, properties: Dict) -> None: # In _capture we only prefetch a couple of fields in Team to avoid fetching too much data save = False if event not in team.event_names: save = True team.event_names.append(event) for key in properties.keys(): if key not in team.event_properties: team.event_properties.append(key) save = True if save: team.save()
def store_names_and_properties(team: Team, event: str, properties: Dict) -> None: # In _capture we only prefetch a couple of fields in Team to avoid fetching too much data save = False if event not in team.event_names: save = True team.event_names.append(event) for key, value in properties.items(): if key not in team.event_properties: team.event_properties.append(key) save = True if isinstance(value, Number) and key not in team.event_properties_numerical: team.event_properties_numerical.append(key) save = True if save: team.save()
def _calculate_funnel(filter: Filter, team_id: int) -> List[Dict[str, Any]]: dashboard_items = DashboardItem.objects.filter(team_id=team_id, filters=filter.to_dict()) dashboard_items.update(refreshing=True) result = Funnel(filter=filter, team=Team(pk=team_id)).run() dashboard_items.update(last_refresh=timezone.now(), refreshing=False) return result
def _calculate_funnel(filter: Filter, key: str, team_id: int) -> List[Dict[str, Any]]: dashboard_items = DashboardItem.objects.filter(team_id=team_id, filters_hash=key) dashboard_items.update(refreshing=True) team = Team(pk=team_id) if is_clickhouse_enabled(): funnel_order_class: Type[ClickhouseFunnelBase] = ClickhouseFunnel if filter.funnel_order_type == FunnelOrderType.UNORDERED: funnel_order_class = ClickhouseFunnelUnordered elif filter.funnel_order_type == FunnelOrderType.STRICT: funnel_order_class = ClickhouseFunnelStrict if filter.funnel_viz_type == FunnelVizType.TRENDS: result = ClickhouseFunnelTrends( team=team, filter=filter, funnel_order_class=funnel_order_class).run() elif filter.funnel_viz_type == FunnelVizType.TIME_TO_CONVERT: result = ClickhouseFunnelTimeToConvert( team=team, filter=filter, funnel_order_class=funnel_order_class).run() else: result = funnel_order_class(team=team, filter=filter).run() else: result = Funnel(filter=filter, team=team).run() dashboard_items.update(last_refresh=timezone.now(), refreshing=False) return result
def _calculate_funnel(filter: Filter, key: str, team_id: int) -> List[Dict[str, Any]]: team = Team(pk=team_id) if is_clickhouse_enabled(): funnel_order_class: Type[ClickhouseFunnelBase] = ClickhouseFunnel if filter.funnel_order_type == FunnelOrderType.UNORDERED: funnel_order_class = ClickhouseFunnelUnordered elif filter.funnel_order_type == FunnelOrderType.STRICT: funnel_order_class = ClickhouseFunnelStrict if filter.funnel_viz_type == FunnelVizType.TRENDS: result = ClickhouseFunnelTrends( team=team, filter=filter, funnel_order_class=funnel_order_class).run() elif filter.funnel_viz_type == FunnelVizType.TIME_TO_CONVERT: result = ClickhouseFunnelTimeToConvert( team=team, filter=filter, funnel_order_class=funnel_order_class).run() else: result = funnel_order_class(team=team, filter=filter).run() else: result = Funnel(filter=filter, team=team).run() return result
def _calculate_trends(filter: Filter, key: str, team_id: int) -> List[Dict[str, Any]]: actions = Action.objects.filter(team_id=team_id) actions = actions.prefetch_related(Prefetch("steps", queryset=ActionStep.objects.order_by("id"))) dashboard_items = DashboardItem.objects.filter(team_id=team_id, filters_hash=key) dashboard_items.update(refreshing=True) result = Trends().run(filter, Team(pk=team_id)) dashboard_items.update(last_refresh=timezone.now(), refreshing=False) return result
def _calculate_by_filter(filter: FilterType, key: str, team_id: int, cache_type: CacheType) -> List[Dict[str, Any]]: dashboard_items = DashboardItem.objects.filter(team_id=team_id, filters_hash=key) dashboard_items.update(refreshing=True) insight_class = CACHE_TYPE_TO_INSIGHT_CLASS[cache_type] result = insight_class().run(filter, Team(pk=team_id)) dashboard_items.update(last_refresh=timezone.now(), refreshing=False) return result
def _calculate_funnel(filter: Filter, key: str, team_id: int) -> List[Dict[str, Any]]: dashboard_items = DashboardItem.objects.filter(team_id=team_id, filters_hash=key) dashboard_items.update(refreshing=True) if is_clickhouse_enabled(): insight_class = import_from("ee.clickhouse.queries.clickhouse_funnel", "ClickhouseFunnel") else: insight_class = import_from("posthog.queries.funnel", "Funnel") result = insight_class(filter=filter, team=Team(pk=team_id)).run() dashboard_items.update(last_refresh=timezone.now(), refreshing=False) return result
def _calculate_funnel(filter: Filter, key: str, team_id: int) -> List[Dict[str, Any]]: team = Team(pk=team_id) if filter.funnel_viz_type == FunnelVizType.TRENDS: result = ClickhouseFunnelTrends(team=team, filter=filter).run() elif filter.funnel_viz_type == FunnelVizType.TIME_TO_CONVERT: result = ClickhouseFunnelTimeToConvert(team=team, filter=filter).run() else: funnel_order_class = get_funnel_order_class(filter) result = funnel_order_class(team=team, filter=filter).run() return result
def update_cache_item(key: str, cache_type: CacheType, payload: dict) -> None: result: Optional[Union[List, Dict]] = None filter_dict = json.loads(payload["filter"]) team_id = int(payload["team_id"]) filter = get_filter(data=filter_dict, team=Team(pk=team_id)) if cache_type == CacheType.FUNNEL: result = _calculate_funnel(filter, key, team_id) else: result = _calculate_by_filter(filter, key, team_id, cache_type) if result: cache.set(key, {"result": result, "type": cache_type, "last_refresh": timezone.now()}, CACHED_RESULTS_TTL)
def _calculate_by_filter(filter: FilterType, key: str, team_id: int, cache_type: CacheType) -> List[Dict[str, Any]]: dashboard_items = DashboardItem.objects.filter(team_id=team_id, filters_hash=key) dashboard_items.update(refreshing=True) if is_clickhouse_enabled(): insight_class_path = CH_TYPE_TO_IMPORT[cache_type] else: insight_class_path = TYPE_TO_IMPORT[cache_type] insight_class = import_from(insight_class_path[0], insight_class_path[1]) result = insight_class().run(filter, Team(pk=team_id)) dashboard_items.update(last_refresh=timezone.now(), refreshing=False) return result
def update_cache_item(key: str, cache_type: str, payload: dict) -> None: result: Optional[Union[List, Dict]] = None filter_dict = json.loads(payload["filter"]) filter = get_filter(data=filter_dict, team=Team(pk=payload["team_id"])) if cache_type == CacheType.TRENDS: result = _calculate_trends(filter, key, int(payload["team_id"])) elif cache_type == CacheType.FUNNEL: result = _calculate_funnel(filter, key, int(payload["team_id"])) if result: cache.set(key, { "result": result, "details": payload, "type": cache_type }, CACHED_RESULTS_TTL)
def _calculate_funnel(filter: Filter, key: str, team_id: int) -> List[Dict[str, Any]]: dashboard_items = DashboardItem.objects.filter(team_id=team_id, filters_hash=key) dashboard_items.update(refreshing=True) insight_class: Union[Type[Funnel]] if is_clickhouse_enabled(): if filter.funnel_viz_type == FunnelVizType.TRENDS: insight_class = ClickhouseFunnelTrends elif filter.funnel_viz_type == FunnelVizType.TIME_TO_CONVERT: insight_class = ClickhouseFunnelTimeToConvert else: insight_class = ClickhouseFunnel else: insight_class = Funnel result = insight_class(filter=filter, team=Team(pk=team_id)).run() dashboard_items.update(last_refresh=timezone.now(), refreshing=False) return result
def update_cache_item(key: str, cache_type: CacheType, payload: dict) -> List[Dict[str, Any]]: result: Optional[Union[List, Dict]] = None filter_dict = json.loads(payload["filter"]) team_id = int(payload["team_id"]) filter = get_filter(data=filter_dict, team=Team(pk=team_id)) dashboard_items = Insight.objects.filter(team_id=team_id, filters_hash=key) dashboard_items.update(refreshing=True) if cache_type == CacheType.FUNNEL: result = _calculate_funnel(filter, key, team_id) else: result = _calculate_by_filter(filter, key, team_id, cache_type) cache.set(key, { "result": result, "type": cache_type, "last_refresh": timezone.now() }, CACHED_RESULTS_TTL) dashboard_items.update(last_refresh=timezone.now(), refreshing=False) return result
def update_cache_item(key: str, cache_type: CacheType, payload: dict) -> List[Dict[str, Any]]: timer = statsd.timer("update_cache_item_timer").start() result: Optional[Union[List, Dict]] = None filter_dict = json.loads(payload["filter"]) team_id = int(payload["team_id"]) filter = get_filter(data=filter_dict, team=Team(pk=team_id)) # Doing the filtering like this means we'll update _all_ Insights with the same filters hash dashboard_items = Insight.objects.filter(team_id=team_id, filters_hash=key) dashboard_items.update(refreshing=True) try: if cache_type == CacheType.FUNNEL: result = _calculate_funnel(filter, key, team_id) else: result = _calculate_by_filter(filter, key, team_id, cache_type) cache.set(key, { "result": result, "type": cache_type, "last_refresh": timezone.now() }, settings.CACHED_RESULTS_TTL) except Exception as e: timer.stop() statsd.incr("update_cache_item_error") dashboard_items.filter(refresh_attempt=None).update(refresh_attempt=0) dashboard_items.update(refreshing=False, refresh_attempt=F("refresh_attempt") + 1) raise e timer.stop() statsd.incr("update_cache_item_success") dashboard_items.update(last_refresh=timezone.now(), refreshing=False, refresh_attempt=0) return result
def create_anonymous_users_ch(team: Team, base_url: str) -> None: with open(Path("posthog/demo_data.json").resolve(), "r") as demo_data_file: demo_data = json.load(demo_data_file) demo_data_index = 0 days_ago = 7 for index in range(0, 100): if index > 0 and index % 14 == 0: days_ago -= 1 date = now() - relativedelta(days=days_ago) browser = random.choice(["Chrome", "Safari", "Firefox"]) distinct_id = generate_clickhouse_uuid() person = Person.objects.create(team_id=team.pk, distinct_ids=[distinct_id], properties={"is_demo": True}) event_uuid = uuid4() create_event( team=team, event="$pageview", distinct_id=distinct_id, properties={ "$current_url": base_url, "$browser": browser, "$lib": "web", }, timestamp=date, event_uuid=event_uuid, ) if index % 3 == 0: update_person_properties(team_id=team.pk, id=person.uuid, properties=demo_data[demo_data_index]) update_person_is_identified(team_id=team.pk, id=person.uuid, is_identified=True) demo_data_index += 1 elements = [ Element( tag_name="a", href="/demo/1", attr_class=["btn", "btn-success"], attr_id="sign-up", text="Sign up", ), Element(tag_name="form", attr_class=["form"]), Element(tag_name="div", attr_class=["container"]), Element(tag_name="body"), Element(tag_name="html"), ] event_uuid = uuid4() elements_hash = create_elements(elements=elements, team=team, event_uuid=event_uuid) create_event( team=team, distinct_id=distinct_id, event="$autocapture", properties={ "$current_url": base_url, "$browser": browser, "$lib": "web", "$event_type": "click", }, timestamp=date + relativedelta(seconds=14), elements_hash=elements_hash, event_uuid=event_uuid, ) event_uuid = uuid4() create_event( event="$pageview", team=team, distinct_id=distinct_id, properties={ "$current_url": "%s/1" % base_url, "$browser": browser, "$lib": "web", }, timestamp=date + relativedelta(seconds=15), event_uuid=event_uuid, ) if index % 4 == 0: elements = [ Element( tag_name="button", attr_class=["btn", "btn-success"], text="Sign up!", ), Element(tag_name="form", attr_class=["form"]), Element(tag_name="div", attr_class=["container"]), Element(tag_name="body"), Element(tag_name="html"), ] event_uuid = uuid4() elements_hash = create_elements(elements=elements, team=team, event_uuid=event_uuid) create_event( team=team, event="$autocapture", distinct_id=distinct_id, properties={ "$current_url": "%s/1" % base_url, "$browser": browser, "$lib": "web", "$event_type": "click", }, timestamp=date + relativedelta(seconds=29), elements_hash=elements_hash, event_uuid=event_uuid, ) event_uuid = uuid4() create_event( event="$pageview", team=team, distinct_id=distinct_id, properties={ "$current_url": "%s/2" % base_url, "$browser": browser, "$lib": "web", }, timestamp=date + relativedelta(seconds=30), event_uuid=event_uuid, ) if index % 5 == 0: elements = [ Element( tag_name="button", attr_class=["btn", "btn-success"], text="Pay $10", ), Element(tag_name="form", attr_class=["form"]), Element(tag_name="div", attr_class=["container"]), Element(tag_name="body"), Element(tag_name="html"), ] event_uuid = uuid4() elements_hash = create_elements(elements=elements, team=team, event_uuid=event_uuid) create_event( team=team, event="$autocapture", distinct_id=distinct_id, properties={ "$current_url": "%s/2" % base_url, "$browser": browser, "$lib": "web", "$event_type": "click", }, timestamp=date + relativedelta(seconds=59), elements_hash=elements_hash, event_uuid=event_uuid, ) event_uuid = uuid4() create_event( event="purchase", team=team, distinct_id=distinct_id, properties={"price": 10}, timestamp=date + relativedelta(seconds=60), event_uuid=event_uuid, ) event_uuid = uuid4() create_event( event="$pageview", team=team, distinct_id=distinct_id, properties={ "$current_url": "%s/3" % base_url, "$browser": browser, "$lib": "web", }, timestamp=date + relativedelta(seconds=60), event_uuid=event_uuid, ) team.event_properties_numerical.append("purchase") team.save()
def _create_anonymous_users(team: Team, base_url: str) -> None: with open(Path("posthog/demo_data.json").resolve(), "r") as demo_data_file: demo_data = json.load(demo_data_file) Person.objects.bulk_create([Person(team=team, properties={"is_demo": True}) for _ in range(0, 100)]) distinct_ids: List[PersonDistinctId] = [] events: List[Event] = [] days_ago = 7 demo_data_index = 0 for index, person in enumerate(Person.objects.filter(team=team)): if index > 0 and index % 14 == 0: days_ago -= 1 distinct_id = str(UUIDT()) distinct_ids.append(PersonDistinctId(team=team, person=person, distinct_id=distinct_id)) # Add first user more 3 distinct id's if index == 0: for _ in range(0, 3): distinct_ids.append(PersonDistinctId(team=team, person=person, distinct_id=str(UUIDT()))) date = now() - relativedelta(days=days_ago) browser = random.choice(["Chrome", "Safari", "Firefox"]) events.append( Event( team=team, event="$pageview", distinct_id=distinct_id, properties={"$current_url": base_url, "$browser": browser, "$lib": "web"}, timestamp=date, ) ) if index % 3 == 0: person.properties.update(demo_data[demo_data_index]) person.is_identified = True person.save() demo_data_index += 1 Event.objects.create( team=team, distinct_id=distinct_id, event="$autocapture", properties={"$current_url": base_url, "$browser": browser, "$lib": "web", "$event_type": "click",}, timestamp=date + relativedelta(seconds=14), elements=[ Element( tag_name="a", href="/demo/1", attr_class=["btn", "btn-success"], attr_id="sign-up", text="Sign up", ), Element(tag_name="form", attr_class=["form"]), Element(tag_name="div", attr_class=["container"]), Element(tag_name="body"), Element(tag_name="html"), ], ) events.append( Event( event="$pageview", team=team, distinct_id=distinct_id, properties={"$current_url": "%s/1" % base_url, "$browser": browser, "$lib": "web",}, timestamp=date + relativedelta(seconds=15), ) ) if index % 4 == 0: Event.objects.create( team=team, event="$autocapture", distinct_id=distinct_id, properties={ "$current_url": "%s/1" % base_url, "$browser": browser, "$lib": "web", "$event_type": "click", }, timestamp=date + relativedelta(seconds=29), elements=[ Element(tag_name="button", attr_class=["btn", "btn-success"], text="Sign up!",), Element(tag_name="form", attr_class=["form"]), Element(tag_name="div", attr_class=["container"]), Element(tag_name="body"), Element(tag_name="html"), ], ) events.append( Event( event="$pageview", team=team, distinct_id=distinct_id, properties={"$current_url": "%s/2" % base_url, "$browser": browser, "$lib": "web",}, timestamp=date + relativedelta(seconds=30), ) ) if index % 5 == 0: Event.objects.create( team=team, event="$autocapture", distinct_id=distinct_id, properties={ "$current_url": "%s/2" % base_url, "$browser": browser, "$lib": "web", "$event_type": "click", }, timestamp=date + relativedelta(seconds=59), elements=[ Element(tag_name="button", attr_class=["btn", "btn-success"], text="Pay $10",), Element(tag_name="form", attr_class=["form"]), Element(tag_name="div", attr_class=["container"]), Element(tag_name="body"), Element(tag_name="html"), ], ) events.append( Event( event="purchase", team=team, distinct_id=distinct_id, properties={"price": 10}, timestamp=date + relativedelta(seconds=60), ) ) events.append( Event( event="$pageview", team=team, distinct_id=distinct_id, properties={"$current_url": "%s/3" % base_url, "$browser": browser, "$lib": "web",}, timestamp=date + relativedelta(seconds=60), ) ) team.event_properties_numerical.append("purchase") team.save() PersonDistinctId.objects.bulk_create(distinct_ids) Event.objects.bulk_create(events)
def get_effective_membership_level( self, team: Team) -> Optional[OrganizationMembership.Level]: return team.get_effective_membership_level( self.context["request"].user.id)
def get_event_properties_with_usage(self, instance: Team) -> List: return instance.get_latest_event_properties_with_usage()