def _save_team(team: Team, event_names: Dict[str, Dict], event_properties: Dict[str, Dict]) -> None:
def _sort(to_sort: List) -> List:
return sorted(to_sort, key=lambda item: (item.get("usage_count", 0), item.get("volume", 0)), reverse=True)
team.event_names_with_usage = _sort([val for _, val in event_names.items()])
team.event_properties_with_usage = _sort([val for _, val in event_properties.items()])
team.save()
def store_names_and_properties(team: Team, event: str,
properties: Dict) -> None:
# In _capture we only prefetch a couple of fields in Team to avoid fetching too much data
save = False
if not team.ingested_event:
# First event for the team captured
for user in Team.objects.get(pk=team.pk).users.all():
posthoganalytics.capture(user.distinct_id,
"first team event ingested",
{"team": str(team.uuid)})
team.ingested_event = True
save = True
if event not in team.event_names:
save = True
team.event_names.append(event)
for key, value in properties.items():
if key not in team.event_properties:
team.event_properties.append(key)
save = True
if isinstance(value,
Number) and key not in team.event_properties_numerical:
team.event_properties_numerical.append(key)
save = True
if save:
team.save()
def _calculate_by_filter(filter: FilterType, key: str, team_id: int,
cache_type: CacheType) -> List[Dict[str, Any]]:
insight_class = CACHE_TYPE_TO_INSIGHT_CLASS[cache_type]
if cache_type == CacheType.PATHS:
result = insight_class(filter,
Team(pk=team_id)).run(filter, Team(pk=team_id))
else:
result = insight_class().run(filter, Team(pk=team_id))
return result
def _store_names_and_properties(team: Team, event: str, properties: Dict) -> None:
# In _capture we only prefetch a couple of fields in Team to avoid fetching too much data
save = False
if event not in team.event_names:
save = True
team.event_names.append(event)
for key in properties.keys():
if key not in team.event_properties:
team.event_properties.append(key)
save = True
if save:
team.save()
def store_names_and_properties(team: Team, event: str, properties: Dict) -> None:
# In _capture we only prefetch a couple of fields in Team to avoid fetching too much data
save = False
if event not in team.event_names:
save = True
team.event_names.append(event)
for key, value in properties.items():
if key not in team.event_properties:
team.event_properties.append(key)
save = True
if isinstance(value, Number) and key not in team.event_properties_numerical:
team.event_properties_numerical.append(key)
save = True
if save:
team.save()
def _calculate_funnel(filter: Filter, team_id: int) -> List[Dict[str, Any]]:
dashboard_items = DashboardItem.objects.filter(team_id=team_id,
filters=filter.to_dict())
dashboard_items.update(refreshing=True)
result = Funnel(filter=filter, team=Team(pk=team_id)).run()
dashboard_items.update(last_refresh=timezone.now(), refreshing=False)
return result
def _calculate_funnel(filter: Filter, key: str,
team_id: int) -> List[Dict[str, Any]]:
dashboard_items = DashboardItem.objects.filter(team_id=team_id,
filters_hash=key)
dashboard_items.update(refreshing=True)
team = Team(pk=team_id)
if is_clickhouse_enabled():
funnel_order_class: Type[ClickhouseFunnelBase] = ClickhouseFunnel
if filter.funnel_order_type == FunnelOrderType.UNORDERED:
funnel_order_class = ClickhouseFunnelUnordered
elif filter.funnel_order_type == FunnelOrderType.STRICT:
funnel_order_class = ClickhouseFunnelStrict
if filter.funnel_viz_type == FunnelVizType.TRENDS:
result = ClickhouseFunnelTrends(
team=team,
filter=filter,
funnel_order_class=funnel_order_class).run()
elif filter.funnel_viz_type == FunnelVizType.TIME_TO_CONVERT:
result = ClickhouseFunnelTimeToConvert(
team=team,
filter=filter,
funnel_order_class=funnel_order_class).run()
else:
result = funnel_order_class(team=team, filter=filter).run()
else:
result = Funnel(filter=filter, team=team).run()
dashboard_items.update(last_refresh=timezone.now(), refreshing=False)
return result
def _calculate_funnel(filter: Filter, key: str,
team_id: int) -> List[Dict[str, Any]]:
team = Team(pk=team_id)
if is_clickhouse_enabled():
funnel_order_class: Type[ClickhouseFunnelBase] = ClickhouseFunnel
if filter.funnel_order_type == FunnelOrderType.UNORDERED:
funnel_order_class = ClickhouseFunnelUnordered
elif filter.funnel_order_type == FunnelOrderType.STRICT:
funnel_order_class = ClickhouseFunnelStrict
if filter.funnel_viz_type == FunnelVizType.TRENDS:
result = ClickhouseFunnelTrends(
team=team,
filter=filter,
funnel_order_class=funnel_order_class).run()
elif filter.funnel_viz_type == FunnelVizType.TIME_TO_CONVERT:
result = ClickhouseFunnelTimeToConvert(
team=team,
filter=filter,
funnel_order_class=funnel_order_class).run()
else:
result = funnel_order_class(team=team, filter=filter).run()
else:
result = Funnel(filter=filter, team=team).run()
return result
def _calculate_trends(filter: Filter, key: str, team_id: int) -> List[Dict[str, Any]]:
actions = Action.objects.filter(team_id=team_id)
actions = actions.prefetch_related(Prefetch("steps", queryset=ActionStep.objects.order_by("id")))
dashboard_items = DashboardItem.objects.filter(team_id=team_id, filters_hash=key)
dashboard_items.update(refreshing=True)
result = Trends().run(filter, Team(pk=team_id))
dashboard_items.update(last_refresh=timezone.now(), refreshing=False)
return result
def _calculate_by_filter(filter: FilterType, key: str, team_id: int,
cache_type: CacheType) -> List[Dict[str, Any]]:
dashboard_items = DashboardItem.objects.filter(team_id=team_id,
filters_hash=key)
dashboard_items.update(refreshing=True)
insight_class = CACHE_TYPE_TO_INSIGHT_CLASS[cache_type]
result = insight_class().run(filter, Team(pk=team_id))
dashboard_items.update(last_refresh=timezone.now(), refreshing=False)
return result
def _calculate_funnel(filter: Filter, key: str, team_id: int) -> List[Dict[str, Any]]:
dashboard_items = DashboardItem.objects.filter(team_id=team_id, filters_hash=key)
dashboard_items.update(refreshing=True)
if is_clickhouse_enabled():
insight_class = import_from("ee.clickhouse.queries.clickhouse_funnel", "ClickhouseFunnel")
else:
insight_class = import_from("posthog.queries.funnel", "Funnel")
result = insight_class(filter=filter, team=Team(pk=team_id)).run()
dashboard_items.update(last_refresh=timezone.now(), refreshing=False)
return result
def _calculate_funnel(filter: Filter, key: str,
team_id: int) -> List[Dict[str, Any]]:
team = Team(pk=team_id)
if filter.funnel_viz_type == FunnelVizType.TRENDS:
result = ClickhouseFunnelTrends(team=team, filter=filter).run()
elif filter.funnel_viz_type == FunnelVizType.TIME_TO_CONVERT:
result = ClickhouseFunnelTimeToConvert(team=team, filter=filter).run()
else:
funnel_order_class = get_funnel_order_class(filter)
result = funnel_order_class(team=team, filter=filter).run()
return result
def update_cache_item(key: str, cache_type: CacheType, payload: dict) -> None:
result: Optional[Union[List, Dict]] = None
filter_dict = json.loads(payload["filter"])
team_id = int(payload["team_id"])
filter = get_filter(data=filter_dict, team=Team(pk=team_id))
if cache_type == CacheType.FUNNEL:
result = _calculate_funnel(filter, key, team_id)
else:
result = _calculate_by_filter(filter, key, team_id, cache_type)
if result:
cache.set(key, {"result": result, "type": cache_type, "last_refresh": timezone.now()}, CACHED_RESULTS_TTL)
def _calculate_by_filter(filter: FilterType, key: str, team_id: int, cache_type: CacheType) -> List[Dict[str, Any]]:
dashboard_items = DashboardItem.objects.filter(team_id=team_id, filters_hash=key)
dashboard_items.update(refreshing=True)
if is_clickhouse_enabled():
insight_class_path = CH_TYPE_TO_IMPORT[cache_type]
else:
insight_class_path = TYPE_TO_IMPORT[cache_type]
insight_class = import_from(insight_class_path[0], insight_class_path[1])
result = insight_class().run(filter, Team(pk=team_id))
dashboard_items.update(last_refresh=timezone.now(), refreshing=False)
return result
def update_cache_item(key: str, cache_type: str, payload: dict) -> None:
result: Optional[Union[List, Dict]] = None
filter_dict = json.loads(payload["filter"])
filter = get_filter(data=filter_dict, team=Team(pk=payload["team_id"]))
if cache_type == CacheType.TRENDS:
result = _calculate_trends(filter, key, int(payload["team_id"]))
elif cache_type == CacheType.FUNNEL:
result = _calculate_funnel(filter, key, int(payload["team_id"]))
if result:
cache.set(key, {
"result": result,
"details": payload,
"type": cache_type
}, CACHED_RESULTS_TTL)
def _calculate_funnel(filter: Filter, key: str,
team_id: int) -> List[Dict[str, Any]]:
dashboard_items = DashboardItem.objects.filter(team_id=team_id,
filters_hash=key)
dashboard_items.update(refreshing=True)
insight_class: Union[Type[Funnel]]
if is_clickhouse_enabled():
if filter.funnel_viz_type == FunnelVizType.TRENDS:
insight_class = ClickhouseFunnelTrends
elif filter.funnel_viz_type == FunnelVizType.TIME_TO_CONVERT:
insight_class = ClickhouseFunnelTimeToConvert
else:
insight_class = ClickhouseFunnel
else:
insight_class = Funnel
result = insight_class(filter=filter, team=Team(pk=team_id)).run()
dashboard_items.update(last_refresh=timezone.now(), refreshing=False)
return result
def update_cache_item(key: str, cache_type: CacheType,
payload: dict) -> List[Dict[str, Any]]:
result: Optional[Union[List, Dict]] = None
filter_dict = json.loads(payload["filter"])
team_id = int(payload["team_id"])
filter = get_filter(data=filter_dict, team=Team(pk=team_id))
dashboard_items = Insight.objects.filter(team_id=team_id, filters_hash=key)
dashboard_items.update(refreshing=True)
if cache_type == CacheType.FUNNEL:
result = _calculate_funnel(filter, key, team_id)
else:
result = _calculate_by_filter(filter, key, team_id, cache_type)
cache.set(key, {
"result": result,
"type": cache_type,
"last_refresh": timezone.now()
}, CACHED_RESULTS_TTL)
dashboard_items.update(last_refresh=timezone.now(), refreshing=False)
return result
def update_cache_item(key: str, cache_type: CacheType,
payload: dict) -> List[Dict[str, Any]]:
timer = statsd.timer("update_cache_item_timer").start()
result: Optional[Union[List, Dict]] = None
filter_dict = json.loads(payload["filter"])
team_id = int(payload["team_id"])
filter = get_filter(data=filter_dict, team=Team(pk=team_id))
# Doing the filtering like this means we'll update _all_ Insights with the same filters hash
dashboard_items = Insight.objects.filter(team_id=team_id, filters_hash=key)
dashboard_items.update(refreshing=True)
try:
if cache_type == CacheType.FUNNEL:
result = _calculate_funnel(filter, key, team_id)
else:
result = _calculate_by_filter(filter, key, team_id, cache_type)
cache.set(key, {
"result": result,
"type": cache_type,
"last_refresh": timezone.now()
}, settings.CACHED_RESULTS_TTL)
except Exception as e:
timer.stop()
statsd.incr("update_cache_item_error")
dashboard_items.filter(refresh_attempt=None).update(refresh_attempt=0)
dashboard_items.update(refreshing=False,
refresh_attempt=F("refresh_attempt") + 1)
raise e
timer.stop()
statsd.incr("update_cache_item_success")
dashboard_items.update(last_refresh=timezone.now(),
refreshing=False,
refresh_attempt=0)
return result
def create_anonymous_users_ch(team: Team, base_url: str) -> None:
with open(Path("posthog/demo_data.json").resolve(), "r") as demo_data_file:
demo_data = json.load(demo_data_file)
demo_data_index = 0
days_ago = 7
for index in range(0, 100):
if index > 0 and index % 14 == 0:
days_ago -= 1
date = now() - relativedelta(days=days_ago)
browser = random.choice(["Chrome", "Safari", "Firefox"])
distinct_id = generate_clickhouse_uuid()
person = Person.objects.create(team_id=team.pk,
distinct_ids=[distinct_id],
properties={"is_demo": True})
event_uuid = uuid4()
create_event(
team=team,
event="$pageview",
distinct_id=distinct_id,
properties={
"$current_url": base_url,
"$browser": browser,
"$lib": "web",
},
timestamp=date,
event_uuid=event_uuid,
)
if index % 3 == 0:
update_person_properties(team_id=team.pk,
id=person.uuid,
properties=demo_data[demo_data_index])
update_person_is_identified(team_id=team.pk,
id=person.uuid,
is_identified=True)
demo_data_index += 1
elements = [
Element(
tag_name="a",
href="/demo/1",
attr_class=["btn", "btn-success"],
attr_id="sign-up",
text="Sign up",
),
Element(tag_name="form", attr_class=["form"]),
Element(tag_name="div", attr_class=["container"]),
Element(tag_name="body"),
Element(tag_name="html"),
]
event_uuid = uuid4()
elements_hash = create_elements(elements=elements,
team=team,
event_uuid=event_uuid)
create_event(
team=team,
distinct_id=distinct_id,
event="$autocapture",
properties={
"$current_url": base_url,
"$browser": browser,
"$lib": "web",
"$event_type": "click",
},
timestamp=date + relativedelta(seconds=14),
elements_hash=elements_hash,
event_uuid=event_uuid,
)
event_uuid = uuid4()
create_event(
event="$pageview",
team=team,
distinct_id=distinct_id,
properties={
"$current_url": "%s/1" % base_url,
"$browser": browser,
"$lib": "web",
},
timestamp=date + relativedelta(seconds=15),
event_uuid=event_uuid,
)
if index % 4 == 0:
elements = [
Element(
tag_name="button",
attr_class=["btn", "btn-success"],
text="Sign up!",
),
Element(tag_name="form", attr_class=["form"]),
Element(tag_name="div", attr_class=["container"]),
Element(tag_name="body"),
Element(tag_name="html"),
]
event_uuid = uuid4()
elements_hash = create_elements(elements=elements,
team=team,
event_uuid=event_uuid)
create_event(
team=team,
event="$autocapture",
distinct_id=distinct_id,
properties={
"$current_url": "%s/1" % base_url,
"$browser": browser,
"$lib": "web",
"$event_type": "click",
},
timestamp=date + relativedelta(seconds=29),
elements_hash=elements_hash,
event_uuid=event_uuid,
)
event_uuid = uuid4()
create_event(
event="$pageview",
team=team,
distinct_id=distinct_id,
properties={
"$current_url": "%s/2" % base_url,
"$browser": browser,
"$lib": "web",
},
timestamp=date + relativedelta(seconds=30),
event_uuid=event_uuid,
)
if index % 5 == 0:
elements = [
Element(
tag_name="button",
attr_class=["btn", "btn-success"],
text="Pay $10",
),
Element(tag_name="form", attr_class=["form"]),
Element(tag_name="div", attr_class=["container"]),
Element(tag_name="body"),
Element(tag_name="html"),
]
event_uuid = uuid4()
elements_hash = create_elements(elements=elements,
team=team,
event_uuid=event_uuid)
create_event(
team=team,
event="$autocapture",
distinct_id=distinct_id,
properties={
"$current_url": "%s/2" % base_url,
"$browser": browser,
"$lib": "web",
"$event_type": "click",
},
timestamp=date + relativedelta(seconds=59),
elements_hash=elements_hash,
event_uuid=event_uuid,
)
event_uuid = uuid4()
create_event(
event="purchase",
team=team,
distinct_id=distinct_id,
properties={"price": 10},
timestamp=date + relativedelta(seconds=60),
event_uuid=event_uuid,
)
event_uuid = uuid4()
create_event(
event="$pageview",
team=team,
distinct_id=distinct_id,
properties={
"$current_url": "%s/3" % base_url,
"$browser": browser,
"$lib": "web",
},
timestamp=date + relativedelta(seconds=60),
event_uuid=event_uuid,
)
team.event_properties_numerical.append("purchase")
team.save()
def _create_anonymous_users(team: Team, base_url: str) -> None:
with open(Path("posthog/demo_data.json").resolve(), "r") as demo_data_file:
demo_data = json.load(demo_data_file)
Person.objects.bulk_create([Person(team=team, properties={"is_demo": True}) for _ in range(0, 100)])
distinct_ids: List[PersonDistinctId] = []
events: List[Event] = []
days_ago = 7
demo_data_index = 0
for index, person in enumerate(Person.objects.filter(team=team)):
if index > 0 and index % 14 == 0:
days_ago -= 1
distinct_id = str(UUIDT())
distinct_ids.append(PersonDistinctId(team=team, person=person, distinct_id=distinct_id))
# Add first user more 3 distinct id's
if index == 0:
for _ in range(0, 3):
distinct_ids.append(PersonDistinctId(team=team, person=person, distinct_id=str(UUIDT())))
date = now() - relativedelta(days=days_ago)
browser = random.choice(["Chrome", "Safari", "Firefox"])
events.append(
Event(
team=team,
event="$pageview",
distinct_id=distinct_id,
properties={"$current_url": base_url, "$browser": browser, "$lib": "web"},
timestamp=date,
)
)
if index % 3 == 0:
person.properties.update(demo_data[demo_data_index])
person.is_identified = True
person.save()
demo_data_index += 1
Event.objects.create(
team=team,
distinct_id=distinct_id,
event="$autocapture",
properties={"$current_url": base_url, "$browser": browser, "$lib": "web", "$event_type": "click",},
timestamp=date + relativedelta(seconds=14),
elements=[
Element(
tag_name="a",
href="/demo/1",
attr_class=["btn", "btn-success"],
attr_id="sign-up",
text="Sign up",
),
Element(tag_name="form", attr_class=["form"]),
Element(tag_name="div", attr_class=["container"]),
Element(tag_name="body"),
Element(tag_name="html"),
],
)
events.append(
Event(
event="$pageview",
team=team,
distinct_id=distinct_id,
properties={"$current_url": "%s/1" % base_url, "$browser": browser, "$lib": "web",},
timestamp=date + relativedelta(seconds=15),
)
)
if index % 4 == 0:
Event.objects.create(
team=team,
event="$autocapture",
distinct_id=distinct_id,
properties={
"$current_url": "%s/1" % base_url,
"$browser": browser,
"$lib": "web",
"$event_type": "click",
},
timestamp=date + relativedelta(seconds=29),
elements=[
Element(tag_name="button", attr_class=["btn", "btn-success"], text="Sign up!",),
Element(tag_name="form", attr_class=["form"]),
Element(tag_name="div", attr_class=["container"]),
Element(tag_name="body"),
Element(tag_name="html"),
],
)
events.append(
Event(
event="$pageview",
team=team,
distinct_id=distinct_id,
properties={"$current_url": "%s/2" % base_url, "$browser": browser, "$lib": "web",},
timestamp=date + relativedelta(seconds=30),
)
)
if index % 5 == 0:
Event.objects.create(
team=team,
event="$autocapture",
distinct_id=distinct_id,
properties={
"$current_url": "%s/2" % base_url,
"$browser": browser,
"$lib": "web",
"$event_type": "click",
},
timestamp=date + relativedelta(seconds=59),
elements=[
Element(tag_name="button", attr_class=["btn", "btn-success"], text="Pay $10",),
Element(tag_name="form", attr_class=["form"]),
Element(tag_name="div", attr_class=["container"]),
Element(tag_name="body"),
Element(tag_name="html"),
],
)
events.append(
Event(
event="purchase",
team=team,
distinct_id=distinct_id,
properties={"price": 10},
timestamp=date + relativedelta(seconds=60),
)
)
events.append(
Event(
event="$pageview",
team=team,
distinct_id=distinct_id,
properties={"$current_url": "%s/3" % base_url, "$browser": browser, "$lib": "web",},
timestamp=date + relativedelta(seconds=60),
)
)
team.event_properties_numerical.append("purchase")
team.save()
PersonDistinctId.objects.bulk_create(distinct_ids)
Event.objects.bulk_create(events)
def get_effective_membership_level(
self, team: Team) -> Optional[OrganizationMembership.Level]:
return team.get_effective_membership_level(
self.context["request"].user.id)
def get_event_properties_with_usage(self, instance: Team) -> List:
return instance.get_latest_event_properties_with_usage()
