def __call__(self, request: HttpRequest):
""" Install monkey-patch on demand.
If monkey-patch has not been run in for this process (assuming multiple preforked processes),
then do it now.
"""
from ee.clickhouse import client
route = resolve(request.path)
client._request_information = {
"save":
(is_ee_enabled() and request.user.pk
and (request.user.is_staff or is_impersonated_session(request)
or settings.DEBUG)),
"user_id":
request.user.pk,
"kind":
"request",
"id":
f"{route.route} ({route.func.__name__})",
}
response: HttpResponse = self.get_response(request)
client._request_information = None
return response
def insert_users_by_list(self, items: List[str]) -> None:
"""
Items can be distinct_id or email
"""
batchsize = 1000
use_clickhouse = is_ee_enabled()
if use_clickhouse:
from ee.clickhouse.models.cohort import insert_static_cohort
try:
cursor = connection.cursor()
for i in range(0, len(items), batchsize):
batch = items[i : i + batchsize]
persons_query = (
Person.objects.filter(team_id=self.team_id)
.filter(Q(persondistinctid__distinct_id__in=batch) | Q(properties__email__in=batch))
.exclude(cohort__id=self.id)
)
if use_clickhouse:
insert_static_cohort([p for p in persons_query.values_list("uuid", flat=True)], self.pk, self.team)
sql, params = persons_query.distinct("pk").only("pk").query.sql_with_params()
query = UPDATE_QUERY.format(
cohort_id=self.pk,
values_query=sql.replace('FROM "posthog_person"', ', {} FROM "posthog_person"'.format(self.pk), 1,),
)
cursor.execute(query, params)
self.is_calculating = False
self.last_calculation = timezone.now()
self.errors_calculating = 0
self.save()
except Exception:
self.is_calculating = False
self.errors_calculating = F("errors_calculating") + 1
self.save()
capture_exception()
def demo(request):
user = request.user
organization = user.organization
try:
team = organization.teams.get(name=TEAM_NAME)
except Team.DoesNotExist:
team = Team.objects.create_with_data(organization=organization,
name=TEAM_NAME,
ingested_event=True,
completed_snippet_onboarding=True)
_create_anonymous_users(team=team,
base_url=request.build_absolute_uri("/demo"))
_create_funnel(team=team, base_url=request.build_absolute_uri("/demo"))
_recalculate(team=team)
user.current_team = team
user.save()
if "$pageview" not in team.event_names:
team.event_names.append("$pageview")
team.save()
if is_ee_enabled():
from ee.clickhouse.demo import create_anonymous_users_ch
from ee.clickhouse.models.event import get_events_by_team
result = get_events_by_team(team_id=team.pk)
if not result:
create_anonymous_users_ch(
team=team, base_url=request.build_absolute_uri("/demo"))
return render_template("demo.html",
request=request,
context={"api_token": team.api_token})
def insert_cohort_from_query(cohort_id: int, insight_type: str,
filter_data: Dict[str, Any],
entity_data: Dict[str, Any]) -> None:
if is_ee_enabled():
from ee.clickhouse.queries.clickhouse_stickiness import insert_stickiness_people_into_cohort
from ee.clickhouse.queries.util import get_earliest_timestamp
from ee.clickhouse.views.actions import insert_entity_people_into_cohort
from ee.clickhouse.views.cohort import insert_cohort_people_into_pg
from posthog.models.entity import Entity
from posthog.models.filters.filter import Filter
from posthog.models.filters.stickiness_filter import StickinessFilter
cohort = Cohort.objects.get(pk=cohort_id)
entity = Entity(data=entity_data)
if insight_type == INSIGHT_STICKINESS:
_stickiness_filter = StickinessFilter(
data=filter_data,
team=cohort.team,
get_earliest_timestamp=get_earliest_timestamp)
insert_stickiness_people_into_cohort(cohort, entity,
_stickiness_filter)
else:
_filter = Filter(data=filter_data)
insert_entity_people_into_cohort(cohort, entity, _filter)
insert_cohort_people_into_pg(cohort=cohort)
def create(self, site_url: Optional[str] = None, *args: Any, **kwargs: Any):
with transaction.atomic():
if kwargs.get("elements"):
if kwargs.get("team"):
kwargs["elements_hash"] = ElementGroup.objects.create(
team=kwargs["team"], elements=kwargs.pop("elements")
).hash
else:
kwargs["elements_hash"] = ElementGroup.objects.create(
team_id=kwargs["team_id"], elements=kwargs.pop("elements")
).hash
event = super().create(*args, **kwargs)
# Matching actions to events can get very expensive to do as events are streaming in
# In a few cases we have had it OOM Postgres with the query it is running
# Short term solution is to have this be configurable to be run in batch
if not settings.ASYNC_EVENT_ACTION_MAPPING:
should_post_webhook = False
relations = []
for action in event.actions:
relations.append(action.events.through(action_id=action.pk, event_id=event.pk))
if is_ee_enabled():
continue # avoiding duplication here - in EE hooks are handled by webhooks_ee.py
action.on_perform(event)
if action.post_to_slack:
should_post_webhook = True
Action.events.through.objects.bulk_create(relations, ignore_conflicts=True)
team = kwargs.get("team", event.team)
if (
should_post_webhook and team and team.slack_incoming_webhook and not is_ee_enabled()
): # ee will handle separately
celery.current_app.send_task("posthog.tasks.webhooks.post_event_to_webhook", (event.pk, site_url))
return event
def calculate_people(self, use_clickhouse=is_ee_enabled()):
try:
if not use_clickhouse:
self.is_calculating = True
self.save()
persons_query = self._clickhouse_persons_query(
) if use_clickhouse else self._postgres_persons_query()
try:
sql, params = persons_query.distinct("pk").only(
"pk").query.sql_with_params()
except EmptyResultSet:
query = DELETE_QUERY.format(cohort_id=self.pk)
params = {}
else:
query = "{}{}".format(DELETE_QUERY, UPDATE_QUERY).format(
cohort_id=self.pk,
values_query=sql.replace(
'FROM "posthog_person"',
', {} FROM "posthog_person"'.format(self.pk),
1,
),
)
cursor = connection.cursor()
with transaction.atomic():
cursor.execute(query, params)
self.is_calculating = False
self.last_calculation = timezone.now()
self.save()
except:
capture_exception()
def preflight_check(request: HttpRequest) -> JsonResponse:
response = {
"django": True,
"redis": is_redis_alive() or settings.TEST,
"plugins": is_plugin_server_alive() or settings.TEST,
"celery": is_celery_alive() or settings.TEST,
"db": is_postgres_alive(),
"initiated": User.objects.exists() if not settings.E2E_TESTING else
False, # Enables E2E testing of signup flow
"cloud": settings.MULTI_TENANCY,
"available_social_auth_providers":
get_available_social_auth_providers(),
}
if request.user.is_authenticated:
response = {
**response,
"ee_available": settings.EE_AVAILABLE,
"ee_enabled": is_ee_enabled(),
"db_backend": settings.PRIMARY_DB.value,
"available_timezones": get_available_timezones_with_offsets(),
"opt_out_capture": os.environ.get("OPT_OUT_CAPTURE", False),
"posthog_version": VERSION,
"email_service_available":
is_email_available(with_absolute_urls=True),
"is_debug": settings.DEBUG,
"is_event_property_usage_enabled":
settings.ASYNC_EVENT_PROPERTY_USAGE,
"licensed_users_available": get_licensed_users_available(),
"site_url": settings.SITE_URL,
}
return JsonResponse(response)
def create(self, *args: Any, **kwargs: Any):
site_url = kwargs.get("site_url")
with transaction.atomic():
if kwargs.get("elements"):
if kwargs.get("team"):
kwargs["elements_hash"] = ElementGroup.objects.create(
team=kwargs["team"], elements=kwargs.pop("elements")
).hash
else:
kwargs["elements_hash"] = ElementGroup.objects.create(
team_id=kwargs["team_id"], elements=kwargs.pop("elements")
).hash
event = super().create(*args, **kwargs)
# DEPRECATED: ASYNC_EVENT_ACTION_MAPPING is the main approach now, as it works with the plugin server
if not settings.ASYNC_EVENT_ACTION_MAPPING:
should_post_webhook = False
relations = []
for action in event.actions:
relations.append(action.events.through(action_id=action.pk, event_id=event.pk))
if is_ee_enabled():
continue # avoiding duplication here - in EE hooks are handled by webhooks_ee.py
action.on_perform(event)
if action.post_to_slack:
should_post_webhook = True
Action.events.through.objects.bulk_create(relations, ignore_conflicts=True)
team = kwargs.get("team", event.team)
if (
should_post_webhook and team and team.slack_incoming_webhook and not is_ee_enabled()
): # ee will handle separately
celery.current_app.send_task("posthog.tasks.webhooks.post_event_to_webhook", (event.pk, site_url))
return event
def handle(self, *args, **options):
from django.test.runner import DiscoverRunner as TestRunner
test_runner = TestRunner(interactive=False)
test_runner.setup_databases()
test_runner.setup_test_environment()
if is_ee_enabled():
from infi.clickhouse_orm import Database # type: ignore
from posthog.settings import (
CLICKHOUSE_DATABASE,
CLICKHOUSE_HTTP_URL,
CLICKHOUSE_PASSWORD,
CLICKHOUSE_USER,
CLICKHOUSE_VERIFY,
)
database = Database(
CLICKHOUSE_DATABASE,
db_url=CLICKHOUSE_HTTP_URL,
username=CLICKHOUSE_USER,
password=CLICKHOUSE_PASSWORD,
verify_ssl_cert=CLICKHOUSE_VERIFY,
)
try:
database.create_database()
except:
pass
database.migrate("ee.clickhouse.migrations")
def demo(request: Request):
user = request.user
organization = user.organization
try:
team = organization.teams.get(is_demo=True)
except Team.DoesNotExist:
team = create_demo_team(organization, user, request)
user.current_team = team
user.save()
if "$pageview" not in team.event_names:
team.event_names.append("$pageview")
team.event_names_with_usage.append({
"event": "$pageview",
"usage_count": None,
"volume": None
})
team.save()
if is_ee_enabled(): # :TRICKY: Lazily backfill missing event data.
from ee.clickhouse.models.event import get_events_by_team
result = get_events_by_team(team_id=team.pk)
if not result:
create_demo_data(team, dashboards=False)
return render_template("demo.html",
request=request,
context={"api_token": team.api_token})
def test_pagination(self):
person_factory(team=self.team, distinct_ids=["1"])
for idx in range(0, 150):
event_factory(
team=self.team,
event="some event",
distinct_id="1",
timestamp=timezone.now() - relativedelta(months=11) +
relativedelta(days=idx, seconds=idx),
)
response = self.client.get("/api/event/?distinct_id=1").json()
self.assertEqual(len(response["results"]), 100)
self.assertIn("http://testserver/api/event/?distinct_id=1&before=",
response["next"])
page2 = self.client.get(response["next"]).json()
from posthog.ee import is_ee_enabled
if is_ee_enabled():
from ee.clickhouse.client import sync_execute
self.assertEqual(
sync_execute("select count(*) from events")[0][0], 150)
self.assertEqual(len(page2["results"]), 50)
def test_capture_new_person(self) -> None:
self._create_user("tim")
action1 = Action.objects.create(team=self.team)
ActionStep.objects.create(action=action1, selector="a", event="$autocapture")
action2 = Action.objects.create(team=self.team)
ActionStep.objects.create(action=action2, selector="a", event="$autocapture")
team_id = self.team.pk
self.team.ingested_event = True # avoid sending `first team event ingested` to PostHog
self.team.save()
num_queries = (
41 # TODO: #4070 temporary; 17 + 24 from running synchronously sync_event_and_properties_definitions
)
if is_ee_enabled(): # extra queries to check for REST hooks
num_queries += 4
with self.assertNumQueries(num_queries):
process_event(
2,
"127.0.0.1",
"",
{
"event": "$autocapture",
"properties": {
"distinct_id": 2,
"token": self.team.api_token,
"$elements": [
{"tag_name": "a", "nth_child": 1, "nth_of_type": 2, "attr__class": "btn btn-sm",},
{"tag_name": "div", "nth_child": 1, "nth_of_type": 2, "$el_text": "💻",},
],
},
},
team_id,
now().isoformat(),
now().isoformat(),
)
self.assertEqual(Person.objects.get().distinct_ids, ["2"])
event = get_events()[0]
self.assertEqual(event.event, "$autocapture")
elements = get_elements(event.id)
self.assertEqual(elements[0].tag_name, "a")
self.assertEqual(elements[0].attr_class, ["btn", "btn-sm"])
self.assertEqual(elements[1].order, 1)
self.assertEqual(elements[1].text, "💻")
self.assertEqual(event.distinct_id, "2")
team = Team.objects.get()
self.assertEqual(team.event_names, ["$autocapture"])
self.assertEqual(
team.event_names_with_usage, [{"event": "$autocapture", "volume": None, "usage_count": None,}]
)
self.assertEqual(team.event_properties, ["distinct_id", "token", "$ip"])
self.assertEqual(
team.event_properties_with_usage,
[
{"key": "distinct_id", "usage_count": None, "volume": None},
{"key": "token", "usage_count": None, "volume": None},
{"key": "$ip", "usage_count": None, "volume": None},
],
)
def fetch_plugin_log_entries(
*,
team_id: Optional[int] = None,
plugin_config_id: Optional[int] = None,
after: Optional[timezone.datetime] = None,
before: Optional[timezone.datetime] = None,
search: Optional[str] = None,
limit: Optional[int] = None,
) -> List[Union[PluginLogEntry, PluginLogEntryRaw]]:
if is_ee_enabled():
clickhouse_where_parts: List[str] = []
clickhouse_kwargs: Dict[str, Any] = {}
if team_id is not None:
clickhouse_where_parts.append("team_id = %(team_id)s")
clickhouse_kwargs["team_id"] = team_id
if plugin_config_id is not None:
clickhouse_where_parts.append(
"plugin_config_id = %(plugin_config_id)s")
clickhouse_kwargs["plugin_config_id"] = plugin_config_id
if after is not None:
clickhouse_where_parts.append(
"timestamp > toDateTime64(%(after)s, 6)")
clickhouse_kwargs["after"] = after.isoformat().replace(
"+00:00", "")
if before is not None:
clickhouse_where_parts.append(
"timestamp < toDateTime64(%(before)s, 6)")
clickhouse_kwargs["before"] = before.isoformat().replace(
"+00:00", "")
if search:
clickhouse_where_parts.append("message ILIKE %(search)s")
clickhouse_kwargs["search"] = f"%{search}%"
clickhouse_query = f"""
SELECT id, team_id, plugin_id, plugin_config_id, timestamp, source, type, message, instance_id FROM plugin_log_entries
WHERE {' AND '.join(clickhouse_where_parts)} ORDER BY timestamp DESC {f'LIMIT {limit}' if limit else ''}
"""
return [
PluginLogEntryRaw(*result) for result in cast(
list, sync_execute(clickhouse_query, clickhouse_kwargs))
]
else:
filter_kwargs: Dict[str, Any] = {}
if team_id is not None:
filter_kwargs["team_id"] = team_id
if plugin_config_id is not None:
filter_kwargs["plugin_config_id"] = plugin_config_id
if after is not None:
filter_kwargs["timestamp__gt"] = after
if before is not None:
filter_kwargs["timestamp__lt"] = before
if search:
filter_kwargs["message__icontains"] = search
query = PluginLogEntry.objects.order_by("-timestamp").filter(
**filter_kwargs)
if limit:
query = query[:limit]
return list(query)
def setup_periodic_tasks(sender, **kwargs):
if not settings.DEBUG:
sender.add_periodic_task(1.0, redis_celery_queue_depth.s(), name="1 sec queue probe", priority=0)
# Heartbeat every 10sec to make sure the worker is alive
sender.add_periodic_task(10.0, redis_heartbeat.s(), name="10 sec heartbeat", priority=0)
# update events table partitions twice a week
sender.add_periodic_task(
crontab(day_of_week="mon,fri", hour=0, minute=0), update_event_partitions.s(), # check twice a week
)
if getattr(settings, "MULTI_TENANCY", False) and not is_ee_enabled():
sender.add_periodic_task(crontab(minute=0, hour="*/12"), run_session_recording_retention.s())
# send weekly status report on non-PostHog Cloud instances
if not getattr(settings, "MULTI_TENANCY", False):
sender.add_periodic_task(crontab(day_of_week="mon", hour=0, minute=0), status_report.s())
# Cloud (posthog-production) cron jobs
if getattr(settings, "MULTI_TENANCY", False):
sender.add_periodic_task(crontab(hour=0, minute=0), calculate_billing_daily_usage.s()) # every day midnight UTC
# send weekly email report (~ 8:00 SF / 16:00 UK / 17:00 EU)
sender.add_periodic_task(crontab(day_of_week="mon", hour=15, minute=0), send_weekly_email_report.s())
sender.add_periodic_task(crontab(day_of_week="fri", hour=0, minute=0), clean_stale_partials.s())
sender.add_periodic_task(90, check_cached_items.s(), name="check dashboard items")
if is_ee_enabled():
sender.add_periodic_task(120, clickhouse_lag.s(), name="clickhouse table lag")
sender.add_periodic_task(120, clickhouse_row_count.s(), name="clickhouse events table row count")
sender.add_periodic_task(120, clickhouse_part_count.s(), name="clickhouse table parts count")
sender.add_periodic_task(60, calculate_cohort.s(), name="recalculate cohorts")
if settings.ASYNC_EVENT_ACTION_MAPPING:
sender.add_periodic_task(
(60 * ACTION_EVENT_MAPPING_INTERVAL_MINUTES),
calculate_event_action_mappings.s(),
name="calculate event action mappings",
expires=(60 * ACTION_EVENT_MAPPING_INTERVAL_MINUTES),
)
def clickhouse_lag():
if is_ee_enabled() and settings.EE_AVAILABLE:
from ee.clickhouse.client import sync_execute
for table in CLICKHOUSE_TABLES:
QUERY = """select max(_timestamp) observed_ts, now() now_ts, now() - max(_timestamp) as lag from {table};"""
query = QUERY.format(table=table)
lag = sync_execute(query)[0][2]
g = statsd.Gauge("%s_posthog_celery" % (settings.STATSD_PREFIX,))
g.send("clickhouse_{table}_table_lag_seconds".format(table=table), lag)
else:
pass
def clickhouse_row_count():
if is_ee_enabled() and settings.EE_AVAILABLE:
from ee.clickhouse.client import sync_execute
for table in CLICKHOUSE_TABLES:
QUERY = """select count(1) freq from {table};"""
query = QUERY.format(table=table)
rows = sync_execute(query)[0][0]
g = statsd.Gauge("%s_posthog_celery" % (settings.STATSD_PREFIX,))
g.send("clickhouse_{table}_table_row_count".format(table=table), rows)
else:
pass
def _get_events_volume(team: Team) -> List[Tuple[str, int]]:
timestamp = now() - timedelta(days=30)
if is_ee_enabled():
from ee.clickhouse.client import sync_execute
from ee.clickhouse.sql.events import GET_EVENTS_VOLUME
return sync_execute(GET_EVENTS_VOLUME, {"team_id": team.pk, "timestamp": timestamp},)
return (
Event.objects.filter(team=team, timestamp__gt=timestamp)
.values("event")
.annotate(count=Count("id"))
.values_list("event", "count")
)
def _get_properties_volume(team: Team) -> List[Tuple[str, int]]:
timestamp = now() - timedelta(days=30)
if is_ee_enabled():
from ee.clickhouse.client import sync_execute
from ee.clickhouse.sql.events import GET_PROPERTIES_VOLUME
return sync_execute(GET_PROPERTIES_VOLUME, {"team_id": team.pk, "timestamp": timestamp},)
cursor = connection.cursor()
cursor.execute(
"SELECT json_build_array(jsonb_object_keys(properties)) ->> 0 as key1, count(1) FROM posthog_event WHERE team_id = %s AND timestamp > %s group by key1 order by count desc",
[team.pk, timestamp],
)
return cursor.fetchall()
def bulk_import_events(self):
if is_ee_enabled():
from ee.clickhouse.demo import bulk_create_events, bulk_create_session_recording_events
bulk_create_events(self.events, team=self.team)
bulk_create_session_recording_events(self.snapshots,
team_id=self.team.pk)
else:
Event.objects.bulk_create(
[Event(**kw, team=self.team) for kw in self.events])
SessionRecordingEvent.objects.bulk_create([
SessionRecordingEvent(**kw, team=self.team)
for kw in self.snapshots
])
def calculate_actions_from_last_calculation() -> None:
if is_ee_enabled(): # In EE, actions are not precalculated
return
start_time_overall = time.time()
for action in cast(
Sequence[Action],
Action.objects.filter(is_calculating=False, deleted=False)):
start_time = time.time()
action.calculate_events(start=action.last_calculated_at)
total_time = time.time() - start_time
logger.info(
f"Calculating action {action.pk} took {total_time:.2f} seconds")
total_time_overall = time.time() - start_time_overall
logger.info(
f"Calculated new event-action pairs in {total_time_overall:.2f} s")
def _calculate_funnel(filter: Filter, key: str,
team_id: int) -> List[Dict[str, Any]]:
dashboard_items = DashboardItem.objects.filter(team_id=team_id,
filters_hash=key)
dashboard_items.update(refreshing=True)
if is_ee_enabled():
insight_class = import_from("ee.clickhouse.queries.clickhouse_funnel",
"ClickhouseFunnel")
else:
insight_class = import_from("posthog.queries.funnel", "Funnel")
result = insight_class(filter=filter, team=Team(pk=team_id)).run()
dashboard_items.update(last_refresh=timezone.now(), refreshing=False)
return result
def _calculate_by_filter(filter: FilterType, key: str, team_id: int,
cache_type: CacheType) -> List[Dict[str, Any]]:
dashboard_items = DashboardItem.objects.filter(team_id=team_id,
filters_hash=key)
dashboard_items.update(refreshing=True)
if is_ee_enabled():
insight_class_path = CH_TYPE_TO_IMPORT[cache_type]
else:
insight_class_path = TYPE_TO_IMPORT[cache_type]
insight_class = import_from(insight_class_path[0], insight_class_path[1])
result = insight_class().run(filter, Team(pk=team_id))
dashboard_items.update(last_refresh=timezone.now(), refreshing=False)
return result
def clickhouse_part_count():
if is_ee_enabled() and settings.EE_AVAILABLE:
from ee.clickhouse.client import sync_execute
QUERY = """
select table, count(1) freq
from system.parts
group by table
order by freq desc;
"""
rows = sync_execute(QUERY)
for (table, parts) in rows:
g = statsd.Gauge("%s_posthog_celery" % (settings.STATSD_PREFIX,))
g.send("clickhouse_{table}_table_parts_count".format(table=table), parts)
else:
pass
def create_demo_team(organization: Organization, user: User, request: Request) -> Team:
team = Team.objects.create_with_data(
organization=organization, name=TEAM_NAME, ingested_event=True, completed_snippet_onboarding=True, is_demo=True,
)
_create_anonymous_users(team=team, base_url=request.build_absolute_uri("/demo"))
_create_funnel(team=team, base_url=request.build_absolute_uri("/demo"))
FeatureFlag.objects.create(
team=team, rollout_percentage=100, name="Sign Up CTA", key="sign-up-cta", created_by=user,
)
_recalculate(team=team)
if is_ee_enabled():
from ee.clickhouse.demo import create_anonymous_users_ch
create_anonymous_users_ch(team=team, base_url=request.build_absolute_uri("/demo"))
return team
def clickhouse_row_count():
if is_ee_enabled() and settings.EE_AVAILABLE:
from statshog.defaults.django import statsd
from ee.clickhouse.client import sync_execute
for table in CLICKHOUSE_TABLES:
try:
QUERY = """select count(1) freq from {table};"""
query = QUERY.format(table=table)
rows = sync_execute(query)[0][0]
statsd.gauge(f"posthog_celery_clickhouse_table_row_count",
rows,
tags={"table": table})
except:
pass
else:
pass
def clickhouse_part_count():
if is_ee_enabled() and settings.EE_AVAILABLE:
from statshog.defaults.django import statsd
from ee.clickhouse.client import sync_execute
QUERY = """
select table, count(1) freq
from system.parts
group by table
order by freq desc;
"""
rows = sync_execute(QUERY)
for (table, parts) in rows:
statsd.gauge(f"posthog_celery_clickhouse_table_parts_count",
parts,
tags={"table": table})
else:
pass
def clickhouse_lag():
if is_ee_enabled() and settings.EE_AVAILABLE:
from statshog.defaults.django import statsd
from ee.clickhouse.client import sync_execute
for table in CLICKHOUSE_TABLES:
try:
QUERY = (
"""select max(_timestamp) observed_ts, now() now_ts, now() - max(_timestamp) as lag from {table};"""
)
query = QUERY.format(table=table)
lag = sync_execute(query)[0][2]
statsd.gauge("posthog_celery_clickhouse__table_lag_seconds",
lag,
tags={"table": table})
except:
pass
else:
pass
def clickhouse_mutation_count():
if is_ee_enabled() and settings.EE_AVAILABLE:
from ee.clickhouse.client import sync_execute
QUERY = """
SELECT
table,
count(1) AS freq
FROM system.mutations
GROUP BY table
ORDER BY freq DESC
"""
rows = sync_execute(QUERY)
for (table, muts) in rows:
g = statsd.Gauge("%s_posthog_celery" % (settings.STATSD_PREFIX, ))
g.send(
"clickhouse_{table}_table_mutations_count".format(table=table),
muts)
else:
pass
def clickhouse_mutation_count():
if is_ee_enabled() and settings.EE_AVAILABLE:
from statshog.defaults.django import statsd
from ee.clickhouse.client import sync_execute
QUERY = """
SELECT
table,
count(1) AS freq
FROM system.mutations
GROUP BY table
ORDER BY freq DESC
"""
rows = sync_execute(QUERY)
for (table, muts) in rows:
statsd.gauge(f"posthog_celery_clickhouse_table_mutations_count",
muts,
tags={"table": table})
else:
pass
# sent_at - timestamp == now - x
# x = now + (timestamp - sent_at)
try:
# timestamp and sent_at must both be in the same format: either both with or both without timezones
# otherwise we can't get a diff to add to now
return now + (parser.isoparse(data["timestamp"]) - sent_at)
except TypeError as e:
pass
return parser.isoparse(data["timestamp"])
now_datetime = now
if data.get("offset"):
return now_datetime - relativedelta(microseconds=data["offset"] * 1000)
return now_datetime
if is_ee_enabled():
def process_event_ee(
distinct_id: str,
ip: str,
site_url: str,
data: dict,
team_id: int,
now: datetime.datetime,
sent_at: Optional[datetime.datetime],
) -> None:
timer = statsd.Timer("%s_posthog_cloud" % (settings.STATSD_PREFIX, ))
timer.start()
properties = data.get("properties", {})
if data.get("$set"):
properties["$set"] = data["$set"]
