def __call__(self, request: HttpRequest): """ Install monkey-patch on demand. If monkey-patch has not been run in for this process (assuming multiple preforked processes), then do it now. """ from ee.clickhouse import client route = resolve(request.path) client._request_information = { "save": (is_ee_enabled() and request.user.pk and (request.user.is_staff or is_impersonated_session(request) or settings.DEBUG)), "user_id": request.user.pk, "kind": "request", "id": f"{route.route} ({route.func.__name__})", } response: HttpResponse = self.get_response(request) client._request_information = None return response
def insert_users_by_list(self, items: List[str]) -> None: """ Items can be distinct_id or email """ batchsize = 1000 use_clickhouse = is_ee_enabled() if use_clickhouse: from ee.clickhouse.models.cohort import insert_static_cohort try: cursor = connection.cursor() for i in range(0, len(items), batchsize): batch = items[i : i + batchsize] persons_query = ( Person.objects.filter(team_id=self.team_id) .filter(Q(persondistinctid__distinct_id__in=batch) | Q(properties__email__in=batch)) .exclude(cohort__id=self.id) ) if use_clickhouse: insert_static_cohort([p for p in persons_query.values_list("uuid", flat=True)], self.pk, self.team) sql, params = persons_query.distinct("pk").only("pk").query.sql_with_params() query = UPDATE_QUERY.format( cohort_id=self.pk, values_query=sql.replace('FROM "posthog_person"', ', {} FROM "posthog_person"'.format(self.pk), 1,), ) cursor.execute(query, params) self.is_calculating = False self.last_calculation = timezone.now() self.errors_calculating = 0 self.save() except Exception: self.is_calculating = False self.errors_calculating = F("errors_calculating") + 1 self.save() capture_exception()
def demo(request): user = request.user organization = user.organization try: team = organization.teams.get(name=TEAM_NAME) except Team.DoesNotExist: team = Team.objects.create_with_data(organization=organization, name=TEAM_NAME, ingested_event=True, completed_snippet_onboarding=True) _create_anonymous_users(team=team, base_url=request.build_absolute_uri("/demo")) _create_funnel(team=team, base_url=request.build_absolute_uri("/demo")) _recalculate(team=team) user.current_team = team user.save() if "$pageview" not in team.event_names: team.event_names.append("$pageview") team.save() if is_ee_enabled(): from ee.clickhouse.demo import create_anonymous_users_ch from ee.clickhouse.models.event import get_events_by_team result = get_events_by_team(team_id=team.pk) if not result: create_anonymous_users_ch( team=team, base_url=request.build_absolute_uri("/demo")) return render_template("demo.html", request=request, context={"api_token": team.api_token})
def insert_cohort_from_query(cohort_id: int, insight_type: str, filter_data: Dict[str, Any], entity_data: Dict[str, Any]) -> None: if is_ee_enabled(): from ee.clickhouse.queries.clickhouse_stickiness import insert_stickiness_people_into_cohort from ee.clickhouse.queries.util import get_earliest_timestamp from ee.clickhouse.views.actions import insert_entity_people_into_cohort from ee.clickhouse.views.cohort import insert_cohort_people_into_pg from posthog.models.entity import Entity from posthog.models.filters.filter import Filter from posthog.models.filters.stickiness_filter import StickinessFilter cohort = Cohort.objects.get(pk=cohort_id) entity = Entity(data=entity_data) if insight_type == INSIGHT_STICKINESS: _stickiness_filter = StickinessFilter( data=filter_data, team=cohort.team, get_earliest_timestamp=get_earliest_timestamp) insert_stickiness_people_into_cohort(cohort, entity, _stickiness_filter) else: _filter = Filter(data=filter_data) insert_entity_people_into_cohort(cohort, entity, _filter) insert_cohort_people_into_pg(cohort=cohort)
def create(self, site_url: Optional[str] = None, *args: Any, **kwargs: Any): with transaction.atomic(): if kwargs.get("elements"): if kwargs.get("team"): kwargs["elements_hash"] = ElementGroup.objects.create( team=kwargs["team"], elements=kwargs.pop("elements") ).hash else: kwargs["elements_hash"] = ElementGroup.objects.create( team_id=kwargs["team_id"], elements=kwargs.pop("elements") ).hash event = super().create(*args, **kwargs) # Matching actions to events can get very expensive to do as events are streaming in # In a few cases we have had it OOM Postgres with the query it is running # Short term solution is to have this be configurable to be run in batch if not settings.ASYNC_EVENT_ACTION_MAPPING: should_post_webhook = False relations = [] for action in event.actions: relations.append(action.events.through(action_id=action.pk, event_id=event.pk)) if is_ee_enabled(): continue # avoiding duplication here - in EE hooks are handled by webhooks_ee.py action.on_perform(event) if action.post_to_slack: should_post_webhook = True Action.events.through.objects.bulk_create(relations, ignore_conflicts=True) team = kwargs.get("team", event.team) if ( should_post_webhook and team and team.slack_incoming_webhook and not is_ee_enabled() ): # ee will handle separately celery.current_app.send_task("posthog.tasks.webhooks.post_event_to_webhook", (event.pk, site_url)) return event
def calculate_people(self, use_clickhouse=is_ee_enabled()): try: if not use_clickhouse: self.is_calculating = True self.save() persons_query = self._clickhouse_persons_query( ) if use_clickhouse else self._postgres_persons_query() try: sql, params = persons_query.distinct("pk").only( "pk").query.sql_with_params() except EmptyResultSet: query = DELETE_QUERY.format(cohort_id=self.pk) params = {} else: query = "{}{}".format(DELETE_QUERY, UPDATE_QUERY).format( cohort_id=self.pk, values_query=sql.replace( 'FROM "posthog_person"', ', {} FROM "posthog_person"'.format(self.pk), 1, ), ) cursor = connection.cursor() with transaction.atomic(): cursor.execute(query, params) self.is_calculating = False self.last_calculation = timezone.now() self.save() except: capture_exception()
def preflight_check(request: HttpRequest) -> JsonResponse: response = { "django": True, "redis": is_redis_alive() or settings.TEST, "plugins": is_plugin_server_alive() or settings.TEST, "celery": is_celery_alive() or settings.TEST, "db": is_postgres_alive(), "initiated": User.objects.exists() if not settings.E2E_TESTING else False, # Enables E2E testing of signup flow "cloud": settings.MULTI_TENANCY, "available_social_auth_providers": get_available_social_auth_providers(), } if request.user.is_authenticated: response = { **response, "ee_available": settings.EE_AVAILABLE, "ee_enabled": is_ee_enabled(), "db_backend": settings.PRIMARY_DB.value, "available_timezones": get_available_timezones_with_offsets(), "opt_out_capture": os.environ.get("OPT_OUT_CAPTURE", False), "posthog_version": VERSION, "email_service_available": is_email_available(with_absolute_urls=True), "is_debug": settings.DEBUG, "is_event_property_usage_enabled": settings.ASYNC_EVENT_PROPERTY_USAGE, "licensed_users_available": get_licensed_users_available(), "site_url": settings.SITE_URL, } return JsonResponse(response)
def create(self, *args: Any, **kwargs: Any): site_url = kwargs.get("site_url") with transaction.atomic(): if kwargs.get("elements"): if kwargs.get("team"): kwargs["elements_hash"] = ElementGroup.objects.create( team=kwargs["team"], elements=kwargs.pop("elements") ).hash else: kwargs["elements_hash"] = ElementGroup.objects.create( team_id=kwargs["team_id"], elements=kwargs.pop("elements") ).hash event = super().create(*args, **kwargs) # DEPRECATED: ASYNC_EVENT_ACTION_MAPPING is the main approach now, as it works with the plugin server if not settings.ASYNC_EVENT_ACTION_MAPPING: should_post_webhook = False relations = [] for action in event.actions: relations.append(action.events.through(action_id=action.pk, event_id=event.pk)) if is_ee_enabled(): continue # avoiding duplication here - in EE hooks are handled by webhooks_ee.py action.on_perform(event) if action.post_to_slack: should_post_webhook = True Action.events.through.objects.bulk_create(relations, ignore_conflicts=True) team = kwargs.get("team", event.team) if ( should_post_webhook and team and team.slack_incoming_webhook and not is_ee_enabled() ): # ee will handle separately celery.current_app.send_task("posthog.tasks.webhooks.post_event_to_webhook", (event.pk, site_url)) return event
def handle(self, *args, **options): from django.test.runner import DiscoverRunner as TestRunner test_runner = TestRunner(interactive=False) test_runner.setup_databases() test_runner.setup_test_environment() if is_ee_enabled(): from infi.clickhouse_orm import Database # type: ignore from posthog.settings import ( CLICKHOUSE_DATABASE, CLICKHOUSE_HTTP_URL, CLICKHOUSE_PASSWORD, CLICKHOUSE_USER, CLICKHOUSE_VERIFY, ) database = Database( CLICKHOUSE_DATABASE, db_url=CLICKHOUSE_HTTP_URL, username=CLICKHOUSE_USER, password=CLICKHOUSE_PASSWORD, verify_ssl_cert=CLICKHOUSE_VERIFY, ) try: database.create_database() except: pass database.migrate("ee.clickhouse.migrations")
def demo(request: Request): user = request.user organization = user.organization try: team = organization.teams.get(is_demo=True) except Team.DoesNotExist: team = create_demo_team(organization, user, request) user.current_team = team user.save() if "$pageview" not in team.event_names: team.event_names.append("$pageview") team.event_names_with_usage.append({ "event": "$pageview", "usage_count": None, "volume": None }) team.save() if is_ee_enabled(): # :TRICKY: Lazily backfill missing event data. from ee.clickhouse.models.event import get_events_by_team result = get_events_by_team(team_id=team.pk) if not result: create_demo_data(team, dashboards=False) return render_template("demo.html", request=request, context={"api_token": team.api_token})
def test_pagination(self): person_factory(team=self.team, distinct_ids=["1"]) for idx in range(0, 150): event_factory( team=self.team, event="some event", distinct_id="1", timestamp=timezone.now() - relativedelta(months=11) + relativedelta(days=idx, seconds=idx), ) response = self.client.get("/api/event/?distinct_id=1").json() self.assertEqual(len(response["results"]), 100) self.assertIn("http://testserver/api/event/?distinct_id=1&before=", response["next"]) page2 = self.client.get(response["next"]).json() from posthog.ee import is_ee_enabled if is_ee_enabled(): from ee.clickhouse.client import sync_execute self.assertEqual( sync_execute("select count(*) from events")[0][0], 150) self.assertEqual(len(page2["results"]), 50)
def test_capture_new_person(self) -> None: self._create_user("tim") action1 = Action.objects.create(team=self.team) ActionStep.objects.create(action=action1, selector="a", event="$autocapture") action2 = Action.objects.create(team=self.team) ActionStep.objects.create(action=action2, selector="a", event="$autocapture") team_id = self.team.pk self.team.ingested_event = True # avoid sending `first team event ingested` to PostHog self.team.save() num_queries = ( 41 # TODO: #4070 temporary; 17 + 24 from running synchronously sync_event_and_properties_definitions ) if is_ee_enabled(): # extra queries to check for REST hooks num_queries += 4 with self.assertNumQueries(num_queries): process_event( 2, "127.0.0.1", "", { "event": "$autocapture", "properties": { "distinct_id": 2, "token": self.team.api_token, "$elements": [ {"tag_name": "a", "nth_child": 1, "nth_of_type": 2, "attr__class": "btn btn-sm",}, {"tag_name": "div", "nth_child": 1, "nth_of_type": 2, "$el_text": "💻",}, ], }, }, team_id, now().isoformat(), now().isoformat(), ) self.assertEqual(Person.objects.get().distinct_ids, ["2"]) event = get_events()[0] self.assertEqual(event.event, "$autocapture") elements = get_elements(event.id) self.assertEqual(elements[0].tag_name, "a") self.assertEqual(elements[0].attr_class, ["btn", "btn-sm"]) self.assertEqual(elements[1].order, 1) self.assertEqual(elements[1].text, "💻") self.assertEqual(event.distinct_id, "2") team = Team.objects.get() self.assertEqual(team.event_names, ["$autocapture"]) self.assertEqual( team.event_names_with_usage, [{"event": "$autocapture", "volume": None, "usage_count": None,}] ) self.assertEqual(team.event_properties, ["distinct_id", "token", "$ip"]) self.assertEqual( team.event_properties_with_usage, [ {"key": "distinct_id", "usage_count": None, "volume": None}, {"key": "token", "usage_count": None, "volume": None}, {"key": "$ip", "usage_count": None, "volume": None}, ], )
def fetch_plugin_log_entries( *, team_id: Optional[int] = None, plugin_config_id: Optional[int] = None, after: Optional[timezone.datetime] = None, before: Optional[timezone.datetime] = None, search: Optional[str] = None, limit: Optional[int] = None, ) -> List[Union[PluginLogEntry, PluginLogEntryRaw]]: if is_ee_enabled(): clickhouse_where_parts: List[str] = [] clickhouse_kwargs: Dict[str, Any] = {} if team_id is not None: clickhouse_where_parts.append("team_id = %(team_id)s") clickhouse_kwargs["team_id"] = team_id if plugin_config_id is not None: clickhouse_where_parts.append( "plugin_config_id = %(plugin_config_id)s") clickhouse_kwargs["plugin_config_id"] = plugin_config_id if after is not None: clickhouse_where_parts.append( "timestamp > toDateTime64(%(after)s, 6)") clickhouse_kwargs["after"] = after.isoformat().replace( "+00:00", "") if before is not None: clickhouse_where_parts.append( "timestamp < toDateTime64(%(before)s, 6)") clickhouse_kwargs["before"] = before.isoformat().replace( "+00:00", "") if search: clickhouse_where_parts.append("message ILIKE %(search)s") clickhouse_kwargs["search"] = f"%{search}%" clickhouse_query = f""" SELECT id, team_id, plugin_id, plugin_config_id, timestamp, source, type, message, instance_id FROM plugin_log_entries WHERE {' AND '.join(clickhouse_where_parts)} ORDER BY timestamp DESC {f'LIMIT {limit}' if limit else ''} """ return [ PluginLogEntryRaw(*result) for result in cast( list, sync_execute(clickhouse_query, clickhouse_kwargs)) ] else: filter_kwargs: Dict[str, Any] = {} if team_id is not None: filter_kwargs["team_id"] = team_id if plugin_config_id is not None: filter_kwargs["plugin_config_id"] = plugin_config_id if after is not None: filter_kwargs["timestamp__gt"] = after if before is not None: filter_kwargs["timestamp__lt"] = before if search: filter_kwargs["message__icontains"] = search query = PluginLogEntry.objects.order_by("-timestamp").filter( **filter_kwargs) if limit: query = query[:limit] return list(query)
def setup_periodic_tasks(sender, **kwargs): if not settings.DEBUG: sender.add_periodic_task(1.0, redis_celery_queue_depth.s(), name="1 sec queue probe", priority=0) # Heartbeat every 10sec to make sure the worker is alive sender.add_periodic_task(10.0, redis_heartbeat.s(), name="10 sec heartbeat", priority=0) # update events table partitions twice a week sender.add_periodic_task( crontab(day_of_week="mon,fri", hour=0, minute=0), update_event_partitions.s(), # check twice a week ) if getattr(settings, "MULTI_TENANCY", False) and not is_ee_enabled(): sender.add_periodic_task(crontab(minute=0, hour="*/12"), run_session_recording_retention.s()) # send weekly status report on non-PostHog Cloud instances if not getattr(settings, "MULTI_TENANCY", False): sender.add_periodic_task(crontab(day_of_week="mon", hour=0, minute=0), status_report.s()) # Cloud (posthog-production) cron jobs if getattr(settings, "MULTI_TENANCY", False): sender.add_periodic_task(crontab(hour=0, minute=0), calculate_billing_daily_usage.s()) # every day midnight UTC # send weekly email report (~ 8:00 SF / 16:00 UK / 17:00 EU) sender.add_periodic_task(crontab(day_of_week="mon", hour=15, minute=0), send_weekly_email_report.s()) sender.add_periodic_task(crontab(day_of_week="fri", hour=0, minute=0), clean_stale_partials.s()) sender.add_periodic_task(90, check_cached_items.s(), name="check dashboard items") if is_ee_enabled(): sender.add_periodic_task(120, clickhouse_lag.s(), name="clickhouse table lag") sender.add_periodic_task(120, clickhouse_row_count.s(), name="clickhouse events table row count") sender.add_periodic_task(120, clickhouse_part_count.s(), name="clickhouse table parts count") sender.add_periodic_task(60, calculate_cohort.s(), name="recalculate cohorts") if settings.ASYNC_EVENT_ACTION_MAPPING: sender.add_periodic_task( (60 * ACTION_EVENT_MAPPING_INTERVAL_MINUTES), calculate_event_action_mappings.s(), name="calculate event action mappings", expires=(60 * ACTION_EVENT_MAPPING_INTERVAL_MINUTES), )
def clickhouse_lag(): if is_ee_enabled() and settings.EE_AVAILABLE: from ee.clickhouse.client import sync_execute for table in CLICKHOUSE_TABLES: QUERY = """select max(_timestamp) observed_ts, now() now_ts, now() - max(_timestamp) as lag from {table};""" query = QUERY.format(table=table) lag = sync_execute(query)[0][2] g = statsd.Gauge("%s_posthog_celery" % (settings.STATSD_PREFIX,)) g.send("clickhouse_{table}_table_lag_seconds".format(table=table), lag) else: pass
def clickhouse_row_count(): if is_ee_enabled() and settings.EE_AVAILABLE: from ee.clickhouse.client import sync_execute for table in CLICKHOUSE_TABLES: QUERY = """select count(1) freq from {table};""" query = QUERY.format(table=table) rows = sync_execute(query)[0][0] g = statsd.Gauge("%s_posthog_celery" % (settings.STATSD_PREFIX,)) g.send("clickhouse_{table}_table_row_count".format(table=table), rows) else: pass
def _get_events_volume(team: Team) -> List[Tuple[str, int]]: timestamp = now() - timedelta(days=30) if is_ee_enabled(): from ee.clickhouse.client import sync_execute from ee.clickhouse.sql.events import GET_EVENTS_VOLUME return sync_execute(GET_EVENTS_VOLUME, {"team_id": team.pk, "timestamp": timestamp},) return ( Event.objects.filter(team=team, timestamp__gt=timestamp) .values("event") .annotate(count=Count("id")) .values_list("event", "count") )
def _get_properties_volume(team: Team) -> List[Tuple[str, int]]: timestamp = now() - timedelta(days=30) if is_ee_enabled(): from ee.clickhouse.client import sync_execute from ee.clickhouse.sql.events import GET_PROPERTIES_VOLUME return sync_execute(GET_PROPERTIES_VOLUME, {"team_id": team.pk, "timestamp": timestamp},) cursor = connection.cursor() cursor.execute( "SELECT json_build_array(jsonb_object_keys(properties)) ->> 0 as key1, count(1) FROM posthog_event WHERE team_id = %s AND timestamp > %s group by key1 order by count desc", [team.pk, timestamp], ) return cursor.fetchall()
def bulk_import_events(self): if is_ee_enabled(): from ee.clickhouse.demo import bulk_create_events, bulk_create_session_recording_events bulk_create_events(self.events, team=self.team) bulk_create_session_recording_events(self.snapshots, team_id=self.team.pk) else: Event.objects.bulk_create( [Event(**kw, team=self.team) for kw in self.events]) SessionRecordingEvent.objects.bulk_create([ SessionRecordingEvent(**kw, team=self.team) for kw in self.snapshots ])
def calculate_actions_from_last_calculation() -> None: if is_ee_enabled(): # In EE, actions are not precalculated return start_time_overall = time.time() for action in cast( Sequence[Action], Action.objects.filter(is_calculating=False, deleted=False)): start_time = time.time() action.calculate_events(start=action.last_calculated_at) total_time = time.time() - start_time logger.info( f"Calculating action {action.pk} took {total_time:.2f} seconds") total_time_overall = time.time() - start_time_overall logger.info( f"Calculated new event-action pairs in {total_time_overall:.2f} s")
def _calculate_funnel(filter: Filter, key: str, team_id: int) -> List[Dict[str, Any]]: dashboard_items = DashboardItem.objects.filter(team_id=team_id, filters_hash=key) dashboard_items.update(refreshing=True) if is_ee_enabled(): insight_class = import_from("ee.clickhouse.queries.clickhouse_funnel", "ClickhouseFunnel") else: insight_class = import_from("posthog.queries.funnel", "Funnel") result = insight_class(filter=filter, team=Team(pk=team_id)).run() dashboard_items.update(last_refresh=timezone.now(), refreshing=False) return result
def _calculate_by_filter(filter: FilterType, key: str, team_id: int, cache_type: CacheType) -> List[Dict[str, Any]]: dashboard_items = DashboardItem.objects.filter(team_id=team_id, filters_hash=key) dashboard_items.update(refreshing=True) if is_ee_enabled(): insight_class_path = CH_TYPE_TO_IMPORT[cache_type] else: insight_class_path = TYPE_TO_IMPORT[cache_type] insight_class = import_from(insight_class_path[0], insight_class_path[1]) result = insight_class().run(filter, Team(pk=team_id)) dashboard_items.update(last_refresh=timezone.now(), refreshing=False) return result
def clickhouse_part_count(): if is_ee_enabled() and settings.EE_AVAILABLE: from ee.clickhouse.client import sync_execute QUERY = """ select table, count(1) freq from system.parts group by table order by freq desc; """ rows = sync_execute(QUERY) for (table, parts) in rows: g = statsd.Gauge("%s_posthog_celery" % (settings.STATSD_PREFIX,)) g.send("clickhouse_{table}_table_parts_count".format(table=table), parts) else: pass
def create_demo_team(organization: Organization, user: User, request: Request) -> Team: team = Team.objects.create_with_data( organization=organization, name=TEAM_NAME, ingested_event=True, completed_snippet_onboarding=True, is_demo=True, ) _create_anonymous_users(team=team, base_url=request.build_absolute_uri("/demo")) _create_funnel(team=team, base_url=request.build_absolute_uri("/demo")) FeatureFlag.objects.create( team=team, rollout_percentage=100, name="Sign Up CTA", key="sign-up-cta", created_by=user, ) _recalculate(team=team) if is_ee_enabled(): from ee.clickhouse.demo import create_anonymous_users_ch create_anonymous_users_ch(team=team, base_url=request.build_absolute_uri("/demo")) return team
def clickhouse_row_count(): if is_ee_enabled() and settings.EE_AVAILABLE: from statshog.defaults.django import statsd from ee.clickhouse.client import sync_execute for table in CLICKHOUSE_TABLES: try: QUERY = """select count(1) freq from {table};""" query = QUERY.format(table=table) rows = sync_execute(query)[0][0] statsd.gauge(f"posthog_celery_clickhouse_table_row_count", rows, tags={"table": table}) except: pass else: pass
def clickhouse_part_count(): if is_ee_enabled() and settings.EE_AVAILABLE: from statshog.defaults.django import statsd from ee.clickhouse.client import sync_execute QUERY = """ select table, count(1) freq from system.parts group by table order by freq desc; """ rows = sync_execute(QUERY) for (table, parts) in rows: statsd.gauge(f"posthog_celery_clickhouse_table_parts_count", parts, tags={"table": table}) else: pass
def clickhouse_lag(): if is_ee_enabled() and settings.EE_AVAILABLE: from statshog.defaults.django import statsd from ee.clickhouse.client import sync_execute for table in CLICKHOUSE_TABLES: try: QUERY = ( """select max(_timestamp) observed_ts, now() now_ts, now() - max(_timestamp) as lag from {table};""" ) query = QUERY.format(table=table) lag = sync_execute(query)[0][2] statsd.gauge("posthog_celery_clickhouse__table_lag_seconds", lag, tags={"table": table}) except: pass else: pass
def clickhouse_mutation_count(): if is_ee_enabled() and settings.EE_AVAILABLE: from ee.clickhouse.client import sync_execute QUERY = """ SELECT table, count(1) AS freq FROM system.mutations GROUP BY table ORDER BY freq DESC """ rows = sync_execute(QUERY) for (table, muts) in rows: g = statsd.Gauge("%s_posthog_celery" % (settings.STATSD_PREFIX, )) g.send( "clickhouse_{table}_table_mutations_count".format(table=table), muts) else: pass
def clickhouse_mutation_count(): if is_ee_enabled() and settings.EE_AVAILABLE: from statshog.defaults.django import statsd from ee.clickhouse.client import sync_execute QUERY = """ SELECT table, count(1) AS freq FROM system.mutations GROUP BY table ORDER BY freq DESC """ rows = sync_execute(QUERY) for (table, muts) in rows: statsd.gauge(f"posthog_celery_clickhouse_table_mutations_count", muts, tags={"table": table}) else: pass
# sent_at - timestamp == now - x # x = now + (timestamp - sent_at) try: # timestamp and sent_at must both be in the same format: either both with or both without timezones # otherwise we can't get a diff to add to now return now + (parser.isoparse(data["timestamp"]) - sent_at) except TypeError as e: pass return parser.isoparse(data["timestamp"]) now_datetime = now if data.get("offset"): return now_datetime - relativedelta(microseconds=data["offset"] * 1000) return now_datetime if is_ee_enabled(): def process_event_ee( distinct_id: str, ip: str, site_url: str, data: dict, team_id: int, now: datetime.datetime, sent_at: Optional[datetime.datetime], ) -> None: timer = statsd.Timer("%s_posthog_cloud" % (settings.STATSD_PREFIX, )) timer.start() properties = data.get("properties", {}) if data.get("$set"): properties["$set"] = data["$set"]