def values(self, request: request.Request, **kwargs) -> response.Response:
key = request.GET.get("key")
value = request.GET.get("value")
flattened = []
if key:
timer = statsd.timer("get_person_property_values_for_key_timer").start()
try:
result = get_person_property_values_for_key(key, self.team, value)
statsd.incr(
"get_person_property_values_for_key_success",
tags={"key": key, "value": value, "team_id": self.team.id},
)
except Exception as e:
statsd.incr(
"get_person_property_values_for_key_error",
tags={"error": str(e), "key": key, "value": value, "team_id": self.team.id},
)
raise e
finally:
timer.stop()
for (value, count) in result:
try:
# Try loading as json for dicts or arrays
flattened.append({"name": convert_property_value(json.loads(value)), "count": count}) # type: ignore
except json.decoder.JSONDecodeError:
flattened.append({"name": convert_property_value(value), "count": count})
return response.Response(flattened)
def update_cache_item(key: str, cache_type: CacheType,
payload: dict) -> List[Dict[str, Any]]:
timer = statsd.timer("update_cache_item_timer").start()
result: Optional[Union[List, Dict]] = None
filter_dict = json.loads(payload["filter"])
team_id = int(payload["team_id"])
filter = get_filter(data=filter_dict, team=Team(pk=team_id))
# Doing the filtering like this means we'll update _all_ Insights with the same filters hash
dashboard_items = Insight.objects.filter(team_id=team_id, filters_hash=key)
dashboard_items.update(refreshing=True)
try:
if cache_type == CacheType.FUNNEL:
result = _calculate_funnel(filter, key, team_id)
else:
result = _calculate_by_filter(filter, key, team_id, cache_type)
cache.set(key, {
"result": result,
"type": cache_type,
"last_refresh": timezone.now()
}, settings.CACHED_RESULTS_TTL)
except Exception as e:
timer.stop()
statsd.incr("update_cache_item_error")
dashboard_items.filter(refresh_attempt=None).update(refresh_attempt=0)
dashboard_items.update(refreshing=False,
refresh_attempt=F("refresh_attempt") + 1)
raise e
timer.stop()
statsd.incr("update_cache_item_success")
dashboard_items.update(last_refresh=timezone.now(),
refreshing=False,
refresh_attempt=0)
return result
def get_event(request):
timer = statsd.timer("posthog_cloud_event_endpoint").start()
now = timezone.now()
try:
data = load_data_from_request(request)
except RequestParsingError as error:
capture_exception(
error
) # We still capture this on Sentry to identify actual potential bugs
return cors_response(
request,
generate_exception_response("capture",
f"Malformed request data: {error}",
code="invalid_payload"),
)
if not data:
return cors_response(
request,
generate_exception_response(
"capture",
"No data found. Make sure to use a POST request when sending the payload in the body of the request.",
code="no_data",
),
)
sent_at = _get_sent_at(data, request)
token = _get_token(data, request)
if not token:
return cors_response(
request,
generate_exception_response(
"capture",
"API key not provided. You can find your project API key in PostHog project settings.",
type="authentication_error",
code="missing_api_key",
status_code=status.HTTP_401_UNAUTHORIZED,
),
)
team = Team.objects.get_team_from_token(token)
if team is None:
try:
project_id = _get_project_id(data, request)
except ValueError:
return cors_response(
request,
generate_exception_response("capture",
"Invalid Project ID.",
code="invalid_project",
attr="project_id"),
)
if not project_id:
return cors_response(
request,
generate_exception_response(
"capture",
"Project API key invalid. You can find your project API key in PostHog project settings.",
type="authentication_error",
code="invalid_api_key",
status_code=status.HTTP_401_UNAUTHORIZED,
),
)
user = User.objects.get_from_personal_api_key(token)
if user is None:
return cors_response(
request,
generate_exception_response(
"capture",
"Invalid Personal API key.",
type="authentication_error",
code="invalid_personal_api_key",
status_code=status.HTTP_401_UNAUTHORIZED,
),
)
team = user.teams.get(id=project_id)
if isinstance(data, dict):
if data.get("batch"): # posthog-python and posthog-ruby
data = data["batch"]
assert data is not None
elif "engage" in request.path_info: # JS identify call
data["event"] = "$identify" # make sure it has an event name
if isinstance(data, list):
events = data
else:
events = [data]
try:
events = preprocess_session_recording_events(events)
except ValueError as e:
return cors_response(
request,
generate_exception_response("capture",
f"Invalid payload: {e}",
code="invalid_payload"))
for event in events:
try:
distinct_id = _get_distinct_id(event)
except KeyError:
return cors_response(
request,
generate_exception_response(
"capture",
"You need to set user distinct ID field `distinct_id`.",
code="required",
attr="distinct_id",
),
)
except ValueError:
return cors_response(
request,
generate_exception_response(
"capture",
"Distinct ID field `distinct_id` must have a non-empty value.",
code="required",
attr="distinct_id",
),
)
if not event.get("event"):
return cors_response(
request,
generate_exception_response(
"capture",
"You need to set user event name, field `event`.",
code="required",
attr="event"),
)
if not event.get("properties"):
event["properties"] = {}
_ensure_web_feature_flags_in_properties(event, team, distinct_id)
event_uuid = UUIDT()
ip = None if team.anonymize_ips else get_ip_address(request)
if is_ee_enabled():
statsd.incr("posthog_cloud_plugin_server_ingestion")
log_event(
distinct_id=distinct_id,
ip=ip,
site_url=request.build_absolute_uri("/")[:-1],
data=event,
team_id=team.pk,
now=now,
sent_at=sent_at,
event_uuid=event_uuid,
)
else:
task_name = "posthog.tasks.process_event.process_event_with_plugins"
celery_queue = settings.PLUGINS_CELERY_QUEUE
celery_app.send_task(
name=task_name,
queue=celery_queue,
args=[
distinct_id,
ip,
request.build_absolute_uri("/")[:-1],
event,
team.pk,
now.isoformat(),
sent_at,
],
)
timer.stop()
statsd.incr(f"posthog_cloud_raw_endpoint_success",
tags={
"endpoint": "capture",
})
return cors_response(request, JsonResponse({"status": 1}))
def get_event(request):
timer = statsd.timer("posthog_cloud_event_endpoint").start()
now = timezone.now()
data, error_response = get_data(request)
if error_response:
return error_response
sent_at = _get_sent_at(data, request)
token = get_token(data, request)
if not token:
return cors_response(
request,
generate_exception_response(
"capture",
"API key not provided. You can find your project API key in PostHog project settings.",
type="authentication_error",
code="missing_api_key",
status_code=status.HTTP_401_UNAUTHORIZED,
),
)
ingestion_context, db_error, error_response = get_event_ingestion_context(request, data, token)
if error_response:
return error_response
send_events_to_dead_letter_queue = False
if db_error:
send_events_to_dead_letter_queue = True
if isinstance(data, dict):
if data.get("batch"): # posthog-python and posthog-ruby
data = data["batch"]
assert data is not None
elif "engage" in request.path_info: # JS identify call
data["event"] = "$identify" # make sure it has an event name
if isinstance(data, list):
events = data
else:
events = [data]
try:
events = preprocess_session_recording_events(events)
except ValueError as e:
return cors_response(
request, generate_exception_response("capture", f"Invalid payload: {e}", code="invalid_payload")
)
site_url = request.build_absolute_uri("/")[:-1]
ip = None if not ingestion_context or ingestion_context.anonymize_ips else get_ip_address(request)
for event in events:
event_uuid = UUIDT()
distinct_id = get_distinct_id(event)
if not distinct_id:
continue
payload_uuid = event.get("uuid", None)
if payload_uuid:
if UUIDT.is_valid_uuid(payload_uuid):
event_uuid = UUIDT(uuid_str=payload_uuid)
else:
statsd.incr("invalid_event_uuid")
event = parse_event(event, distinct_id, ingestion_context)
if not event:
continue
if send_events_to_dead_letter_queue:
kafka_event = parse_kafka_event_data(
distinct_id=distinct_id,
ip=None,
site_url=site_url,
team_id=None,
now=now,
event_uuid=event_uuid,
data=event,
sent_at=sent_at,
)
log_event_to_dead_letter_queue(
data,
event["event"],
kafka_event,
f"Unable to fetch team from Postgres. Error: {db_error}",
"django_server_capture_endpoint",
)
continue
try:
capture_internal(event, distinct_id, ip, site_url, now, sent_at, ingestion_context.team_id, event_uuid) # type: ignore
except Exception as e:
timer.stop()
capture_exception(e, {"data": data})
statsd.incr(
"posthog_cloud_raw_endpoint_failure", tags={"endpoint": "capture",},
)
return cors_response(
request,
generate_exception_response(
"capture",
"Unable to store event. Please try again. If you are the owner of this app you can check the logs for further details.",
code="server_error",
type="server_error",
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
),
)
timer.stop()
statsd.incr(
"posthog_cloud_raw_endpoint_success", tags={"endpoint": "capture",},
)
return cors_response(request, JsonResponse({"status": 1}))
def post_event_to_webhook_ee(self: Task, event: Dict[str, Any], team_id: int, site_url: str) -> None:
if not site_url:
site_url = settings.SITE_URL
timer = statsd.timer("posthog_cloud_hooks_processed_for_event").start()
team = Team.objects.select_related("organization").get(pk=team_id)
elements_list = chain_to_elements(event.get("elements_chain", ""))
ephemeral_postgres_event = Event.objects.create(
event=event["event"],
distinct_id=event["distinct_id"],
properties=event["properties"],
team=team,
site_url=site_url,
**({"timestamp": event["timestamp"]} if event["timestamp"] else {}),
**({"elements": elements_list})
)
try:
is_zapier_available = team.organization.is_feature_available("zapier")
actionFilters = {"team_id": team_id, "deleted": False}
if not is_zapier_available:
if not team.slack_incoming_webhook:
return # Exit this task if neither Zapier nor webhook URL are available
else:
actionFilters["post_to_slack"] = True # We only need to fire for actions that are posted to webhook URL
for action in cast(Sequence[Action], Action.objects.filter(**actionFilters).all()):
try:
# Wrapped in len to evaluate right away
qs = len(Event.objects.filter(pk=ephemeral_postgres_event.pk).query_db_by_action(action))
except DataError as e:
# Ignore invalid regex errors, which are user mistakes
if not "invalid regular expression" in str(e):
capture_exception(e)
continue
except:
capture_exception()
continue
if not qs:
continue
# REST hooks
if is_zapier_available:
action.on_perform(ephemeral_postgres_event)
# webhooks
if team.slack_incoming_webhook and action.post_to_slack:
message_text, message_markdown = get_formatted_message(action, ephemeral_postgres_event, site_url)
if determine_webhook_type(team) == "slack":
message = {
"text": message_text,
"blocks": [{"type": "section", "text": {"type": "mrkdwn", "text": message_markdown}}],
}
else:
message = {
"text": message_markdown,
}
statsd.incr("posthog_cloud_hooks_web_fired")
requests.post(team.slack_incoming_webhook, verify=False, json=message)
except:
raise
finally:
timer.stop()
ephemeral_postgres_event.delete()
def get_event(request):
timer = statsd.timer("posthog_cloud_event_endpoint").start()
now = timezone.now()
data, error_response = get_data(request)
if error_response:
return error_response
sent_at = _get_sent_at(data, request)
token = get_token(data, request)
if not token:
return cors_response(
request,
generate_exception_response(
"capture",
"API key not provided. You can find your project API key in PostHog project settings.",
type="authentication_error",
code="missing_api_key",
status_code=status.HTTP_401_UNAUTHORIZED,
),
)
team, db_error, error_response = get_team(request, data, token)
if error_response:
return error_response
send_events_to_dead_letter_queue = False
if db_error and is_clickhouse_enabled():
send_events_to_dead_letter_queue = True
if isinstance(data, dict):
if data.get("batch"): # posthog-python and posthog-ruby
data = data["batch"]
assert data is not None
elif "engage" in request.path_info: # JS identify call
data["event"] = "$identify" # make sure it has an event name
if isinstance(data, list):
events = data
else:
events = [data]
try:
events = preprocess_session_recording_events(events)
except ValueError as e:
return cors_response(
request,
generate_exception_response("capture",
f"Invalid payload: {e}",
code="invalid_payload"))
site_url = request.build_absolute_uri("/")[:-1]
ip = None if not team or team.anonymize_ips else get_ip_address(request)
for event in events:
event_uuid = UUIDT()
distinct_id = get_distinct_id(event)
if not distinct_id:
continue
event = parse_event(event, distinct_id, team)
if not event:
continue
if send_events_to_dead_letter_queue:
kafka_event = parse_kafka_event_data(
distinct_id=distinct_id,
ip=None,
site_url=site_url,
team_id=None,
now=now,
event_uuid=event_uuid,
data=event,
sent_at=sent_at,
)
log_event_to_dead_letter_queue(
data,
event["event"],
kafka_event,
f"Unable to fetch team from Postgres. Error: {db_error}",
"django_server_capture_endpoint",
)
continue
statsd.incr("posthog_cloud_plugin_server_ingestion")
capture_internal(event, distinct_id, ip, site_url, now, sent_at,
team.pk, event_uuid) # type: ignore
timer.stop()
statsd.incr(
f"posthog_cloud_raw_endpoint_success",
tags={
"endpoint": "capture",
},
)
return cors_response(request, JsonResponse({"status": 1}))