def create_group(
team_id: int,
group_type_index: GroupTypeIndex,
group_key: str,
properties: Optional[Dict] = {},
timestamp: Optional[datetime.datetime] = None,
):
if not timestamp:
timestamp = now()
data = {
"group_type_index": group_type_index,
"group_key": group_key,
"team_id": team_id,
"group_properties": json.dumps(properties),
"created_at": timestamp.strftime("%Y-%m-%d %H:%M:%S.%f"),
"_timestamp": timestamp.strftime("%Y-%m-%d %H:%M:%S"),
}
p = ClickhouseProducer()
p.produce(topic=KAFKA_GROUPS, sql=INSERT_GROUP_SQL, data=data)
Group.objects.create(
team_id=team_id,
group_type_index=group_type_index,
group_key=group_key,
group_properties=properties,
version=0,
)
def create_person(
team_id: int,
uuid: Optional[str] = None,
properties: Optional[Dict] = {},
sync: bool = False,
is_identified: bool = False,
timestamp: Optional[datetime.datetime] = None,
) -> str:
if uuid:
uuid = str(uuid)
else:
uuid = str(UUIDT())
if not timestamp:
timestamp = now()
data = {
"id": str(uuid),
"team_id": team_id,
"properties": json.dumps(properties),
"is_identified": int(is_identified),
"timestamp": timestamp.strftime("%Y-%m-%d %H:%M:%S"),
}
p = ClickhouseProducer()
p.produce(topic=KAFKA_PERSON, sql=INSERT_PERSON_SQL, data=data, sync=sync)
return uuid
def create_session_recording_event(
uuid: uuid.UUID,
team_id: int,
distinct_id: str,
session_id: str,
window_id: str,
timestamp: Union[datetime.datetime, str],
snapshot_data: dict,
) -> str:
timestamp = cast_timestamp_or_now(timestamp)
snapshot_data_json = json.dumps(snapshot_data)
data = {
"uuid": str(uuid),
"team_id": team_id,
"distinct_id": distinct_id,
"session_id": session_id,
"window_id": window_id,
"snapshot_data": snapshot_data_json,
"timestamp": timestamp,
"created_at": timestamp,
}
if len(snapshot_data_json) <= MAX_KAFKA_MESSAGE_LENGTH:
p = ClickhouseProducer()
p.produce(sql=INSERT_SESSION_RECORDING_EVENT_SQL, topic=KAFKA_SESSION_RECORDING_EVENTS, data=data)
elif len(snapshot_data_json) <= MAX_INSERT_LENGTH:
sync_execute(INSERT_SESSION_RECORDING_EVENT_SQL, data, settings={"max_query_size": MAX_INSERT_LENGTH})
else:
capture_exception(Exception(f"Session recording event data too large - {len(snapshot_data_json)}"))
return str(uuid)
def create_event(
event_uuid: uuid.UUID,
event: str,
team: Team,
distinct_id: str,
timestamp: Optional[Union[timezone.datetime, str]] = None,
properties: Optional[Dict] = {},
elements: Optional[List[Element]] = None,
site_url: Optional[str] = None,
) -> str:
if not timestamp:
timestamp = timezone.now()
assert timestamp is not None
# clickhouse specific formatting
if isinstance(timestamp, str):
timestamp = isoparse(timestamp)
else:
timestamp = timestamp.astimezone(pytz.utc)
elements_chain = ""
if elements and len(elements) > 0:
elements_chain = elements_to_string(elements=elements)
pb_event = events_pb2.Event()
pb_event.uuid = str(event_uuid)
pb_event.event = event
pb_event.properties = json.dumps(properties)
pb_event.timestamp = timestamp.strftime("%Y-%m-%d %H:%M:%S.%f")
pb_event.team_id = team.pk
pb_event.distinct_id = str(distinct_id)
pb_event.elements_chain = elements_chain
pb_event.created_at = timestamp.strftime("%Y-%m-%d %H:%M:%S.%f")
p = ClickhouseProducer()
p.produce_proto(sql=INSERT_EVENT_SQL, topic=KAFKA_EVENTS, data=pb_event)
if team.slack_incoming_webhook or team.organization.is_feature_available("zapier"):
try:
celery.current_app.send_task(
"ee.tasks.webhooks_ee.post_event_to_webhook_ee",
(
{
"event": event,
"properties": properties,
"distinct_id": distinct_id,
"timestamp": timestamp,
"elements_chain": elements_chain,
},
team.pk,
site_url,
),
)
except:
capture_exception()
return str(event_uuid)
def create_person_distinct_id(id: int, team_id: int, distinct_id: str,
person_id: str) -> None:
data = {
"id": id,
"distinct_id": distinct_id,
"person_id": person_id,
"team_id": team_id
}
p = ClickhouseProducer()
p.produce(topic=KAFKA_PERSON_UNIQUE_ID,
sql=INSERT_PERSON_DISTINCT_ID,
data=data)
def create_event(
event_uuid: uuid.UUID,
event: str,
team: Team,
distinct_id: str,
timestamp: Optional[Union[timezone.datetime, str]] = None,
properties: Optional[Dict] = {},
elements: Optional[List[Element]] = None,
site_url: Optional[str] = None,
) -> str:
if not timestamp:
timestamp = timezone.now()
assert timestamp is not None
# clickhouse specific formatting
if isinstance(timestamp, str):
timestamp = isoparse(timestamp)
else:
timestamp = timestamp.astimezone(pytz.utc)
elements_chain = ""
if elements and len(elements) > 0:
elements_chain = elements_to_string(elements=elements)
pb_event = events_pb2.Event()
pb_event.uuid = str(event_uuid)
pb_event.event = event
pb_event.properties = json.dumps(properties)
pb_event.timestamp = timestamp.strftime("%Y-%m-%d %H:%M:%S.%f")
pb_event.team_id = team.pk
pb_event.distinct_id = str(distinct_id)
pb_event.elements_chain = elements_chain
pb_event.created_at = timestamp.strftime("%Y-%m-%d %H:%M:%S.%f")
p = ClickhouseProducer()
p.produce_proto(sql=INSERT_EVENT_SQL, topic=KAFKA_EVENTS, data=pb_event)
return str(event_uuid)
def create_person_distinct_id(team_id: int,
distinct_id: str,
person_id: str,
version=0,
sign=1) -> None:
data = {
"distinct_id": distinct_id,
"person_id": person_id,
"team_id": team_id,
"_sign": sign
}
p = ClickhouseProducer()
p.produce(topic=KAFKA_PERSON_UNIQUE_ID,
sql=INSERT_PERSON_DISTINCT_ID,
data=data)
if sign == 1:
p.produce(
topic=KAFKA_PERSON_DISTINCT_ID,
sql=INSERT_PERSON_DISTINCT_ID2,
data={
"distinct_id": distinct_id,
"person_id": person_id,
"team_id": team_id,
"version": version,
},
)
def create_event(
event_uuid: uuid.UUID,
event: str,
team: Team,
distinct_id: str,
timestamp: Optional[Union[timezone.datetime, str]] = None,
properties: Optional[Dict] = {},
elements: Optional[List[Element]] = None,
) -> str:
if not timestamp:
timestamp = timezone.now()
assert timestamp is not None
# clickhouse specific formatting
if isinstance(timestamp, str):
timestamp = isoparse(timestamp)
else:
timestamp = timestamp.astimezone(pytz.utc)
elements_chain = ""
if elements and len(elements) > 0:
elements_chain = elements_to_string(elements=elements)
data = {
"uuid": str(event_uuid),
"event": event,
"properties": json.dumps(properties),
"timestamp": timestamp.strftime("%Y-%m-%d %H:%M:%S.%f"),
"team_id": team.pk,
"distinct_id": distinct_id,
"created_at": timestamp.strftime("%Y-%m-%d %H:%M:%S.%f"),
"elements_chain": elements_chain,
}
p = ClickhouseProducer()
p.produce(sql=INSERT_EVENT_SQL, topic=KAFKA_EVENTS, data=data)
return str(event_uuid)
def create_session_recording_event(
uuid: uuid.UUID,
team_id: int,
distinct_id: str,
session_id: str,
timestamp: Union[datetime.datetime, str],
snapshot_data: dict,
) -> str:
timestamp = cast_timestamp_or_now(timestamp)
data = {
"uuid": str(uuid),
"team_id": team_id,
"distinct_id": distinct_id,
"session_id": session_id,
"snapshot_data": json.dumps(snapshot_data),
"timestamp": timestamp,
"created_at": timestamp,
}
p = ClickhouseProducer()
p.produce(sql=INSERT_SESSION_RECORDING_EVENT_SQL,
topic=KAFKA_SESSION_RECORDING_EVENTS,
data=data)
return str(uuid)