def create_event(
event_uuid: uuid.UUID,
event: str,
team: Team,
distinct_id: str,
timestamp: Optional[Union[datetime, str]],
properties: Optional[Dict] = {},
elements_hash: Optional[str] = "",
elements: Optional[List[Element]] = None,
) -> None:
if not timestamp:
timestamp = datetime.now()
# clickhouse specific formatting
if isinstance(timestamp, str):
timestamp = isoparse(timestamp)
else:
timestamp = timestamp.astimezone(pytz.utc)
if elements and not elements_hash:
elements_hash = create_elements(event_uuid=event_uuid, elements=elements, team=team)
data = {
"uuid": str(event_uuid),
"event": event,
"properties": json.dumps(properties),
"timestamp": timestamp.strftime("%Y-%m-%d %H:%M:%S.%f"),
"team_id": team.pk,
"distinct_id": distinct_id,
"elements_hash": elements_hash,
"created_at": timestamp.strftime("%Y-%m-%d %H:%M:%S.%f"),
}
p = ClickhouseProducer()
p.produce(sql=INSERT_EVENT_SQL, topic=KAFKA_EVENTS, data=data)
def _capture_ee(
event_uuid: UUID,
person_uuid: UUID,
ip: str,
site_url: str,
team_id: int,
event: str,
distinct_id: str,
properties: Dict,
timestamp: datetime.datetime,
) -> None:
elements = properties.get("$elements")
elements_list = []
if elements:
del properties["$elements"]
elements_list = [
Element(
text=el["$el_text"][0:400] if el.get("$el_text") else None,
tag_name=el["tag_name"],
href=el["attr__href"][0:2048] if el.get("attr__href") else None,
attr_class=el["attr__class"].split(" ") if el.get("attr__class") else None,
attr_id=el.get("attr__id"),
nth_child=el.get("nth_child"),
nth_of_type=el.get("nth_of_type"),
attributes={key: value for key, value in el.items() if key.startswith("attr__")},
)
for index, el in enumerate(elements)
]
team = Team.objects.only("slack_incoming_webhook", "event_names", "event_properties", "anonymize_ips").get(
pk=team_id
)
if not team.anonymize_ips and "$ip" not in properties:
properties["$ip"] = ip
store_names_and_properties(team=team, event=event, properties=properties)
# determine/create elements
elements_hash = create_elements(event_uuid=event_uuid, elements=elements_list, team=team)
# # determine create events
create_event(
event_uuid=event_uuid,
event=event,
properties=properties,
timestamp=timestamp,
team=team,
elements_hash=elements_hash,
distinct_id=distinct_id,
)
emit_omni_person(
event_uuid=event_uuid,
uuid=person_uuid,
team_id=team_id,
distinct_id=distinct_id,
timestamp=timestamp,
properties=properties,
)
def test_create_cache(self) -> None:
self.assertEqual(len(get_all_elements()), 0)
create_elements(
event_uuid=uuid.uuid4(),
team=self.team,
elements=[
Element(tag_name="a",
href="/a-url",
nth_child=1,
nth_of_type=0),
Element(tag_name="button", nth_child=0, nth_of_type=0),
Element(tag_name="div", nth_child=0, nth_of_type=0),
Element(
tag_name="div",
nth_child=0,
nth_of_type=0,
attr_id="nested",
),
],
use_cache=True,
)
self.assertEqual(len(get_all_elements()), 4)
create_elements(
event_uuid=uuid.uuid4(),
team=self.team,
elements=[
Element(tag_name="a",
href="/a-url",
nth_child=1,
nth_of_type=0),
Element(tag_name="button", nth_child=0, nth_of_type=0),
Element(tag_name="div", nth_child=0, nth_of_type=0),
Element(
tag_name="div",
nth_child=0,
nth_of_type=0,
attr_id="nested",
),
],
use_cache=True,
)
self.assertEqual(len(get_all_elements()), 4)
def test_create_elements(self) -> None:
elements_hash_1 = create_elements(
event_uuid=uuid.uuid4(),
team=self.team,
elements=[
Element(tag_name="a",
href="/a-url",
nth_child=1,
nth_of_type=0),
Element(tag_name="button", nth_child=0, nth_of_type=0),
Element(tag_name="div", nth_child=0, nth_of_type=0),
Element(
tag_name="div",
nth_child=0,
nth_of_type=0,
attr_id="nested",
),
],
use_cache=False,
)
self.assertEqual(len(get_all_elements()), 4)
elements_hash_2 = create_elements(
event_uuid=uuid.uuid4(),
team=self.team,
elements=[
Element(tag_name="a",
href="/a-url",
nth_child=1,
nth_of_type=0),
Element(tag_name="button", nth_child=0, nth_of_type=0),
Element(tag_name="div", nth_child=0, nth_of_type=0),
Element(
tag_name="div",
nth_child=0,
nth_of_type=0,
attr_id="nested",
),
],
use_cache=False,
)
self.assertEqual(elements_hash_1, elements_hash_2)
self.assertGreater(len(get_all_elements(final=False)), 4)
self.assertEqual(len(get_all_elements(final=True)), 4)
elements = get_elements_by_elements_hash(elements_hash=elements_hash_1,
team_id=self.team.pk)
self.assertEqual(len(elements), 4)
self.assertEqual(elements[0]["tag_name"], "a")
self.assertEqual(elements[1]["tag_name"], "button")
self.assertEqual(elements[2]["tag_name"], "div")
self.assertEqual(elements[3]["tag_name"], "div")
self.assertEqual(elements[0]["order"], 0)
self.assertEqual(elements[1]["order"], 1)
self.assertEqual(elements[2]["order"], 2)
self.assertEqual(elements[3]["order"], 3)
self.assertGreater(len(get_all_elements()), 4)
sync_execute("OPTIMIZE TABLE elements FINAL")
self.assertEqual(len(get_all_elements()), 4)
def create_anonymous_users_ch(team: Team, base_url: str) -> None:
with open(Path("posthog/demo_data.json").resolve(), "r") as demo_data_file:
demo_data = json.load(demo_data_file)
demo_data_index = 0
days_ago = 7
for index in range(0, 100):
if index > 0 and index % 14 == 0:
days_ago -= 1
date = now() - relativedelta(days=days_ago)
browser = random.choice(["Chrome", "Safari", "Firefox"])
distinct_id = generate_clickhouse_uuid()
person = Person.objects.create(team_id=team.pk,
distinct_ids=[distinct_id],
properties={"is_demo": True})
event_uuid = uuid4()
create_event(
team=team,
event="$pageview",
distinct_id=distinct_id,
properties={
"$current_url": base_url,
"$browser": browser,
"$lib": "web",
},
timestamp=date,
event_uuid=event_uuid,
)
if index % 3 == 0:
update_person_properties(team_id=team.pk,
id=person.uuid,
properties=demo_data[demo_data_index])
update_person_is_identified(team_id=team.pk,
id=person.uuid,
is_identified=True)
demo_data_index += 1
elements = [
Element(
tag_name="a",
href="/demo/1",
attr_class=["btn", "btn-success"],
attr_id="sign-up",
text="Sign up",
),
Element(tag_name="form", attr_class=["form"]),
Element(tag_name="div", attr_class=["container"]),
Element(tag_name="body"),
Element(tag_name="html"),
]
event_uuid = uuid4()
elements_hash = create_elements(elements=elements,
team=team,
event_uuid=event_uuid)
create_event(
team=team,
distinct_id=distinct_id,
event="$autocapture",
properties={
"$current_url": base_url,
"$browser": browser,
"$lib": "web",
"$event_type": "click",
},
timestamp=date + relativedelta(seconds=14),
elements_hash=elements_hash,
event_uuid=event_uuid,
)
event_uuid = uuid4()
create_event(
event="$pageview",
team=team,
distinct_id=distinct_id,
properties={
"$current_url": "%s/1" % base_url,
"$browser": browser,
"$lib": "web",
},
timestamp=date + relativedelta(seconds=15),
event_uuid=event_uuid,
)
if index % 4 == 0:
elements = [
Element(
tag_name="button",
attr_class=["btn", "btn-success"],
text="Sign up!",
),
Element(tag_name="form", attr_class=["form"]),
Element(tag_name="div", attr_class=["container"]),
Element(tag_name="body"),
Element(tag_name="html"),
]
event_uuid = uuid4()
elements_hash = create_elements(elements=elements,
team=team,
event_uuid=event_uuid)
create_event(
team=team,
event="$autocapture",
distinct_id=distinct_id,
properties={
"$current_url": "%s/1" % base_url,
"$browser": browser,
"$lib": "web",
"$event_type": "click",
},
timestamp=date + relativedelta(seconds=29),
elements_hash=elements_hash,
event_uuid=event_uuid,
)
event_uuid = uuid4()
create_event(
event="$pageview",
team=team,
distinct_id=distinct_id,
properties={
"$current_url": "%s/2" % base_url,
"$browser": browser,
"$lib": "web",
},
timestamp=date + relativedelta(seconds=30),
event_uuid=event_uuid,
)
if index % 5 == 0:
elements = [
Element(
tag_name="button",
attr_class=["btn", "btn-success"],
text="Pay $10",
),
Element(tag_name="form", attr_class=["form"]),
Element(tag_name="div", attr_class=["container"]),
Element(tag_name="body"),
Element(tag_name="html"),
]
event_uuid = uuid4()
elements_hash = create_elements(elements=elements,
team=team,
event_uuid=event_uuid)
create_event(
team=team,
event="$autocapture",
distinct_id=distinct_id,
properties={
"$current_url": "%s/2" % base_url,
"$browser": browser,
"$lib": "web",
"$event_type": "click",
},
timestamp=date + relativedelta(seconds=59),
elements_hash=elements_hash,
event_uuid=event_uuid,
)
event_uuid = uuid4()
create_event(
event="purchase",
team=team,
distinct_id=distinct_id,
properties={"price": 10},
timestamp=date + relativedelta(seconds=60),
event_uuid=event_uuid,
)
event_uuid = uuid4()
create_event(
event="$pageview",
team=team,
distinct_id=distinct_id,
properties={
"$current_url": "%s/3" % base_url,
"$browser": browser,
"$lib": "web",
},
timestamp=date + relativedelta(seconds=60),
event_uuid=event_uuid,
)
team.event_properties_numerical.append("purchase")
team.save()
