def create_event( event_uuid: uuid.UUID, event: str, team: Team, distinct_id: str, timestamp: Optional[Union[datetime, str]], properties: Optional[Dict] = {}, elements_hash: Optional[str] = "", elements: Optional[List[Element]] = None, ) -> None: if not timestamp: timestamp = datetime.now() # clickhouse specific formatting if isinstance(timestamp, str): timestamp = isoparse(timestamp) else: timestamp = timestamp.astimezone(pytz.utc) if elements and not elements_hash: elements_hash = create_elements(event_uuid=event_uuid, elements=elements, team=team) data = { "uuid": str(event_uuid), "event": event, "properties": json.dumps(properties), "timestamp": timestamp.strftime("%Y-%m-%d %H:%M:%S.%f"), "team_id": team.pk, "distinct_id": distinct_id, "elements_hash": elements_hash, "created_at": timestamp.strftime("%Y-%m-%d %H:%M:%S.%f"), } p = ClickhouseProducer() p.produce(sql=INSERT_EVENT_SQL, topic=KAFKA_EVENTS, data=data)
def _capture_ee( event_uuid: UUID, person_uuid: UUID, ip: str, site_url: str, team_id: int, event: str, distinct_id: str, properties: Dict, timestamp: datetime.datetime, ) -> None: elements = properties.get("$elements") elements_list = [] if elements: del properties["$elements"] elements_list = [ Element( text=el["$el_text"][0:400] if el.get("$el_text") else None, tag_name=el["tag_name"], href=el["attr__href"][0:2048] if el.get("attr__href") else None, attr_class=el["attr__class"].split(" ") if el.get("attr__class") else None, attr_id=el.get("attr__id"), nth_child=el.get("nth_child"), nth_of_type=el.get("nth_of_type"), attributes={key: value for key, value in el.items() if key.startswith("attr__")}, ) for index, el in enumerate(elements) ] team = Team.objects.only("slack_incoming_webhook", "event_names", "event_properties", "anonymize_ips").get( pk=team_id ) if not team.anonymize_ips and "$ip" not in properties: properties["$ip"] = ip store_names_and_properties(team=team, event=event, properties=properties) # determine/create elements elements_hash = create_elements(event_uuid=event_uuid, elements=elements_list, team=team) # # determine create events create_event( event_uuid=event_uuid, event=event, properties=properties, timestamp=timestamp, team=team, elements_hash=elements_hash, distinct_id=distinct_id, ) emit_omni_person( event_uuid=event_uuid, uuid=person_uuid, team_id=team_id, distinct_id=distinct_id, timestamp=timestamp, properties=properties, )
def test_create_cache(self) -> None: self.assertEqual(len(get_all_elements()), 0) create_elements( event_uuid=uuid.uuid4(), team=self.team, elements=[ Element(tag_name="a", href="/a-url", nth_child=1, nth_of_type=0), Element(tag_name="button", nth_child=0, nth_of_type=0), Element(tag_name="div", nth_child=0, nth_of_type=0), Element( tag_name="div", nth_child=0, nth_of_type=0, attr_id="nested", ), ], use_cache=True, ) self.assertEqual(len(get_all_elements()), 4) create_elements( event_uuid=uuid.uuid4(), team=self.team, elements=[ Element(tag_name="a", href="/a-url", nth_child=1, nth_of_type=0), Element(tag_name="button", nth_child=0, nth_of_type=0), Element(tag_name="div", nth_child=0, nth_of_type=0), Element( tag_name="div", nth_child=0, nth_of_type=0, attr_id="nested", ), ], use_cache=True, ) self.assertEqual(len(get_all_elements()), 4)
def test_create_elements(self) -> None: elements_hash_1 = create_elements( event_uuid=uuid.uuid4(), team=self.team, elements=[ Element(tag_name="a", href="/a-url", nth_child=1, nth_of_type=0), Element(tag_name="button", nth_child=0, nth_of_type=0), Element(tag_name="div", nth_child=0, nth_of_type=0), Element( tag_name="div", nth_child=0, nth_of_type=0, attr_id="nested", ), ], use_cache=False, ) self.assertEqual(len(get_all_elements()), 4) elements_hash_2 = create_elements( event_uuid=uuid.uuid4(), team=self.team, elements=[ Element(tag_name="a", href="/a-url", nth_child=1, nth_of_type=0), Element(tag_name="button", nth_child=0, nth_of_type=0), Element(tag_name="div", nth_child=0, nth_of_type=0), Element( tag_name="div", nth_child=0, nth_of_type=0, attr_id="nested", ), ], use_cache=False, ) self.assertEqual(elements_hash_1, elements_hash_2) self.assertGreater(len(get_all_elements(final=False)), 4) self.assertEqual(len(get_all_elements(final=True)), 4) elements = get_elements_by_elements_hash(elements_hash=elements_hash_1, team_id=self.team.pk) self.assertEqual(len(elements), 4) self.assertEqual(elements[0]["tag_name"], "a") self.assertEqual(elements[1]["tag_name"], "button") self.assertEqual(elements[2]["tag_name"], "div") self.assertEqual(elements[3]["tag_name"], "div") self.assertEqual(elements[0]["order"], 0) self.assertEqual(elements[1]["order"], 1) self.assertEqual(elements[2]["order"], 2) self.assertEqual(elements[3]["order"], 3) self.assertGreater(len(get_all_elements()), 4) sync_execute("OPTIMIZE TABLE elements FINAL") self.assertEqual(len(get_all_elements()), 4)
def create_anonymous_users_ch(team: Team, base_url: str) -> None: with open(Path("posthog/demo_data.json").resolve(), "r") as demo_data_file: demo_data = json.load(demo_data_file) demo_data_index = 0 days_ago = 7 for index in range(0, 100): if index > 0 and index % 14 == 0: days_ago -= 1 date = now() - relativedelta(days=days_ago) browser = random.choice(["Chrome", "Safari", "Firefox"]) distinct_id = generate_clickhouse_uuid() person = Person.objects.create(team_id=team.pk, distinct_ids=[distinct_id], properties={"is_demo": True}) event_uuid = uuid4() create_event( team=team, event="$pageview", distinct_id=distinct_id, properties={ "$current_url": base_url, "$browser": browser, "$lib": "web", }, timestamp=date, event_uuid=event_uuid, ) if index % 3 == 0: update_person_properties(team_id=team.pk, id=person.uuid, properties=demo_data[demo_data_index]) update_person_is_identified(team_id=team.pk, id=person.uuid, is_identified=True) demo_data_index += 1 elements = [ Element( tag_name="a", href="/demo/1", attr_class=["btn", "btn-success"], attr_id="sign-up", text="Sign up", ), Element(tag_name="form", attr_class=["form"]), Element(tag_name="div", attr_class=["container"]), Element(tag_name="body"), Element(tag_name="html"), ] event_uuid = uuid4() elements_hash = create_elements(elements=elements, team=team, event_uuid=event_uuid) create_event( team=team, distinct_id=distinct_id, event="$autocapture", properties={ "$current_url": base_url, "$browser": browser, "$lib": "web", "$event_type": "click", }, timestamp=date + relativedelta(seconds=14), elements_hash=elements_hash, event_uuid=event_uuid, ) event_uuid = uuid4() create_event( event="$pageview", team=team, distinct_id=distinct_id, properties={ "$current_url": "%s/1" % base_url, "$browser": browser, "$lib": "web", }, timestamp=date + relativedelta(seconds=15), event_uuid=event_uuid, ) if index % 4 == 0: elements = [ Element( tag_name="button", attr_class=["btn", "btn-success"], text="Sign up!", ), Element(tag_name="form", attr_class=["form"]), Element(tag_name="div", attr_class=["container"]), Element(tag_name="body"), Element(tag_name="html"), ] event_uuid = uuid4() elements_hash = create_elements(elements=elements, team=team, event_uuid=event_uuid) create_event( team=team, event="$autocapture", distinct_id=distinct_id, properties={ "$current_url": "%s/1" % base_url, "$browser": browser, "$lib": "web", "$event_type": "click", }, timestamp=date + relativedelta(seconds=29), elements_hash=elements_hash, event_uuid=event_uuid, ) event_uuid = uuid4() create_event( event="$pageview", team=team, distinct_id=distinct_id, properties={ "$current_url": "%s/2" % base_url, "$browser": browser, "$lib": "web", }, timestamp=date + relativedelta(seconds=30), event_uuid=event_uuid, ) if index % 5 == 0: elements = [ Element( tag_name="button", attr_class=["btn", "btn-success"], text="Pay $10", ), Element(tag_name="form", attr_class=["form"]), Element(tag_name="div", attr_class=["container"]), Element(tag_name="body"), Element(tag_name="html"), ] event_uuid = uuid4() elements_hash = create_elements(elements=elements, team=team, event_uuid=event_uuid) create_event( team=team, event="$autocapture", distinct_id=distinct_id, properties={ "$current_url": "%s/2" % base_url, "$browser": browser, "$lib": "web", "$event_type": "click", }, timestamp=date + relativedelta(seconds=59), elements_hash=elements_hash, event_uuid=event_uuid, ) event_uuid = uuid4() create_event( event="purchase", team=team, distinct_id=distinct_id, properties={"price": 10}, timestamp=date + relativedelta(seconds=60), event_uuid=event_uuid, ) event_uuid = uuid4() create_event( event="$pageview", team=team, distinct_id=distinct_id, properties={ "$current_url": "%s/3" % base_url, "$browser": browser, "$lib": "web", }, timestamp=date + relativedelta(seconds=60), event_uuid=event_uuid, ) team.event_properties_numerical.append("purchase") team.save()