def test_team_that_should_not_set_up_billing(self):
organization, team, user = self.create_org_team_user()
OrganizationBilling.objects.create(
organization=organization,
should_setup_billing=False,
)
self.client.force_login(user)
for _ in range(0, 3):
# Create some events on CH
create_event(
team=team,
event="$pageview",
distinct_id="distinct_id",
event_uuid=uuid.uuid4(),
)
response = self.client.post("/api/user/")
self.assertEqual(response.status_code, status.HTTP_200_OK)
response_data: Dict = response.json()
self.assertEqual(
response_data["billing"],
{
"plan": None,
"current_usage": {
"value": 3,
"formatted": "3"
}
},
)
def capture_event(event: EventData):
"""
Creates an event, given an event dict. Currently just puts this data
directly into clickhouse, but could be created via api to get better parity
with real world, and could provide the abstraction over if we are using
clickhouse or postgres as the primary backend
"""
# NOTE: I'm switching on PRIMARY_DB here although I would like to move this
# behind an app interface rather than have that detail in the tests. It
# shouldn't be required to understand the datastore used for us to test.
if settings.PRIMARY_DB == "clickhouse":
# NOTE: I'm moving this import here as currently in the CI we're
# removing the `ee/` directory from the FOSS build
from ee.clickhouse.models.event import create_event
team = Team.objects.get(id=event.team_id)
create_event(
event_uuid=uuid4(),
team=team,
distinct_id=event.distinct_id,
timestamp=event.timestamp,
event=event.event,
properties=event.properties,
)
else:
Event.objects.create(**dataclasses.asdict(event))
def test_event_usage_cache_is_reset_at_beginning_of_month(self):
organization, team, user = self.create_org_team_user()
self.client.force_login(user)
for _ in range(0, 3):
# Create some events on CH
create_event(
team=team,
event="$pageview",
distinct_id="distinct_id",
event_uuid=uuid.uuid4(),
)
response = self.client.post("/api/user/")
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.json()["billing"]["current_usage"]["value"],
3)
# Check that result was cached
cache_key = f"monthly_usage_{organization.id}"
self.assertEqual(cache.get(cache_key), 3)
# Even though default caching time is 12 hours, the result is only cached until beginning of next month
self.assertEqual(
cache._expire_info.get(cache.make_key(cache_key)),
1546300800.0,
) # 1546300800 = Jan 1, 2019 00:00 UTC
def test_send_to_slack(self, patch_post_to_slack):
self.team.slack_incoming_webhook = "http://slack.com/hook"
_create_action(team=self.team, name="user paid", post_to_slack=True)
_now = now()
create_event(
event_uuid=uuid4(),
team=self.team,
distinct_id="test",
event="user paid",
site_url="http://testserver",
timestamp=_now,
)
self.assertEqual(patch_post_to_slack.call_count, 1)
patch_post_to_slack.assert_has_calls(
[
call(
"ee.tasks.webhooks_ee.post_event_to_webhook_ee",
(
{
"event": "user paid",
"properties": {},
"distinct_id": "test",
"timestamp": _now,
"elements_list": None,
},
self.team.pk,
"http://testserver",
),
)
]
)
def _capture_ee(
event_uuid: UUID,
person_uuid: UUID,
ip: str,
site_url: str,
team_id: int,
event: str,
distinct_id: str,
properties: Dict,
timestamp: datetime.datetime,
) -> None:
elements = properties.get("$elements")
elements_list = []
if elements:
del properties["$elements"]
elements_list = [
Element(
text=el["$el_text"][0:400] if el.get("$el_text") else None,
tag_name=el["tag_name"],
href=el["attr__href"][0:2048] if el.get("attr__href") else None,
attr_class=el["attr__class"].split(" ") if el.get("attr__class") else None,
attr_id=el.get("attr__id"),
nth_child=el.get("nth_child"),
nth_of_type=el.get("nth_of_type"),
attributes={key: value for key, value in el.items() if key.startswith("attr__")},
)
for index, el in enumerate(elements)
]
team = Team.objects.only("slack_incoming_webhook", "event_names", "event_properties", "anonymize_ips").get(
pk=team_id
)
if not team.anonymize_ips and "$ip" not in properties:
properties["$ip"] = ip
store_names_and_properties(team=team, event=event, properties=properties)
# determine/create elements
elements_hash = create_elements(event_uuid=event_uuid, elements=elements_list, team=team)
# # determine create events
create_event(
event_uuid=event_uuid,
event=event,
properties=properties,
timestamp=timestamp,
team=team,
elements_hash=elements_hash,
distinct_id=distinct_id,
)
emit_omni_person(
event_uuid=event_uuid,
uuid=person_uuid,
team_id=team_id,
distinct_id=distinct_id,
timestamp=timestamp,
properties=properties,
)
def create_event_clickhouse(distinct_id: str, event: str, lib: str, created_at: datetime, team: Team) -> None:
create_event(
event_uuid=uuid4(),
team=team,
distinct_id=distinct_id,
event=event,
timestamp=created_at,
properties={"$lib": lib},
)
def _create_event(distinct_id: str, event: str, lib: str, created_at: datetime, team: Team):
create_event(
event_uuid=uuid4(),
event=event,
distinct_id=distinct_id,
timestamp=created_at,
team=team,
properties={"$lib": lib},
)
def _capture_ee(
event_uuid: UUID,
person_uuid: UUID,
ip: str,
site_url: str,
team_id: int,
event: str,
distinct_id: str,
properties: Dict,
timestamp: datetime.datetime,
) -> None:
elements = properties.get("$elements")
elements_list = []
if elements:
del properties["$elements"]
elements_list = [
Element(
text=el["$el_text"][0:400] if el.get("$el_text") else None,
tag_name=el["tag_name"],
href=el["attr__href"][0:2048] if el.get("attr__href") else None,
attr_class=el["attr__class"].split(" ") if el.get("attr__class") else None,
attr_id=el.get("attr__id"),
nth_child=el.get("nth_child"),
nth_of_type=el.get("nth_of_type"),
attributes={key: value for key, value in el.items() if key.startswith("attr__")},
)
for index, el in enumerate(elements)
]
team = Team.objects.select_related("organization").get(pk=team_id)
if not team.anonymize_ips and "$ip" not in properties:
properties["$ip"] = ip
event = sanitize_event_name(event)
_add_missing_feature_flags(properties, team, distinct_id)
store_names_and_properties(team=team, event=event, properties=properties)
if not Person.objects.distinct_ids_exist(team_id=team_id, distinct_ids=[str(distinct_id)]):
# Catch race condition where in between getting and creating,
# another request already created this user
try:
Person.objects.create(team_id=team_id, distinct_ids=[str(distinct_id)])
except IntegrityError:
pass
# # determine create events
create_event(
event_uuid=event_uuid,
event=event,
properties=properties,
timestamp=timestamp,
team=team,
distinct_id=distinct_id,
elements=elements_list,
site_url=site_url,
)
def event_factory(self, team: Team, quantity: int = 1):
for _ in range(0, quantity):
create_event(
team=team,
event=random.choice(
["$pageview", "$autocapture", "order completed"]),
distinct_id=f"distinct_id_{random.randint(100,999)}",
event_uuid=uuid.uuid4(),
)
def _case_reverse_order(self, interval, user, start_date):
create_event(uuid.uuid4(), "step five", self.team, user, start_date)
create_event(uuid.uuid4(), "step four", self.team, user,
self._add_interval(interval, 1, start_date))
create_event(uuid.uuid4(), "step three", self.team, user,
self._add_interval(interval, 2, start_date))
create_event(uuid.uuid4(), "step two", self.team, user,
self._add_interval(interval, 3, start_date))
create_event(uuid.uuid4(), "step one", self.team, user,
self._add_interval(interval, 4, start_date))
def test_get_event_by_id(self):
event_id: Union[str, int] = 12345
if settings.PRIMARY_DB == AnalyticsDBMS.CLICKHOUSE:
from ee.clickhouse.models.event import create_event
event_id = "01793986-dc4b-0000-93e8-1fb646df3a93"
Event(
pk=create_event(
team=self.team,
event="event",
distinct_id="1",
timestamp=timezone.now(),
event_uuid=uuid.UUID(event_id),
)
)
else:
event_factory(team=self.team, event="event", distinct_id="1", timestamp=timezone.now(), id=event_id)
response = self.client.get(f"/api/projects/{self.team.id}/events/{event_id}",)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.json()["event"], "event")
response = self.client.get(f"/api/projects/{self.team.id}/events/123456",)
# EE will inform the user the ID passed is not a valid UUID
self.assertIn(response.status_code, [status.HTTP_404_NOT_FOUND, status.HTTP_400_BAD_REQUEST])
response = self.client.get(f"/api/projects/{self.team.id}/events/im_a_string_not_an_integer",)
self.assertIn(response.status_code, [status.HTTP_404_NOT_FOUND, status.HTTP_400_BAD_REQUEST])
def _create_event(distinct_id: str,
event: str,
lib: str,
timestamp: Union[datetime, str],
team: Team,
properties: Dict = {}) -> None:
create_event(
event_uuid=uuid4(),
team=team,
distinct_id=distinct_id,
event=event,
timestamp=timestamp,
properties={
"$lib": lib,
**properties
},
)
def capture_event(event: EventData):
"""
Creates an event, given an event dict. Currently just puts this data
directly into clickhouse, but could be created via api to get better parity
with real world, and could provide the abstraction over if we are using
clickhouse or postgres as the primary backend
"""
from ee.clickhouse.models.event import create_event
team = Team.objects.get(id=event.team_id)
create_event(
event_uuid=uuid4(),
team=team,
distinct_id=event.distinct_id,
timestamp=event.timestamp,
event=event.event,
properties=event.properties,
)
def _generate_ch_data(self, team, n_events, n_days):
distinct_ids = []
for i in range(0, n_events):
distinct_id = str(UUIDT())
distinct_ids.append(distinct_id)
Person.objects.create(team=team, distinct_ids=[distinct_id], properties={"is_demo": True})
for i in range(0, n_events):
event_uuid = uuid4()
plan = random.choice(PRICING_TIERS)
create_event(
event="$purchase",
team=team,
distinct_id=distinct_ids[i],
properties={"plan": plan[0], "purchase_value": plan[1],},
timestamp=now() - relativedelta(days=random.randint(0, n_days)),
event_uuid=event_uuid,
)
def test_delete_events(self):
create_event(uuid4(), "event1", self.teams[0], "1")
create_event(uuid4(), "event2", self.teams[1], "2")
create_event(uuid4(), "event3", self.teams[2], "3")
delete_teams_data([self.teams[0].pk, self.teams[1].pk])
self.assertEqual(self.select_remaining("events", "event"), ["event3"])
def _insert_many_events(self, start_date):
step_one = self.number + 1
step_two = round(step_one / 2)
step_three = round(step_one / 3)
step_four = round(step_one / 4)
step_five = round(step_one / 5)
for i in range(1, step_one):
create_event(uuid.uuid4(), "step one", self.team, f"user_{i}",
start_date)
for i in range(1, step_two):
create_event(uuid.uuid4(), "step two", self.team, f"user_{i}",
self._add_interval("days", 1, start_date))
for i in range(1, step_three):
create_event(uuid.uuid4(), "step three", self.team, f"user_{i}",
self._add_interval("days", 2, start_date))
for i in range(1, step_four):
create_event(uuid.uuid4(), "step four", self.team, f"user_{i}",
self._add_interval("days", 3, start_date))
for i in range(1, step_five):
create_event(uuid.uuid4(), "step five", self.team, f"user_{i}",
self._add_interval("days", 4, start_date))
def _insert_events(self):
step_one = self.number + 1
step_two = round(step_one / 2)
step_three = round(step_one / 3)
step_four = round(step_one / 4)
step_five = round(step_one / 5)
for i in range(1, step_one):
create_event(uuid.uuid4(), "step one", self.team, f"user_{i}",
"2021-05-01 00:00:00")
for i in range(1, step_two):
create_event(uuid.uuid4(), "step two", self.team, f"user_{i}",
"2021-05-03 00:00:00")
for i in range(1, step_three):
create_event(uuid.uuid4(), "step three", self.team, f"user_{i}",
"2021-05-05 00:00:00")
for i in range(1, step_four):
create_event(uuid.uuid4(), "step four", self.team, f"user_{i}",
"2021-05-07 00:00:00")
for i in range(1, step_five):
create_event(uuid.uuid4(), "step five", self.team, f"user_{i}",
"2021-05-09 00:00:00")
def _create_event(**kwargs):
kwargs.update({"event_uuid": uuid4()})
create_event(**kwargs)
def _create_event(**kwargs):
kwargs["event_uuid"] = uuid4()
create_event(**kwargs)
def _create_event(**kwargs):
kwargs.update({"event_uuid": uuid4()})
return Event(pk=create_event(**kwargs))
def _create_event(**kwargs) -> Event:
pk = uuid4()
kwargs.update({"event_uuid": pk})
create_event(**kwargs)
return Event(pk=str(pk))
def _create_event(**kwargs) -> UUID:
pk = uuid4()
kwargs.update({"event_uuid": pk})
create_event(**kwargs)
return pk
def _create_event(uuid=None, **kwargs):
kwargs.update({"event_uuid": uuid if uuid else uuid4()})
create_event(**kwargs)
def _create_event(**kwargs):
uuid = uuid4()
kwargs.update({"event_uuid": uuid})
create_event(**kwargs)
return Event(id=str(uuid))
def _create_event(**kwargs):
event_uuid = uuid.uuid4()
kwargs.update({"event_uuid": event_uuid})
create_event(**kwargs)
return str(event_uuid)
def create_anonymous_users_ch(team: Team, base_url: str) -> None:
with open(Path("posthog/demo_data.json").resolve(), "r") as demo_data_file:
demo_data = json.load(demo_data_file)
demo_data_index = 0
days_ago = 7
for index in range(0, 100):
if index > 0 and index % 14 == 0:
days_ago -= 1
date = now() - relativedelta(days=days_ago)
browser = random.choice(["Chrome", "Safari", "Firefox"])
distinct_id = generate_clickhouse_uuid()
person = Person.objects.create(team_id=team.pk,
distinct_ids=[distinct_id],
properties={"is_demo": True})
event_uuid = uuid4()
create_event(
team=team,
event="$pageview",
distinct_id=distinct_id,
properties={
"$current_url": base_url,
"$browser": browser,
"$lib": "web",
},
timestamp=date,
event_uuid=event_uuid,
)
if index % 3 == 0:
update_person_properties(team_id=team.pk,
id=person.uuid,
properties=demo_data[demo_data_index])
update_person_is_identified(team_id=team.pk,
id=person.uuid,
is_identified=True)
demo_data_index += 1
elements = [
Element(
tag_name="a",
href="/demo/1",
attr_class=["btn", "btn-success"],
attr_id="sign-up",
text="Sign up",
),
Element(tag_name="form", attr_class=["form"]),
Element(tag_name="div", attr_class=["container"]),
Element(tag_name="body"),
Element(tag_name="html"),
]
event_uuid = uuid4()
elements_hash = create_elements(elements=elements,
team=team,
event_uuid=event_uuid)
create_event(
team=team,
distinct_id=distinct_id,
event="$autocapture",
properties={
"$current_url": base_url,
"$browser": browser,
"$lib": "web",
"$event_type": "click",
},
timestamp=date + relativedelta(seconds=14),
elements_hash=elements_hash,
event_uuid=event_uuid,
)
event_uuid = uuid4()
create_event(
event="$pageview",
team=team,
distinct_id=distinct_id,
properties={
"$current_url": "%s/1" % base_url,
"$browser": browser,
"$lib": "web",
},
timestamp=date + relativedelta(seconds=15),
event_uuid=event_uuid,
)
if index % 4 == 0:
elements = [
Element(
tag_name="button",
attr_class=["btn", "btn-success"],
text="Sign up!",
),
Element(tag_name="form", attr_class=["form"]),
Element(tag_name="div", attr_class=["container"]),
Element(tag_name="body"),
Element(tag_name="html"),
]
event_uuid = uuid4()
elements_hash = create_elements(elements=elements,
team=team,
event_uuid=event_uuid)
create_event(
team=team,
event="$autocapture",
distinct_id=distinct_id,
properties={
"$current_url": "%s/1" % base_url,
"$browser": browser,
"$lib": "web",
"$event_type": "click",
},
timestamp=date + relativedelta(seconds=29),
elements_hash=elements_hash,
event_uuid=event_uuid,
)
event_uuid = uuid4()
create_event(
event="$pageview",
team=team,
distinct_id=distinct_id,
properties={
"$current_url": "%s/2" % base_url,
"$browser": browser,
"$lib": "web",
},
timestamp=date + relativedelta(seconds=30),
event_uuid=event_uuid,
)
if index % 5 == 0:
elements = [
Element(
tag_name="button",
attr_class=["btn", "btn-success"],
text="Pay $10",
),
Element(tag_name="form", attr_class=["form"]),
Element(tag_name="div", attr_class=["container"]),
Element(tag_name="body"),
Element(tag_name="html"),
]
event_uuid = uuid4()
elements_hash = create_elements(elements=elements,
team=team,
event_uuid=event_uuid)
create_event(
team=team,
event="$autocapture",
distinct_id=distinct_id,
properties={
"$current_url": "%s/2" % base_url,
"$browser": browser,
"$lib": "web",
"$event_type": "click",
},
timestamp=date + relativedelta(seconds=59),
elements_hash=elements_hash,
event_uuid=event_uuid,
)
event_uuid = uuid4()
create_event(
event="purchase",
team=team,
distinct_id=distinct_id,
properties={"price": 10},
timestamp=date + relativedelta(seconds=60),
event_uuid=event_uuid,
)
event_uuid = uuid4()
create_event(
event="$pageview",
team=team,
distinct_id=distinct_id,
properties={
"$current_url": "%s/3" % base_url,
"$browser": browser,
"$lib": "web",
},
timestamp=date + relativedelta(seconds=60),
event_uuid=event_uuid,
)
team.event_properties_numerical.append("purchase")
team.save()
def _create_event(**kwargs):
create_event(event_uuid=uuid4(), **kwargs)
def _create_event(**kwargs):
if "event_uuid" not in kwargs:
kwargs.update({"event_uuid": uuid4()})
create_event(**kwargs)
def bulk_create_events(events: List[Dict], **kw):
for event_data in events:
create_event(**event_data, **kw, event_uuid=uuid4()) # type: ignore
def _create_event(**kwargs):
pk = uuid4()
kwargs.update({"event_uuid": pk})
create_event(**kwargs)