async def test_onesided_individual_conversation(writer: TwitterDataWriter):
conversation_id = "one-sided conversation"
talking_user = DOG_RATES
start_time = "2010-06-06T01:00:00.543Z"
end_time = "2010-09-06T01:00:00.345Z"
messages = generate_messages(
5, start_time, end_time, conversation_id, talking_user, MAIN_USER_ID
)
for message in messages:
writer.add_message(message)
await writer.finalize()
assert writer.execute(
"select * from conversations where id=?;", (conversation_id,)
).fetchone() == (
conversation_id,
"individual",
None,
5,
0,
int(talking_user),
start_time,
end_time,
0,
None,
2,
0,
)
assert check_dog_rates(writer) == 5
assert writer.execute(
"select * from participants where conversation=? and participant=?;",
(conversation_id, int(talking_user)),
).fetchone() == (
int(talking_user),
conversation_id,
5,
None,
None,
None,
)
assert writer.execute(
"select * from participants where conversation=? and participant=?;",
(conversation_id, MAIN_USER_ID),
).fetchone() == (
MAIN_USER_ID,
conversation_id,
0,
None,
None,
None,
)
async def test_othersided_individual_conversation(writer: TwitterDataWriter):
start_time = "2015-10-10T07:07:07.777Z"
end_time = "2020-10-10T07:07:07.777Z"
conversation_id = "you-talking"
messages = generate_messages(
8, start_time, end_time, conversation_id, MAIN_USER_ID, DOG_RATES
)
for message in messages:
writer.add_message(message)
await writer.finalize()
assert writer.execute(
"select * from conversations where id=?;", (conversation_id,)
).fetchone() == (
conversation_id,
"individual",
None,
8,
8,
DOG_RATES,
start_time,
end_time,
1,
None,
2,
0,
)
assert check_dog_rates(writer) == 0
assert writer.execute(
"select * from participants where conversation=? and participant=?;",
(conversation_id, DOG_RATES),
).fetchone() == (
DOG_RATES,
conversation_id,
0,
None,
None,
None,
)
assert writer.execute(
"select * from participants where conversation=? and participant=?;",
(conversation_id, MAIN_USER_ID),
).fetchone() == (
MAIN_USER_ID,
conversation_id,
8,
None,
None,
None,
)
def test_known_participant_join(writer: TwitterDataWriter,
messages: deque[dict]):
join_event = messages.popleft()
writer.add_message(join_event, True)
check_conversation(writer, join_event, True)
check_user(writer, join_event["initiatingUserId"])
check_participant(writer, join_event["initiatingUserId"],
join_event["conversationId"])
for user in join_event["userIds"]:
assert ("start_time",
join_event["createdAt"]) in writer.participant_events[(
user, join_event["conversationId"])]
check_user(writer, user)
assert writer.execute(
"select * from participants where participant=? and conversation=?;",
(int(user), join_event["conversationId"]),
).fetchone() == (
int(user),
join_event["conversationId"],
None,
# join_event["createdAt"],
None,
None,
int(join_event["initiatingUserId"]),
)
def check_conversation(writer: TwitterDataWriter, message: dict,
group_dm: bool):
if group_dm:
other_person = None
elif int(message["recipientId"]) == MAIN_USER_ID:
other_person = int(message["senderId"])
else:
other_person = int(message["recipientId"])
assert writer.execute(
"""select * from conversations where id=?;""",
(message["conversationId"], ),
).fetchone() == (
message["conversationId"],
"group" if group_dm else "individual",
None,
None,
None,
other_person,
None,
None,
1,
None,
None,
None,
)
check_participant(writer, MAIN_USER_ID, message["conversationId"])
def test_self_being_added(writer: TwitterDataWriter, messages: deque[dict]):
message = messages.popleft()
writer.add_message(message, True)
assert writer.execute("select * from conversations where id=?;",
(message["conversationId"], )).fetchone() == (
message["conversationId"],
"group",
None,
None,
None,
None,
message["createdAt"],
None,
0,
int(message["initiatingUserId"]),
None,
None,
)
check_user(writer, message["initiatingUserId"])
check_participant(writer, message["initiatingUserId"],
message["conversationId"])
assert ("start_time", message["createdAt"]) in writer.participant_events[(
str(MAIN_USER_ID), message["conversationId"])]
assert writer.execute(
"select * from participants where participant=? and conversation=?;",
(MAIN_USER_ID, message["conversationId"]),
).fetchone() == (
MAIN_USER_ID,
message["conversationId"],
None,
None,
None,
int(message["initiatingUserId"]),
)
for user in message["participantsSnapshot"]:
check_user(writer, user)
assert (
"start_time",
"0000-00-00T00:00:00.000Z",
) in writer.participant_events[(user, message["conversationId"])]
assert (writer.execute(
"""select start_time
from participants where conversation=? and participant=?;""",
(message["conversationId"], int(user)),
).fetchone() == (None, ))
def test_add_message_with_media(writer: TwitterDataWriter,
messages: deque[dict]):
image_message = messages.popleft()
message_add_with_checks(writer, image_message)
image_url = image_message["mediaUrls"][0]
image_url_comps = image_url.replace("https://", "").split("/")
assert writer.execute("select * from media where message=?",
(int(image_message["id"]), )).fetchone() == (
int(image_url_comps[3]),
image_url,
"image",
image_url_comps[-1],
int(image_url_comps[2]),
0,
)
video_message = messages.popleft()
message_add_with_checks(writer, video_message)
video_url = video_message["mediaUrls"][0]
video_url_comps = video_url.replace("https://", "").split("/")
assert writer.execute("select * from media where message=?;",
(int(video_message["id"]), )).fetchone() == (
int(video_url_comps[2]),
video_url,
"video",
video_url_comps[-1],
int(video_message["id"]),
0,
)
gif_message = messages.popleft()
message_add_with_checks(writer, gif_message, True)
gif_url = gif_message["mediaUrls"][0]
gif_url_comps = gif_url.replace("https://", "").split("/")
assert writer.execute("select * from media where message=?;",
(int(gif_message["id"]), )).fetchone() == (
int(gif_url_comps[2]),
gif_url,
"gif",
gif_url_comps[-1],
int(gif_message["id"]),
1,
)
def check_participant(writer: TwitterDataWriter, user: str, conversation: str):
assert writer.execute(
"select * from participants where participant=? and conversation=?;",
(int(user), conversation),
).fetchone() == (
int(user),
conversation,
None,
None,
None,
None,
)
def check_dog_rates(writer: TwitterDataWriter):
"""checks that a user matching the description of the dog_rates twitter account
has been created; this account is the one that even the dummied-out http client
will return data for. returns the number of messages that that user has been
recording sending."""
row = writer.execute(
"""select
number_of_messages, loaded_full_data, handle
from users where id=?;""",
(DOG_RATES,),
).fetchone()
assert row[1:3] == (1, "dog_rates")
return row[0]
def test_add_message_with_link(writer: TwitterDataWriter,
messages: deque[dict]):
message = messages.popleft()
message_add_with_checks(writer, message)
link = message["urls"][0]
added_link = writer.execute("select * from links where message=?;",
(int(message["id"]), )).fetchone()
assert added_link == (
link["expanded"],
link["display"],
link["url"],
int(message["id"]),
)
def check_name_update(writer: TwitterDataWriter, name_update: dict):
writer.add_message(name_update, True)
check_conversation(writer, name_update, True)
check_user(writer, name_update["initiatingUserId"])
check_participant(writer, name_update["initiatingUserId"],
name_update["conversationId"])
assert writer.execute(
"select * from name_updates where conversation=?;",
(name_update["conversationId"], ),
).fetchone() == (
name_update["createdAt"],
int(name_update["initiatingUserId"]),
name_update["name"],
name_update["conversationId"],
)
def test_add_message_with_reactions(writer: TwitterDataWriter,
messages: deque[dict]):
message = messages.popleft()
message_add_with_checks(writer, message)
added_reactions = writer.execute(
"select * from reactions where message=?;",
(int(message["id"]), ),
).fetchall()
assert len(added_reactions) == 2
for reaction in message["reactions"]:
assert (
reaction["reactionKey"],
reaction["createdAt"],
int(reaction["senderId"]),
int(message["id"]),
) in added_reactions
def check_user(writer: TwitterDataWriter, user_id: str):
assert writer.execute(
"select * from users where id=?",
(int(user_id), ),
).fetchone() == (
int(user_id),
None,
None,
None,
0,
None,
None,
None,
None,
None,
None,
None,
)
def test_join_after_leave(writer: TwitterDataWriter, messages: deque[dict]):
leave_event = messages.popleft()
writer.add_message(leave_event, True)
check_conversation(writer, leave_event, True)
for user in leave_event["userIds"]:
check_user(writer, user)
assert ("end_time",
leave_event["createdAt"]) in writer.participant_events[(
user, leave_event["conversationId"])]
join_event = messages.popleft()
writer.add_message(join_event, True)
check_conversation(writer, join_event, True)
check_participant(writer, join_event["initiatingUserId"],
join_event["conversationId"])
user = join_event["userIds"][0]
check_user(writer, user)
# todo: with separate fixtures for each test's data this wouldn't have to be
# hard-coded; it can be taken from the data for the above tests
assert ("end_time",
"2016-02-19T01:38:06.141Z") in writer.participant_events[(
user, join_event["conversationId"])]
assert ("start_time",
join_event["createdAt"]) in writer.participant_events[(
user, join_event["conversationId"])]
assert writer.execute(
"select * from participants where participant=? and conversation=?;",
(int(user), join_event["conversationId"]),
).fetchone() == (
int(user),
join_event["conversationId"],
None,
None,
None,
16573941,
)
def message_add_with_checks(writer: TwitterDataWriter,
message: dict,
group_dm: bool = False):
old_added_messages = writer.added_messages
old_added_users = writer.added_users
old_conversations = writer.added_conversations
new_conversation = (message["conversationId"]
not in writer.added_conversations_cache)
new_users = 0
if (not group_dm
) and message["recipientId"] not in writer.added_users_cache:
new_users += 1
if message["senderId"] not in writer.added_users_cache:
new_users += 1
for reactor in set(x["senderId"] for x in message["reactions"]):
if (reactor not in writer.added_users_cache
and reactor != message["senderId"]
and (group_dm or reactor != message["recipientId"])):
new_users += 1
writer.add_message(message, group_dm)
# check that cache was updated
if new_conversation:
assert writer.added_conversations == old_conversations + 1
assert writer.added_messages == old_added_messages + 1
assert writer.added_users == old_added_users + new_users
assert group_dm or message["recipientId"] in writer.added_users_cache
assert message["senderId"] in writer.added_users_cache
assert (
message["senderId"],
message["conversationId"],
) in writer.added_participants_cache
for reaction in message["reactions"]:
assert (
reaction["senderId"],
message["conversationId"],
) in writer.added_participants_cache
assert (group_dm or (
message["recipientId"],
message["conversationId"],
) in writer.added_participants_cache)
# check that conversation record was added
check_conversation(writer, message, group_dm)
# check that message record was added
assert writer.execute(
"select sent_time, id, sender, content from messages where id=?",
(int(message["id"]), ),
).fetchone() == (
message["createdAt"],
int(message["id"]),
int(message["senderId"]),
message["text"],
)
# check that text search record was added
first_word_match = re.search(r"(^|\s)(\w+)($|\s)", message["text"])
if first_word_match:
first_word = first_word_match.group(0).strip()
print("test searching for word: ", first_word)
assert writer.execute(
f"""select id from messages_text_search where
messages_text_search='{first_word}' and id=?;""",
(int(message["id"]), ),
).fetchone()
else: # pragma: no cover
print("no searchable words found in text")
# check that user records were added
check_user(writer, message["senderId"])
if not group_dm:
check_user(writer, message["recipientId"])
# check that participant records were added
check_participant(writer, message["senderId"], message["conversationId"])
if not group_dm:
check_participant(writer, message["recipientId"],
message["conversationId"])
async def test_simple_group_conversation(writer: TwitterDataWriter):
starts = (
"2010-05-05T12:40:01.000Z",
"2010-05-05T13:40:01.000Z",
"2010-05-05T15:50:01.000Z",
)
ends = (
"2011-05-05T12:40:01.000Z",
"2012-05-05T13:40:01.000Z",
"2013-05-05T15:50:01.000Z",
)
message_counts = (10, 12, 15)
users = (MAIN_USER_ID, DOG_RATES, OBAMA)
conversation_id = "simple-group"
messages = sorted(
generate_conversation(message_counts, starts, ends, conversation_id, users)
+ [
{
"type": "conversationNameUpdate",
"initiatingUserId": users[1],
"name": "bim bam boom",
"createdAt": "2011-02-08T18:43:50.249Z",
"conversationId": conversation_id,
}
],
key=lambda x: x["createdAt"],
)
for message in messages:
writer.add_message(message, True)
await writer.finalize()
assert writer.execute(
"select * from conversations where id=?;", (conversation_id,)
).fetchone() == (
conversation_id,
"group",
None,
sum(message_counts),
message_counts[0],
None,
starts[0],
ends[2],
1,
None,
len(users),
1,
)
assert check_dog_rates(writer) == message_counts[1]
for i, user in enumerate(users):
assert writer.execute(
"select * from participants where participant=? and conversation=?;",
(int(user), conversation_id),
).fetchone() == (
int(user),
conversation_id,
message_counts[i],
None,
None,
None,
)
async def test_conversation_ended_by_various_events(writer: TwitterDataWriter):
starts = (
"2010-05-05T12:40:01.000Z",
"2010-05-05T13:40:01.000Z",
"2010-05-05T15:50:01.000Z",
)
ends = (
"2011-05-05T12:40:01.000Z",
"2012-05-05T13:40:01.000Z",
"2013-05-05T15:50:01.000Z",
)
message_counts = (78, 102, 52)
users = (MAIN_USER_ID, DOG_RATES, OBAMA)
base_conversation = lambda cname: generate_conversation(
message_counts, starts, ends, cname, users
)
conversation_end_time = "2019-12-20T10:51:37.176Z"
all_messages = []
added_to_id = "added-to"
added_to = base_conversation(added_to_id) + [
{
"type": "joinConversation",
"conversationId": added_to_id,
"initiatingUserId": str(DOG_RATES),
"participantsSnapshot": [str(OBAMA)],
"createdAt": conversation_end_time,
}
]
all_messages += added_to
someone_joined_id = "someone-joined"
someone_joined = base_conversation(someone_joined_id) + [
{
"type": "participantsJoin",
"conversationId": someone_joined_id,
"userIds": [str(OBAMA)],
"initiatingUserId": str(DOG_RATES),
"createdAt": conversation_end_time,
}
]
all_messages += someone_joined
someone_left_id = "someone-left"
someone_left = base_conversation(someone_left_id) + [
{
"type": "participantsLeave",
"conversationId": someone_left_id,
"userIds": [str(AMAZINGPHIL)],
"createdAt": conversation_end_time,
}
]
all_messages += someone_left
name_update_id = "name-updated"
name_update = base_conversation(name_update_id) + [
{
"type": "conversationNameUpdate",
"conversationId": name_update_id,
"initiatingUserId": str(OBAMA),
"name": "something's turning over",
"createdAt": conversation_end_time,
}
]
all_messages += name_update
for message in all_messages:
writer.add_message(message, True)
await writer.finalize()
for conversation_id in (
added_to_id,
someone_joined_id,
someone_left_id,
name_update_id,
):
assert writer.execute(
"select last_time from conversations where id=?;", (conversation_id,)
).fetchone() == (conversation_end_time,)
def test_db_initial_state(writer: TwitterDataWriter):
assert writer.account == "test"
assert writer.account_id == MAIN_USER_ID
assert writer.execute("select * from me;").fetchone()[0] == MAIN_USER_ID
