def message_base_to_event(message: Union[MessageBase, Message],
event_type: EventTypes = EventTypes.MESSAGE,
group: GroupBase = None):
event = {
"event_type": event_type,
"group_id": message.group_id,
"sender_id": str(message.user_id),
"message_id": message.message_id,
"message_payload": message.message_payload,
"message_type": message.message_type,
"updated_at": to_int(to_ts(message.updated_at, allow_none=True)),
"created_at": to_int(to_ts(message.created_at)),
}
# if the 'message' variable is Message instead of MessageBase, there's no file_id available; (e.g. for /edit)
if hasattr(message, "file_id"):
event["file_id"] = message.file_id
if group is not None:
group_dict = group_base_to_event(group)
del group_dict["event_type"]
event["group"] = group_dict
return event
def message_base_to_message(message: MessageBase) -> Message:
message_dict = message.dict()
message_dict["updated_at"] = to_ts(message_dict["updated_at"],
allow_none=True)
message_dict["created_at"] = to_ts(message_dict["created_at"],
allow_none=True)
return Message(**message_dict)
def stats_to_event_dict(user_stats):
stats_dict = user_stats.dict()
stats_dict["last_read"] = int(to_ts(stats_dict["last_read"]) * 1000)
stats_dict["last_sent"] = int(to_ts(stats_dict["last_sent"]) * 1000)
stats_dict["delete_before"] = int(
to_ts(stats_dict["delete_before"]) * 1000)
stats_dict["join_time"] = int(to_ts(stats_dict["join_time"]) * 1000)
if stats_dict["highlight_time"]:
stats_dict["highlight_time"] = int(
to_ts(stats_dict["highlight_time"]) * 1000)
if stats_dict["last_updated_time"]:
stats_dict["last_updated_time"] = int(
to_ts(stats_dict["last_updated_time"]) * 1000)
if stats_dict["first_sent"]:
stats_dict["first_sent"] = int(to_ts(stats_dict["first_sent"]) * 1000)
if stats_dict["receiver_highlight_time"]:
stats_dict["receiver_highlight_time"] = \
int(to_ts(stats_dict["receiver_highlight_time"]) * 1000)
# not needed in the mqtt event
del stats_dict["user_id"]
del stats_dict["group_id"]
return stats_dict
def to_user_group_stats(user_stats: UserGroupStatsBase) -> UserGroupStats:
delete_before = to_ts(user_stats.delete_before)
last_updated_time = to_ts(user_stats.last_updated_time)
last_sent = to_ts(user_stats.last_sent, allow_none=True)
last_read = to_ts(user_stats.last_read, allow_none=True)
first_sent = to_ts(user_stats.first_sent, allow_none=True)
join_time = to_ts(user_stats.join_time, allow_none=True)
highlight_time = to_ts(user_stats.highlight_time, allow_none=True)
# try using the counter column on the stats table instead of actually counting
"""
unread_amount = self.env.storage.count_messages_in_group_since(
group_id, user_stats.last_read
)
"""
return UserGroupStats(
user_id=user_stats.user_id,
group_id=user_stats.group_id,
unread=user_stats.unread_count,
join_time=join_time,
receiver_unread=-1, # TODO: should be count for other user here as well?
last_read_time=last_read,
last_sent_time=last_sent,
delete_before=delete_before,
first_sent=first_sent,
rating=user_stats.rating,
highlight_time=highlight_time,
hide=user_stats.hide,
pin=user_stats.pin,
deleted=user_stats.deleted,
bookmark=user_stats.bookmark,
last_updated_time=last_updated_time,
)
def update_user_stats_to_now(self,
group_id: str,
user_id: int = BaseTest.USER_ID):
now = arrow.utcnow().datetime
now_ts = to_ts(now)
raw_response = self.client.put(
f"/v1/groups/{group_id}/user/{user_id}/update",
json={"last_read_time": now_ts},
)
self.assertEqual(raw_response.status_code, 200)
return float(now_ts)
def group_base_to_group(group: GroupBase,
users: Dict[int, float],
user_count: int,
message_amount: int = -1) -> Group:
group_dict = group.dict()
users = [
GroupJoinTime(
user_id=user_id,
join_time=join_time,
) for user_id, join_time in users.items()
]
users.sort(key=lambda user: user.join_time, reverse=True)
group_dict["updated_at"] = to_ts(group_dict["updated_at"], allow_none=True)
group_dict["created_at"] = to_ts(group_dict["created_at"])
group_dict["last_message_time"] = to_ts(group_dict["last_message_time"])
group_dict["first_message_time"] = to_ts(group_dict["first_message_time"])
group_dict["users"] = users
group_dict["user_count"] = user_count
group_dict["message_amount"] = message_amount
return Group(**group_dict)
def highlight_group_for_user(self,
group_id: str,
user_id: int,
highlight_time: float = None) -> None:
if highlight_time is None:
now_plus_2_days = arrow.utcnow().shift(days=2).datetime
now_plus_2_days = to_ts(now_plus_2_days)
highlight_time = now_plus_2_days
raw_response = self.client.put(
f"/v1/groups/{group_id}/user/{user_id}/update",
json={"highlight_time": highlight_time},
)
self.assertEqual(raw_response.status_code, 200)
async def get_user_stats(self, user_id: int, query: UserStatsQuery,
db: Session) -> UserStats:
# if the user has more than 100 groups with unread messages in
# it won't matter if the count is exact or not, just forget about
# the super old ones (if a user reads a group, another unread
# group will be selected next time for this query anyway)
sub_query = GroupQuery(
per_page=100,
only_unread=query.only_unread,
count_unread=query.count_unread,
hidden=query.hidden,
)
user_groups: List[UserGroupBase] = self.env.db.get_groups_for_user(
user_id, sub_query, db)
if query.count_unread:
unread_amount = 0
n_unread_groups = 0
for user_group in user_groups:
if user_group.unread > 0:
unread_amount += user_group.unread
n_unread_groups += 1
else:
unread_amount = -1
n_unread_groups = -1
group_amounts = self.env.db.count_group_types_for_user(
user_id, sub_query, db)
group_amounts = dict(group_amounts)
last_sent_group_id, last_sent_time = self.env.db.get_last_sent_for_user(
user_id, db)
if last_sent_time is None:
last_sent_time = self.long_ago
last_sent_time_ts = to_ts(last_sent_time)
# TODO: what about last_update_time? it's in the model
return UserStats(
user_id=user_id,
unread_amount=unread_amount,
unread_groups_amount=n_unread_groups,
group_amount=group_amounts.get(GroupTypes.GROUP, 0),
one_to_one_amount=group_amounts.get(GroupTypes.ONE_TO_ONE, 0),
last_sent_time=last_sent_time_ts,
last_sent_group_id=last_sent_group_id,
)
async def join_group(self, group_id: str, query: JoinGroupQuery, db: Session) -> None:
now = utcnow_dt()
now_ts = to_ts(now)
user_ids_and_last_read = {
user_id: float(now_ts)
for user_id in query.users
}
self.env.db.set_group_updated_at(group_id, now, db)
self.env.db.update_user_stats_on_join_or_create_group(
group_id, user_ids_and_last_read, now, db
)
self.create_action_log(query.action_log, db, group_id=group_id)
def group_base_to_event(group: GroupBase, user_ids: List[int] = None) -> dict:
group_dict = {
"event_type":
EventTypes.GROUP,
"group_id":
group.group_id,
"name":
group.name,
"description":
group.description,
"updated_at":
to_int(to_ts(group.updated_at, allow_none=True)),
"created_at":
to_int(to_ts(group.created_at)),
"last_message_time":
to_int(to_ts(group.last_message_time, allow_none=True)),
"last_message_overview":
group.last_message_overview,
"last_message_type":
group.last_message_type,
"last_message_user_id":
str(group.last_message_user_id),
"status":
group.status,
"group_type":
group.group_type,
"owner_id":
str(group.owner_id),
"meta":
group.meta
}
if user_ids is not None:
group_dict["user_ids"] = [str(uid) for uid in user_ids]
return group_dict
def group_base_to_user_group(
group_base: GroupBase,
stats_base: UserGroupStatsBase,
receiver_stats_base: UserGroupStatsBase,
users: Dict[int, float],
user_count: int,
receiver_unread: int,
unread: int,
) -> UserGroup:
group = group_base_to_group(group_base, users, user_count)
stats_dict = stats_base.__dict__
stats_dict["unread"] = unread
stats_dict["receiver_unread"] = receiver_unread
stats_dict["receiver_highlight_time"] = to_ts(
stats_base.receiver_highlight_time)
if receiver_stats_base is not None:
stats_dict["receiver_delete_before"] = to_ts(
receiver_stats_base.delete_before)
stats_dict["receiver_hide"] = receiver_stats_base.hide
stats_dict["receiver_deleted"] = receiver_stats_base.deleted
stats_dict["last_read_time"] = to_ts(stats_base.last_read)
stats_dict["last_sent_time"] = to_ts(stats_base.last_sent)
stats_dict["join_time"] = to_ts(stats_base.join_time)
stats_dict["delete_before"] = to_ts(stats_base.delete_before)
stats_dict["highlight_time"] = to_ts(stats_base.highlight_time,
allow_none=True)
stats_dict["first_sent"] = to_ts(stats_base.first_sent, allow_none=True)
stats_dict["last_updated_time"] = to_ts(stats_base.last_updated_time)
stats = UserGroupStats(**stats_dict)
return UserGroup(
group=group,
stats=stats,
)
def test_receiver_highlight_time(self):
self.assert_groups_for_user(0)
group_message = self.send_1v1_message(
user_id=BaseTest.USER_ID, receiver_id=BaseTest.OTHER_USER_ID)
stats = self.groups_for_user(BaseTest.USER_ID)[0]["stats"]
self.assertEqual(self.long_ago, stats["receiver_highlight_time"])
now_plus_2_days = arrow.utcnow().shift(days=2).datetime
now_plus_2_days = to_ts(now_plus_2_days)
self.highlight_group_for_user(group_message["group_id"],
user_id=BaseTest.OTHER_USER_ID,
highlight_time=now_plus_2_days)
stats = self.groups_for_user(BaseTest.USER_ID)[0]["stats"]
self.assertEqual(now_plus_2_days, stats["receiver_highlight_time"])
stats = self.groups_for_user(BaseTest.OTHER_USER_ID)[0]["stats"]
self.assertEqual(now_plus_2_days, stats["highlight_time"])
async def create_new_group(
self, user_id: int, query: CreateGroupQuery, db: Session
) -> Group:
now = utcnow_dt()
now_ts = to_ts(now)
group_base = self.env.db.create_group(user_id, query, now, db)
users = {user_id: float(now_ts)}
if query.users is not None and query.users:
users.update({user_id: float(now_ts) for user_id in query.users})
self.env.db.update_user_stats_on_join_or_create_group(
group_base.group_id, users, now, db
)
group = group_base_to_group(
group=group_base, users=users, user_count=len(users),
)
# notify users they're in a new group
self.env.client_publisher.group_change(group_base, list(users.keys()))
return group
def to_rfc3339(date: dt):
return timestamp_to_rfc3339_utcoffset(to_ts(date))
async def get_message_count_for_user_in_group(
group_id: str, user_id: int, query: Optional[OnlySenderQuery] = None, db: Session = Depends(get_db)
) -> MessageCount:
"""
Count the number of messages in a group since a user's `delete_before`.
If `only_sender` is set to False (default value), the messages for all
users in the groups will be counted. If set to True, only messages sent
by the specified `user_id` will be counted. When set to True, only
messages send by this user _after_ his/her `delete_before` and _before_
his/her `last_sent_time` will be counted.
Note: setting `only_sender=true` is slow. Around 2 seconds for a group
of 6k messages. This is because we can not filter by `user_id` in
Cassandra, and have to instead batch query for all messages in the group
and filter out and count afterwards.
**Potential error codes in response:**
* `600`: if the user is not in the group,
* `601`: if the group does not exist,
* `250`: if an unknown error occurred.
"""
try:
group_info: UserGroupStatsBase = environ.env.db.get_user_stats_in_group(group_id, user_id, db)
# can't filter by user id in cassandra without restricting 'created_at', so
# use the cached value from the rdbms
if query and query.only_sender:
# can return both None and -1; -1 means we've checked the db before, but it has not
# yet been counted, to avoid checking the db every time a new message is sent
message_count = environ.env.db.get_sent_message_count(group_id, user_id, db)
# if it hasn't been counted before, count from cassandra in batches (could be slow)
if message_count is None or message_count == -1:
# until isn't inclusive, so the last message sent won't be counted otherwise;
until = group_info.last_sent
until += timedelta(milliseconds=1)
message_count = environ.env.storage.count_messages_in_group_from_user_since(
group_id,
user_id,
until=until,
since=group_info.delete_before
)
environ.env.db.set_sent_message_count(group_id, user_id, message_count, db)
else:
message_count = environ.env.storage.count_messages_in_group_since(
group_id, group_info.delete_before
)
return MessageCount(
group_id=group_id,
user_id=user_id,
delete_before=to_ts(group_info.delete_before),
message_count=message_count
)
except NoSuchGroupException as e:
log_error_and_raise_known(ErrorCodes.NO_SUCH_GROUP, sys.exc_info(), e)
except UserNotInGroupException as e:
log_error_and_raise_known(ErrorCodes.USER_NOT_IN_GROUP, sys.exc_info(), e)
except Exception as e:
log_error_and_raise_unknown(sys.exc_info(), e)
