def test_update_categories_when_actionlog_entry_missing(
db, default_account, message, imapuid
):
message.categories_changes = True
db.session.commit()
update_message_metadata(db.session, imapuid.account, message, False)
assert message.categories == {imapuid.folder.category}
def resync_uids_impl(self):
# NOTE: first, let's check if the UIVDALIDITY change was spurious, if
# it is, just discard it and go on, if it isn't, drop the relevant
# entries (filtering by account and folder IDs) from the imapuid table,
# download messages, if necessary - in case a message has changed UID -
# update UIDs, and discard orphaned messages. -siro
with mailsync_session_scope() as db_session:
folder_info = db_session.query(ImapFolderInfo). \
filter_by(account_id=self.account_id,
folder_id=self.folder_id).one()
cached_uidvalidity = folder_info.uidvalidity
with self.conn_pool.get() as crispin_client:
crispin_client.select_folder(self.folder_name,
lambda *args: True)
uidvalidity = crispin_client.selected_uidvalidity
if uidvalidity <= cached_uidvalidity:
log.debug('UIDVALIDITY unchanged')
return
invalid_uids = db_session.query(ImapUid). \
filter_by(account_id=self.account_id,
folder_id=self.folder_id)
data_sha256_message = {uid.message.data_sha256: uid.message
for uid in invalid_uids}
for uid in invalid_uids:
db_session.delete(uid)
# NOTE: this is necessary (and OK since it doesn't persist any
# data) to maintain the order between UIDs deletion and
# insertion. Without this, I was seeing constraints violation
# on the imapuid table. -siro
db_session.flush()
remote_uids = crispin_client.all_uids()
for remote_uid in remote_uids:
raw_message = crispin_client.uids([remote_uid])[0]
data_sha256 = sha256(raw_message.body).hexdigest()
if data_sha256 in data_sha256_message:
message = data_sha256_message[data_sha256]
# Create a new imapuid
uid = ImapUid(msg_uid=raw_message.uid,
message_id=message.id,
account_id=self.account_id,
folder_id=self.folder_id)
uid.update_flags(raw_message.flags)
db_session.add(uid)
# Update the existing message's metadata too
common.update_message_metadata(db_session, uid)
del data_sha256_message[data_sha256]
else:
self.download_and_commit_uids(crispin_client,
[remote_uid])
self.heartbeat_status.publish()
# FIXME: do we want to throttle the account when recovering
# from UIDVALIDITY changes? -siro
for message in data_sha256_message.itervalues():
db_session.delete(message)
folder_info.uidvalidity = uidvalidity
folder_info.highestmodseq = None
def __deduplicate_message_object_creation(self, db_session, raw_messages,
account):
"""
We deduplicate messages based on g_msgid: if we've previously saved a
Message object for this raw message, we don't create a new one. But we
do create a new ImapUid, associate it to the message, and update flags
and categories accordingly.
Note: we could do this prior to downloading the actual message
body, but that's really more complicated than it's worth. This
operation is not super common unless you're regularly moving lots
of messages to trash or spam, and even then the overhead of just
downloading the body is generally not that high.
"""
new_g_msgids = {msg.g_msgid for msg in raw_messages}
existing_g_msgids = g_msgids(self.namespace_id,
db_session,
in_=new_g_msgids)
brand_new_messages = [
m for m in raw_messages if m.g_msgid not in existing_g_msgids
]
previously_synced_messages = [
m for m in raw_messages if m.g_msgid in existing_g_msgids
]
if previously_synced_messages:
log.info('saving new uids for existing messages',
count=len(previously_synced_messages))
account = Account.get(self.account_id, db_session)
folder = Folder.get(self.folder_id, db_session)
for raw_message in previously_synced_messages:
message_obj = db_session.query(Message).filter(
Message.namespace_id == self.namespace_id,
Message.g_msgid == raw_message.g_msgid).first()
if message_obj is None:
log.warning('Message disappeared while saving new uid',
g_msgid=raw_message.g_msgid,
uid=raw_message.uid)
brand_new_messages.append(raw_message)
continue
already_have_uid = ((raw_message.uid, self.folder_id)
in {(u.msg_uid, u.folder_id)
for u in message_obj.imapuids})
if already_have_uid:
log.warning('Skipping existing UID for message',
uid=raw_message.uid,
message_id=message_obj.id)
continue
uid = ImapUid(account=account,
folder=folder,
msg_uid=raw_message.uid,
message=message_obj)
uid.update_flags(raw_message.flags)
uid.update_labels(raw_message.g_labels)
common.update_message_metadata(db_session, account,
message_obj, uid.is_draft)
db_session.commit()
return brand_new_messages
def __deduplicate_message_object_creation(self, db_session, raw_messages,
account):
"""
We deduplicate messages based on g_msgid: if we've previously saved a
Message object for this raw message, we don't create a new one. But we
do create a new ImapUid, associate it to the message, and update flags
and categories accordingly.
Note: we could do this prior to downloading the actual message
body, but that's really more complicated than it's worth. This
operation is not super common unless you're regularly moving lots
of messages to trash or spam, and even then the overhead of just
downloading the body is generally not that high.
"""
new_g_msgids = {msg.g_msgid for msg in raw_messages}
existing_g_msgids = g_msgids(self.namespace_id, db_session,
in_=new_g_msgids)
brand_new_messages = [m for m in raw_messages if m.g_msgid not in
existing_g_msgids]
previously_synced_messages = [m for m in raw_messages if m.g_msgid in
existing_g_msgids]
if previously_synced_messages:
log.info('saving new uids for existing messages',
count=len(previously_synced_messages))
account = Account.get(self.account_id, db_session)
folder = Folder.get(self.folder_id, db_session)
for raw_message in previously_synced_messages:
message_obj = db_session.query(Message).filter(
Message.namespace_id == self.namespace_id,
Message.g_msgid == raw_message.g_msgid).first()
if message_obj is None:
log.warning(
'Message disappeared while saving new uid',
g_msgid=raw_message.g_msgid,
uid=raw_message.uid)
brand_new_messages.append(raw_message)
continue
already_have_uid = (
(raw_message.uid, self.folder_id) in
{(u.msg_uid, u.folder_id) for u in message_obj.imapuids}
)
if already_have_uid:
log.warning('Skipping existing UID for message',
uid=raw_message.uid, message_id=message_obj.id)
continue
uid = ImapUid(account=account,
folder=folder,
msg_uid=raw_message.uid,
message=message_obj)
uid.update_flags(raw_message.flags)
uid.update_labels(raw_message.g_labels)
common.update_message_metadata(
db_session, account, message_obj, uid.is_draft)
db_session.commit()
return brand_new_messages
def add_custom_label(db, default_account, message):
assert len(message.imapuids) == 1
imapuid = message.imapuids[0]
existing = [c.name for c in imapuid.categories][0]
imapuid.update_labels(["<3"])
db.session.commit()
assert set([c.name for c in imapuid.categories]) == set([existing, ""])
update_message_metadata(db.session, default_account, message, False)
db.session.commit()
return message
def add_inbox_label(db, default_account, message):
assert len(message.imapuids) == 1
imapuid = message.imapuids[0]
assert set([c.name for c in imapuid.categories]) == set(["all"])
imapuid.update_labels(["\\Inbox"])
db.session.commit()
assert set([c.name for c in imapuid.categories]) == set(["all", "inbox"])
update_message_metadata(db.session, default_account, message, False)
db.session.commit()
return message
def add_custom_label(db, default_account, message):
assert len(message.imapuids) == 1
imapuid = message.imapuids[0]
existing = [c.name for c in imapuid.categories][0]
imapuid.update_labels(['<3'])
db.session.commit()
assert set([c.name for c in imapuid.categories]) == set([existing, ''])
update_message_metadata(db.session, default_account, message, False)
db.session.commit()
return message
def add_inbox_label(db, default_account, message):
assert len(message.imapuids) == 1
imapuid = message.imapuids[0]
assert set([c.name for c in imapuid.categories]) == set(['all'])
imapuid.update_labels(['\\Inbox'])
db.session.commit()
assert set([c.name for c in imapuid.categories]) == set(['all', 'inbox'])
update_message_metadata(db.session, default_account, message, False)
db.session.commit()
return message
def folder_and_message_maps(db, default_account):
folder_map, message_map = {}, {}
for name in ("all", "trash", "spam"):
# Create a folder
display_name = name.capitalize() if name != "all" else "All Mail"
folder = add_fake_folder(db.session, default_account, display_name, name)
thread = add_fake_thread(db.session, default_account.namespace.id)
# Create a message in the folder
message = add_fake_message(db.session, default_account.namespace.id, thread)
add_fake_imapuid(db.session, default_account.id, message, folder, 13)
update_message_metadata(db.session, default_account, message, False)
db.session.commit()
folder_map[name] = folder
message_map[name] = message
return folder_map, message_map
def folder_and_message_maps(db, default_account):
folder_map, message_map = {}, {}
for name in ('all', 'trash', 'spam'):
# Create a folder
display_name = name.capitalize() if name != 'all' else 'All Mail'
folder = add_fake_folder(db.session, default_account, display_name, name)
thread = add_fake_thread(db.session, default_account.namespace.id)
# Create a message in the folder
message = add_fake_message(db.session, default_account.namespace.id,
thread)
add_fake_imapuid(db.session, default_account.id, message, folder, 13)
update_message_metadata(db.session, default_account, message, False)
db.session.commit()
folder_map[name] = folder
message_map[name] = message
return folder_map, message_map
def resync_uids_impl(self):
# NOTE: first, let's check if the UIVDALIDITY change was spurious, if
# it is, just discard it and go on, if it isn't, drop the relevant
# entries (filtering by account and folder IDs) from the imapuid table,
# download messages, if necessary - in case a message has changed UID -
# update UIDs, and discard orphaned messages. -siro
with mailsync_session_scope() as db_session:
folder_info = db_session.query(ImapFolderInfo). \
filter_by(account_id=self.account_id,
folder_id=self.folder_id).one()
cached_uidvalidity = folder_info.uidvalidity
with self.conn_pool.get() as crispin_client:
crispin_client.select_folder(self.folder_name,
lambda *args: True)
uidvalidity = crispin_client.selected_uidvalidity
if uidvalidity <= cached_uidvalidity:
log.debug('UIDVALIDITY unchanged')
return
invalid_uids = db_session.query(ImapUid). \
filter_by(account_id=self.account_id,
folder_id=self.folder_id)
data_sha256_message = {
uid.message.data_sha256: uid.message
for uid in invalid_uids
}
for uid in invalid_uids:
db_session.delete(uid)
# NOTE: this is necessary (and OK since it doesn't persist any
# data) to maintain the order between UIDs deletion and
# insertion. Without this, I was seeing constraints violation
# on the imapuid table. -siro
db_session.flush()
remote_uids = crispin_client.all_uids()
for remote_uid in remote_uids:
raw_message = crispin_client.uids([remote_uid])[0]
data_sha256 = sha256(raw_message.body).hexdigest()
if data_sha256 in data_sha256_message:
message = data_sha256_message[data_sha256]
# Create a new imapuid
uid = ImapUid(msg_uid=raw_message.uid,
message_id=message.id,
account_id=self.account_id,
folder_id=self.folder_id)
uid.update_flags(raw_message.flags)
db_session.add(uid)
# Update the existing message's metadata too
common.update_message_metadata(db_session, uid)
del data_sha256_message[data_sha256]
else:
self.download_and_commit_uids(crispin_client,
[remote_uid])
self.heartbeat_status.publish()
# FIXME: do we want to throttle the account when recovering
# from UIDVALIDITY changes? -siro
for message in data_sha256_message.itervalues():
db_session.delete(message)
folder_info.uidvalidity = uidvalidity
folder_info.highestmodseq = None
def test_update_categories_when_actionlog_entry_missing(db, default_account, message, imapuid):
message.categories_changes = True
db.session.commit()
update_message_metadata(db.session, imapuid.account, message, False)
assert message.categories == {imapuid.folder.category}
def add_new_imapuids(crispin_client, remote_g_metadata, syncmanager_lock,
uids):
"""
Add ImapUid entries only for (already-downloaded) messages.
If a message has already been downloaded via another folder, we only need
to add `ImapUid` accounting for the current folder. `Message` objects
etc. have already been created.
"""
flags = crispin_client.flags(uids)
with syncmanager_lock:
with mailsync_session_scope() as db_session:
# Since we prioritize download for messages in certain threads, we
# may already have ImapUid entries despite calling this method.
local_folder_uids = {
uid
for uid, in db_session.query(ImapUid.msg_uid).join(Folder).
filter(ImapUid.account_id == crispin_client.account_id,
Folder.name == crispin_client.selected_folder_name,
ImapUid.msg_uid.in_(uids))
}
uids = [uid for uid in uids if uid not in local_folder_uids]
if uids:
acc = db_session.query(GmailAccount).get(
crispin_client.account_id)
# Collate message objects to relate the new imapuids to
imapuid_for = dict([
(metadata.msgid, uid)
for (uid, metadata) in remote_g_metadata.items()
if uid in uids
])
imapuid_g_msgids = [
remote_g_metadata[uid].msgid for uid in uids
]
message_for = dict([
(imapuid_for[m.g_msgid], m)
for m in db_session.query(Message).join(ImapThread).filter(
Message.g_msgid.in_(imapuid_g_msgids),
ImapThread.namespace_id == acc.namespace.id)
])
# Stop Folder.find_or_create()'s query from triggering a flush.
with db_session.no_autoflush as session:
new_imapuids = [
ImapUid(account=acc,
folder=Folder.find_or_create(
db_session, acc,
crispin_client.selected_folder_name),
msg_uid=uid,
message=message_for[uid]) for uid in uids
if uid in message_for
]
for uid in new_imapuids:
# Skip uids which have disappeared in the meantime
if uid.msg_uid in flags:
uid.update_flags(flags[uid.msg_uid].flags)
uid.update_labels(flags[uid.msg_uid].labels)
common.update_message_metadata(session, uid)
db_session.add_all(new_imapuids)
db_session.commit()
def add_new_imapuids(crispin_client, remote_g_metadata, syncmanager_lock,
uids):
"""
Add ImapUid entries only for (already-downloaded) messages.
If a message has already been downloaded via another folder, we only need
to add `ImapUid` accounting for the current folder. `Message` objects
etc. have already been created.
"""
flags = crispin_client.flags(uids)
with syncmanager_lock:
with mailsync_session_scope() as db_session:
# Since we prioritize download for messages in certain threads, we
# may already have ImapUid entries despite calling this method.
local_folder_uids = {uid for uid, in
db_session.query(ImapUid.msg_uid).join(Folder)
.filter(
ImapUid.account_id ==
crispin_client.account_id,
Folder.name ==
crispin_client.selected_folder_name,
ImapUid.msg_uid.in_(uids))}
uids = [uid for uid in uids if uid not in local_folder_uids]
if uids:
acc = db_session.query(GmailAccount).get(
crispin_client.account_id)
# Collate message objects to relate the new imapuids to
imapuid_for = dict([(metadata.msgid, uid) for (uid, metadata)
in remote_g_metadata.items()
if uid in uids])
imapuid_g_msgids = [remote_g_metadata[uid].msgid for uid in
uids]
message_for = dict([(imapuid_for[m.g_msgid], m) for m in
db_session.query(Message).join(ImapThread)
.filter(
Message.g_msgid.in_(imapuid_g_msgids),
ImapThread.namespace_id ==
acc.namespace.id)])
# Stop Folder.find_or_create()'s query from triggering a flush.
with db_session.no_autoflush as session:
new_imapuids = [ImapUid(
account=acc,
folder=Folder.find_or_create(
db_session, acc,
crispin_client.selected_folder_name),
msg_uid=uid, message=message_for[uid]) for uid in uids
if uid in message_for]
for uid in new_imapuids:
# Skip uids which have disappeared in the meantime
if uid.msg_uid in flags:
uid.update_flags(flags[uid.msg_uid].flags)
uid.update_labels(flags[uid.msg_uid].labels)
common.update_message_metadata(session, uid)
db_session.add_all(new_imapuids)
db_session.commit()