def test_soft_delete(db, config):
from inbox.server.models.tables.base import Folder, Message
from inbox.server.models.tables.imap import ImapUid
f = Folder(name='DOES NOT EXIST', account_id=ACCOUNT_ID)
db.session.add(f)
db.session.flush()
m = Message(thread_id=1, received_date=datetime.datetime.utcnow(),
size=0, sanitized_body="", snippet="")
u = ImapUid(message=m, imapaccount_id=ACCOUNT_ID, folder_id=f.id,
msg_uid=9999, extra_flags="")
db.session.add_all([m, u])
f.mark_deleted()
u.mark_deleted()
db.session.commit()
m_id = m.id
# bypass custom query method to confirm creation
db.new_session(ignore_soft_deletes=False)
f = db.session.query(Folder).filter_by(name='DOES NOT EXIST').one()
assert f, "Can't find Folder object"
assert f.deleted_at is not None, "Folder not marked as deleted"
db.new_session(ignore_soft_deletes=True)
with pytest.raises(NoResultFound):
folders = db.session.query(Folder).filter(
Folder.name == 'DOES NOT EXIST').one()
count = db.session.query(Folder).filter(
Folder.name == 'DOES NOT EXIST').count()
assert count == 0, "Shouldn't find any deleted folders!"
m = db.session.query(Message).filter_by(id=m_id).one()
assert not m.imapuids, "imapuid was deleted!"
def save_folder_names(log, account, folder_names, db_session):
"""
Create Folder objects & map special folder names on Account objects.
Folders that belong to an account and no longer exist in `folder_names`
ARE DELETED.
"""
# NOTE: We don't do anything like canonicalizing to lowercase because
# different backends may be case-sensitive or not. Code that references
# saved folder names should canonicalize if needed when doing comparisons.
assert 'inbox' in folder_names, 'Account {} has no detected inbox folder'\
.format(account.email_address)
folders = dict([
(f.name.lower(), f)
for f in db_session.query(Folder).filter_by(account=account).all()
])
for tag in [
'inbox', 'drafts', 'sent', 'spam', 'trash', 'starred', 'important',
'archive', 'all'
]:
if tag in folder_names:
if folder_names[tag].lower() not in folders:
folder = Folder.create(account, folder_names[tag], tag)
attr_name = '{}_folder'.format(tag)
setattr(
account, attr_name,
verify_folder_name(account.id, getattr(account, attr_name),
folder))
if folder_names[tag].lower() in folders:
del folders[folder_names[tag].lower()]
# Gmail labels, user-created IMAP/EAS folders, etc.
if 'extra' in folder_names:
for name in folder_names['extra']:
name = name[:MAX_FOLDER_NAME_LENGTH]
if name.lower() not in folders:
folder = Folder.create(account, name)
db_session.add(folder)
if name.lower() in folders:
del folders[name.lower()]
# This may cascade to FolderItems and ImapUid (ONLY), which is what we
# want--doing the update here short-circuits us syncing that change later.
log.info("Folders were deleted from the remote: {}".format(folders.keys()))
for folder in folders.values():
db_session.delete(folder)
db_session.commit()
def add_new_imapuid(db_session, log, gmessage, folder_name, acc):
""" Add ImapUid object for this GMessage if we don't already have one.
Parameters
----------
message : GMessage
Message to add ImapUid for.
folder_name : str
Which folder to add the ImapUid in.
acc : ImapAccount
Which account to associate the message with. (Not looking this up
within this function is a db access optimization.)
"""
if not db_session.query(ImapUid.msg_uid).join(Folder).filter(
Folder.name == folder_name,
ImapUid.msg_uid == gmessage.uid).all():
message = db_session.query(Message).filter_by(
g_msgid=gmessage.g_metadata.msgid).one()
new_imapuid = ImapUid(
imapaccount=acc,
folder=Folder.find_or_create(db_session, acc, folder_name),
msg_uid=gmessage.uid, message=message)
new_imapuid.update_imap_flags(gmessage.flags, gmessage.labels)
db_session.add(new_imapuid)
db_session.commit()
else:
log.debug("Skipping {} imapuid creation for UID {}".format(
folder_name, gmessage.uid))
def local_copy(db_session, account, thread_id, from_folder, to_folder):
""" Copy thread in the local datastore (*not* the account backend).
NOT idempotent.
"""
if from_folder == to_folder:
return
with db_write_lock(account.namespace.id):
listings = {
item.folder.name: item
for item in db_session.query(FolderItem).join(Folder).join(Thread).
filter(Thread.namespace_id == account.namespace.id,
FolderItem.thread_id == thread_id,
Folder.name.in_([from_folder, to_folder])).all()
}
if from_folder not in listings:
raise LocalActionError(
"thread {} does not exist in folder {}".format(
thread_id, from_folder))
elif to_folder not in listings:
thread = listings[from_folder].thread
folder = Folder.find_or_create(db_session,
thread.namespace.account, to_folder)
thread.folders.add(folder)
db_session.commit()
def local_copy(db_session, account, thread_id, from_folder, to_folder):
""" Copy thread in the local datastore (*not* the account backend).
NOT idempotent.
"""
if from_folder == to_folder:
return
with db_write_lock(account.namespace.id):
listings = {item.folder.name: item for item in
db_session.query(FolderItem).join(Folder).join(Thread)
.filter(
Thread.namespace_id == account.namespace.id,
FolderItem.thread_id == thread_id,
Folder.name.in_([from_folder, to_folder]))
.all()}
if from_folder not in listings:
raise LocalActionError("thread {} does not exist in folder {}"
.format(thread_id, from_folder))
elif to_folder not in listings:
thread = listings[from_folder].thread
folder = Folder.find_or_create(db_session,
thread.namespace.account,
to_folder)
thread.folders.add(folder)
db_session.commit()
def add_new_imapuid(db_session, log, gmessage, folder_name, acc):
""" Add ImapUid object for this GMessage if we don't already have one.
Parameters
----------
message : GMessage
Message to add ImapUid for.
folder_name : str
Which folder to add the ImapUid in.
acc : ImapAccount
Which account to associate the message with. (Not looking this up
within this function is a db access optimization.)
"""
if not db_session.query(ImapUid.msg_uid).join(Folder).filter(
Folder.name == folder_name,
ImapUid.msg_uid == gmessage.uid).all():
message = db_session.query(Message).filter_by(
g_msgid=gmessage.g_metadata.msgid).one()
new_imapuid = ImapUid(
imapaccount=acc,
folder=Folder.find_or_create(db_session, acc, folder_name),
msg_uid=gmessage.uid, message=message)
new_imapuid.update_imap_flags(gmessage.flags, gmessage.labels)
db_session.add(new_imapuid)
db_session.commit()
else:
log.debug("Skipping {} imapuid creation for UID {}".format(
folder_name, gmessage.uid))
def save_folder_names(log, account, folder_names, db_session):
"""
Create Folder objects & map special folder names on Account objects.
Folders that belong to an account and no longer exist in `folder_names`
ARE DELETED.
"""
# NOTE: We don't do anything like canonicalizing to lowercase because
# different backends may be case-sensitive or not. Code that references
# saved folder names should canonicalize if needed when doing comparisons.
assert 'inbox' in folder_names, 'Account {} has no detected inbox folder'\
.format(account.email_address)
folders = dict([(f.name.lower(), f) for f in
db_session.query(Folder).filter_by(account=account).all()])
for tag in ['inbox', 'drafts', 'sent', 'spam', 'trash', 'starred',
'important', 'archive', 'all']:
if tag in folder_names:
if folder_names[tag].lower() not in folders:
folder = Folder.create(account, folder_names[tag], tag)
attr_name = '{}_folder'.format(tag)
setattr(account, attr_name, verify_folder_name(
account.id, getattr(account, attr_name), folder))
if folder_names[tag].lower() in folders:
del folders[folder_names[tag].lower()]
# Gmail labels, user-created IMAP/EAS folders, etc.
if 'extra' in folder_names:
for name in folder_names['extra']:
name = name[:MAX_FOLDER_NAME_LENGTH]
if name.lower() not in folders:
folder = Folder.create(account, name)
db_session.add(folder)
if name.lower() in folders:
del folders[name.lower()]
# This may cascade to FolderItems and ImapUid (ONLY), which is what we
# want--doing the update here short-circuits us syncing that change later.
log.info("Folders were deleted from the remote: {}".format(folders.keys()))
for folder in folders.values():
db_session.delete(folder)
db_session.commit()
def test_soft_delete(db, config):
from inbox.server.models.tables.base import Folder, Message
from inbox.server.models.tables.imap import ImapUid
f = Folder(name='DOES NOT EXIST', account_id=ACCOUNT_ID)
db.session.add(f)
db.session.flush()
m = Message(thread_id=1,
received_date=datetime.datetime.utcnow(),
size=0,
sanitized_body="",
snippet="")
u = ImapUid(message=m,
imapaccount_id=ACCOUNT_ID,
folder_id=f.id,
msg_uid=9999,
extra_flags="")
db.session.add_all([m, u])
f.mark_deleted()
u.mark_deleted()
db.session.commit()
m_id = m.id
# bypass custom query method to confirm creation
db.new_session(ignore_soft_deletes=False)
f = db.session.query(Folder).filter_by(name='DOES NOT EXIST').one()
assert f, "Can't find Folder object"
assert f.deleted_at is not None, "Folder not marked as deleted"
db.new_session(ignore_soft_deletes=True)
with pytest.raises(NoResultFound):
folders = db.session.query(Folder).filter(
Folder.name == 'DOES NOT EXIST').one()
count = db.session.query(Folder).filter(
Folder.name == 'DOES NOT EXIST').count()
assert count == 0, "Shouldn't find any deleted folders!"
m = db.session.query(Message).filter_by(id=m_id).one()
assert not m.imapuids, "imapuid was deleted!"
def create_db_objects(account_id, db_session, log, folder_name, raw_messages,
msg_create_fn):
new_uids = []
# TODO: Detect which namespace to add message to. (shared folders)
# Look up message thread,
acc = db_session.query(Account).get(account_id)
folder = Folder.find_or_create(db_session, acc, folder_name)
for msg in raw_messages:
uid = msg_create_fn(db_session, log, acc, folder, msg)
if uid is not None:
new_uids.append(uid)
# imapuid, message, thread, labels
return new_uids
def create_db_objects(account_id, db_session, log, folder_name, raw_messages,
msg_create_fn):
new_uids = []
# TODO: Detect which namespace to add message to. (shared folders)
# Look up message thread,
acc = db_session.query(Account).get(account_id)
folder = Folder.find_or_create(db_session, acc, folder_name)
for msg in raw_messages:
uid = msg_create_fn(db_session, log, acc, folder, msg)
if uid is not None:
new_uids.append(uid)
# imapuid, message, thread, labels
return new_uids
def add_new_imapuids(crispin_client, log, db_session, remote_g_metadata,
syncmanager_lock, uids):
""" Add ImapUid entries only for (already-downloaded) messages.
If a message has already been downloaded via another folder, we only need
to add `ImapUid` accounting for the current folder. `Message` objects
etc. have already been created.
"""
flags = crispin_client.flags(uids)
with syncmanager_lock:
log.debug("add_new_imapuids acquired syncmanager_lock")
# Since we prioritize download for messages in certain threads, we may
# already have ImapUid entries despite calling this method.
local_folder_uids = {uid for uid, in
db_session.query(ImapUid.msg_uid).join(Folder)
.filter(
Folder.name ==
crispin_client.selected_folder_name,
ImapUid.msg_uid.in_(uids))}
uids = [uid for uid in uids if uid not in local_folder_uids]
if uids:
# collate message objects to relate the new imapuids
imapuid_uid_for = dict([(metadata.msgid, uid) for (uid, metadata)
in remote_g_metadata.items()
if uid in uids])
imapuid_g_msgids = [remote_g_metadata[uid].msgid for uid in uids]
message_for = dict([(imapuid_uid_for[mm.g_msgid], mm) for mm in
db_session.query(Message).filter(
Message.g_msgid.in_(imapuid_g_msgids))])
acc = db_session.query(ImapAccount).get(crispin_client.account_id)
# Folder.find_or_create()'s query will otherwise trigger a flush.
with db_session.no_autoflush:
new_imapuids = [ImapUid(
imapaccount=acc,
folder=Folder.find_or_create(
db_session, acc, crispin_client.selected_folder_name),
msg_uid=uid, message=message_for[uid]) for uid in uids]
for item in new_imapuids:
item.update_imap_flags(flags[item.msg_uid].flags,
flags[item.msg_uid].labels)
db_session.add_all(new_imapuids)
db_session.commit()
def add_new_imapuids(crispin_client, log, db_session, remote_g_metadata,
syncmanager_lock, uids):
""" Add ImapUid entries only for (already-downloaded) messages.
If a message has already been downloaded via another folder, we only need
to add `ImapUid` accounting for the current folder. `Message` objects
etc. have already been created.
"""
flags = crispin_client.flags(uids)
with syncmanager_lock:
log.debug("add_new_imapuids acquired syncmanager_lock")
# Since we prioritize download for messages in certain threads, we may
# already have ImapUid entries despite calling this method.
local_folder_uids = {uid for uid, in
db_session.query(ImapUid.msg_uid).join(Folder)
.filter(
Folder.name ==
crispin_client.selected_folder_name,
ImapUid.msg_uid.in_(uids))}
uids = [uid for uid in uids if uid not in local_folder_uids]
if uids:
# collate message objects to relate the new imapuids
imapuid_uid_for = dict([(metadata.msgid, uid) for (uid, metadata)
in remote_g_metadata.items()
if uid in uids])
imapuid_g_msgids = [remote_g_metadata[uid].msgid for uid in uids]
message_for = dict([(imapuid_uid_for[mm.g_msgid], mm) for mm in
db_session.query(Message).filter(
Message.g_msgid.in_(imapuid_g_msgids))])
acc = db_session.query(ImapAccount).get(crispin_client.account_id)
# Folder.find_or_create()'s query will otherwise trigger a flush.
with db_session.no_autoflush:
new_imapuids = [ImapUid(
imapaccount=acc,
folder=Folder.find_or_create(
db_session, acc, crispin_client.selected_folder_name),
msg_uid=uid, message=message_for[uid]) for uid in uids]
for item in new_imapuids:
item.update_imap_flags(flags[item.msg_uid].flags,
flags[item.msg_uid].labels)
db_session.add_all(new_imapuids)
db_session.commit()
def add_gmail_attrs(db_session, log, new_uid, flags, folder, g_thrid, g_msgid,
g_labels, created):
""" Gmail-specific post-create-message bits."""
new_uid.message.g_msgid = g_msgid
# NOTE: g_thrid == g_msgid on the first message in the thread :)
new_uid.message.g_thrid = g_thrid
new_uid.update_imap_flags(flags, g_labels)
# If we don't disable autoflush here, the thread query may flush a
# message to the database with a NULL thread_id, causing a crash.
with db_session.no_autoflush:
thread = new_uid.message.thread = ImapThread.from_gmail_message(
db_session, new_uid.imapaccount.namespace, new_uid.message)
# make sure this thread has all the correct labels
existing_labels = {folder.name.lower() for folder in thread.folders}
# convert things like \Inbox -> Inbox, \Important -> Important
new_labels = {l.lstrip('\\') for l in g_labels} | {folder.name}
# The IMAP folder name for the inbox on Gmail is INBOX, but there's ALSO a
# flag called '\Inbox' on all messages in it... that only appears when you
# look at the message with a folder OTHER than INBOX selected. Standardize
# on keeping \Inbox in our database.
if 'Inbox' in new_labels or 'INBOX' in new_labels:
new_labels.discard('INBOX')
new_labels.discard('Inbox')
new_labels.add('Inbox')
# NOTE: Gmail labels are case-insensitive, though we store them in the
# original case in the db to not confuse users when displayed.
new_labels_ci = {l.lower() for l in new_labels}
# Remove labels that have been deleted -- note that the \Inbox, \Sent,
# \Important, and \Drafts labels are per-message, not per-thread, but since
# we always work at the thread level, _we_ apply the label to the whole
# thread.
thread.folders = {folder for folder in thread.folders if
folder.name.lower() in new_labels_ci or
folder.name.lower() in ('inbox', 'sent', 'drafts',
'important')}
# add new labels
for label in new_labels:
if label.lower() not in existing_labels:
# The problem here is that Gmail's attempt to squash labels and
# IMAP folders into the same abstraction doesn't work perfectly. In
# particular, there is a '[Gmail]/Sent' folder, but *also* a 'Sent'
# label, and so on. We handle this by only maintaining one folder
# object that encapsulates both of these.
if label == 'Sent':
thread.folders.add(thread.namespace.account.sent_folder)
elif label == 'Draft':
thread.folders.add(thread.namespace.account.drafts_folder)
elif label == 'Starred':
thread.folders.add(thread.namespace.account.starred_folder)
elif label == 'Important':
thread.folders.add(thread.namespace.account.important_folder)
else:
folder = Folder.find_or_create(db_session,
thread.namespace.account,
label)
thread.folders.add(folder)
# Reconciliation for Sent Mail folder:
if ('sent' in new_labels_ci and not created and
new_uid.message.inbox_uid):
if not thread.id:
db_session.flush()
reconcile_gmail_message(db_session, log, new_uid.message.inbox_uid,
new_uid.message, thread.id, g_thrid)
return new_uid
def add_gmail_attrs(db_session, log, new_uid, flags, folder, g_thrid, g_msgid,
g_labels, created):
""" Gmail-specific post-create-message bits."""
new_uid.message.g_msgid = g_msgid
# NOTE: g_thrid == g_msgid on the first message in the thread :)
new_uid.message.g_thrid = g_thrid
new_uid.update_imap_flags(flags, g_labels)
# If we don't disable autoflush here, the thread query may flush a
# message to the database with a NULL thread_id, causing a crash.
with db_session.no_autoflush:
thread = new_uid.message.thread = ImapThread.from_gmail_message(
db_session, new_uid.imapaccount.namespace, new_uid.message)
# make sure this thread has all the correct labels
existing_labels = {folder.name.lower() for folder in thread.folders}
# convert things like \Inbox -> Inbox, \Important -> Important
new_labels = {l.lstrip('\\') for l in g_labels} | {folder.name}
# The IMAP folder name for the inbox on Gmail is INBOX, but there's ALSO a
# flag called '\Inbox' on all messages in it... that only appears when you
# look at the message with a folder OTHER than INBOX selected. Standardize
# on keeping \Inbox in our database.
if 'Inbox' in new_labels or 'INBOX' in new_labels:
new_labels.discard('INBOX')
new_labels.discard('Inbox')
new_labels.add('Inbox')
# NOTE: Gmail labels are case-insensitive, though we store them in the
# original case in the db to not confuse users when displayed.
new_labels_ci = {l.lower() for l in new_labels}
# Remove labels that have been deleted -- note that the \Inbox, \Sent,
# \Important, and \Drafts labels are per-message, not per-thread, but since
# we always work at the thread level, _we_ apply the label to the whole
# thread.
thread.folders = {folder for folder in thread.folders if
folder.name.lower() in new_labels_ci or
folder.name.lower() in ('inbox', 'sent', 'drafts',
'important')}
# add new labels
for label in new_labels:
if label.lower() not in existing_labels:
# The problem here is that Gmail's attempt to squash labels and
# IMAP folders into the same abstraction doesn't work perfectly. In
# particular, there is a '[Gmail]/Sent' folder, but *also* a 'Sent'
# label, and so on. We handle this by only maintaining one folder
# object that encapsulates both of these.
if label == 'Sent':
thread.folders.add(thread.namespace.account.sent_folder)
elif label == 'Draft':
thread.folders.add(thread.namespace.account.drafts_folder)
elif label == 'Starred':
thread.folders.add(thread.namespace.account.starred_folder)
elif label == 'Important':
thread.folders.add(thread.namespace.account.important_folder)
else:
folder = Folder.find_or_create(db_session,
thread.namespace.account,
label)
thread.folders.add(folder)
# Reconciliation for Sent Mail folder:
if ('sent' in new_labels_ci and not created and
new_uid.message.inbox_uid):
if not thread.id:
db_session.flush()
reconcile_gmail_message(db_session, log, new_uid.message.inbox_uid,
new_uid.message, thread.id, g_thrid)
return new_uid