def user_console(user_email_address):
with session_scope() as db_session:
account = db_session.query(Account).filter_by(
email_address=user_email_address).one()
crispin_client = new_crispin(account.id, account.provider,
conn_pool_size=1)
with crispin_client.pool.get() as c:
crispin_client.select_folder(crispin_client.folder_names(c)['all'],
uidvalidity_cb(db_session, account),
c)
server_uids = crispin_client.all_uids(c)
banner = """
You can access the crispin instance with the 'crispin_client' variable.
You can access the IMAPClient connection with the 'c' variable.
AllMail message UIDs are in 'server_uids'.
You can refresh the session with 'refresh_crispin()'.
IMAPClient docs are at:
http://imapclient.readthedocs.org/en/latest/#imapclient-class-reference
"""
IPython.embed(banner1=banner)
def download_queued_threads(crispin_client, db_session, log, folder_name,
message_download_stack, status_cb,
syncmanager_lock):
""" Download threads until `message_download_stack` is empty.
UIDs and g_metadata that come out of `message_download_stack` are for
the _folder that threads are being expanded in_.
Threads are downloaded in the order they come out of the stack, which
_ought_ to be putting newest threads at the top. Messages are
downloaded newest-to-oldest in thread. (Threads are expanded to all
messages in the email archive that belong to the threads corresponding
to the given uids.)
"""
num_total_messages = message_download_stack.qsize()
log.info("{} messages found initially (unsorted by thread)"
.format(num_total_messages))
log.info("Expanding threads and downloading messages.")
# We still need the original crispin connection for progress reporting,
# so the easiest thing to do here with the current pooling setup is to
# create a new crispin client for querying All Mail.
with connection_pool(crispin_client.account_id).get() as \
all_mail_crispin_client:
all_mail_crispin_client.select_folder(
crispin_client.folder_names()['all'],
uidvalidity_cb(db_session, crispin_client.account_id))
# Since we do thread expansion, for any given thread, even if we
# already have the UID in the given GMessage downloaded, we may not
# have _every_ message in the thread. We have to expand it and make
# sure we have all messages.
acc = db_session.query(ImapAccount).get(crispin_client.account_id)
while not message_download_stack.empty():
message = message_download_stack.get_nowait()
# Don't try to re-download any messages that are in the same
# thread. (Putting this _before_ the download to guarantee no
# context switches happen in the meantime; we _should_ re-download
# if another message arrives on the thread.)
processed_msgs = [m for m in message_download_stack.queue if
m.g_metadata.thrid ==
message.g_metadata.thrid]
processed_msgs.append(message)
message_download_stack.queue = [
m for m in message_download_stack.queue if m.g_metadata.thrid
!= message.g_metadata.thrid]
thread_uids = all_mail_crispin_client.expand_threads(
[message.g_metadata.thrid])
thread_g_metadata = all_mail_crispin_client.g_metadata(
thread_uids)
download_thread(all_mail_crispin_client, db_session, log,
syncmanager_lock, thread_g_metadata,
message.g_metadata.thrid, thread_uids)
# In theory we only ever have one Greenlet modifying ImapUid
# entries for a non-All Mail folder, but grab the lock anyway
# to be safe.
with syncmanager_lock:
log.debug("download_queued_threads acquired syncmanager_lock")
# Since we download msgs from All Mail, we need to separately
# make sure we have ImapUids recorded for this folder (used in
# progress tracking, queuing, and delete detection).
log.debug("Adding {} imapuid rows for {} processed messages"
.format(folder_name, len(processed_msgs)))
for msg in processed_msgs:
add_new_imapuid(db_session, log, msg, folder_name, acc)
report_progress(crispin_client, db_session, log, folder_name,
message_download_stack.qsize(), status_cb)
log.info("Message download queue emptied")
def check_new_g_thrids(account_id, provider, folder_name, log,
message_download_stack, poll_frequency,
syncmanager_lock):
""" Check for new X-GM-THRIDs and add them to the download stack.
We do this by comparing local UID lists to remote UID lists, maintaining
the invariant that (stack uids)+(local uids) == (remote uids).
We also remove local messages that have disappeared from the remote, since
it's totally probable that users will be archiving mail as the initial
sync goes on.
We grab a new IMAP connection from the pool for this to isolate its
actions from whatever the main greenlet may be doing.
Runs until killed. (Intended to be run in a greenlet.)
"""
with connection_pool(account_id).get() as crispin_client:
with session_scope(ignore_soft_deletes=False) as db_session:
crispin_client.select_folder(folder_name,
uidvalidity_cb(
db_session,
crispin_client.account_id))
while True:
log.info("Checking for new/deleted messages during initial sync.")
remote_uids = set(crispin_client.all_uids())
# We lock this section to make sure no messages are being modified
# in the database while we make sure the queue is in a good state.
with syncmanager_lock:
log.debug("check_new_g_thrids acquired syncmanager_lock")
with session_scope(ignore_soft_deletes=False) as db_session:
local_uids = set(account.all_uids(account_id, db_session,
folder_name))
stack_uids = {gm.uid for gm in
message_download_stack.queue}
local_with_pending_uids = local_uids | stack_uids
deleted_uids = remove_deleted_uids(
account_id, db_session, log, folder_name, local_uids,
remote_uids)
log.info("Removed {} deleted UIDs from {}".format(
len(deleted_uids), folder_name))
# filter out messages that have disappeared on the remote side
new_message_download_stack = [gm for gm in
message_download_stack.queue
if gm.uid in remote_uids]
# add in any new uids from the remote
new_uids = [uid for uid in remote_uids if uid not in
local_with_pending_uids]
flags = crispin_client.flags(new_uids)
g_metadata = crispin_client.g_metadata(new_uids)
log.info("Adding {} new messages to the download queue for {}"
.format(min(len(flags), len(g_metadata)), folder_name))
for new_uid in new_uids:
# could have disappeared from the folder in the meantime
if new_uid in flags and new_uid in g_metadata:
new_message_download_stack.append(
GMessage(new_uid, g_metadata[new_uid],
flags[new_uid].flags,
flags[new_uid].labels))
message_download_stack.queue = sorted(
new_message_download_stack, key=lambda m: m.uid)
log.info("Idling on {0} with {1} timeout".format(
folder_name, poll_frequency))
crispin_client.conn.idle()
crispin_client.conn.idle_check(timeout=poll_frequency)
crispin_client.conn.idle_done()
log.info("IDLE on {0} detected changes or timeout reached"
.format(folder_name))
def download_queued_threads(crispin_client, db_session, log, folder_name,
message_download_stack, status_cb,
syncmanager_lock):
""" Download threads until `message_download_stack` is empty.
UIDs and g_metadata that come out of `message_download_stack` are for
the _folder that threads are being expanded in_.
Threads are downloaded in the order they come out of the stack, which
_ought_ to be putting newest threads at the top. Messages are
downloaded newest-to-oldest in thread. (Threads are expanded to all
messages in the email archive that belong to the threads corresponding
to the given uids.)
"""
num_total_messages = message_download_stack.qsize()
log.info("{} messages found initially (unsorted by thread)"
.format(num_total_messages))
log.info("Expanding threads and downloading messages.")
# We still need the original crispin connection for progress reporting,
# so the easiest thing to do here with the current pooling setup is to
# create a new crispin client for querying All Mail.
with connection_pool(crispin_client.account_id).get() as \
all_mail_crispin_client:
all_mail_crispin_client.select_folder(
crispin_client.folder_names()['all'],
uidvalidity_cb(db_session, crispin_client.account_id))
# Since we do thread expansion, for any given thread, even if we
# already have the UID in the given GMessage downloaded, we may not
# have _every_ message in the thread. We have to expand it and make
# sure we have all messages.
acc = db_session.query(ImapAccount).get(crispin_client.account_id)
while not message_download_stack.empty():
message = message_download_stack.get_nowait()
# Don't try to re-download any messages that are in the same
# thread. (Putting this _before_ the download to guarantee no
# context switches happen in the meantime; we _should_ re-download
# if another message arrives on the thread.)
processed_msgs = [m for m in message_download_stack.queue if
m.g_metadata.thrid ==
message.g_metadata.thrid]
processed_msgs.append(message)
message_download_stack.queue = [
m for m in message_download_stack.queue if m.g_metadata.thrid
!= message.g_metadata.thrid]
thread_uids = all_mail_crispin_client.expand_threads(
[message.g_metadata.thrid])
thread_g_metadata = all_mail_crispin_client.g_metadata(
thread_uids)
download_thread(all_mail_crispin_client, db_session, log,
syncmanager_lock, thread_g_metadata,
message.g_metadata.thrid, thread_uids)
# In theory we only ever have one Greenlet modifying ImapUid
# entries for a non-All Mail folder, but grab the lock anyway
# to be safe.
with syncmanager_lock:
log.debug("download_queued_threads acquired syncmanager_lock")
# Since we download msgs from All Mail, we need to separately
# make sure we have ImapUids recorded for this folder (used in
# progress tracking, queuing, and delete detection).
log.debug("Adding {} imapuid rows for {} processed messages"
.format(folder_name, len(processed_msgs)))
for msg in processed_msgs:
add_new_imapuid(db_session, log, msg, folder_name, acc)
report_progress(crispin_client, db_session, log, folder_name,
message_download_stack.qsize(), status_cb)
log.info("Message download queue emptied")
def check_new_g_thrids(account_id, provider, folder_name, log,
message_download_stack, poll_frequency,
syncmanager_lock):
""" Check for new X-GM-THRIDs and add them to the download stack.
We do this by comparing local UID lists to remote UID lists, maintaining
the invariant that (stack uids)+(local uids) == (remote uids).
We also remove local messages that have disappeared from the remote, since
it's totally probable that users will be archiving mail as the initial
sync goes on.
We grab a new IMAP connection from the pool for this to isolate its
actions from whatever the main greenlet may be doing.
Runs until killed. (Intended to be run in a greenlet.)
"""
with connection_pool(account_id).get() as crispin_client:
with session_scope(ignore_soft_deletes=False) as db_session:
crispin_client.select_folder(folder_name,
uidvalidity_cb(
db_session,
crispin_client.account_id))
while True:
log.info("Checking for new/deleted messages during initial sync.")
remote_uids = set(crispin_client.all_uids())
# We lock this section to make sure no messages are being modified
# in the database while we make sure the queue is in a good state.
with syncmanager_lock:
log.debug("check_new_g_thrids acquired syncmanager_lock")
with session_scope(ignore_soft_deletes=False) as db_session:
local_uids = set(account.all_uids(account_id, db_session,
folder_name))
stack_uids = {gm.uid for gm in
message_download_stack.queue}
local_with_pending_uids = local_uids | stack_uids
deleted_uids = remove_deleted_uids(
account_id, db_session, log, folder_name, local_uids,
remote_uids)
log.info("Removed {} deleted UIDs from {}".format(
len(deleted_uids), folder_name))
# filter out messages that have disappeared on the remote side
new_message_download_stack = [gm for gm in
message_download_stack.queue
if gm.uid in remote_uids]
# add in any new uids from the remote
new_uids = [uid for uid in remote_uids if uid not in
local_with_pending_uids]
flags = crispin_client.flags(new_uids)
g_metadata = crispin_client.g_metadata(new_uids)
log.info("Adding {} new messages to the download queue for {}"
.format(min(len(flags), len(g_metadata)), folder_name))
for new_uid in new_uids:
# could have disappeared from the folder in the meantime
if new_uid in flags and new_uid in g_metadata:
new_message_download_stack.append(
GMessage(new_uid, g_metadata[new_uid],
flags[new_uid].flags,
flags[new_uid].labels))
message_download_stack.queue = sorted(
new_message_download_stack, key=lambda m: m.uid)
log.info("Idling on {0} with {1} timeout".format(
folder_name, poll_frequency))
crispin_client.conn.idle()
crispin_client.conn.idle_check(timeout=poll_frequency)
crispin_client.conn.idle_done()
log.info("IDLE on {0} detected changes or timeout reached"
.format(folder_name))
def download_queued_threads(crispin_client, db_session, log, folder_name,
message_download_stack, status_cb,
syncmanager_lock, c):
""" Download threads until `message_download_stack` is empty.
UIDs and g_metadata that come out of `message_download_stack` are for
the _folder that threads are being expanded in_.
Threads are downloaded in the order they come out of the stack, which
_ought_ to be putting newest threads at the top. Messages are
downloaded newest-to-oldest in thread. (Threads are expanded to all
messages in the email archive that belong to the threads corresponding
to the given uids.)
"""
num_total_messages = message_download_stack.qsize()
log.info("{} messages found initially (unsorted by thread)"
.format(num_total_messages))
# We still need the original crispin connection for progress reporting,
# so the easiest thing to do here with the current pooling setup is to
# create a new crispin client for querying All Mail.
all_mail_crispin_client = new_crispin(crispin_client.account_id,
crispin_client.PROVIDER,
conn_pool_size=1)
log.info("Expanding threads and downloading messages.")
with all_mail_crispin_client.pool.get() as all_mail_c:
all_mail_crispin_client.select_folder(
crispin_client.folder_names(c)['all'],
uidvalidity_cb(db_session, crispin_client.account_id), all_mail_c)
# Since we do thread expansion, for any given thread, even if we
# already have the UID in the given GMessage downloaded, we may not
# have _every_ message in the thread. We have to expand it and make
# sure we have all messages.
acc = db_session.query(ImapAccount).join(Namespace).filter_by(
id=crispin_client.account_id).one()
while not message_download_stack.empty():
message = message_download_stack.get_nowait()
# Don't try to re-download any messages that are in the same
# thread. (Putting this _before_ the download to guarantee no
# context switches happen in the meantime; we _should_ re-download
# if another message arrives on the thread.)
processed_msgs = [m for m in message_download_stack.queue if
m.g_metadata.thrid ==
message.g_metadata.thrid]
processed_msgs.append(message)
message_download_stack.queue = [
m for m in message_download_stack.queue if m.g_metadata.thrid
!= message.g_metadata.thrid]
thread_uids = all_mail_crispin_client.expand_threads(
[message.g_metadata.thrid], all_mail_c)
thread_g_metadata = all_mail_crispin_client.g_metadata(
thread_uids, all_mail_c)
download_thread(all_mail_crispin_client, db_session, log,
syncmanager_lock, thread_g_metadata,
message.g_metadata.thrid, thread_uids, all_mail_c)
# Since we download msgs from All Mail, we need to separately make
# sure we have ImapUids recorded for this folder (used in progress
# tracking and delete detection).
for msg in processed_msgs:
add_new_imapuid(db_session, msg, folder_name, acc)
report_progress(crispin_client, db_session, log, folder_name,
message_download_stack.qsize(), status_cb, c)
log.info("Message download queue emptied")