Example #1
0
File: imap.py Project: jre21/inbox
def base_initial_sync(crispin_client, db_session, log, folder_name, shared_state, initial_sync_fn):
    """ Downloads entire messages.

    This function may be retried as many times as you like; it will pick up
    where it left off, delete removed messages if things disappear between
    restarts, and only complete once we have all the UIDs in the given folder
    locally.

    This function also starts up a secondary greenlet that checks for new
    messages periodically, to deal with the case of very large folders---it's
    a bad experience for the user to keep receiving old mail but not receive
    new mail! We use a LIFO queue to make sure we're downloading newest mail
    first.
    """
    log.info("Starting initial sync for {0}".format(folder_name))

    local_uids = account.all_uids(crispin_client.account_id, db_session, folder_name)

    uid_download_stack = LifoQueue()

    with crispin_client.pool.get() as c:
        crispin_client.select_folder(folder_name, uidvalidity_cb(db_session, crispin_client.account_id), c)

        initial_sync_fn(crispin_client, db_session, log, folder_name, shared_state, local_uids, uid_download_stack, c)

    verify_db(crispin_client, db_session)

    return "poll"
Example #2
0
File: imap.py Project: jre21/inbox
def highestmodseq_update(
    crispin_client, db_session, log, folder_name, last_highestmodseq, status_cb, highestmodseq_fn, syncmanager_lock, c
):
    account_id = crispin_client.account_id
    new_highestmodseq = crispin_client.selected_highestmodseq
    new_uidvalidity = crispin_client.selected_uidvalidity
    log.info("Starting highestmodseq update on {0} (current HIGHESTMODSEQ: {1})".format(folder_name, new_highestmodseq))
    local_uids = account.all_uids(account_id, db_session, folder_name)
    changed_uids = crispin_client.new_and_updated_uids(last_highestmodseq, c)
    remote_uids = crispin_client.all_uids(c)

    if changed_uids:
        new, updated = new_or_updated(changed_uids, local_uids)
        log.info("{0} new and {1} updated UIDs".format(len(new), len(updated)))
        local_uids += new
        deleted_uids = remove_deleted_uids(
            account_id, db_session, log, folder_name, local_uids, remote_uids, syncmanager_lock, c
        )

        local_uids = set(local_uids) - deleted_uids
        update_metadata(crispin_client, db_session, log, folder_name, updated, syncmanager_lock, c)

        highestmodseq_fn(
            crispin_client, db_session, log, folder_name, changed_uids, local_uids, status_cb, syncmanager_lock, c
        )
    else:
        log.info("No new or updated messages")

    remove_deleted_uids(
        crispin_client.account_id, db_session, log, folder_name, local_uids, remote_uids, syncmanager_lock, c
    )
    account.update_uidvalidity(account_id, db_session, folder_name, new_uidvalidity, new_highestmodseq)
    db_session.commit()
Example #3
0
File: imap.py Project: jre21/inbox
def check_new_uids(account_id, provider, folder_name, log, uid_download_stack, poll_frequency, syncmanager_lock):
    """ Check for new UIDs and add them to the download stack.

    We do this by comparing local UID lists to remote UID lists, maintaining
    the invariant that (stack uids)+(local uids) == (remote uids).

    We also remove local messages that have disappeared from the remote, since
    it's totally probable that users will be archiving mail as the initial
    sync goes on.

    We grab a new IMAP connection from the pool for this to isolate its
    actions from whatever the main greenlet may be doing.

    Runs until killed. (Intended to be run in a greenlet.)
    """
    log.info("Spinning up new UID-check poller for {}".format(folder_name))
    # can't mix and match crispin clients when playing with different folders
    crispin_client = new_crispin(account_id, provider, conn_pool_size=1)
    with crispin_client.pool.get() as c:
        with session_scope() as db_session:
            crispin_client.select_folder(folder_name, uidvalidity_cb(db_session, crispin_client.account_id), c)
        while True:
            remote_uids = set(crispin_client.all_uids(c))
            # We lock this section to make sure no messages are being
            # downloaded while we make sure the queue is in a good state.
            with syncmanager_lock:
                with session_scope() as db_session:
                    local_uids = set(account.all_uids(account_id, db_session, folder_name))
                    stack_uids = set(uid_download_stack.queue)
                    local_with_pending_uids = local_uids | stack_uids
                    deleted_uids = remove_deleted_uids(
                        account_id, db_session, log, folder_name, local_uids, remote_uids, syncmanager_lock, c
                    )
                    # XXX This double-grabs syncmanager_lock, does that cause
                    # a deadlock?
                    log.info("Removed {} deleted UIDs from {}".format(len(deleted_uids), folder_name))

                # filter out messages that have disappeared on the remote side
                new_uid_download_stack = {u for u in uid_download_stack.queue if u in remote_uids}

                # add in any new uids from the remote
                for uid in remote_uids:
                    if uid not in local_with_pending_uids:
                        new_uid_download_stack.add(uid)

                uid_download_stack.queue = sorted(new_uid_download_stack, key=int)
            sleep(poll_frequency)
Example #4
0
def check_new_g_thrids(account_id, provider, folder_name, log,
                       message_download_stack, poll_frequency,
                       syncmanager_lock):
    """ Check for new X-GM-THRIDs and add them to the download stack.

    We do this by comparing local UID lists to remote UID lists, maintaining
    the invariant that (stack uids)+(local uids) == (remote uids).

    We also remove local messages that have disappeared from the remote, since
    it's totally probable that users will be archiving mail as the initial
    sync goes on.

    We grab a new IMAP connection from the pool for this to isolate its
    actions from whatever the main greenlet may be doing.

    Runs until killed. (Intended to be run in a greenlet.)
    """
    with connection_pool(account_id).get() as crispin_client:
        with session_scope(ignore_soft_deletes=False) as db_session:
            crispin_client.select_folder(folder_name,
                                         uidvalidity_cb(
                                             db_session,
                                             crispin_client.account_id))
        while True:
            log.info("Checking for new/deleted messages during initial sync.")
            remote_uids = set(crispin_client.all_uids())
            # We lock this section to make sure no messages are being modified
            # in the database while we make sure the queue is in a good state.
            with syncmanager_lock:
                log.debug("check_new_g_thrids acquired syncmanager_lock")
                with session_scope(ignore_soft_deletes=False) as db_session:
                    local_uids = set(account.all_uids(account_id, db_session,
                                                      folder_name))
                    stack_uids = {gm.uid for gm in
                                  message_download_stack.queue}
                    local_with_pending_uids = local_uids | stack_uids
                    deleted_uids = remove_deleted_uids(
                        account_id, db_session, log, folder_name, local_uids,
                        remote_uids)
                    log.info("Removed {} deleted UIDs from {}".format(
                        len(deleted_uids), folder_name))

                # filter out messages that have disappeared on the remote side
                new_message_download_stack = [gm for gm in
                                              message_download_stack.queue
                                              if gm.uid in remote_uids]

                # add in any new uids from the remote
                new_uids = [uid for uid in remote_uids if uid not in
                            local_with_pending_uids]
                flags = crispin_client.flags(new_uids)
                g_metadata = crispin_client.g_metadata(new_uids)
                log.info("Adding {} new messages to the download queue for {}"
                         .format(min(len(flags), len(g_metadata)), folder_name))
                for new_uid in new_uids:
                    # could have disappeared from the folder in the meantime
                    if new_uid in flags and new_uid in g_metadata:
                        new_message_download_stack.append(
                            GMessage(new_uid, g_metadata[new_uid],
                                     flags[new_uid].flags,
                                     flags[new_uid].labels))

                message_download_stack.queue = sorted(
                    new_message_download_stack, key=lambda m: m.uid)

            log.info("Idling on {0} with {1} timeout".format(
                folder_name, poll_frequency))
            crispin_client.conn.idle()
            crispin_client.conn.idle_check(timeout=poll_frequency)
            crispin_client.conn.idle_done()
            log.info("IDLE on {0} detected changes or timeout reached"
                     .format(folder_name))
Example #5
0
File: gmail.py Project: caitp/inbox
def check_new_g_thrids(account_id, provider, folder_name, log,
                       message_download_stack, poll_frequency,
                       syncmanager_lock):
    """ Check for new X-GM-THRIDs and add them to the download stack.

    We do this by comparing local UID lists to remote UID lists, maintaining
    the invariant that (stack uids)+(local uids) == (remote uids).

    We also remove local messages that have disappeared from the remote, since
    it's totally probable that users will be archiving mail as the initial
    sync goes on.

    We grab a new IMAP connection from the pool for this to isolate its
    actions from whatever the main greenlet may be doing.

    Runs until killed. (Intended to be run in a greenlet.)
    """
    with connection_pool(account_id).get() as crispin_client:
        with session_scope(ignore_soft_deletes=False) as db_session:
            crispin_client.select_folder(folder_name,
                                         uidvalidity_cb(
                                             db_session,
                                             crispin_client.account_id))
        while True:
            log.info("Checking for new/deleted messages during initial sync.")
            remote_uids = set(crispin_client.all_uids())
            # We lock this section to make sure no messages are being modified
            # in the database while we make sure the queue is in a good state.
            with syncmanager_lock:
                log.debug("check_new_g_thrids acquired syncmanager_lock")
                with session_scope(ignore_soft_deletes=False) as db_session:
                    local_uids = set(account.all_uids(account_id, db_session,
                                                      folder_name))
                    stack_uids = {gm.uid for gm in
                                  message_download_stack.queue}
                    local_with_pending_uids = local_uids | stack_uids
                    deleted_uids = remove_deleted_uids(
                        account_id, db_session, log, folder_name, local_uids,
                        remote_uids)
                    log.info("Removed {} deleted UIDs from {}".format(
                        len(deleted_uids), folder_name))

                # filter out messages that have disappeared on the remote side
                new_message_download_stack = [gm for gm in
                                              message_download_stack.queue
                                              if gm.uid in remote_uids]

                # add in any new uids from the remote
                new_uids = [uid for uid in remote_uids if uid not in
                            local_with_pending_uids]
                flags = crispin_client.flags(new_uids)
                g_metadata = crispin_client.g_metadata(new_uids)
                log.info("Adding {} new messages to the download queue for {}"
                         .format(min(len(flags), len(g_metadata)), folder_name))
                for new_uid in new_uids:
                    # could have disappeared from the folder in the meantime
                    if new_uid in flags and new_uid in g_metadata:
                        new_message_download_stack.append(
                            GMessage(new_uid, g_metadata[new_uid],
                                     flags[new_uid].flags,
                                     flags[new_uid].labels))

                message_download_stack.queue = sorted(
                    new_message_download_stack, key=lambda m: m.uid)

            log.info("Idling on {0} with {1} timeout".format(
                folder_name, poll_frequency))
            crispin_client.conn.idle()
            crispin_client.conn.idle_check(timeout=poll_frequency)
            crispin_client.conn.idle_done()
            log.info("IDLE on {0} detected changes or timeout reached"
                     .format(folder_name))