Example #1
0
    def check_uid_changes(self, crispin_client, download_stack,
                          async_download):
        crispin_client.select_folder(self.folder_name, uidvalidity_cb)
        new_highestmodseq = crispin_client.selected_highestmodseq
        with mailsync_session_scope() as db_session:
            saved_folder_info = common.get_folder_info(self.account_id,
                                                       db_session,
                                                       self.folder_name)
            # Ensure that we have an initial highestmodseq value stored before
            # we begin polling for changes.
            if saved_folder_info is None or \
                    saved_folder_info.highestmodseq is None:
                assert (crispin_client.selected_uidvalidity is not None
                        and crispin_client.selected_highestmodseq is not None)
                saved_folder_info = common.update_folder_info(
                    crispin_client.account_id, db_session, self.folder_name,
                    crispin_client.selected_uidvalidity,
                    crispin_client.selected_highestmodseq)
            saved_highestmodseq = saved_folder_info.highestmodseq
            if new_highestmodseq == saved_highestmodseq:
                # Don't need to do anything if the highestmodseq hasn't
                # changed.
                return
            elif new_highestmodseq < saved_highestmodseq:
                # This should really never happen, but if it does, handle it.
                log.warning(
                    'got server highestmodseq less than saved '
                    'highestmodseq',
                    new_highestmodseq=new_highestmodseq,
                    saved_highestmodseq=saved_highestmodseq)
                return
        # Highestmodseq has changed, update accordingly.
        new_uidvalidity = crispin_client.selected_uidvalidity
        changed_uids = crispin_client.new_and_updated_uids(saved_highestmodseq)
        remote_uids = crispin_client.all_uids()
        with mailsync_session_scope() as db_session:
            local_uids = common.all_uids(self.account_id, db_session,
                                         self.folder_id)
        stack_uids = set(download_stack.keys())
        local_with_pending_uids = local_uids | stack_uids
        new, updated = new_or_updated(changed_uids, local_with_pending_uids)
        if changed_uids:
            log.info("Changed UIDs",
                     message="new: {} updated: {}".format(
                         len(new), len(updated)),
                     new_uid_count=len(new),
                     updated_uid_count=len(updated))
            self.update_metadata(crispin_client, updated)
            self.highestmodseq_callback(crispin_client, new, updated,
                                        download_stack, async_download)

        with mailsync_session_scope() as db_session:
            with self.syncmanager_lock:
                self.remove_deleted_uids(db_session, local_uids, remote_uids)
            self.update_uid_counts(db_session,
                                   remote_uid_count=len(remote_uids))
            common.update_folder_info(self.account_id, db_session,
                                      self.folder_name, new_uidvalidity,
                                      new_highestmodseq)
            db_session.commit()
Example #2
0
    def check_uid_changes(self, crispin_client, download_stack,
                          async_download):
        crispin_client.select_folder(self.folder_name, uidvalidity_cb)
        new_highestmodseq = crispin_client.selected_highestmodseq
        with mailsync_session_scope() as db_session:
            saved_folder_info = common.get_folder_info(
                self.account_id, db_session, self.folder_name)
            # Ensure that we have an initial highestmodseq value stored before
            # we begin polling for changes.
            if saved_folder_info is None or \
                    saved_folder_info.highestmodseq is None:
                assert (crispin_client.selected_uidvalidity is not None
                        and crispin_client.selected_highestmodseq is
                        not None)
                saved_folder_info = common.update_folder_info(
                    crispin_client.account_id, db_session,
                    self.folder_name,
                    crispin_client.selected_uidvalidity,
                    crispin_client.selected_highestmodseq)
            saved_highestmodseq = saved_folder_info.highestmodseq
            if new_highestmodseq == saved_highestmodseq:
                # Don't need to do anything if the highestmodseq hasn't
                # changed.
                return
            elif new_highestmodseq < saved_highestmodseq:
                # This should really never happen, but if it does, handle it.
                log.warning('got server highestmodseq less than saved '
                            'highestmodseq',
                            new_highestmodseq=new_highestmodseq,
                            saved_highestmodseq=saved_highestmodseq)
                return
            save_folder_names(log, self.account_id,
                              crispin_client.folder_names(), db_session)
        # Highestmodseq has changed, update accordingly.
        new_uidvalidity = crispin_client.selected_uidvalidity
        changed_uids = crispin_client.new_and_updated_uids(saved_highestmodseq)
        remote_uids = crispin_client.all_uids()
        with mailsync_session_scope() as db_session:
            local_uids = common.all_uids(self.account_id, db_session,
                                         self.folder_name)
        stack_uids = {uid for uid, _ in download_stack}
        local_with_pending_uids = local_uids | stack_uids
        new, updated = new_or_updated(changed_uids, local_with_pending_uids)
        if changed_uids:
            log.info("Changed UIDs", message="new: {} updated: {}"
                                             .format(len(new), len(updated)),
                     new_uid_count=len(new), updated_uid_count=len(updated))
            self.update_metadata(crispin_client, updated)
            self.highestmodseq_callback(crispin_client, new, updated,
                                        download_stack, async_download)

        with mailsync_session_scope() as db_session:
            with self.syncmanager_lock:
                self.remove_deleted_uids(db_session, local_uids, remote_uids)
            self.update_uid_counts(db_session,
                                   remote_uid_count=len(remote_uids))
            common.update_folder_info(self.account_id, db_session,
                                      self.folder_name, new_uidvalidity,
                                      new_highestmodseq)
            db_session.commit()
Example #3
0
def highestmodseq_update(crispin_client, log, folder_name, last_highestmodseq,
                         highestmodseq_fn, syncmanager_lock):
    account_id = crispin_client.account_id
    new_highestmodseq = crispin_client.selected_highestmodseq
    new_uidvalidity = crispin_client.selected_uidvalidity
    log.info('starting highestmodseq update',
             current_highestmodseq=new_highestmodseq)
    changed_uids = crispin_client.new_and_updated_uids(last_highestmodseq)
    remote_uids = crispin_client.all_uids()

    local_uids = None
    if changed_uids:
        with session_scope(ignore_soft_deletes=False) as db_session:
            local_uids = account.all_uids(account_id, db_session, folder_name)

        new, updated = new_or_updated(changed_uids, local_uids)
        log.info(new_uid_count=len(new), updated_uid_count=len(updated))

        local_uids += new
        with syncmanager_lock:
            log.debug("highestmodseq_update acquired syncmanager_lock")
            with session_scope(ignore_soft_deletes=False) as db_session:
                deleted_uids = remove_deleted_uids(account_id, db_session, log,
                                                   folder_name, local_uids,
                                                   remote_uids)

        local_uids = set(local_uids) - deleted_uids
        update_metadata(crispin_client, log, folder_name, updated,
                        syncmanager_lock)

        with session_scope(ignore_soft_deletes=False) as db_session:
            update_uid_counts(db_session, log, account_id, folder_name,
                              remote_uid_count=len(remote_uids),
                              download_uid_count=len(new),
                              update_uid_count=len(updated),
                              delete_uid_count=len(deleted_uids))

        highestmodseq_fn(crispin_client, log, folder_name, new,
                         updated, syncmanager_lock)
    else:
        log.info("No new or updated messages")

    with session_scope(ignore_soft_deletes=False) as db_session:
        with syncmanager_lock:
            log.debug("highestmodseq_update acquired syncmanager_lock")
            if local_uids is None:
                local_uids = account.all_uids(
                    account_id, db_session, folder_name)
            deleted_uids = remove_deleted_uids(crispin_client.account_id,
                                               db_session, log, folder_name,
                                               local_uids, remote_uids)
        update_uid_counts(db_session, log, account_id, folder_name,
                          remote_uid_count=len(remote_uids),
                          delete_uid_count=len(deleted_uids))
        account.update_folder_info(account_id, db_session, folder_name,
                                   new_uidvalidity, new_highestmodseq)
        db_session.commit()
Example #4
0
    def highestmodseq_update(self, crispin_client, last_highestmodseq):
        new_highestmodseq = crispin_client.selected_highestmodseq
        new_uidvalidity = crispin_client.selected_uidvalidity
        log.info('starting highestmodseq update',
                 current_highestmodseq=new_highestmodseq)
        changed_uids = crispin_client.new_and_updated_uids(last_highestmodseq)
        remote_uids = crispin_client.all_uids()

        local_uids = None
        if changed_uids:
            with mailsync_session_scope() as db_session:
                local_uids = common.all_uids(self.account_id, db_session,
                                             self.folder_name)

            new, updated = new_or_updated(changed_uids, local_uids)
            log.info(new_uid_count=len(new), updated_uid_count=len(updated))

            local_uids.update(new)
            with self.syncmanager_lock:
                log.debug("highestmodseq_update acquired syncmanager_lock")
                with mailsync_session_scope() as db_session:
                    deleted_uids = self.remove_deleted_uids(
                        db_session, local_uids, remote_uids)

            local_uids = local_uids - deleted_uids
            self.update_metadata(crispin_client, updated)

            with mailsync_session_scope() as db_session:
                self.update_uid_counts(
                    db_session,
                    remote_uid_count=len(remote_uids),
                    download_uid_count=len(new),
                    update_uid_count=len(updated),
                    delete_uid_count=len(deleted_uids))

            self.highestmodseq_callback(crispin_client, new, updated)
        else:
            log.info("No new or updated messages")

        with mailsync_session_scope() as db_session:
            with self.syncmanager_lock:
                log.debug("highestmodseq_update acquired syncmanager_lock")
                if local_uids is None:
                    local_uids = common.all_uids(
                        self.account_id, db_session, self.folder_name)
                deleted_uids = self.remove_deleted_uids(
                    db_session, local_uids, remote_uids)
            self.update_uid_counts(db_session,
                                   remote_uid_count=len(remote_uids),
                                   delete_uid_count=len(deleted_uids))
            common.update_folder_info(self.account_id, db_session,
                                      self.folder_name, new_uidvalidity,
                                      new_highestmodseq)
            db_session.commit()
Example #5
0
    def __update_saved_g_metadata(self, crispin_client, remote_g_metadata,
                                  local_uids):
        """
        If HIGHESTMODSEQ has changed since we saved the X-GM-MSGID cache,
        we need to query for any changes since then and update the saved
        data.

        """
        log.info('Updating cache with latest changes')
        # Any uids we don't already have will be downloaded correctly as usual,
        # but updated uids need to be updated manually.
        # XXX it may actually be faster to just query for X-GM-MSGID for the
        # whole folder rather than getting changed UIDs first; MODSEQ queries
        # are slow on large folders.
        modified = crispin_client.new_and_updated_uids(
            crispin_client.selected_highestmodseq)
        log.info(modified_msg_count=len(modified))
        new, updated = new_or_updated(modified, local_uids)
        log.info(new_uid_count=len(new), updated_uid_count=len(updated))
        if new:
            remote_g_metadata.update(crispin_client.g_metadata(new))
            log.info('Updated cache with new messages')
        else:
            log.info('No new messages to update metadata for')
        # Filter out messages that have disappeared.
        old_len = len(remote_g_metadata)
        current_remote_uids = set(crispin_client.all_uids())
        remote_g_metadata = dict((uid, md) for uid, md in
                                 remote_g_metadata.iteritems() if uid in
                                 current_remote_uids)
        num_removed = old_len - len(remote_g_metadata)
        if num_removed > 0:
            log.info(removed_msg_count=num_removed)
        set_cache(remote_g_metadata_cache_file(self.account_id,
                                               self.folder_name),
                  remote_g_metadata)
        if updated:
            # It's easy and fast to just update these here and now.
            # Bigger chunk because the data being fetched here is very small.
            for uids in chunk(updated, 5 * crispin_client.CHUNK_SIZE):
                self.update_metadata(crispin_client, uids)
            log.info('updated metadata for modified messages',
                     msg_count=len(updated))
            return len(updated)
        else:
            log.info('No modified messages to update metadata for')
            return 0
Example #6
0
    def __update_saved_g_metadata(self, crispin_client, remote_g_metadata,
                                  local_uids):
        """
        If HIGHESTMODSEQ has changed since we saved the X-GM-MSGID cache,
        we need to query for any changes since then and update the saved
        data.

        """
        log.info('Updating cache with latest changes')
        # Any uids we don't already have will be downloaded correctly as usual,
        # but updated uids need to be updated manually.
        # XXX it may actually be faster to just query for X-GM-MSGID for the
        # whole folder rather than getting changed UIDs first; MODSEQ queries
        # are slow on large folders.
        modified = crispin_client.new_and_updated_uids(
            crispin_client.selected_highestmodseq)
        log.info(modified_msg_count=len(modified))
        new, updated = new_or_updated(modified, local_uids)
        log.info(new_uid_count=len(new), updated_uid_count=len(updated))
        if new:
            remote_g_metadata.update(crispin_client.g_metadata(new))
            log.info('Updated cache with new messages')
        else:
            log.info('No new messages to update metadata for')
        # Filter out messages that have disappeared.
        old_len = len(remote_g_metadata)
        current_remote_uids = set(crispin_client.all_uids())
        remote_g_metadata = dict((uid, md)
                                 for uid, md in remote_g_metadata.iteritems()
                                 if uid in current_remote_uids)
        num_removed = old_len - len(remote_g_metadata)
        if num_removed > 0:
            log.info(removed_msg_count=num_removed)
        set_cache(
            remote_g_metadata_cache_file(self.account_id, self.folder_name),
            remote_g_metadata)
        if updated:
            # It's easy and fast to just update these here and now.
            # Bigger chunk because the data being fetched here is very small.
            for uids in chunk(updated, 5 * crispin_client.CHUNK_SIZE):
                self.update_metadata(crispin_client, uids)
            log.info('updated metadata for modified messages',
                     msg_count=len(updated))
            return len(updated)
        else:
            log.info('No modified messages to update metadata for')
            return 0
Example #7
0
def check_flags(crispin_client, db_session, log, folder_name, local_uids,
                syncmanager_lock):
    """ Update message flags if folder has changed on the remote.

    If we have saved validity info for this folder, make sure the folder hasn't
    changed since we saved it. Otherwise we need to query for flag changes too.
    """
    saved_folder_info = account.get_folder_info(crispin_client.account_id,
                                                db_session, folder_name)
    if saved_folder_info is not None:
        last_highestmodseq = saved_folder_info.highestmodseq
        if last_highestmodseq > crispin_client.selected_highestmodseq:
            uids = crispin_client.new_and_updated_uids(last_highestmodseq)
            if uids:
                _, updated = new_or_updated(uids, local_uids)
                update_metadata(crispin_client, db_session, log, folder_name,
                                updated, syncmanager_lock)
Example #8
0
def check_flags(crispin_client, db_session, log, folder_name, local_uids,
                syncmanager_lock):
    """ Update message flags if folder has changed on the remote.

    If we have saved validity info for this folder, make sure the folder hasn't
    changed since we saved it. Otherwise we need to query for flag changes too.
    """
    saved_folder_info = account.get_folder_info(crispin_client.account_id,
                                                db_session, folder_name)
    if saved_folder_info is not None:
        last_highestmodseq = saved_folder_info.highestmodseq
        if last_highestmodseq > crispin_client.selected_highestmodseq:
            uids = crispin_client.new_and_updated_uids(last_highestmodseq)
            if uids:
                _, updated = new_or_updated(uids, local_uids)
                update_metadata(crispin_client, db_session, log, folder_name,
                                updated, syncmanager_lock)
Example #9
0
def highestmodseq_update(crispin_client, db_session, log, folder_name,
                         last_highestmodseq, highestmodseq_fn,
                         syncmanager_lock):
    account_id = crispin_client.account_id
    new_highestmodseq = crispin_client.selected_highestmodseq
    new_uidvalidity = crispin_client.selected_uidvalidity
    log.info("Starting highestmodseq update on {} (current HIGHESTMODSEQ: {})"
             .format(folder_name, new_highestmodseq))
    local_uids = account.all_uids(account_id, db_session, folder_name)
    changed_uids = crispin_client.new_and_updated_uids(last_highestmodseq)
    remote_uids = crispin_client.all_uids()

    if changed_uids:
        new, updated = new_or_updated(changed_uids, local_uids)
        log.info("{0} new and {1} updated UIDs".format(len(new), len(updated)))
        local_uids += new
        with syncmanager_lock:
            log.debug("highestmodseq_update acquired syncmanager_lock")
            deleted_uids = remove_deleted_uids(account_id, db_session, log,
                                               folder_name, local_uids,
                                               remote_uids)

        local_uids = set(local_uids) - deleted_uids
        update_metadata(crispin_client, db_session, log, folder_name,
                        updated, syncmanager_lock)

        update_uid_counts(db_session, log, account_id, folder_name,
                          remote_uid_count=len(remote_uids),
                          download_uid_count=len(new),
                          update_uid_count=len(updated),
                          delete_uid_count=len(deleted_uids))

        highestmodseq_fn(crispin_client, db_session, log, folder_name,
                         changed_uids, local_uids, syncmanager_lock)
    else:
        log.info("No new or updated messages")

    with syncmanager_lock:
        log.debug("highestmodseq_update acquired syncmanager_lock")
        remove_deleted_uids(crispin_client.account_id, db_session, log,
                            folder_name, local_uids, remote_uids)
    account.update_folder_info(account_id, db_session, folder_name,
                               new_uidvalidity, new_highestmodseq)
    db_session.commit()
Example #10
0
def highestmodseq_update(crispin_client, log, folder_name, last_highestmodseq,
                         highestmodseq_fn, syncmanager_lock):
    account_id = crispin_client.account_id
    new_highestmodseq = crispin_client.selected_highestmodseq
    new_uidvalidity = crispin_client.selected_uidvalidity
    log.info('starting highestmodseq update',
             current_highestmodseq=new_highestmodseq)
    changed_uids = crispin_client.new_and_updated_uids(last_highestmodseq)
    remote_uids = crispin_client.all_uids()

    local_uids = None
    if changed_uids:
        with session_scope(ignore_soft_deletes=False) as db_session:
            local_uids = account.all_uids(account_id, db_session, folder_name)

        new, updated = new_or_updated(changed_uids, local_uids)
        log.info(new_uid_count=len(new), updated_uid_count=len(updated))

        local_uids += new
        with syncmanager_lock:
            log.debug("highestmodseq_update acquired syncmanager_lock")
            with session_scope(ignore_soft_deletes=False) as db_session:
                deleted_uids = remove_deleted_uids(account_id, db_session, log,
                                                   folder_name, local_uids,
                                                   remote_uids)

        local_uids = set(local_uids) - deleted_uids
        update_metadata(crispin_client, log, folder_name, updated,
                        syncmanager_lock)

        with session_scope(ignore_soft_deletes=False) as db_session:
            update_uid_counts(db_session,
                              log,
                              account_id,
                              folder_name,
                              remote_uid_count=len(remote_uids),
                              download_uid_count=len(new),
                              update_uid_count=len(updated),
                              delete_uid_count=len(deleted_uids))

        highestmodseq_fn(crispin_client, log, folder_name, new, updated,
                         syncmanager_lock)
    else:
        log.info("No new or updated messages")

    with session_scope(ignore_soft_deletes=False) as db_session:
        with syncmanager_lock:
            log.debug("highestmodseq_update acquired syncmanager_lock")
            if local_uids is None:
                local_uids = account.all_uids(account_id, db_session,
                                              folder_name)
            deleted_uids = remove_deleted_uids(crispin_client.account_id,
                                               db_session, log, folder_name,
                                               local_uids, remote_uids)
        update_uid_counts(db_session,
                          log,
                          account_id,
                          folder_name,
                          remote_uid_count=len(remote_uids),
                          delete_uid_count=len(deleted_uids))
        account.update_folder_info(account_id, db_session, folder_name,
                                   new_uidvalidity, new_highestmodseq)
        db_session.commit()