Пример #1
0
    def highestmodseq_update(self, crispin_client, last_highestmodseq):
        new_highestmodseq = crispin_client.selected_highestmodseq
        new_uidvalidity = crispin_client.selected_uidvalidity
        log.info('starting highestmodseq update',
                 current_highestmodseq=new_highestmodseq)
        changed_uids = crispin_client.new_and_updated_uids(last_highestmodseq)
        remote_uids = crispin_client.all_uids()

        local_uids = None
        if changed_uids:
            with mailsync_session_scope() as db_session:
                local_uids = common.all_uids(self.account_id, db_session,
                                             self.folder_name)

            new, updated = new_or_updated(changed_uids, local_uids)
            log.info(new_uid_count=len(new), updated_uid_count=len(updated))

            local_uids.update(new)
            with self.syncmanager_lock:
                log.debug("highestmodseq_update acquired syncmanager_lock")
                with mailsync_session_scope() as db_session:
                    deleted_uids = self.remove_deleted_uids(
                        db_session, local_uids, remote_uids)

            local_uids = local_uids - deleted_uids
            self.update_metadata(crispin_client, updated)

            with mailsync_session_scope() as db_session:
                self.update_uid_counts(
                    db_session,
                    remote_uid_count=len(remote_uids),
                    download_uid_count=len(new),
                    update_uid_count=len(updated),
                    delete_uid_count=len(deleted_uids))

            self.highestmodseq_callback(crispin_client, new, updated)
        else:
            log.info("No new or updated messages")

        with mailsync_session_scope() as db_session:
            with self.syncmanager_lock:
                log.debug("highestmodseq_update acquired syncmanager_lock")
                if local_uids is None:
                    local_uids = common.all_uids(
                        self.account_id, db_session, self.folder_name)
                deleted_uids = self.remove_deleted_uids(
                    db_session, local_uids, remote_uids)
            self.update_uid_counts(db_session,
                                   remote_uid_count=len(remote_uids),
                                   delete_uid_count=len(deleted_uids))
            common.update_folder_info(self.account_id, db_session,
                                      self.folder_name, new_uidvalidity,
                                      new_highestmodseq)
            db_session.commit()
Пример #2
0
    def __imap_flag_change_poller(self):
        """
        Periodically update message flags for those servers
        who don't support CONDSTORE.
        Runs until killed. (Intended to be run in a greenlet)
        """
        log.info("Spinning up new flags-refresher for ",
                 folder_name=self.folder_name)
        with self.conn_pool.get() as crispin_client:
            with mailsync_session_scope() as db_session:
                crispin_client.select_folder(
                    self.folder_name,
                    uidvalidity_cb(crispin_client.account_id))
            while True:
                remote_uids = set(crispin_client.all_uids())
                local_uids = common.all_uids(self.account_id, db_session,
                                             self.folder_name)
                # STOPSHIP(emfree): sorted does nothing here
                to_refresh = sorted(remote_uids
                                    & local_uids)[-self.refresh_flags_max:]

                self.update_metadata(crispin_client, to_refresh)
                with session_scope(ignore_soft_deletes=True) as db_session:
                    self.update_uid_counts(db_session,
                                           update_uid_count=len(to_refresh))

                sleep(self.poll_frequency)
Пример #3
0
 def check_uid_changes(self, crispin_client, download_stack,
                       async_download):
     remote_uids = set(crispin_client.all_uids())
     with self.syncmanager_lock:
         with mailsync_session_scope() as db_session:
             local_uids = common.all_uids(self.account_id, db_session,
                                          self.folder_id)
             # Download new UIDs.
             stack_uids = set(download_stack.keys())
             local_with_pending_uids = local_uids | stack_uids
             for uid in sorted(remote_uids):
                 if uid not in local_with_pending_uids:
                     download_stack.put(uid, None)
             self.remove_deleted_uids(db_session, local_uids, remote_uids)
     if not async_download:
         self.download_uids(crispin_client, download_stack)
         with mailsync_session_scope() as db_session:
             self.update_uid_counts(
                 db_session,
                 remote_uid_count=len(remote_uids),
                 download_uid_count=len(download_stack))
     to_refresh = sorted(remote_uids &
                         local_uids)[-self.refresh_flags_max:]
     self.update_metadata(crispin_client, to_refresh)
     with mailsync_session_scope() as db_session:
         common.update_folder_info(self.account_id, db_session,
                                   self.folder_name,
                                   crispin_client.selected_uidvalidity,
                                   None,
                                   crispin_client.selected_uidnext)
Пример #4
0
    def initial_sync_impl(self, crispin_client):
        # We wrap the block in a try/finally because the change_poller greenlet
        # needs to be killed when this greenlet is interrupted
        change_poller = None
        try:
            assert crispin_client.selected_folder_name == self.folder_name
            remote_uids = crispin_client.all_uids()
            with self.syncmanager_lock:
                with mailsync_session_scope() as db_session:
                    local_uids = common.all_uids(self.account_id, db_session,
                                                 self.folder_id)
                    self.remove_deleted_uids(db_session, local_uids,
                                             remote_uids)

            new_uids = set(remote_uids) - local_uids
            download_stack = UIDStack()
            for uid in sorted(new_uids):
                download_stack.put(uid, GenericUIDMetadata(self.throttled))

            with mailsync_session_scope() as db_session:
                self.update_uid_counts(
                    db_session,
                    remote_uid_count=len(remote_uids),
                    # This is the initial size of our download_queue
                    download_uid_count=len(new_uids))

            change_poller = spawn(self.poll_for_changes, download_stack)
            bind_context(change_poller, 'changepoller', self.account_id,
                         self.folder_id)
            self.download_uids(crispin_client, download_stack)

        finally:
            if change_poller is not None:
                # schedule change_poller to die
                kill(change_poller)
Пример #5
0
    def initial_sync_impl(self, crispin_client):
        # We wrap the block in a try/finally because the change_poller greenlet
        # needs to be killed when this greenlet is interrupted
        change_poller = None
        try:
            assert crispin_client.selected_folder_name == self.folder_name
            remote_uids = crispin_client.all_uids()
            with self.syncmanager_lock:
                with mailsync_session_scope() as db_session:
                    local_uids = common.all_uids(self.account_id, db_session,
                                                 self.folder_name)
                    deleted_uids = self.remove_deleted_uids(
                        db_session, local_uids, remote_uids)

            local_uids = set(local_uids) - deleted_uids
            new_uids = set(remote_uids) - local_uids
            download_stack = UIDStack(new_uids)

            with mailsync_session_scope() as db_session:
                self.update_uid_counts(
                    db_session,
                    remote_uid_count=len(remote_uids),
                    # This is the initial size of our download_queue
                    download_uid_count=len(new_uids))

            change_poller = spawn(self.poll_for_changes, download_stack)
            self.download_uids(crispin_client, download_stack)

        finally:
            if change_poller is not None:
                change_poller.kill()
Пример #6
0
    def initial_sync_impl(self, crispin_client):
        # We wrap the block in a try/finally because the change_poller greenlet
        # needs to be killed when this greenlet is interrupted
        change_poller = None
        try:
            assert crispin_client.selected_folder_name == self.folder_name
            remote_uids = crispin_client.all_uids()
            with self.syncmanager_lock:
                with mailsync_session_scope() as db_session:
                    local_uids = common.all_uids(self.account_id, db_session,
                                                 self.folder_id)
                    self.remove_deleted_uids(db_session, local_uids,
                                             remote_uids)

            new_uids = set(remote_uids) - local_uids
            download_stack = UIDStack()
            for uid in sorted(new_uids):
                download_stack.put(uid, GenericUIDMetadata(self.throttled))

            with mailsync_session_scope() as db_session:
                self.update_uid_counts(
                    db_session,
                    remote_uid_count=len(remote_uids),
                    # This is the initial size of our download_queue
                    download_uid_count=len(new_uids))

            change_poller = spawn(self.poll_for_changes, download_stack)
            bind_context(change_poller, 'changepoller', self.account_id,
                         self.folder_id)
            self.download_uids(crispin_client, download_stack)

        finally:
            if change_poller is not None:
                # schedule change_poller to die
                kill(change_poller)
Пример #7
0
 def check_uid_changes(self, crispin_client, download_stack,
                       async_download):
     remote_uids = set(crispin_client.all_uids())
     with self.syncmanager_lock:
         with mailsync_session_scope() as db_session:
             local_uids = common.all_uids(self.account_id, db_session,
                                          self.folder_name)
             # Download new UIDs.
             stack_uids = {uid for uid, _ in download_stack}
             local_with_pending_uids = local_uids | stack_uids
             # filter out messages that have disappeared on the remote side
             download_stack.discard([item for item in download_stack if
                                     item[0] not in remote_uids])
             for uid in sorted(remote_uids):
                 if uid not in local_with_pending_uids:
                     download_stack.put(uid, None)
             self.remove_deleted_uids(db_session, local_uids, remote_uids)
     if not async_download:
         self.download_uids(crispin_client, download_stack)
         with mailsync_session_scope() as db_session:
             self.update_uid_counts(
                 db_session,
                 remote_uid_count=len(remote_uids),
                 download_uid_count=download_stack.qsize())
     to_refresh = sorted(remote_uids &
                         local_uids)[-self.refresh_flags_max:]
     self.update_metadata(crispin_client, to_refresh)
Пример #8
0
    def check_uid_changes(self, crispin_client, download_stack,
                          async_download):
        crispin_client.select_folder(self.folder_name, uidvalidity_cb)
        new_highestmodseq = crispin_client.selected_highestmodseq
        with mailsync_session_scope() as db_session:
            saved_folder_info = common.get_folder_info(self.account_id,
                                                       db_session,
                                                       self.folder_name)
            # Ensure that we have an initial highestmodseq value stored before
            # we begin polling for changes.
            if saved_folder_info is None or \
                    saved_folder_info.highestmodseq is None:
                assert (crispin_client.selected_uidvalidity is not None
                        and crispin_client.selected_highestmodseq is not None)
                saved_folder_info = common.update_folder_info(
                    crispin_client.account_id, db_session, self.folder_name,
                    crispin_client.selected_uidvalidity,
                    crispin_client.selected_highestmodseq)
            saved_highestmodseq = saved_folder_info.highestmodseq
            if new_highestmodseq == saved_highestmodseq:
                # Don't need to do anything if the highestmodseq hasn't
                # changed.
                return
            elif new_highestmodseq < saved_highestmodseq:
                # This should really never happen, but if it does, handle it.
                log.warning(
                    'got server highestmodseq less than saved '
                    'highestmodseq',
                    new_highestmodseq=new_highestmodseq,
                    saved_highestmodseq=saved_highestmodseq)
                return
        # Highestmodseq has changed, update accordingly.
        new_uidvalidity = crispin_client.selected_uidvalidity
        changed_uids = crispin_client.new_and_updated_uids(saved_highestmodseq)
        remote_uids = crispin_client.all_uids()
        with mailsync_session_scope() as db_session:
            local_uids = common.all_uids(self.account_id, db_session,
                                         self.folder_id)
        stack_uids = set(download_stack.keys())
        local_with_pending_uids = local_uids | stack_uids
        new, updated = new_or_updated(changed_uids, local_with_pending_uids)
        if changed_uids:
            log.info("Changed UIDs",
                     message="new: {} updated: {}".format(
                         len(new), len(updated)),
                     new_uid_count=len(new),
                     updated_uid_count=len(updated))
            self.update_metadata(crispin_client, updated)
            self.highestmodseq_callback(crispin_client, new, updated,
                                        download_stack, async_download)

        with mailsync_session_scope() as db_session:
            with self.syncmanager_lock:
                self.remove_deleted_uids(db_session, local_uids, remote_uids)
            self.update_uid_counts(db_session,
                                   remote_uid_count=len(remote_uids))
            common.update_folder_info(self.account_id, db_session,
                                      self.folder_name, new_uidvalidity,
                                      new_highestmodseq)
            db_session.commit()
Пример #9
0
    def poll_impl(self, crispin_client):
        crispin_client.select_folder(self.folder_name,
                                     uidvalidity_cb(self.account_id))

        remote_uids = set(crispin_client.all_uids())
        with mailsync_session_scope() as db_session:
            local_uids = common.all_uids(self.account_id, db_session,
                                         self.folder_name)
            deleted_uids = self.remove_deleted_uids(db_session, local_uids,
                                                    remote_uids)

            local_uids -= deleted_uids
            log.info("Removed {} deleted UIDs from {}".format(
                len(deleted_uids), self.folder_name))
            uids_to_download = remote_uids - local_uids

            self.update_uid_counts(db_session,
                                   remote_uid_count=len(remote_uids),
                                   download_uid_count=len(uids_to_download),
                                   delete_uid_count=len(deleted_uids))

        log.info("UIDs to download: {}".format(uids_to_download))
        if uids_to_download:
            self.download_uids(crispin_client,
                               uid_list_to_stack(uids_to_download))

        uids_to_refresh = sorted(remote_uids -
                                 uids_to_download)[-self.refresh_flags_max:]
        log.info('UIDs to refresh: ', uids=uids_to_refresh)
        if uids_to_refresh:
            self.update_metadata(crispin_client, uids_to_refresh)

        sleep(self.poll_frequency)
Пример #10
0
    def check_uid_changes(self, crispin_client, download_stack,
                          async_download):
        crispin_client.select_folder(self.folder_name, uidvalidity_cb)
        new_highestmodseq = crispin_client.selected_highestmodseq
        with mailsync_session_scope() as db_session:
            saved_folder_info = common.get_folder_info(
                self.account_id, db_session, self.folder_name)
            # Ensure that we have an initial highestmodseq value stored before
            # we begin polling for changes.
            if saved_folder_info is None or \
                    saved_folder_info.highestmodseq is None:
                assert (crispin_client.selected_uidvalidity is not None
                        and crispin_client.selected_highestmodseq is
                        not None)
                saved_folder_info = common.update_folder_info(
                    crispin_client.account_id, db_session,
                    self.folder_name,
                    crispin_client.selected_uidvalidity,
                    crispin_client.selected_highestmodseq)
            saved_highestmodseq = saved_folder_info.highestmodseq
            if new_highestmodseq == saved_highestmodseq:
                # Don't need to do anything if the highestmodseq hasn't
                # changed.
                return
            elif new_highestmodseq < saved_highestmodseq:
                # This should really never happen, but if it does, handle it.
                log.warning('got server highestmodseq less than saved '
                            'highestmodseq',
                            new_highestmodseq=new_highestmodseq,
                            saved_highestmodseq=saved_highestmodseq)
                return
            save_folder_names(log, self.account_id,
                              crispin_client.folder_names(), db_session)
        # Highestmodseq has changed, update accordingly.
        new_uidvalidity = crispin_client.selected_uidvalidity
        changed_uids = crispin_client.new_and_updated_uids(saved_highestmodseq)
        remote_uids = crispin_client.all_uids()
        with mailsync_session_scope() as db_session:
            local_uids = common.all_uids(self.account_id, db_session,
                                         self.folder_name)
        stack_uids = {uid for uid, _ in download_stack}
        local_with_pending_uids = local_uids | stack_uids
        new, updated = new_or_updated(changed_uids, local_with_pending_uids)
        if changed_uids:
            log.info("Changed UIDs", message="new: {} updated: {}"
                                             .format(len(new), len(updated)),
                     new_uid_count=len(new), updated_uid_count=len(updated))
            self.update_metadata(crispin_client, updated)
            self.highestmodseq_callback(crispin_client, new, updated,
                                        download_stack, async_download)

        with mailsync_session_scope() as db_session:
            with self.syncmanager_lock:
                self.remove_deleted_uids(db_session, local_uids, remote_uids)
            self.update_uid_counts(db_session,
                                   remote_uid_count=len(remote_uids))
            common.update_folder_info(self.account_id, db_session,
                                      self.folder_name, new_uidvalidity,
                                      new_highestmodseq)
            db_session.commit()
Пример #11
0
 def check_uid_changes(self, crispin_client, download_stack,
                       async_download):
     remote_uids = set(crispin_client.all_uids())
     with self.syncmanager_lock:
         with mailsync_session_scope() as db_session:
             local_uids = common.all_uids(self.account_id, db_session,
                                          self.folder_name)
             # Download new UIDs.
             stack_uids = {uid for uid, _ in download_stack}
             local_with_pending_uids = local_uids | stack_uids
             # filter out messages that have disappeared on the remote side
             download_stack.discard([
                 item for item in download_stack
                 if item[0] not in remote_uids
             ])
             for uid in sorted(remote_uids):
                 if uid not in local_with_pending_uids:
                     download_stack.put(uid, None)
             self.remove_deleted_uids(db_session, local_uids, remote_uids)
     if not async_download:
         self.download_uids(crispin_client, download_stack)
         with mailsync_session_scope() as db_session:
             self.update_uid_counts(
                 db_session,
                 remote_uid_count=len(remote_uids),
                 download_uid_count=download_stack.qsize())
     to_refresh = sorted(remote_uids & local_uids)[-self.refresh_flags_max:]
     self.update_metadata(crispin_client, to_refresh)
Пример #12
0
    def __imap_flag_change_poller(self):
        """
        Periodically update message flags for those servers
        who don't support CONDSTORE.
        Runs until killed. (Intended to be run in a greenlet)
        """
        log.info("Spinning up new flags-refresher for ",
                 folder_name=self.folder_name)
        with self.conn_pool.get() as crispin_client:
            with mailsync_session_scope() as db_session:
                crispin_client.select_folder(self.folder_name,
                                             uidvalidity_cb(
                                                 crispin_client.account_id))
            while True:
                remote_uids = set(crispin_client.all_uids())
                local_uids = common.all_uids(self.account_id, db_session,
                                             self.folder_name)
                # STOPSHIP(emfree): sorted does nothing here
                to_refresh = sorted(remote_uids &
                                    local_uids)[-self.refresh_flags_max:]

                self.update_metadata(crispin_client, to_refresh)
                with session_scope(ignore_soft_deletes=True) as db_session:
                    self.update_uid_counts(db_session,
                                           update_uid_count=len(to_refresh))

                sleep(self.poll_frequency)
Пример #13
0
    def poll_impl(self, crispin_client):
        crispin_client.select_folder(self.folder_name,
                                     uidvalidity_cb(self.account_id))

        remote_uids = set(crispin_client.all_uids())
        with mailsync_session_scope() as db_session:
            local_uids = common.all_uids(
                self.account_id, db_session, self.folder_name)
            deleted_uids = self.remove_deleted_uids(
                db_session, local_uids, remote_uids)

            local_uids -= deleted_uids
            log.info("Removed {} deleted UIDs from {}".format(
                len(deleted_uids), self.folder_name))
            uids_to_download = remote_uids - local_uids

            self.update_uid_counts(db_session,
                                   remote_uid_count=len(remote_uids),
                                   download_uid_count=len(uids_to_download),
                                   delete_uid_count=len(deleted_uids))

        log.info("UIDs to download: {}".format(uids_to_download))
        if uids_to_download:
            self.download_uids(crispin_client,
                               uid_list_to_stack(uids_to_download))

        uids_to_refresh = sorted(remote_uids -
                                 uids_to_download)[-self.refresh_flags_max:]
        log.info('UIDs to refresh: ', uids=uids_to_refresh)
        if uids_to_refresh:
            self.update_metadata(crispin_client, uids_to_refresh)

        sleep(self.poll_frequency)
Пример #14
0
    def initial_sync_impl(self, crispin_client):
        # We wrap the block in a try/finally because the greenlets like
        # change_poller need to be killed when this greenlet is interrupted
        change_poller = None
        try:
            with mailsync_session_scope() as db_session:
                local_uids = common.all_uids(self.account_id, db_session,
                                             self.folder_name)
            remote_uid_count = len(set(crispin_client.all_uids()))
            remote_g_metadata = self.__fetch_g_metadata(
                crispin_client, local_uids)
            remote_uids = sorted(remote_g_metadata.keys(), key=int)
            with self.syncmanager_lock:
                with mailsync_session_scope() as db_session:
                    deleted_uids = self.remove_deleted_uids(
                        db_session, local_uids, remote_uids)

                    local_uids = set(local_uids) - deleted_uids
                    unknown_uids = set(remote_uids) - local_uids
                    self.update_uid_counts(
                        db_session, remote_uid_count=remote_uid_count,
                        download_uid_count=len(unknown_uids))

            download_stack = UIDStack()
            if self.folder_name == crispin_client.folder_names()['inbox']:
                # We don't do an initial dedupe for Inbox because we do thread
                # expansion, which means even if we have a given msgid
                # downloaded, we miiight not have the whole thread. This means
                # that restarts cause duplicate work, but hopefully these
                # folders aren't too huge.
                flags = crispin_client.flags(unknown_uids)
                for uid in sorted(unknown_uids):
                    if uid in flags:
                        download_stack.put(
                            uid,
                            GMessage(uid, remote_g_metadata[uid],
                                     flags[uid].flags, flags[uid].labels,
                                     throttled=self.throttled))
                change_poller = spawn(self.poll_for_changes, download_stack)
                self.__download_queued_threads(crispin_client,
                                               download_stack)
            elif self.folder_name in uid_download_folders(crispin_client):
                full_download = self.__deduplicate_message_download(
                    crispin_client, remote_g_metadata, unknown_uids)
                for uid in sorted(full_download):
                    download_stack.put(uid, None)
                change_poller = spawn(self.poll_for_changes, download_stack)
                self.download_uids(crispin_client, download_stack)
            else:
                raise MailsyncError(
                    'Unknown Gmail sync folder: {}'.format(self.folder_name))

            # Complete X-GM-MSGID mapping is no longer needed after initial
            # sync.
            rm_cache(remote_g_metadata_cache_file(self.account_id,
                                                  self.folder_name))
        finally:
            if change_poller is not None:
                change_poller.kill()
Пример #15
0
    def check_new_uids(self, uid_download_stack):
        """ Check for new UIDs and add them to the download stack.

        We do this by comparing local UID lists to remote UID lists,
        maintaining the invariant that (stack uids)+(local uids) == (remote
        uids).

        We also remove local messages that have disappeared from the remote,
        since it's totally probable that users will be archiving mail as the
        initial sync goes on.

        We grab a new IMAP connection from the pool for this to isolate its
        actions from whatever the main greenlet may be doing.

        Runs until killed. (Intended to be run in a greenlet.)
        """
        log.info("starting new UID-check poller")
        with self.conn_pool.get() as crispin_client:
            crispin_client.select_folder(
                self.folder_name, uidvalidity_cb(crispin_client.account_id))
            while True:
                remote_uids = set(crispin_client.all_uids())
                # We lock this section to make sure no messages are being
                # created while we make sure the queue is in a good state.
                with self.syncmanager_lock:
                    with mailsync_session_scope() as db_session:
                        local_uids = common.all_uids(self.account_id,
                                                     db_session,
                                                     self.folder_name)
                        stack_uids = set(uid_download_stack.queue)
                        local_with_pending_uids = local_uids | stack_uids
                        deleted_uids = self.remove_deleted_uids(
                            db_session, local_uids, remote_uids)
                        log.info('remoted deleted uids',
                                 count=len(deleted_uids))

                        # filter out messages that have disappeared on the
                        # remote side
                        new_uid_download_stack = {
                            u
                            for u in uid_download_stack.queue
                            if u in remote_uids
                        }

                        # add in any new uids from the remote
                        for uid in remote_uids:
                            if uid not in local_with_pending_uids:
                                new_uid_download_stack.add(uid)
                        uid_download_stack.queue = sorted(
                            new_uid_download_stack, key=int)

                        self.update_uid_counts(
                            db_session,
                            remote_uid_count=len(remote_uids),
                            download_uid_count=uid_download_stack.qsize(),
                            delete_uid_count=len(deleted_uids))
                sleep(self.poll_frequency)
Пример #16
0
    def check_new_uids(self, uid_download_stack):
        """ Check for new UIDs and add them to the download stack.

        We do this by comparing local UID lists to remote UID lists,
        maintaining the invariant that (stack uids)+(local uids) == (remote
        uids).

        We also remove local messages that have disappeared from the remote,
        since it's totally probable that users will be archiving mail as the
        initial sync goes on.

        We grab a new IMAP connection from the pool for this to isolate its
        actions from whatever the main greenlet may be doing.

        Runs until killed. (Intended to be run in a greenlet.)
        """
        log.info("starting new UID-check poller")
        with self.conn_pool.get() as crispin_client:
            crispin_client.select_folder(
                self.folder_name, uidvalidity_cb(crispin_client.account_id))
            while True:
                remote_uids = set(crispin_client.all_uids())
                # We lock this section to make sure no messages are being
                # created while we make sure the queue is in a good state.
                with self.syncmanager_lock:
                    with mailsync_session_scope() as db_session:
                        local_uids = common.all_uids(self.account_id,
                                                     db_session,
                                                     self.folder_name)
                        stack_uids = set(uid_download_stack.queue)
                        local_with_pending_uids = local_uids | stack_uids
                        deleted_uids = self.remove_deleted_uids(
                            db_session, local_uids, remote_uids)
                        log.info('remoted deleted uids',
                                 count=len(deleted_uids))

                        # filter out messages that have disappeared on the
                        # remote side
                        new_uid_download_stack = {
                            u for u in uid_download_stack.queue if u in
                            remote_uids}

                        # add in any new uids from the remote
                        for uid in remote_uids:
                            if uid not in local_with_pending_uids:
                                new_uid_download_stack.add(uid)
                        uid_download_stack.queue = sorted(
                            new_uid_download_stack, key=int)

                        self.update_uid_counts(
                            db_session,
                            remote_uid_count=len(remote_uids),
                            download_uid_count=uid_download_stack.qsize(),
                            delete_uid_count=len(deleted_uids))
                sleep(self.poll_frequency)
Пример #17
0
    def initial_sync_impl(self, crispin_client):
        # We wrap the block in a try/finally because the greenlets like
        # change_poller need to be killed when this greenlet is interrupted
        change_poller = None
        try:
            with mailsync_session_scope() as db_session:
                local_uids = common.all_uids(self.account_id, db_session,
                                             self.folder_id)
            remote_uids = sorted(crispin_client.all_uids(), key=int)
            remote_uid_count = len(remote_uids)
            with self.syncmanager_lock:
                with mailsync_session_scope() as db_session:
                    self.remove_deleted_uids(db_session, local_uids,
                                             remote_uids)
                    unknown_uids = set(remote_uids) - local_uids
                    self.update_uid_counts(
                        db_session,
                        remote_uid_count=remote_uid_count,
                        download_uid_count=len(unknown_uids))

            remote_g_metadata = crispin_client.g_metadata(unknown_uids)
            download_stack = UIDStack()
            change_poller = spawn(self.poll_for_changes, download_stack)
            bind_context(change_poller, 'changepoller', self.account_id,
                         self.folder_id)
            if self.is_all_mail(crispin_client):
                # Put UIDs on the stack such that UIDs for messages in the
                # inbox get downloaded first, and such that higher (i.e., more
                # recent) UIDs get downloaded before lower ones.
                inbox_uids = crispin_client.search_uids(['X-GM-LABELS inbox'])
                inbox_uid_set = set(inbox_uids)
                # Note that we have to be checking membership in a /set/ for
                # performance.
                ordered_uids_to_sync = [
                    u for u in sorted(remote_uids) if u not in inbox_uid_set
                ] + sorted(inbox_uids)
                for uid in ordered_uids_to_sync:
                    if uid in remote_g_metadata:
                        metadata = GMetadata(remote_g_metadata[uid].msgid,
                                             remote_g_metadata[uid].thrid,
                                             self.throttled)
                        download_stack.put(uid, metadata)
                self.__download_queued_threads(crispin_client, download_stack)
            else:
                full_download = self.__deduplicate_message_download(
                    crispin_client, remote_g_metadata, unknown_uids)
                for uid in sorted(full_download):
                    download_stack.put(uid, None)
                self.download_uids(crispin_client, download_stack)
        finally:
            if change_poller is not None:
                # schedule change_poller to die
                kill(change_poller)
Пример #18
0
    def initial_sync(self):
        with self.conn_pool.get() as crispin_client:
            uid_download_stack = LifoQueue()
            crispin_client.select_folder(
                self.folder_name, uidvalidity_cb(crispin_client.account_id))

            with mailsync_session_scope() as db_session:
                local_uids = common.all_uids(crispin_client.account_id,
                                             db_session, self.folder_name)

            self.initial_sync_impl(crispin_client, local_uids,
                                   uid_download_stack)
        return 'poll'
Пример #19
0
    def initial_sync(self):
        with self.conn_pool.get() as crispin_client:
            uid_download_stack = LifoQueue()
            crispin_client.select_folder(
                self.folder_name, uidvalidity_cb(crispin_client.account_id))

            with mailsync_session_scope() as db_session:
                local_uids = common.all_uids(crispin_client.account_id,
                                             db_session, self.folder_name)

            self.initial_sync_impl(crispin_client, local_uids,
                                   uid_download_stack)
        return 'poll'
Пример #20
0
    def initial_sync_impl(self, crispin_client):
        # We wrap the block in a try/finally because the greenlets like
        # change_poller need to be killed when this greenlet is interrupted
        change_poller = None
        try:
            with mailsync_session_scope() as db_session:
                local_uids = common.all_uids(self.account_id, db_session,
                                             self.folder_id)
            remote_uids = sorted(crispin_client.all_uids(), key=int)
            remote_uid_count = len(remote_uids)
            with self.syncmanager_lock:
                with mailsync_session_scope() as db_session:
                    self.remove_deleted_uids(db_session, local_uids,
                                             remote_uids)
                    unknown_uids = set(remote_uids) - local_uids
                    self.update_uid_counts(
                        db_session, remote_uid_count=remote_uid_count,
                        download_uid_count=len(unknown_uids))

            remote_g_metadata = crispin_client.g_metadata(unknown_uids)
            download_stack = UIDStack()
            change_poller = spawn(self.poll_for_changes, download_stack)
            bind_context(change_poller, 'changepoller', self.account_id,
                         self.folder_id)
            if self.is_all_mail(crispin_client):
                # Put UIDs on the stack such that UIDs for messages in the
                # inbox get downloaded first, and such that higher (i.e., more
                # recent) UIDs get downloaded before lower ones.
                inbox_uids = crispin_client.search_uids(['X-GM-LABELS inbox'])
                inbox_uid_set = set(inbox_uids)
                # Note that we have to be checking membership in a /set/ for
                # performance.
                ordered_uids_to_sync = [u for u in sorted(remote_uids) if u not
                                        in inbox_uid_set] + sorted(inbox_uids)
                for uid in ordered_uids_to_sync:
                    if uid in remote_g_metadata:
                        metadata = GMetadata(remote_g_metadata[uid].msgid,
                                             remote_g_metadata[uid].thrid,
                                             self.throttled)
                        download_stack.put(uid, metadata)
                self.__download_queued_threads(crispin_client, download_stack)
            else:
                full_download = self.__deduplicate_message_download(
                    crispin_client, remote_g_metadata, unknown_uids)
                for uid in sorted(full_download):
                    download_stack.put(uid, None)
                self.download_uids(crispin_client, download_stack)
        finally:
            if change_poller is not None:
                # schedule change_poller to die
                kill(change_poller)
Пример #21
0
Файл: gmail.py Проект: wmv/inbox
    def initial_sync_impl(self, crispin_client):
        # We wrap the block in a try/finally because the greenlets like
        # change_poller need to be killed when this greenlet is interrupted
        change_poller = None
        try:
            with mailsync_session_scope() as db_session:
                local_uids = common.all_uids(self.account_id, db_session,
                                             self.folder_name)
            remote_uids = sorted(crispin_client.all_uids(), key=int)
            remote_uid_count = len(remote_uids)
            with self.syncmanager_lock:
                with mailsync_session_scope() as db_session:
                    deleted_uids = self.remove_deleted_uids(
                        db_session, local_uids, remote_uids)

                    local_uids = set(local_uids) - deleted_uids
                    unknown_uids = set(remote_uids) - local_uids
                    self.update_uid_counts(
                        db_session,
                        remote_uid_count=remote_uid_count,
                        download_uid_count=len(unknown_uids))

            remote_g_metadata = crispin_client.g_metadata(unknown_uids)
            download_stack = UIDStack()
            change_poller = spawn(self.poll_for_changes, download_stack)
            if self.folder_name in uid_download_folders(crispin_client):
                full_download = self.__deduplicate_message_download(
                    crispin_client, remote_g_metadata, unknown_uids)
                for uid in sorted(full_download):
                    download_stack.put(uid, None)
                self.download_uids(crispin_client, download_stack)
            elif self.folder_name in thread_expand_folders(crispin_client):
                flags = crispin_client.flags(unknown_uids)
                for uid in sorted(unknown_uids):
                    if uid in flags:
                        gmessage = GMessage(uid,
                                            remote_g_metadata[uid],
                                            flags[uid].flags,
                                            flags[uid].labels,
                                            throttled=self.throttled)
                        download_stack.put(uid, gmessage)
                # We always download threads via the 'All Mail' folder.
                crispin_client.select_folder(
                    crispin_client.folder_names()['all'], uidvalidity_cb)
                self.__download_queued_threads(crispin_client, download_stack)
            else:
                raise MailsyncError('Unknown Gmail sync folder: {}'.format(
                    self.folder_name))
        finally:
            if change_poller is not None:
                change_poller.kill()
Пример #22
0
    def initial_sync_impl(self, crispin_client):
        # We wrap the block in a try/finally because the greenlets like
        # change_poller need to be killed when this greenlet is interrupted
        change_poller = None
        try:
            with mailsync_session_scope() as db_session:
                local_uids = common.all_uids(self.account_id, db_session,
                                             self.folder_name)
            remote_uids = sorted(crispin_client.all_uids(), key=int)
            remote_uid_count = len(remote_uids)
            with self.syncmanager_lock:
                with mailsync_session_scope() as db_session:
                    deleted_uids = self.remove_deleted_uids(
                        db_session, local_uids, remote_uids)

                    local_uids = set(local_uids) - deleted_uids
                    unknown_uids = set(remote_uids) - local_uids
                    self.update_uid_counts(
                        db_session, remote_uid_count=remote_uid_count,
                        download_uid_count=len(unknown_uids))

            remote_g_metadata = crispin_client.g_metadata(unknown_uids)
            download_stack = UIDStack()
            change_poller = spawn(self.poll_for_changes, download_stack)
            if self.folder_name in uid_download_folders(crispin_client):
                full_download = self.__deduplicate_message_download(
                    crispin_client, remote_g_metadata, unknown_uids)
                for uid in sorted(full_download):
                    download_stack.put(uid, None)
                self.download_uids(crispin_client, download_stack)
            elif self.folder_name in thread_expand_folders(crispin_client):
                flags = crispin_client.flags(unknown_uids)
                for uid in sorted(unknown_uids):
                    if uid in flags:
                        gmessage = GMessage(uid, remote_g_metadata[uid],
                                            flags[uid].flags,
                                            flags[uid].labels,
                                            throttled=self.throttled)
                        download_stack.put(uid, gmessage)
                # We always download threads via the 'All Mail' folder.
                crispin_client.select_folder(
                    crispin_client.folder_names()['all'], uidvalidity_cb)
                self.__download_queued_threads(crispin_client, download_stack)
            else:
                raise MailsyncError(
                    'Unknown Gmail sync folder: {}'.format(self.folder_name))
        finally:
            if change_poller is not None:
                change_poller.kill()
Пример #23
0
 def check_uid_changes(self, crispin_client, download_stack,
                       async_download):
     remote_uids = set(crispin_client.all_uids())
     with self.syncmanager_lock:
         with mailsync_session_scope() as db_session:
             local_uids = common.all_uids(self.account_id, db_session,
                                          self.folder_id)
             # Download new UIDs.
             stack_uids = set(download_stack.keys())
             local_with_pending_uids = local_uids | stack_uids
             for uid in sorted(remote_uids):
                 if uid not in local_with_pending_uids:
                     download_stack.put(uid, None)
             self.remove_deleted_uids(db_session, local_uids, remote_uids)
     if not async_download:
         self.download_uids(crispin_client, download_stack)
         with mailsync_session_scope() as db_session:
             self.update_uid_counts(db_session,
                                    remote_uid_count=len(remote_uids),
                                    download_uid_count=len(download_stack))
     to_refresh = sorted(remote_uids & local_uids)[-self.refresh_flags_max:]
     self.update_metadata(crispin_client, to_refresh)
Пример #24
0
    def __check_new_g_thrids(self, message_download_stack):
        """
        Check for new X-GM-THRIDs and add them to the download stack.

        We do this by comparing local UID lists to remote UID lists,
        maintaining the invariant that (stack uids)+(local uids) == (remote
        uids).

        We also remove local messages that have disappeared from the remote,
        since it's totally probable that users will be archiving mail as the
        initial sync goes on.

        We grab a new IMAP connection from the pool for this to isolate its
        actions from whatever the main greenlet may be doing.

        Runs until killed. (Intended to be run in a greenlet.)

        """
        with connection_pool(self.account_id).get() as crispin_client:
            crispin_client.select_folder(self.folder_name,
                                         uidvalidity_cb(self.account_id))
            while True:
                log.info('Checking for new/deleted messages during initial '
                         'sync.')
                remote_uids = set(crispin_client.all_uids())
                # We lock this section to make sure no messages are being
                # modified in the database while we make sure the queue is in a
                # good state.
                with self.syncmanager_lock:
                    with mailsync_session_scope() as db_session:
                        local_uids = common.all_uids(self.account_id,
                                                     db_session,
                                                     self.folder_name)
                        stack_uids = {gm.uid for gm in
                                      message_download_stack.queue}
                        local_with_pending_uids = local_uids | stack_uids
                        deleted_uids = self.remove_deleted_uids(
                            db_session, local_uids, remote_uids)
                        log.info(deleted_uid_count=len(deleted_uids))

                    # filter out messages that have disappeared on the remote
                    # side
                    new_message_download_stack = [gm for gm in
                                                  message_download_stack.queue
                                                  if gm.uid in remote_uids]

                    # add in any new uids from the remote
                    new_uids = [uid for uid in remote_uids if uid not in
                                local_with_pending_uids]
                    flags = crispin_client.flags(new_uids)
                    g_metadata = crispin_client.g_metadata(new_uids)
                    log.info('adding new messages to download queue',
                             count=min(len(flags), len(g_metadata)))
                    for new_uid in new_uids:
                        # could have disappeared from the folder in the
                        # meantime
                        if new_uid in flags and new_uid in g_metadata:
                            new_message_download_stack.append(
                                GMessage(new_uid, g_metadata[new_uid],
                                         flags[new_uid].flags,
                                         flags[new_uid].labels))
                    message_download_stack.queue = sorted(
                        new_message_download_stack, key=lambda m: m.uid)

                    with mailsync_session_scope() as db_session:
                        self.update_uid_counts(
                            db_session,
                            remote_uid_count=len(remote_uids),
                            download_uid_count=message_download_stack.qsize(),
                            delete_uid_count=len(deleted_uids))

                log.info('idling', timeout=self.poll_frequency)
                crispin_client.conn.idle()
                crispin_client.conn.idle_check(timeout=self.poll_frequency)
                crispin_client.conn.idle_done()
                log.info('IDLE detected changes or timeout reached')
Пример #25
0
    def __check_new_g_thrids(self, message_download_stack):
        """
        Check for new X-GM-THRIDs and add them to the download stack.

        We do this by comparing local UID lists to remote UID lists,
        maintaining the invariant that (stack uids)+(local uids) == (remote
        uids).

        We also remove local messages that have disappeared from the remote,
        since it's totally probable that users will be archiving mail as the
        initial sync goes on.

        We grab a new IMAP connection from the pool for this to isolate its
        actions from whatever the main greenlet may be doing.

        Runs until killed. (Intended to be run in a greenlet.)

        """
        with connection_pool(self.account_id).get() as crispin_client:
            crispin_client.select_folder(self.folder_name,
                                         uidvalidity_cb(self.account_id))
            while True:
                log.info('Checking for new/deleted messages during initial '
                         'sync.')
                remote_uids = set(crispin_client.all_uids())
                # We lock this section to make sure no messages are being
                # modified in the database while we make sure the queue is in a
                # good state.
                with self.syncmanager_lock:
                    with mailsync_session_scope() as db_session:
                        local_uids = common.all_uids(self.account_id,
                                                     db_session,
                                                     self.folder_name)
                        stack_uids = {
                            gm.uid
                            for gm in message_download_stack.queue
                        }
                        local_with_pending_uids = local_uids | stack_uids
                        deleted_uids = self.remove_deleted_uids(
                            db_session, local_uids, remote_uids)
                        log.info(deleted_uid_count=len(deleted_uids))

                    # filter out messages that have disappeared on the remote
                    # side
                    new_message_download_stack = [
                        gm for gm in message_download_stack.queue
                        if gm.uid in remote_uids
                    ]

                    # add in any new uids from the remote
                    new_uids = [
                        uid for uid in remote_uids
                        if uid not in local_with_pending_uids
                    ]
                    flags = crispin_client.flags(new_uids)
                    g_metadata = crispin_client.g_metadata(new_uids)
                    log.info('adding new messages to download queue',
                             count=min(len(flags), len(g_metadata)))
                    for new_uid in new_uids:
                        # could have disappeared from the folder in the
                        # meantime
                        if new_uid in flags and new_uid in g_metadata:
                            new_message_download_stack.append(
                                GMessage(new_uid, g_metadata[new_uid],
                                         flags[new_uid].flags,
                                         flags[new_uid].labels))
                    message_download_stack.queue = sorted(
                        new_message_download_stack, key=lambda m: m.uid)

                    with mailsync_session_scope() as db_session:
                        self.update_uid_counts(
                            db_session,
                            remote_uid_count=len(remote_uids),
                            download_uid_count=message_download_stack.qsize(),
                            delete_uid_count=len(deleted_uids))

                log.info('idling', timeout=self.poll_frequency)
                crispin_client.conn.idle()
                crispin_client.conn.idle_check(timeout=self.poll_frequency)
                crispin_client.conn.idle_done()
                log.info('IDLE detected changes or timeout reached')