Ejemplo n.º 1
0
    def savemessage(self, uid, content, flags, rtime):
        """Save the message on the Server

        This backend always assigns a new uid, so the uid arg is ignored.

        This function will update the self.messagelist dict to contain
        the new message after sucessfully saving it.

        :param rtime: A timestamp to be used as the mail date
        :returns: the UID of the new message as assigned by the server. If the
                  message is saved, but it's UID can not be found, it will
                  return 0. If the message can't be written (folder is
                  read-only for example) it will return -1."""
        self.ui.debug('imap', 'savemessage: called')

        # already have it, just save modified flags
        if uid > 0 and self.uidexists(uid):
            self.savemessageflags(uid, flags)
            return uid

        retry_left = 2  # succeeded in APPENDING?
        imapobj = self.imapserver.acquireconnection()
        try:
            while retry_left:
                # UIDPLUS extension provides us with an APPENDUID response.
                use_uidplus = 'UIDPLUS' in imapobj.capabilities

                # get the date of the message, so we can pass it to the server.
                date = self.getmessageinternaldate(content, rtime)
                content = re.sub("(?<!\r)\n", "\r\n", content)

                if not use_uidplus:
                    # insert a random unique header that we can fetch later
                    (headername,
                     headervalue) = self.generate_randomheader(content)
                    self.ui.debug('imap', 'savemessage: header is: %s: %s' %\
                                      (headername, headervalue))
                    content = self.savemessage_addheader(
                        content, headername, headervalue)
                if len(content) > 200:
                    dbg_output = "%s...%s" % (content[:150], content[-50:])
                else:
                    dbg_output = content
                self.ui.debug(
                    'imap', "savemessage: date: %s, content: '%s'" %
                    (date, dbg_output))

                try:
                    # Select folder for append and make the box READ-WRITE
                    imapobj.select(self.getfullname())
                except imapobj.readonly:
                    # readonly exception. Return original uid to notify that
                    # we did not save the message. (see savemessage in Base.py)
                    self.ui.msgtoreadonly(self, uid, content, flags)
                    return uid

                #Do the APPEND
                try:
                    (typ,
                     dat) = imapobj.append(self.getfullname(),
                                           imaputil.flagsmaildir2imap(flags),
                                           date, content)
                    retry_left = 0  # Mark as success
                except imapobj.abort, e:
                    # connection has been reset, release connection and retry.
                    retry_left -= 1
                    self.imapserver.releaseconnection(imapobj, True)
                    imapobj = self.imapserver.acquireconnection()
                    if not retry_left:
                        raise OfflineImapError(
                            "Saving msg in folder '%s', "
                            "repository '%s' failed (abort). Server reponded: %s\n"
                            "Message content was: %s" %
                            (self, self.getrepository(), str(e), dbg_output),
                            OfflineImapError.ERROR.MESSAGE)
                    self.ui.error(e, exc_info()[2])
                except imapobj.error, e:  # APPEND failed
                    # If the server responds with 'BAD', append()
                    # raise()s directly.  So we catch that too.
                    # drop conn, it might be bad.
                    self.imapserver.releaseconnection(imapobj, True)
                    imapobj = None
                    raise OfflineImapError(
                        "Saving msg folder '%s', repo '%s'"
                        "failed (error). Server reponded: %s\nMessage content was: "
                        "%s" %
                        (self, self.getrepository(), str(e), dbg_output),
                        OfflineImapError.ERROR.MESSAGE)
Ejemplo n.º 2
0
    def getfolders(self):
        """Return a list of instances of OfflineIMAP representative folder."""

        if self.folders is not None:
            return self.folders
        retval = []
        imapobj = self.imapserver.acquireconnection()
        # check whether to list all folders, or subscribed only
        listfunction = imapobj.list
        if self.getconfboolean('subscribedonly', False):
            listfunction = imapobj.lsub

        try:
            result, listresult = \
                listfunction(directory=self.imapserver.reference, pattern='"*"')
            if result != 'OK':
                raise OfflineImapError(
                    "Could not list the folders for"
                    " repository %s. Server responded: %s" %
                    (self.name, str(listresult)),
                    OfflineImapError.ERROR.FOLDER)
        finally:
            self.imapserver.releaseconnection(imapobj)

        for fldr in listresult:
            if fldr is None or (isinstance(fldr, str) and fldr == ''):
                # Bug in imaplib: empty strings in results from
                # literals. TODO: still relevant?
                continue
            try:
                flags, delim, name = imaputil.imapsplit(fldr)
            except ValueError:
                self.ui.error(
                    "could not correctly parse server response; got: %s" %
                    fldr)
                raise
            flaglist = [x.lower() for x in imaputil.flagsplit(flags)]
            if '\\noselect' in flaglist:
                continue
            retval.append(self.getfoldertype()(self.imapserver, name, self))
        # Add all folderincludes
        if len(self.folderincludes):
            imapobj = self.imapserver.acquireconnection()
            try:
                for foldername in self.folderincludes:
                    try:
                        imapobj.select(imaputil.utf8_IMAP(foldername),
                                       readonly=True)
                    except OfflineImapError as exc:
                        # couldn't select this folderinclude, so ignore folder.
                        if exc.severity > OfflineImapError.ERROR.FOLDER:
                            raise
                        self.ui.error(exc,
                                      exc_info()[2], 'Invalid folderinclude:')
                        continue
                    retval.append(self.getfoldertype()(self.imapserver,
                                                       foldername,
                                                       self,
                                                       decode=False))
            finally:
                self.imapserver.releaseconnection(imapobj)

        if self.foldersort is None:
            # default sorting by case insensitive transposed name
            retval.sort(key=lambda x: str.lower(x.getvisiblename()))
        else:
            # do foldersort in a python3-compatible way
            # http://bytes.com/topic/python/answers/ \
            # 844614-python-3-sorting-comparison-function
            def cmp2key(mycmp):
                """Converts a cmp= function into a key= function
                We need to keep cmp functions for backward compatibility"""
                class K:
                    """
                    Class to compare getvisiblename() between two objects.
                    """
                    def __init__(self, obj, *args):
                        self.obj = obj

                    def __cmp__(self, other):
                        return mycmp(self.obj.getvisiblename(),
                                     other.obj.getvisiblename())

                    def __lt__(self, other):
                        return self.__cmp__(other) < 0

                    def __le__(self, other):
                        return self.__cmp__(other) <= 0

                    def __gt__(self, other):
                        return self.__cmp__(other) > 0

                    def __ge__(self, other):
                        return self.__cmp__(other) >= 0

                    def __eq__(self, other):
                        return self.__cmp__(other) == 0

                    def __ne__(self, other):
                        return self.__cmp__(other) != 0

                return K

            retval.sort(key=cmp2key(self.foldersort))

        self.folders = retval
        return self.folders
Ejemplo n.º 3
0
    def acquireconnection(self):
        """Fetches a connection from the pool, making sure to create a new one
        if needed, to obey the maximum connection limits, etc.
        Opens a connection to the server and returns an appropriate
        object."""

        self.semaphore.acquire()
        self.connectionlock.acquire()
        curThread = currentThread()
        imapobj = None

        if len(self.availableconnections):  # One is available.
            # Try to find one that previously belonged to this thread
            # as an optimization.  Start from the back since that's where
            # they're popped on.
            imapobj = None
            for i in range(len(self.availableconnections) - 1, -1, -1):
                tryobj = self.availableconnections[i]
                if self.lastowner[tryobj] == curThread.ident:
                    imapobj = tryobj
                    del (self.availableconnections[i])
                    break
            if not imapobj:
                imapobj = self.availableconnections[0]
                del (self.availableconnections[0])
            self.assignedconnections.append(imapobj)
            self.lastowner[imapobj] = curThread.ident
            self.connectionlock.release()
            return imapobj

        self.connectionlock.release()  # Release until need to modify data

        # Must be careful here that if we fail we should bail out gracefully
        # and release locks / threads so that the next attempt can try...
        success = 0
        try:
            while not success:
                # Generate a new connection.
                if self.tunnel:
                    self.ui.connecting('tunnel', self.tunnel)
                    imapobj = imaplibutil.IMAP4_Tunnel(
                        self.tunnel,
                        timeout=socket.getdefaulttimeout(),
                        use_socket=self.proxied_socket,
                    )
                    success = 1
                elif self.usessl:
                    self.ui.connecting(self.hostname, self.port)
                    imapobj = imaplibutil.WrappedIMAP4_SSL(
                        self.hostname,
                        self.port,
                        self.sslclientkey,
                        self.sslclientcert,
                        self.sslcacertfile,
                        self.__verifycert,
                        self.sslversion,
                        timeout=socket.getdefaulttimeout(),
                        fingerprint=self.fingerprint,
                        use_socket=self.proxied_socket,
                        tls_level=self.tlslevel,
                    )
                else:
                    self.ui.connecting(self.hostname, self.port)
                    imapobj = imaplibutil.WrappedIMAP4(
                        self.hostname,
                        self.port,
                        timeout=socket.getdefaulttimeout(),
                        use_socket=self.proxied_socket,
                    )

                if not self.preauth_tunnel:
                    try:
                        self.__authn_helper(imapobj)
                        self.goodpassword = self.password
                        success = 1
                    except OfflineImapError as e:
                        self.passworderror = str(e)
                        raise

            # Enable compression
            if self.repos.getconfboolean('usecompression', 0):
                imapobj.enable_compression()

            # update capabilities after login, e.g. gmail serves different ones
            typ, dat = imapobj.capability()
            if dat != [None]:
                imapobj.capabilities = tuple(dat[-1].upper().split())

            if self.delim == None:
                listres = imapobj.list(self.reference, '""')[1]
                if listres == [None] or listres == None:
                    # Some buggy IMAP servers do not respond well to LIST "" ""
                    # Work around them.
                    listres = imapobj.list(self.reference, '"*"')[1]
                if listres == [None] or listres == None:
                    # No Folders were returned. This occurs, e.g. if the
                    # 'reference' prefix does not exist on the mail
                    # server. Raise exception.
                    err = "Server '%s' returned no folders in '%s'"% \
                        (self.repos.getname(), self.reference)
                    self.ui.warn(err)
                    raise Exception(err)
                self.delim, self.root = \
                     imaputil.imapsplit(listres[0])[1:]
                self.delim = imaputil.dequote(self.delim)
                self.root = imaputil.dequote(self.root)

            with self.connectionlock:
                self.assignedconnections.append(imapobj)
                self.lastowner[imapobj] = curThread.ident
            return imapobj
        except Exception as e:
            """If we are here then we did not succeed in getting a
            connection - we should clean up and then re-raise the
            error..."""

            self.semaphore.release()

            severity = OfflineImapError.ERROR.REPO
            if type(e) == gaierror:
                #DNS related errors. Abort Repo sync
                #TODO: special error msg for e.errno == 2 "Name or service not known"?
                reason = "Could not resolve name '%s' for repository "\
                         "'%s'. Make sure you have configured the ser"\
                         "ver name correctly and that you are online."%\
                         (self.hostname, self.repos)
                raise OfflineImapError(reason, severity), None, exc_info()[2]

            elif isinstance(e, SSLError) and e.errno == errno.EPERM:
                # SSL unknown protocol error
                # happens e.g. when connecting via SSL to a non-SSL service
                if self.port != 993:
                    reason = "Could not connect via SSL to host '%s' and non-s"\
                        "tandard ssl port %d configured. Make sure you connect"\
                        " to the correct port."% (self.hostname, self.port)
                else:
                    reason = "Unknown SSL protocol connecting to host '%s' for "\
                         "repository '%s'. OpenSSL responded:\n%s"\
                         % (self.hostname, self.repos, e)
                raise OfflineImapError(reason, severity), None, exc_info()[2]

            elif isinstance(e,
                            socket.error) and e.args[0] == errno.ECONNREFUSED:
                # "Connection refused", can be a non-existing port, or an unauthorized
                # webproxy (open WLAN?)
                reason = "Connection to host '%s:%d' for repository '%s' was "\
                    "refused. Make sure you have the right host and port "\
                    "configured and that you are actually able to access the "\
                    "network."% (self.hostname, self.port, self.repos)
                raise OfflineImapError(reason, severity), None, exc_info()[2]
            # Could not acquire connection to the remote;
            # socket.error(last_error) raised
            if str(e)[:24] == "can't open socket; error":
                raise OfflineImapError("Could not connect to remote server '%s' "\
                    "for repository '%s'. Remote does not answer."
                    % (self.hostname, self.repos),
                    OfflineImapError.ERROR.REPO), None, exc_info()[2]
            else:
                # re-raise all other errors
                raise
Ejemplo n.º 4
0
    def __xoauth2handler(self, response):
        now = datetime.datetime.now()
        if self.oauth2_access_token_expires_at \
                and self.oauth2_access_token_expires_at < now:
            self.oauth2_access_token = None
            self.ui.debug('imap',
                          'xoauth2handler: oauth2_access_token expired')

        if self.oauth2_access_token is None:
            if self.oauth2_request_url is None:
                raise OfflineImapError(
                    "No remote oauth2_request_url for "
                    "repository '%s' specified." % self,
                    OfflineImapError.ERROR.REPO)

            # Generate new access token.
            params = {}
            params['client_id'] = self.oauth2_client_id
            params['client_secret'] = self.oauth2_client_secret
            params['refresh_token'] = self.oauth2_refresh_token
            params['grant_type'] = 'refresh_token'

            self.ui.debug('imap',
                          'xoauth2handler: url "%s"' % self.oauth2_request_url)
            self.ui.debug('imap', 'xoauth2handler: params "%s"' % params)

            original_socket = socket.socket
            socket.socket = self.authproxied_socket
            try:
                response = urllib.request.urlopen(
                    self.oauth2_request_url,
                    urllib.parse.urlencode(params).encode('utf-8')).read()
            except Exception as e:
                try:
                    msg = "%s (configuration is: %s)" % (e, str(params))
                except Exception as eparams:
                    msg = "%s [cannot display configuration: %s]" % (e,
                                                                     eparams)

                self.ui.error(msg)
                raise
            finally:
                socket.socket = original_socket

            resp = json.loads(response)
            self.ui.debug('imap', 'xoauth2handler: response "%s"' % resp)
            if 'error' in resp:
                raise OfflineImapError("xoauth2handler got: %s" % resp,
                                       OfflineImapError.ERROR.REPO)
            self.oauth2_access_token = resp['access_token']
            if 'expires_in' in resp:
                self.oauth2_access_token_expires_at = now + datetime.timedelta(
                    seconds=resp['expires_in'] / 2)

        self.ui.debug(
            'imap', 'xoauth2handler: access_token "%s expires %s"' %
            (self.oauth2_access_token, self.oauth2_access_token_expires_at))
        auth_string = 'user=%s\1auth=Bearer %s\1\1' % (
            self.username, self.oauth2_access_token)
        # auth_string = base64.b64encode(auth_string)
        self.ui.debug('imap', 'xoauth2handler: returning "%s"' % auth_string)
        return auth_string
Ejemplo n.º 5
0
 def getmaxage(self):
     if self.config.getdefault("Account %s" % self.accountname, "maxage",
                               None):
         raise OfflineImapError(
             "maxage is not supported on IMAP-IMAP sync",
             OfflineImapError.ERROR.REPO), None, exc_info()[2]
Ejemplo n.º 6
0
    def cachemessagelist(self, min_date=None, min_uid=None):
        if not self.synclabels:
            return super(GmailFolder, self).cachemessagelist(min_date=min_date,
                                                             min_uid=min_uid)

        self.dropmessagelistcache()

        self.ui.collectingdata(None, self)
        imapobj = self.imapserver.acquireconnection()
        try:
            msgsToFetch = self._msgs_to_fetch(imapobj,
                                              min_date=min_date,
                                              min_uid=min_uid)
            if not msgsToFetch:
                return  # No messages to sync

            # Get the flags and UIDs for these. single-quotes prevent
            # imaplib2 from quoting the sequence.
            #
            # NB: msgsToFetch are sequential numbers, not UID's
            res_type, response = imapobj.fetch("'%s'" % msgsToFetch,
                                               '(FLAGS X-GM-LABELS UID)')
            if res_type != 'OK':
                six.reraise(
                    OfflineImapError,
                    OfflineImapError(
                        "FETCHING UIDs in folder [%s]%s failed. " %
                        (self.getrepository(), self) +
                        "Server responded '[%s] %s'" % (res_type, response),
                        OfflineImapError.ERROR.FOLDER),
                    exc_info()[2])
        finally:
            self.imapserver.releaseconnection(imapobj)

        for messagestr in response:
            # looks like: '1 (FLAGS (\\Seen Old) X-GM-LABELS (\\Inbox \\Favorites) UID 4807)' or None if no msg
            # Discard initial message number.
            if messagestr == None:
                continue
            messagestr = messagestr.split(' ', 1)[1]
            options = imaputil.flags2hash(messagestr)
            if not 'UID' in options:
                self.ui.warn('No UID in message with options %s' %\
                                          str(options),
                                          minor = 1)
            else:
                uid = int(options['UID'])
                self.messagelist[uid] = self.msglist_item_initializer(uid)
                flags = imaputil.flagsimap2maildir(options['FLAGS'])
                m = re.search('\(([^\)]*)\)', options['X-GM-LABELS'])
                if m:
                    labels = set([
                        imaputil.dequote(lb)
                        for lb in imaputil.imapsplit(m.group(1))
                    ])
                else:
                    labels = set()
                labels = labels - self.ignorelabels
                rtime = imaplibutil.Internaldate2epoch(messagestr)
                self.messagelist[uid] = {
                    'uid': uid,
                    'flags': flags,
                    'labels': labels,
                    'time': rtime
                }
Ejemplo n.º 7
0
    def cachemessagelist(self):
        maxage = self.config.getdefaultint("Account %s" % self.accountname,
                                           "maxage", -1)
        maxsize = self.config.getdefaultint("Account %s" % self.accountname,
                                            "maxsize", -1)
        self.messagelist = {}

        imapobj = self.imapserver.acquireconnection()
        try:
            res_type, imapdata = imapobj.select(self.getfullname(), True, True)
            if imapdata == [None] or imapdata[0] == '0':
                # Empty folder, no need to populate message list
                return
            # By default examine all UIDs in this folder
            msgsToFetch = '1:*'

            if (maxage != -1) | (maxsize != -1):
                search_cond = "("

                if (maxage != -1):
                    #find out what the oldest message is that we should look at
                    oldest_struct = time.gmtime(time.time() -
                                                (60 * 60 * 24 * maxage))
                    if oldest_struct[0] < 1900:
                        raise OfflineImapError(
                            "maxage setting led to year %d. "
                            "Abort syncing." % oldest_struct[0],
                            OfflineImapError.ERROR.REPO)
                    search_cond += "SINCE %02d-%s-%d" % (
                        oldest_struct[2], MonthNames[oldest_struct[1]],
                        oldest_struct[0])

                if (maxsize != -1):
                    if (maxage != -1):  # There are two conditions, add space
                        search_cond += " "
                    search_cond += "SMALLER %d" % maxsize

                search_cond += ")"

                res_type, res_data = imapobj.search(None, search_cond)
                if res_type != 'OK':
                    raise OfflineImapError(
                        "SEARCH in folder [%s]%s failed. "
                        "Search string was '%s'. Server responded '[%s] %s'" %
                        (self.getrepository(), self, search_cond, res_type,
                         res_data), OfflineImapError.ERROR.FOLDER)

                # Result UIDs are seperated by space, coalesce into ranges
                msgsToFetch = imaputil.uid_sequence(res_data[0].split())
                if not msgsToFetch:
                    return  # No messages to sync

            # Get the flags and UIDs for these. single-quotes prevent
            # imaplib2 from quoting the sequence.
            res_type, response = imapobj.fetch("'%s'" % msgsToFetch,
                                               '(FLAGS UID)')
            if res_type != 'OK':
                raise OfflineImapError(
                    "FETCHING UIDs in folder [%s]%s failed. "
                    "Server responded '[%s] %s'" %
                    (self.getrepository(), self, res_type, response),
                    OfflineImapError.ERROR.FOLDER)
        finally:
            self.imapserver.releaseconnection(imapobj)

        for messagestr in response:
            # looks like: '1 (FLAGS (\\Seen Old) UID 4807)' or None if no msg
            # Discard initial message number.
            if messagestr == None:
                continue
            messagestr = messagestr.split(' ', 1)[1]
            options = imaputil.flags2hash(messagestr)
            if not 'UID' in options:
                self.ui.warn('No UID in message with options %s' %\
                                          str(options),
                                          minor = 1)
            else:
                uid = long(options['UID'])
                flags = imaputil.flagsimap2maildir(options['FLAGS'])
                rtime = imaplibutil.Internaldate2epoch(messagestr)
                self.messagelist[uid] = {
                    'uid': uid,
                    'flags': flags,
                    'time': rtime
                }
Ejemplo n.º 8
0
    def __savemessage_fetchheaders(self, imapobj, headername, headervalue):
        """ We fetch all new mail headers and search for the right
        X-OfflineImap line by hand. The response from the server has form:
        (
          'OK',
          [
            (
              '185 (RFC822.HEADER {1789}',
              '... mail headers ...'
            ),
            ' UID 2444)',
            (
              '186 (RFC822.HEADER {1789}',
              '... 2nd mail headers ...'
            ),
            ' UID 2445)'
          ]
        )
        We need to locate the UID just after mail headers containing our
        X-OfflineIMAP line.

        Returns UID when found, 0 when not found."""

        self.ui.debug('imap', '__savemessage_fetchheaders called for %s: %s'% \
                 (headername, headervalue))

        # run "fetch X:* rfc822.header"
        # since we stored the mail we are looking for just recently, it would
        # not be optimal to fetch all messages. So we'll find highest message
        # UID in our local messagelist and search from there (exactly from
        # UID+1). That works because UIDs are guaranteed to be unique and
        # ascending.

        if self.getmessagelist():
            start = 1 + max(self.getmessagelist().keys())
        else:
            # Folder was empty - start from 1
            start = 1

        # Imaplib quotes all parameters of a string type. That must not happen
        # with the range X:*. So we use bytearray to stop imaplib from getting
        # in our way

        result = imapobj.uid('FETCH', bytearray('%d:*' % start),
                             'rfc822.header')
        if result[0] != 'OK':
            raise OfflineImapError(
                'Error fetching mail headers: %s' % '. '.join(result[1]),
                OfflineImapError.ERROR.MESSAGE)

        result = result[1]

        found = 0
        for item in result:
            if found == 0 and type(item) == type(()):
                # Walk just tuples
                if re.search("(?:^|\\r|\\n)%s:\s*%s(?:\\r|\\n)" %
                             (headername, headervalue),
                             item[1],
                             flags=re.IGNORECASE):
                    found = 1
            elif found == 1:
                if type(item) == type(""):
                    uid = re.search("UID\s+(\d+)", item, flags=re.IGNORECASE)
                    if uid:
                        return int(uid.group(1))
                    else:
                        self.ui.warn(
                            "Can't parse FETCH response, can't find UID: %s",
                            result.__repr__())
                else:
                    self.ui.warn(
                        "Can't parse FETCH response, we awaited string: %s",
                        result.__repr__())

        return 0
Ejemplo n.º 9
0
    def _fetch_from_imap(self, uids, retry_num=1):
        """Fetches data from IMAP server.

        Arguments:
        - uids: message UIDS (OfflineIMAP3: First UID returned only)
        - retry_num: number of retries to make

        Returns: data obtained by this query."""

        imapobj = self.imapserver.acquireconnection()
        try:
            query = "(%s)" % (" ".join(self.imap_query))
            fails_left = retry_num  # Retry on dropped connection.
            while fails_left:
                try:
                    imapobj.select(self.getfullIMAPname(), readonly=True)
                    res_type, data = imapobj.uid('fetch', uids, query)
                    break
                except imapobj.abort as e:
                    fails_left -= 1
                    # self.ui.error() will show the original traceback.
                    if fails_left <= 0:
                        message = ("%s, while fetching msg %r in folder %r."
                                   " Max retry reached (%d)" %
                                   (e, uids, self.name, retry_num))
                        raise OfflineImapError(message,
                                               OfflineImapError.ERROR.MESSAGE)
                    self.ui.error("%s. While fetching msg %r in folder %r."
                                  " Query: %s Retrying (%d/%d)" %
                                  (e, uids, self.name, query,
                                   retry_num - fails_left, retry_num))
                    # Release dropped connection, and get a new one.
                    self.imapserver.releaseconnection(imapobj, True)
                    imapobj = self.imapserver.acquireconnection()
        finally:
            # The imapobj here might be different than the one created before
            # the ``try`` clause. So please avoid transforming this to a nice
            # ``with`` without taking this into account.
            self.imapserver.releaseconnection(imapobj)

        # Ensure to not consider unsolicited FETCH responses caused by flag
        # changes from concurrent connections.  These appear as strings in
        # 'data' (the BODY response appears as a tuple).  This should leave
        # exactly one response.
        if res_type == 'OK':
            data = [res for res in data if not isinstance(res, bytes)]

        # Could not fetch message.  Note: it is allowed by rfc3501 to return any
        # data for the UID FETCH command.
        if data == [None] or res_type != 'OK' or len(data) != 1:
            severity = OfflineImapError.ERROR.MESSAGE
            reason = "IMAP server '%s' failed to fetch messages UID '%s'. " \
                     "Server responded: %s %s" % (self.getrepository(), uids,
                                                  res_type, data)
            if data == [None] or len(data) < 1:
                # IMAP server did not find a message with this UID.
                reason = "IMAP server '%s' does not have a message " \
                         "with UID '%s'" % (self.getrepository(), uids)
            raise OfflineImapError(reason, severity)

        # JI: In offlineimap, this function returned a tuple of strings for each
        # fetched UID, offlineimap3 calls to the imap object return bytes and so
        # originally a fixed, utf-8 conversion was done and *only* the first
        # response (d[0]) was returned.  Note that this alters the behavior
        # between code bases.  However, it seems like a single UID is the intent
        # of this function so retaining the modfication here for now.
        #
        # TODO: Can we assume the server response containing the meta data is
        # always 'utf-8' encoded?  Assuming yes for now.
        #
        # Convert responses, d[0][0], into a 'utf-8' string (from bytes) and
        # Convert email, d[0][1], into a message object (from bytes)

        ndata0 = data[0][0].decode('utf-8')
        ndata1 = self.parser['8bit-RFC'].parsebytes(data[0][1])
        ndata = [ndata0, ndata1]

        return ndata
Ejemplo n.º 10
0
    def acquireconnection(self):
        """Fetches a connection from the pool, making sure to create a new one
        if needed, to obey the maximum connection limits, etc.
        Opens a connection to the server and returns an appropriate
        object."""

        self.semaphore.acquire()
        self.connectionlock.acquire()
        imapobj = None

        if len(self.availableconnections):  # One is available.
            # Try to find one that previously belonged to this thread
            # as an optimization.  Start from the back since that's where
            # they're popped on.
            imapobj = None
            for i in range(len(self.availableconnections) - 1, -1, -1):
                tryobj = self.availableconnections[i]
                if self.lastowner[tryobj] == get_ident():
                    imapobj = tryobj
                    del (self.availableconnections[i])
                    break
            if not imapobj:
                imapobj = self.availableconnections[0]
                del (self.availableconnections[0])
            self.assignedconnections.append(imapobj)
            self.lastowner[imapobj] = get_ident()
            self.connectionlock.release()
            return imapobj

        self.connectionlock.release()  # Release until need to modify data
        """ Must be careful here that if we fail we should bail out gracefully
        and release locks / threads so that the next attempt can try...
        """
        success = 0
        try:
            while not success:
                # Generate a new connection.
                if self.tunnel:
                    self.ui.connecting('tunnel', self.tunnel)
                    imapobj = imaplibutil.IMAP4_Tunnel(
                        self.tunnel, timeout=socket.getdefaulttimeout())
                    success = 1
                elif self.usessl:
                    self.ui.connecting(self.hostname, self.port)
                    fingerprint = self.repos.get_ssl_fingerprint()
                    imapobj = imaplibutil.WrappedIMAP4_SSL(
                        self.hostname,
                        self.port,
                        self.sslclientkey,
                        self.sslclientcert,
                        self.sslcacertfile,
                        self.verifycert,
                        timeout=socket.getdefaulttimeout(),
                        fingerprint=fingerprint)
                else:
                    self.ui.connecting(self.hostname, self.port)
                    imapobj = imaplibutil.WrappedIMAP4(
                        self.hostname,
                        self.port,
                        timeout=socket.getdefaulttimeout())

                if not self.tunnel:
                    try:
                        # Try GSSAPI and continue if it fails
                        if 'AUTH=GSSAPI' in imapobj.capabilities and have_gss:
                            self.connectionlock.acquire()
                            self.ui.debug('imap',
                                          'Attempting GSSAPI authentication')
                            try:
                                imapobj.authenticate('GSSAPI', self.gssauth)
                            except imapobj.error, val:
                                self.gssapi = False
                                self.ui.debug('imap',
                                              'GSSAPI Authentication failed')
                            else:
                                self.gssapi = True
                                kerberos.authGSSClientClean(self.gss_vc)
                                self.gss_vc = None
                                self.gss_step = self.GSS_STATE_STEP
                                #if we do self.password = None then the next attempt cannot try...
                                #self.password = None
                            self.connectionlock.release()

                        if not self.gssapi:
                            if 'STARTTLS' in imapobj.capabilities and not\
                                    self.usessl:
                                self.ui.debug('imap',
                                              'Using STARTTLS connection')
                                imapobj.starttls()

                            if 'AUTH=CRAM-MD5' in imapobj.capabilities:
                                self.ui.debug(
                                    'imap',
                                    'Attempting CRAM-MD5 authentication')
                                try:
                                    imapobj.authenticate(
                                        'CRAM-MD5', self.md5handler)
                                except imapobj.error, val:
                                    self.plainauth(imapobj)
                            else:
                                # Use plaintext login, unless
                                # LOGINDISABLED (RFC2595)
                                if 'LOGINDISABLED' in imapobj.capabilities:
                                    raise OfflineImapError(
                                        "Plaintext login "
                                        "disabled by server. Need to use SSL?",
                                        OfflineImapError.ERROR.REPO)
                                self.plainauth(imapobj)
                        # Would bail by here if there was a failure.
                        success = 1
                        self.goodpassword = self.password
                    except imapobj.error, val:
                        self.passworderror = str(val)
                        raise
Ejemplo n.º 11
0
    def savemessage(self, uid, msg, flags, rtime):
        """Save the message on the Server

        This backend always assigns a new uid, so the uid arg is ignored.

        This function will update the self.messagelist dict to contain
        the new message after sucessfully saving it.

        See folder/Base for details. Note that savemessage() does not
        check against dryrun settings, so you need to ensure that
        savemessage is never called in a dryrun mode.

        :param uid: Message UID
        :param msg: Message Object
        :param flags: Message flags
        :param rtime: A timestamp to be used as the mail date
        :returns: the UID of the new message as assigned by the server. If the
                  message is saved, but it's UID can not be found, it will
                  return 0. If the message can't be written (folder is
                  read-only for example) it will return -1."""

        self.ui.savemessage('imap', uid, flags, self)

        # Already have it, just save modified flags.
        if uid > 0 and self.uidexists(uid):
            self.savemessageflags(uid, flags)
            return uid

        # Filter user requested headers before uploading to the IMAP server
        self.deletemessageheaders(msg, self.filterheaders)

        # Should just be able to set the policy, to use CRLF in msg output
        output_policy = self.policy['8bit-RFC']

        # Get the date of the message, so we can pass it to the server.
        date = self.__getmessageinternaldate(msg, rtime)

        # Message-ID is handy for debugging messages.
        msg_id = self.getmessageheader(msg, "message-id")
        if not msg_id:
            msg_id = '[unknown message-id]'

        retry_left = 2  # succeeded in APPENDING?
        imapobj = self.imapserver.acquireconnection()
        # NB: in the finally clause for this try we will release
        # NB: the acquired imapobj, so don't do that twice unless
        # NB: you will put another connection to imapobj.  If you
        # NB: really do need to release connection manually, set
        # NB: imapobj to None.
        try:
            while retry_left:
                # XXX: we can mangle message only once, out of the loop
                # UIDPLUS extension provides us with an APPENDUID response.
                use_uidplus = 'UIDPLUS' in imapobj.capabilities

                if not use_uidplus:
                    # Insert a random unique header that we can fetch later.
                    (headername,
                     headervalue) = self.__generate_randomheader(msg)
                    self.ui.debug(
                        'imap', 'savemessage: header is: %s: %s' %
                        (headername, headervalue))
                    self.addmessageheader(msg, headername, headervalue)

                if self.ui.is_debugging('imap'):
                    # Optimization: don't create the debugging objects unless needed
                    msg_s = msg.as_string(policy=output_policy)
                    if len(msg_s) > 200:
                        dbg_output = "%s...%s" % (msg_s[:150], msg_s[-50:])
                    else:
                        dbg_output = msg_s
                    self.ui.debug(
                        'imap', "savemessage: date: %s, content: '%s'" %
                        (date, dbg_output))

                try:
                    # Select folder for append and make the box READ-WRITE.
                    imapobj.select(self.getfullIMAPname())
                except imapobj.readonly:
                    # readonly exception. Return original uid to notify that
                    # we did not save the message. (see savemessage in Base.py)
                    self.ui.msgtoreadonly(self, uid)
                    return uid

                # Do the APPEND.
                try:
                    (typ,
                     dat) = imapobj.append(self.getfullIMAPname(),
                                           imaputil.flagsmaildir2imap(flags),
                                           date,
                                           msg.as_bytes(policy=output_policy))
                    # This should only catch 'NO' responses since append()
                    # will raise an exception for 'BAD' responses:
                    if typ != 'OK':
                        # For example, Groupwise IMAP server
                        # can return something like:
                        #
                        #   NO APPEND The 1500 MB storage limit \
                        #   has been exceeded.
                        #
                        # In this case, we should immediately abort
                        # the repository sync and continue
                        # with the next account.
                        err_msg = \
                            "Saving msg (%s) in folder '%s', " \
                            "repository '%s' failed (abort). " \
                            "Server responded: %s %s\n" % \
                            (msg_id, self, self.getrepository(), typ, dat)
                        raise OfflineImapError(err_msg,
                                               OfflineImapError.ERROR.REPO)
                    retry_left = 0  # Mark as success.
                except imapobj.abort as e:
                    # Connection has been reset, release connection and retry.
                    retry_left -= 1
                    self.imapserver.releaseconnection(imapobj, True)
                    imapobj = self.imapserver.acquireconnection()
                    if not retry_left:
                        raise OfflineImapError(
                            "Saving msg (%s) in folder '%s', "
                            "repository '%s' failed (abort). "
                            "Server responded: %s\n"
                            "Message content was: %s" %
                            (msg_id, self, self.getrepository(), str(e),
                             dbg_output), OfflineImapError.ERROR.MESSAGE,
                            exc_info()[2])

                    # XXX: is this still needed?
                    self.ui.error(e, exc_info()[2])
                except imapobj.error as e:  # APPEND failed
                    # If the server responds with 'BAD', append()
                    # raise()s directly.  So we catch that too.
                    # drop conn, it might be bad.
                    self.imapserver.releaseconnection(imapobj, True)
                    imapobj = None
                    raise OfflineImapError(
                        "Saving msg (%s) folder '%s', repo '%s'"
                        "failed (error). Server responded: %s\n"
                        "Message content was: %s" %
                        (msg_id, self, self.getrepository(), str(e),
                         dbg_output), OfflineImapError.ERROR.MESSAGE,
                        exc_info()[2])

            # Checkpoint. Let it write out stuff, etc. Eg searches for
            # just uploaded messages won't work if we don't do this.
            (typ, dat) = imapobj.check()
            assert (typ == 'OK')

            # Get the new UID, do we use UIDPLUS?
            if use_uidplus:
                # Get new UID from the APPENDUID response, it could look
                # like OK [APPENDUID 38505 3955] APPEND completed with
                # 38505 bein folder UIDvalidity and 3955 the new UID.
                # note: we would want to use .response() here but that
                # often seems to return [None], even though we have
                # data. TODO
                resp = imapobj._get_untagged_response('APPENDUID')
                if resp == [None] or resp is None:
                    self.ui.warn(
                        "Server supports UIDPLUS but got no APPENDUID "
                        "appending a message. Got: %s." % str(resp))
                    return 0
                try:
                    # Convert the UID from [b'4 1532'] to ['4 1532']
                    s_uid = [x.decode('utf-8') for x in resp]
                    # Now, read the UID field
                    uid = int(s_uid[-1].split(' ')[1])
                except ValueError:
                    uid = 0  # Definetly not what we should have.
                except Exception:
                    raise OfflineImapError(
                        "Unexpected response: %s" % str(resp),
                        OfflineImapError.ERROR.MESSAGE)
                if uid == 0:
                    self.ui.warn("savemessage: Server supports UIDPLUS, but"
                                 " we got no usable UID back. APPENDUID "
                                 "reponse was '%s'" % str(resp))
            else:
                try:
                    # We don't use UIDPLUS.
                    uid = self.__savemessage_searchforheader(
                        imapobj, headername, headervalue)
                    # See docs for savemessage in Base.py for explanation
                    # of this and other return values.
                    if uid == 0:
                        self.ui.debug(
                            'imap', 'savemessage: attempt to get new UID '
                            'UID failed. Search headers manually.')
                        uid = self.__savemessage_fetchheaders(
                            imapobj, headername, headervalue)
                        self.ui.warn("savemessage: Searching mails for new "
                                     "Message-ID failed. "
                                     "Could not determine new UID on %s." %
                                     self.getname())
                # Something wrong happened while trying to get the UID. Explain
                # the error might be about the 'get UID' process not necesseraly
                # the APPEND.
                except Exception:
                    self.ui.warn(
                        "%s: could not determine the UID while we got "
                        "no error while appending the "
                        "email with '%s: %s'" %
                        (self.getname(), headername, headervalue))
                    raise
        finally:
            if imapobj:
                self.imapserver.releaseconnection(imapobj)

        if uid:  # Avoid UID FETCH 0 crash happening later on.
            self.messagelist[uid] = self.msglist_item_initializer(uid)
            self.messagelist[uid]['flags'] = flags

        self.ui.debug('imap', 'savemessage: returning new UID %d' % uid)
        return uid
Ejemplo n.º 12
0
    def savemessage(self, uid, content, flags, rtime):
        """Save the message on the Server

        This backend always assigns a new uid, so the uid arg is ignored.

        This function will update the self.messagelist dict to contain
        the new message after sucessfully saving it.

        See folder/Base for details. Note that savemessage() does not
        check against dryrun settings, so you need to ensure that
        savemessage is never called in a dryrun mode.

        :param rtime: A timestamp to be used as the mail date
        :returns: the UID of the new message as assigned by the server. If the
                  message is saved, but it's UID can not be found, it will
                  return 0. If the message can't be written (folder is
                  read-only for example) it will return -1."""
        self.ui.savemessage('imap', uid, flags, self)

        # already have it, just save modified flags
        if uid > 0 and self.uidexists(uid):
            self.savemessageflags(uid, flags)
            return uid

        retry_left = 2  # succeeded in APPENDING?
        imapobj = self.imapserver.acquireconnection()
        try:
            while retry_left:
                # UIDPLUS extension provides us with an APPENDUID response.
                use_uidplus = 'UIDPLUS' in imapobj.capabilities

                # get the date of the message, so we can pass it to the server.
                date = self.getmessageinternaldate(content, rtime)
                content = re.sub("(?<!\r)\n", "\r\n", content)

                if not use_uidplus:
                    # insert a random unique header that we can fetch later
                    (headername,
                     headervalue) = self.generate_randomheader(content)
                    self.ui.debug('imap', 'savemessage: header is: %s: %s' %\
                                      (headername, headervalue))
                    content = self.savemessage_addheader(
                        content, headername, headervalue)
                if len(content) > 200:
                    dbg_output = "%s...%s" % (content[:150], content[-50:])
                else:
                    dbg_output = content
                self.ui.debug(
                    'imap', "savemessage: date: %s, content: '%s'" %
                    (date, dbg_output))

                try:
                    # Select folder for append and make the box READ-WRITE
                    imapobj.select(self.getfullname())
                except imapobj.readonly:
                    # readonly exception. Return original uid to notify that
                    # we did not save the message. (see savemessage in Base.py)
                    self.ui.msgtoreadonly(self, uid, content, flags)
                    return uid

                #Do the APPEND
                try:
                    (typ,
                     dat) = imapobj.append(self.getfullname(),
                                           imaputil.flagsmaildir2imap(flags),
                                           date, content)
                    retry_left = 0  # Mark as success
                except imapobj.abort as e:
                    # connection has been reset, release connection and retry.
                    retry_left -= 1
                    self.imapserver.releaseconnection(imapobj, True)
                    imapobj = self.imapserver.acquireconnection()
                    if not retry_left:
                        raise OfflineImapError(
                            "Saving msg in folder '%s', "
                            "repository '%s' failed (abort). Server reponded: %s\n"
                            "Message content was: %s" %
                            (self, self.getrepository(), str(e), dbg_output),
                            OfflineImapError.ERROR.MESSAGE)
                    self.ui.error(e, exc_info()[2])
                except imapobj.error as e:  # APPEND failed
                    # If the server responds with 'BAD', append()
                    # raise()s directly.  So we catch that too.
                    # drop conn, it might be bad.
                    self.imapserver.releaseconnection(imapobj, True)
                    imapobj = None
                    raise OfflineImapError(
                        "Saving msg folder '%s', repo '%s'"
                        "failed (error). Server reponded: %s\nMessage content was: "
                        "%s" %
                        (self, self.getrepository(), str(e), dbg_output),
                        OfflineImapError.ERROR.MESSAGE)
            # Checkpoint. Let it write out stuff, etc. Eg searches for
            # just uploaded messages won't work if we don't do this.
            (typ, dat) = imapobj.check()
            assert (typ == 'OK')

            # get the new UID. Test for APPENDUID response even if the
            # server claims to not support it, as e.g. Gmail does :-(
            if use_uidplus or imapobj._get_untagged_response(
                    'APPENDUID', True):
                # get new UID from the APPENDUID response, it could look
                # like OK [APPENDUID 38505 3955] APPEND completed with
                # 38505 bein folder UIDvalidity and 3955 the new UID.
                # note: we would want to use .response() here but that
                # often seems to return [None], even though we have
                # data. TODO
                resp = imapobj._get_untagged_response('APPENDUID')
                if resp == [None]:
                    self.ui.warn(
                        "Server supports UIDPLUS but got no APPENDUID "
                        "appending a message.")
                    return 0
                uid = long(resp[-1].split(' ')[1])
                if uid == 0:
                    self.ui.warn(
                        "savemessage: Server supports UIDPLUS, but"
                        " we got no usable uid back. APPENDUID reponse was "
                        "'%s'" % str(resp))
            else:
                # we don't support UIDPLUS
                uid = self.savemessage_searchforheader(imapobj, headername,
                                                       headervalue)
                # See docs for savemessage in Base.py for explanation
                # of this and other return values
                if uid == 0:
                    self.ui.debug(
                        'imap', 'savemessage: attempt to get new UID '
                        'UID failed. Search headers manually.')
                    uid = self.savemessage_fetchheaders(
                        imapobj, headername, headervalue)
                    self.ui.warn(
                        'imap', "savemessage: Searching mails for new "
                        "Message-ID failed. Could not determine new UID.")
        finally:
            self.imapserver.releaseconnection(imapobj)

        if uid:  # avoid UID FETCH 0 crash happening later on
            self.messagelist[uid] = {'uid': uid, 'flags': flags}

        self.ui.debug('imap', 'savemessage: returning new UID %d' % uid)
        return uid
Ejemplo n.º 13
0
    def savemessagelabels(self, uid, labels, ignorelabels=None):
        """Change a message's labels to `labels`.

        Note that this function does not check against dryrun settings,
        so you need to ensure that it is never called in a dryrun mode."""

        if ignorelabels is None:
            ignorelabels = set()

        filename = self.messagelist[uid]['filename']
        filepath = os.path.join(self.getfullname(), filename)

        file = open(filepath, 'rt')
        content = file.read()
        file.close()

        oldlabels = set()
        for hstr in self.getmessageheaderlist(content, self.labelsheader):
            oldlabels.update(
                imaputil.labels_from_header(self.labelsheader, hstr))

        labels = labels - ignorelabels
        ignoredlabels = oldlabels & ignorelabels
        oldlabels = oldlabels - ignorelabels

        # Nothing to change.
        if labels == oldlabels:
            return

        # Change labels into content.
        labels_str = imaputil.format_labels_string(
            self.labelsheader, sorted(labels | ignoredlabels))

        # First remove old labels header, and then add the new one.
        content = self.deletemessageheaders(content, self.labelsheader)
        content = self.addmessageheader(content, '\n', self.labelsheader,
                                        labels_str)

        mtime = int(os.stat(filepath).st_mtime)

        # Write file with new labels to a unique file in tmp.
        messagename = self.new_message_filename(uid, set())
        tmpname = self.save_to_tmp_file(messagename, content)
        tmppath = os.path.join(self.getfullname(), tmpname)

        # Move to actual location.
        try:
            os.rename(tmppath, filepath)
        except OSError as e:
            raise OfflineImapError(
                "Can't rename file '%s' to '%s': %s" %
                (tmppath, filepath, e[1]), OfflineImapError.ERROR.FOLDER,
                exc_info()[2])

        # If utime_from_header=true, we don't want to change the mtime.
        if self._utime_from_header and mtime:
            os.utime(filepath, (mtime, mtime))

        # save the new mtime and labels
        self.messagelist[uid]['mtime'] = int(os.stat(filepath).st_mtime)
        self.messagelist[uid]['labels'] = labels
Ejemplo n.º 14
0
    def __savemessage_fetchheaders(self, imapobj, headername, headervalue):
        """ We fetch all new mail headers and search for the right
        X-OfflineImap line by hand. The response from the server has form:
        (
          'OK',
          [
            (
              '185 (RFC822.HEADER {1789}',
              '... mail headers ...'
            ),
            ' UID 2444)',
            (
              '186 (RFC822.HEADER {1789}',
              '... 2nd mail headers ...'
            ),
            ' UID 2445)'
          ]
        )
        We need to locate the UID just after mail headers containing our
        X-OfflineIMAP line.

        Returns UID when found, 0 when not found."""

        self.ui.debug(
            'imap', '__savemessage_fetchheaders called for %s: %s' %
            (headername, headervalue))

        # Run "fetch X:* rfc822.header".
        # Since we stored the mail we are looking for just recently, it would
        # not be optimal to fetch all messages. So we'll find highest message
        # UID in our local messagelist and search from there (exactly from
        # UID+1). That works because UIDs are guaranteed to be unique and
        # ascending.

        if self.getmessagelist():
            start = 1 + max(self.getmessagelist().keys())
        else:
            # Folder was empty - start from 1.
            start = 1

        result = imapobj.uid('FETCH', '%d:*' % start, 'rfc822.header')
        if result[0] != 'OK':
            msg = 'Error fetching mail headers: %s' % '. '.join(result[1])
            raise OfflineImapError(msg, OfflineImapError.ERROR.MESSAGE)

        # result is like:
        # [
        #    ('185 (RFC822.HEADER {1789}', '... mail headers ...'),
        #      ' UID 2444)',
        #    ('186 (RFC822.HEADER {1789}', '... 2nd mail headers ...'),
        #      ' UID 2445)'
        # ]
        result = result[1]

        found = None
        # item is like:
        # ('185 (RFC822.HEADER {1789}', '... mail headers ...'), ' UID 2444)'
        for item in result:
            if found is None and type(item) == tuple:
                # Decode the value
                item = [x.decode('utf-8') for x in item]

                # Walk just tuples.
                if re.search("(?:^|\\r|\\n)%s:\s*%s(?:\\r|\\n)" %
                             (headername, headervalue),
                             item[1],
                             flags=re.IGNORECASE):
                    found = item[0]
            elif found is not None:
                if isinstance(item, bytes):
                    item = item.decode('utf-8')
                    uid = re.search("UID\s+(\d+)", item, flags=re.IGNORECASE)
                    if uid:
                        return int(uid.group(1))
                    else:
                        # This parsing is for Davmail.
                        # https://github.com/OfflineIMAP/offlineimap/issues/479
                        # item is like:
                        # ')'
                        # and item[0] stored in "found" is like:
                        # '1694 (UID 1694 RFC822.HEADER {1294}'
                        uid = re.search("\d+\s+\(UID\s+(\d+)",
                                        found,
                                        flags=re.IGNORECASE)
                        if uid:
                            return int(uid.group(1))

                        self.ui.warn("Can't parse FETCH response, "
                                     "can't find UID in %s" % item)
                        self.ui.debug('imap', "Got: %s" % repr(result))
                else:
                    self.ui.warn("Can't parse FETCH response, "
                                 "we awaited string: %s" % repr(item))

        return 0
Ejemplo n.º 15
0
    def __sync(self):
        """Synchronize the account once, then return.

        Assumes that `self.remoterepos`, `self.localrepos`, and
        `self.statusrepos` has already been populated, so it should only
        be called from the :meth:`syncrunner` function."""

        folderthreads = []

        hook_env = {
            'OIMAP_ACCOUNT_NAME': self.getname(),
        }

        self.callhook('presynchook', hook_env)

        if self.utf_8_support and self.remoterepos.getdecodefoldernames():
            raise OfflineImapError(
                "Configuration mismatch in account " +
                "'%s'. " % self.getname() +
                "\nAccount setting 'utf8foldernames' and repository " +
                "setting 'decodefoldernames'\nmay not be used at the " +
                "same time. This account has not been synchronized.\n" +
                "Please check the configuration and documentation.",
                OfflineImapError.ERROR.REPO)

        quickconfig = self.getconfint('quick', 0)
        if quickconfig < 0:
            quick = True
        elif quickconfig > 0:
            if self.quicknum == 0 or self.quicknum > quickconfig:
                self.quicknum = 1
                quick = False
            else:
                self.quicknum = self.quicknum + 1
                quick = True
        else:
            quick = False

        try:
            startedThread = False
            remoterepos = self.remoterepos
            localrepos = self.localrepos
            statusrepos = self.statusrepos

            # Init repos with list of folders, so we have them (and the
            # folder delimiter etc).
            remoterepos.getfolders()
            localrepos.getfolders()

            remoterepos.sync_folder_structure(localrepos, statusrepos)
            # Replicate the folderstructure between REMOTE to LOCAL.
            if not localrepos.getconfboolean('readonly', False):
                self.ui.syncfolders(remoterepos, localrepos)

            # Iterate through all folders on the remote repo and sync.
            for remotefolder in remoterepos.getfolders():
                # Check for CTRL-C or SIGTERM.
                if Account.abort_NOW_signal.is_set():
                    break

                if not remotefolder.sync_this:
                    self.ui.debug(
                        '', "Not syncing filtered folder '%s'"
                        "[%s]" % (remotefolder.getname(), remoterepos))
                    continue  # Ignore filtered folder.

                # The remote folder names must not have the local sep char in
                # their names since this would cause troubles while converting
                # the name back (from local to remote).
                sep = localrepos.getsep()
                if (sep != os.path.sep and sep != remoterepos.getsep()
                        and sep in remotefolder.getname()):
                    self.ui.warn(
                        '', "Ignoring folder '%s' due to unsupported "
                        "'%s' character serving as local separator." %
                        (remotefolder.getname(), localrepos.getsep()))
                    continue  # Ignore unsupported folder name.

                localfolder = self.get_local_folder(remotefolder)
                if not localfolder.sync_this:
                    self.ui.debug(
                        '', "Not syncing filtered folder '%s'"
                        "[%s]" %
                        (localfolder.getname(), localfolder.repository))
                    continue  # Ignore filtered folder.

                if not globals.options.singlethreading:
                    thread = InstanceLimitedThread(
                        limitNamespace="%s%s" %
                        (FOLDER_NAMESPACE, self.remoterepos.getname()),
                        target=syncfolder,
                        name="Folder %s [acc: %s]" %
                        (remotefolder.getexplainedname(), self),
                        args=(self, remotefolder, quick))
                    thread.start()
                    folderthreads.append(thread)
                else:
                    syncfolder(self, remotefolder, quick)
                startedThread = True
            # Wait for all threads to finish.
            for thr in folderthreads:
                thr.join()
            if startedThread is True:
                mbnames.writeIntermediateFile(
                    self.name)  # Write out mailbox names.
            else:
                msg = "Account {}: no folder to sync (folderfilter issue?)".format(
                    self)
                raise OfflineImapError(msg, OfflineImapError.ERROR.REPO)
            localrepos.forgetfolders()
            remoterepos.forgetfolders()
        except:
            # Error while syncing. Drop all connections that we have, they
            # might be bogus by now (e.g. after suspend).
            localrepos.dropconnections()
            remoterepos.dropconnections()
            raise
        else:
            # Sync went fine. Hold or drop depending on config.
            localrepos.holdordropconnections()
            remoterepos.holdordropconnections()

        self.callhook('postsynchook', hook_env)
Ejemplo n.º 16
0
    def _fetch_from_imap(self, uids, retry_num=1):
        """Fetches data from IMAP server.

        Arguments:
        - uids: message UIDS
        - retry_num: number of retries to make

        Returns: data obtained by this query."""

        imapobj = self.imapserver.acquireconnection()
        try:
            query = "(%s)" % (" ".join(self.imap_query))
            fails_left = retry_num  # Retry on dropped connection.
            while fails_left:
                try:
                    imapobj.select(self.getfullIMAPname(), readonly=True)
                    res_type, data = imapobj.uid('fetch', uids, query)
                    break
                except imapobj.abort as e:
                    fails_left -= 1
                    # self.ui.error() will show the original traceback.
                    if fails_left <= 0:
                        message = ("%s, while fetching msg %r in folder %r."
                                   " Max retry reached (%d)" %
                                   (e, uids, self.name, retry_num))
                        raise OfflineImapError(message,
                                               OfflineImapError.ERROR.MESSAGE)
                    self.ui.error("%s. While fetching msg %r in folder %r."
                                  " Query: %s Retrying (%d/%d)" %
                                  (e, uids, self.name, query,
                                   retry_num - fails_left, retry_num))
                    # Release dropped connection, and get a new one.
                    self.imapserver.releaseconnection(imapobj, True)
                    imapobj = self.imapserver.acquireconnection()
        finally:
            # The imapobj here might be different than the one created before
            # the ``try`` clause. So please avoid transforming this to a nice
            # ``with`` without taking this into account.
            self.imapserver.releaseconnection(imapobj)

        # Ensure to not consider unsolicited FETCH responses caused by flag
        # changes from concurrent connections.  These appear as strings in
        # 'data' (the BODY response appears as a tuple).  This should leave
        # exactly one response.
        if res_type == 'OK':
            data = [res for res in data if not isinstance(res, bytes)]

        # Could not fetch message.  Note: it is allowed by rfc3501 to return any
        # data for the UID FETCH command.
        if data == [None] or res_type != 'OK' or len(data) != 1:
            severity = OfflineImapError.ERROR.MESSAGE
            reason = "IMAP server '%s' failed to fetch messages UID '%s'. " \
                     "Server responded: %s %s" % (self.getrepository(), uids,
                                                  res_type, data)
            if data == [None] or len(data) < 1:
                # IMAP server did not find a message with this UID.
                reason = "IMAP server '%s' does not have a message " \
                         "with UID '%s'" % (self.getrepository(), uids)
            raise OfflineImapError(reason, severity)

        # Convert bytes to str
        ndata0 = data[0][0].decode('utf-8')
        ndata1 = data[0][1].decode('utf-8', errors='replace')
        ndata = [ndata0, ndata1]

        return ndata
Ejemplo n.º 17
0
def syncfolder(account, remotefolder, quick):
    """Synchronizes given remote folder for the specified account.

    Filtered folders on the remote side will not invoke this function.

    When called in concurrently for the same localfolder, syncs are
    serialized."""
    def acquire_mutex():
        account_name = account.getname()
        localfolder_name = localfolder.getfullname()

        with SYNC_MUTEXES_LOCK:
            if SYNC_MUTEXES.get(account_name) is None:
                SYNC_MUTEXES[account_name] = {}
            # The localfolder full name is good to uniquely identify the sync
            # transaction.
            if SYNC_MUTEXES[account_name].get(localfolder_name) is None:
                #XXX: This lock could be an external file lock so we can remove
                # the lock at the account level.
                SYNC_MUTEXES[account_name][localfolder_name] = Lock()

        # Acquire the lock.
        SYNC_MUTEXES[account_name][localfolder_name].acquire()

    def release_mutex():
        SYNC_MUTEXES[account.getname()][localfolder.getfullname()].release()

    def check_uid_validity():
        # If either the local or the status folder has messages and
        # there is a UID validity problem, warn and abort.  If there are
        # no messages, UW IMAPd loses UIDVALIDITY.  But we don't really
        # need it if both local folders are empty.  So, in that case,
        # just save it off.
        if localfolder.getmessagecount() > 0 or statusfolder.getmessagecount(
        ) > 0:
            if not localfolder.check_uidvalidity():
                ui.validityproblem(localfolder)
                localfolder.repository.restore_atime()
                return
            if not remotefolder.check_uidvalidity():
                ui.validityproblem(remotefolder)
                localrepos.restore_atime()
                return
        else:
            # Both folders empty, just save new UIDVALIDITY.
            localfolder.save_uidvalidity()
            remotefolder.save_uidvalidity()

    def cachemessagelists_upto_date(date):
        """Returns messages with uid > min(uids of messages newer than date)."""

        remotefolder.cachemessagelist(
            min_date=time.gmtime(time.mktime(date) + 24 * 60 * 60))
        uids = remotefolder.getmessageuidlist()
        localfolder.dropmessagelistcache()
        if len(uids) > 0:
            # Reload the remote message list from min_uid. This avoid issues for
            # old messages, which has been added from local on any previous run
            # (IOW, message is older than maxage _and_ has high enough UID).
            remotefolder.dropmessagelistcache()
            remotefolder.cachemessagelist(min_uid=min(uids))
            localfolder.cachemessagelist(min_uid=min(uids))
        else:
            # Remote folder UIDs list is empty for the given range. We still
            # might have valid local UIDs for this range (e.g.: new local
            # emails).
            localfolder.cachemessagelist(min_date=date)
            uids = localfolder.getmessageuidlist()
            # Take care to only consider positive uids. Negative UIDs might be
            # present due to new emails.
            uids = [uid for uid in uids if uid > 0]
            if len(uids) > 0:
                # Update the remote cache list for this new min(uids).
                remotefolder.dropmessagelistcache()
                remotefolder.cachemessagelist(min_uid=min(uids))

    def cachemessagelists_startdate(new, partial, date):
        """Retrieve messagelists when startdate has been set for
        the folder 'partial'.

        Idea: suppose you want to clone the messages after date in one
        account (partial) to a new one (new). If new is empty, then copy
        messages in partial newer than date to new, and keep track of the
        min uid. On subsequent syncs, sync all the messages in new against
        those after that min uid in partial. This is a partial replacement
        for maxage in the IMAP-IMAP sync case, where maxage doesn't work:
        the UIDs of the messages in localfolder might not be in the same
        order as those of corresponding messages in remotefolder, so if L in
        local corresponds to R in remote, the ranges [L, ...] and [R, ...]
        might not correspond. But, if we're cloning a folder into a new one,
        [min_uid, ...] does correspond to [1, ...].

        This is just for IMAP-IMAP. For Maildir-IMAP, use maxage instead."""

        new.cachemessagelist()
        min_uid = partial.retrieve_min_uid()
        if min_uid == None:  # min_uid file didn't exist
            if len(new.getmessageuidlist()) > 0:
                raise OfflineImapError(
                    "To use startdate on Repository %s, "
                    "Repository %s must be empty" %
                    (partial.repository.name, new.repository.name),
                    OfflineImapError.ERROR.MESSAGE)
            else:
                partial.cachemessagelist(min_date=date)
                # messagelist.keys() instead of getuidmessagelist() because in
                # the UID mapped case we want the actual local UIDs, not their
                # remote counterparts.
                positive_uids = [
                    uid for uid in list(partial.messagelist.keys()) if uid > 0
                ]
                if len(positive_uids) > 0:
                    min_uid = min(positive_uids)
                else:
                    min_uid = 1
                partial.save_min_uid(min_uid)
        else:
            partial.cachemessagelist(min_uid=min_uid)

    remoterepos = account.remoterepos
    localrepos = account.localrepos
    statusrepos = account.statusrepos

    ui = getglobalui()
    ui.registerthread(account)
    try:
        # Load local folder.
        localfolder = account.get_local_folder(remotefolder)

        # Acquire the mutex to start syncing.
        acquire_mutex()

        # Add the folder to the mbnames mailboxes.
        mbnames.add(account.name, localrepos.getlocalroot(),
                    localfolder.getname())

        # Load status folder.
        statusfolder = statusrepos.getfolder(
            remotefolder.getvisiblename().replace(remoterepos.getsep(),
                                                  statusrepos.getsep()))
        statusfolder.openfiles()
        statusfolder.cachemessagelist()

        # Load local folder.
        ui.syncingfolder(remoterepos, remotefolder, localrepos, localfolder)

        # Retrieve messagelists, taking into account age-restriction
        # options.
        maxage = localfolder.getmaxage()
        localstart = localfolder.getstartdate()
        remotestart = remotefolder.getstartdate()
        if (maxage != None) + (localstart != None) + (remotestart != None) > 1:
            six.reraise(
                OfflineImapError,
                OfflineImapError(
                    "You can set at most one of the "
                    "following: maxage, startdate (for the local "
                    "folder), startdate (for the remote folder)",
                    OfflineImapError.ERROR.REPO),
                exc_info()[2])
        if (maxage != None or localstart or remotestart) and quick:
            # IMAP quickchanged isn't compatible with options that
            # involve restricting the messagelist, since the "quick"
            # check can only retrieve a full list of UIDs in the folder.
            ui.warn("Quick syncs (-q) not supported in conjunction "
                    "with maxage or startdate; ignoring -q.")
        if maxage != None:
            cachemessagelists_upto_date(maxage)
            check_uid_validity()
        elif localstart != None:
            cachemessagelists_startdate(remotefolder, localfolder, localstart)
            check_uid_validity()
        elif remotestart != None:
            cachemessagelists_startdate(localfolder, remotefolder, remotestart)
            check_uid_validity()
        else:
            localfolder.cachemessagelist()
            if quick:
                if (not localfolder.quickchanged(statusfolder)
                        and not remotefolder.quickchanged(statusfolder)):
                    ui.skippingfolder(remotefolder)
                    localrepos.restore_atime()
                    return
            check_uid_validity()
            remotefolder.cachemessagelist()

        # Synchronize remote changes.
        if not localrepos.getconfboolean('readonly', False):
            ui.syncingmessages(remoterepos, remotefolder, localrepos,
                               localfolder)
            remotefolder.syncmessagesto(localfolder, statusfolder)
        else:
            ui.debug(
                '', "Not syncing to read-only repository '%s'" %
                localrepos.getname())

        # Synchronize local changes.
        if not remoterepos.getconfboolean('readonly', False):
            ui.syncingmessages(localrepos, localfolder, remoterepos,
                               remotefolder)
            localfolder.syncmessagesto(remotefolder, statusfolder)
        else:
            ui.debug(
                '', "Not syncing to read-only repository '%s'" %
                remoterepos.getname())

        statusfolder.save()
        localrepos.restore_atime()
    except (KeyboardInterrupt, SystemExit):
        raise
    except OfflineImapError as e:
        # Bubble up severe Errors, skip folder otherwise.
        if e.severity > OfflineImapError.ERROR.FOLDER:
            raise
        else:
            ui.error(e,
                     exc_info()[2],
                     msg="Aborting sync, folder '%s' "
                     "[acc: '%s']" % (localfolder, account))
    except Exception as e:
        ui.error(
            e,
            msg="ERROR in syncfolder for %s folder %s: %s" %
            (account, remotefolder.getvisiblename(), traceback.format_exc()))
    finally:
        for folder in ["statusfolder", "localfolder", "remotefolder"]:
            if folder in locals():
                locals()[folder].dropmessagelistcache()
        statusfolder.closefiles()
        # Release the mutex of this sync transaction.
        release_mutex()
Ejemplo n.º 18
0
    def savemessage(self, uid, content, flags, rtime):
        """Save the message on the Server

        This backend always assigns a new uid, so the uid arg is ignored.

        This function will update the self.messagelist dict to contain
        the new message after sucessfully saving it.

        See folder/Base for details. Note that savemessage() does not
        check against dryrun settings, so you need to ensure that
        savemessage is never called in a dryrun mode.

        :param rtime: A timestamp to be used as the mail date
        :returns: the UID of the new message as assigned by the server. If the
                  message is saved, but it's UID can not be found, it will
                  return 0. If the message can't be written (folder is
                  read-only for example) it will return -1."""

        self.ui.savemessage('imap', uid, flags, self)

        # already have it, just save modified flags
        if uid > 0 and self.uidexists(uid):
            self.savemessageflags(uid, flags)
            return uid

        content = self.deletemessageheaders(content, self.filterheaders)

        # Use proper CRLF all over the message
        content = re.sub("(?<!\r)\n", CRLF, content)

        # get the date of the message, so we can pass it to the server.
        date = self.__getmessageinternaldate(content, rtime)

        # Message-ID is handy for debugging messages
        msg_id = self.getmessageheader(content, "message-id")
        if not msg_id:
            msg_id = '[unknown message-id]'

        retry_left = 2  # succeeded in APPENDING?
        imapobj = self.imapserver.acquireconnection()
        # NB: in the finally clause for this try we will release
        # NB: the acquired imapobj, so don't do that twice unless
        # NB: you will put another connection to imapobj.  If you
        # NB: really do need to release connection manually, set
        # NB: imapobj to None.
        try:
            while retry_left:
                # XXX: we can mangle message only once, out of the loop
                # UIDPLUS extension provides us with an APPENDUID response.
                use_uidplus = 'UIDPLUS' in imapobj.capabilities

                if not use_uidplus:
                    # insert a random unique header that we can fetch later
                    (headername,
                     headervalue) = self.__generate_randomheader(content)
                    self.ui.debug(
                        'imap', 'savemessage: header is: %s: %s' %
                        (headername, headervalue))
                    content = self.addmessageheader(content, CRLF, headername,
                                                    headervalue)

                if len(content) > 200:
                    dbg_output = "%s...%s" % (content[:150], content[-50:])
                else:
                    dbg_output = content
                self.ui.debug(
                    'imap', "savemessage: date: %s, content: '%s'" %
                    (date, dbg_output))

                try:
                    # Select folder for append and make the box READ-WRITE
                    imapobj.select(self.getfullname())
                except imapobj.readonly:
                    # readonly exception. Return original uid to notify that
                    # we did not save the message. (see savemessage in Base.py)
                    self.ui.msgtoreadonly(self, uid, content, flags)
                    return uid

                #Do the APPEND
                try:
                    (typ,
                     dat) = imapobj.append(self.getfullname(),
                                           imaputil.flagsmaildir2imap(flags),
                                           date, content)
                    # This should only catch 'NO' responses since append()
                    # will raise an exception for 'BAD' responses:
                    if typ != 'OK':
                        # For example, Groupwise IMAP server can return something like:
                        #
                        #   NO APPEND The 1500 MB storage limit has been exceeded.
                        #
                        # In this case, we should immediately abort the repository sync
                        # and continue with the next account.
                        msg = \
                            "Saving msg (%s) in folder '%s', repository '%s' failed (abort). " \
                            "Server responded: %s %s\n"% \
                            (msg_id, self, self.getrepository(), typ, dat)
                        raise OfflineImapError(msg,
                                               OfflineImapError.ERROR.REPO)
                    retry_left = 0  # Mark as success
                except imapobj.abort as e:
                    # connection has been reset, release connection and retry.
                    retry_left -= 1
                    self.imapserver.releaseconnection(imapobj, True)
                    imapobj = self.imapserver.acquireconnection()
                    if not retry_left:
                        raise OfflineImapError("Saving msg (%s) in folder '%s', "
                              "repository '%s' failed (abort). Server responded: %s\n"
                              "Message content was: %s"%
                              (msg_id, self, self.getrepository(), str(e), dbg_output),
                                               OfflineImapError.ERROR.MESSAGE), \
                              None, exc_info()[2]
                    # XXX: is this still needed?
                    self.ui.error(e, exc_info()[2])
                except imapobj.error as e:  # APPEND failed
                    # If the server responds with 'BAD', append()
                    # raise()s directly.  So we catch that too.
                    # drop conn, it might be bad.
                    self.imapserver.releaseconnection(imapobj, True)
                    imapobj = None
                    raise OfflineImapError(
                        "Saving msg (%s) folder '%s', repo '%s'"
                        "failed (error). Server responded: %s\nMessage content was: "
                        "%s" % (msg_id, self, self.getrepository(), str(e),
                                dbg_output),
                        OfflineImapError.ERROR.MESSAGE), None, exc_info()[2]
            # Checkpoint. Let it write out stuff, etc. Eg searches for
            # just uploaded messages won't work if we don't do this.
            (typ, dat) = imapobj.check()
            assert (typ == 'OK')

            # get the new UID, do we use UIDPLUS?
            if use_uidplus:
                # get new UID from the APPENDUID response, it could look
                # like OK [APPENDUID 38505 3955] APPEND completed with
                # 38505 bein folder UIDvalidity and 3955 the new UID.
                # note: we would want to use .response() here but that
                # often seems to return [None], even though we have
                # data. TODO
                resp = imapobj._get_untagged_response('APPENDUID')
                if resp == [None] or resp is None:
                    self.ui.warn(
                        "Server supports UIDPLUS but got no APPENDUID "
                        "appending a message.")
                    return 0
                uid = long(resp[-1].split(' ')[1])
                if uid == 0:
                    self.ui.warn(
                        "savemessage: Server supports UIDPLUS, but"
                        " we got no usable uid back. APPENDUID reponse was "
                        "'%s'" % str(resp))
            else:
                # we don't support UIDPLUS
                uid = self.__savemessage_searchforheader(
                    imapobj, headername, headervalue)
                # See docs for savemessage in Base.py for explanation
                # of this and other return values
                if uid == 0:
                    self.ui.debug(
                        'imap', 'savemessage: attempt to get new UID '
                        'UID failed. Search headers manually.')
                    uid = self.__savemessage_fetchheaders(
                        imapobj, headername, headervalue)
                    self.ui.warn(
                        'imap', "savemessage: Searching mails for new "
                        "Message-ID failed. Could not determine new UID.")
        finally:
            if imapobj: self.imapserver.releaseconnection(imapobj)

        if uid:  # avoid UID FETCH 0 crash happening later on
            self.messagelist[uid] = self.msglist_item_initializer(uid)
            self.messagelist[uid]['flags'] = flags

        self.ui.debug('imap', 'savemessage: returning new UID %d' % uid)
        return uid
Ejemplo n.º 19
0
            self.semaphore.release()

            #Make sure that this can be retried the next time...
            self.passworderror = None
            if (self.connectionlock.locked()):
                self.connectionlock.release()

            severity = OfflineImapError.ERROR.REPO
            if type(e) == gaierror:
                #DNS related errors. Abort Repo sync
                #TODO: special error msg for e.errno == 2 "Name or service not known"?
                reason = "Could not resolve name '%s' for repository "\
                         "'%s'. Make sure you have configured the ser"\
                         "ver name correctly and that you are online."%\
                         (self.hostname, self.reposname)
                raise OfflineImapError(reason, severity)

            elif SSLError and isinstance(e, SSLError) and e.errno == 1:
                # SSL unknown protocol error
                # happens e.g. when connecting via SSL to a non-SSL service
                if self.port != 443:
                    reason = "Could not connect via SSL to host '%s' and non-s"\
                        "tandard ssl port %d configured. Make sure you connect"\
                        " to the correct port." % (self.hostname, self.port)
                else:
                    reason = "Unknown SSL protocol connecting to host '%s' for"\
                         "repository '%s'. OpenSSL responded:\n%s"\
                         % (self.hostname, self.reposname, e)
                raise OfflineImapError(reason, severity)

            elif isinstance(e,
Ejemplo n.º 20
0
class IMAPRepository(BaseRepository):
    def __init__(self, reposname, account):
        """Initialize an IMAPRepository object."""
        BaseRepository.__init__(self, reposname, account)
        # self.ui is being set by the BaseRepository
        self._host = None
        self.imapserver = imapserver.IMAPServer(self)
        self.folders = None

    def startkeepalive(self):
        keepalivetime = self.getkeepalive()
        if not keepalivetime: return
        self.kaevent = Event()
        self.kathread = ExitNotifyThread(target=self.imapserver.keepalive,
                                         name="Keep alive " + self.getname(),
                                         args=(keepalivetime, self.kaevent))
        self.kathread.setDaemon(1)
        self.kathread.start()

    def stopkeepalive(self):
        if not hasattr(self, 'kaevent'):
            # Keepalive is not active.
            return

        self.kaevent.set()
        del self.kathread
        del self.kaevent

    def holdordropconnections(self):
        if not self.getholdconnectionopen():
            self.dropconnections()

    def dropconnections(self):
        self.imapserver.close()

    def getholdconnectionopen(self):
        if self.getidlefolders():
            return 1
        return self.getconfboolean("holdconnectionopen", 0)

    def getkeepalive(self):
        num = self.getconfint("keepalive", 0)
        if num == 0 and self.getidlefolders():
            return 29 * 60
        else:
            return num

    def getsep(self):
        return self.imapserver.delim

    def gethost(self):
        """Return the configured hostname to connect to

        :returns: hostname as string or throws Exception"""
        if self._host:  # use cached value if possible
            return self._host

        # 1) check for remotehosteval setting
        if self.config.has_option(self.getsection(), 'remotehosteval'):
            host = self.getconf('remotehosteval')
            try:
                host = self.localeval.eval(host)
            except Exception, e:
                raise OfflineImapError("remotehosteval option for repository "\
                                       "'%s' failed:\n%s" % (self, e),
                                       OfflineImapError.ERROR.REPO)
            if host:
                self._host = host
                return self._host
        # 2) check for plain remotehost setting
        host = self.getconf('remotehost', None)
        if host != None:
            self._host = host
            return self._host

        # no success
        raise OfflineImapError("No remote host for repository "\
                                   "'%s' specified." % self,
                               OfflineImapError.ERROR.REPO)
Ejemplo n.º 21
0
    def _msgs_to_fetch(self, imapobj):
        """
        Determines sequence numbers of messages to be fetched.

        Message sequence numbers (MSNs) are more easily compacted
        into ranges which makes transactions slightly faster.

        Arguments:
        - imapobj: instance of IMAPlib

        Returns: range(s) for messages or None if no messages
        are to be fetched.

        """
        res_type, imapdata = imapobj.select(self.getfullname(), True, True)
        if imapdata == [None] or imapdata[0] == '0':
            # Empty folder, no need to populate message list
            return None

        # By default examine all messages in this folder
        msgsToFetch = '1:*'

        maxage = self.config.getdefaultint("Account %s" % self.accountname,
                                           "maxage", -1)
        maxsize = self.config.getdefaultint("Account %s" % self.accountname,
                                            "maxsize", -1)

        # Build search condition
        if (maxage != -1) | (maxsize != -1):
            search_cond = "("

            if (maxage != -1):
                #find out what the oldest message is that we should look at
                oldest_struct = time.gmtime(time.time() -
                                            (60 * 60 * 24 * maxage))
                if oldest_struct[0] < 1900:
                    raise OfflineImapError(
                        "maxage setting led to year %d. "
                        "Abort syncing." % oldest_struct[0],
                        OfflineImapError.ERROR.REPO)
                search_cond += "SINCE %02d-%s-%d" % (
                    oldest_struct[2], MonthNames[oldest_struct[1]],
                    oldest_struct[0])

            if (maxsize != -1):
                if (maxage != -1):  # There are two conditions, add space
                    search_cond += " "
                search_cond += "SMALLER %d" % maxsize

            search_cond += ")"

            res_type, res_data = imapobj.search(None, search_cond)
            if res_type != 'OK':
                raise OfflineImapError(
                    "SEARCH in folder [%s]%s failed. "
                    "Search string was '%s'. Server responded '[%s] %s'" %
                    (self.getrepository(), self, search_cond, res_type,
                     res_data), OfflineImapError.ERROR.FOLDER)

            # Resulting MSN are separated by space, coalesce into ranges
            msgsToFetch = imaputil.uid_sequence(res_data[0].split())

        return msgsToFetch
Ejemplo n.º 22
0
def syncfolder(account, remotefolder, quick):
    """Synchronizes given remote folder for the specified account.

    Filtered folders on the remote side will not invoke this function."""
    def check_uid_validity(localfolder, remotefolder, statusfolder):
        # If either the local or the status folder has messages and
        # there is a UID validity problem, warn and abort.  If there are
        # no messages, UW IMAPd loses UIDVALIDITY.  But we don't really
        # need it if both local folders are empty.  So, in that case,
        # just save it off.
        if localfolder.getmessagecount() > 0 or statusfolder.getmessagecount(
        ) > 0:
            if not localfolder.check_uidvalidity():
                ui.validityproblem(localfolder)
                localfolder.repository.restore_atime()
                return
            if not remotefolder.check_uidvalidity():
                ui.validityproblem(remotefolder)
                localrepos.restore_atime()
                return
        else:
            # Both folders empty, just save new UIDVALIDITY
            localfolder.save_uidvalidity()
            remotefolder.save_uidvalidity()

    def save_min_uid(folder, min_uid):
        uidfile = folder.get_min_uid_file()
        fd = open(uidfile, 'wt')
        fd.write(str(min_uid) + "\n")
        fd.close()

    def cachemessagelists_upto_date(localfolder, remotefolder, date):
        """ Returns messages with uid > min(uids of messages newer than date)."""

        localfolder.cachemessagelist(min_date=date)
        check_uid_validity(localfolder, remotefolder, statusfolder)
        # local messagelist had date restriction applied already. Restrict
        # sync to messages with UIDs >= min_uid from this list.
        #
        # local messagelist might contain new messages (with uid's < 0).
        positive_uids = [
            uid for uid in localfolder.getmessageuidlist() if uid > 0
        ]
        if len(positive_uids) > 0:
            remotefolder.cachemessagelist(min_uid=min(positive_uids))
        else:
            # No messages with UID > 0 in range in localfolder.
            # date restriction was applied with respect to local dates but
            # remote folder timezone might be different from local, so be
            # safe and make sure the range isn't bigger than in local.
            remotefolder.cachemessagelist(
                min_date=time.gmtime(time.mktime(date) + 24 * 60 * 60))

    def cachemessagelists_startdate(new, partial, date):
        """ Retrieve messagelists when startdate has been set for
        the folder 'partial'.

        Idea: suppose you want to clone the messages after date in one
        account (partial) to a new one (new). If new is empty, then copy
        messages in partial newer than date to new, and keep track of the
        min uid. On subsequent syncs, sync all the messages in new against
        those after that min uid in partial. This is a partial replacement
        for maxage in the IMAP-IMAP sync case, where maxage doesn't work:
        the UIDs of the messages in localfolder might not be in the same
        order as those of corresponding messages in remotefolder, so if L in
        local corresponds to R in remote, the ranges [L, ...] and [R, ...]
        might not correspond. But, if we're cloning a folder into a new one,
        [min_uid, ...] does correspond to [1, ...].

        This is just for IMAP-IMAP. For Maildir-IMAP, use maxage instead.
        """

        new.cachemessagelist()
        min_uid = partial.retrieve_min_uid()
        if min_uid == None:  # min_uid file didn't exist
            if len(new.getmessageuidlist()) > 0:
                raise OfflineImapError(
                    "To use startdate on Repository %s, "
                    "Repository %s must be empty" %
                    (partial.repository.name, new.repository.name),
                    OfflineImapError.ERROR.MESSAGE)
            else:
                partial.cachemessagelist(min_date=date)
                # messagelist.keys() instead of getuidmessagelist() because in
                # the UID mapped case we want the actual local UIDs, not their
                # remote counterparts
                positive_uids = [
                    uid for uid in list(partial.messagelist.keys()) if uid > 0
                ]
                if len(positive_uids) > 0:
                    min_uid = min(positive_uids)
                else:
                    min_uid = 1
                save_min_uid(partial, min_uid)
        else:
            partial.cachemessagelist(min_uid=min_uid)

    remoterepos = account.remoterepos
    localrepos = account.localrepos
    statusrepos = account.statusrepos

    ui = getglobalui()
    ui.registerthread(account)
    try:
        # Load local folder.
        localfolder = account.get_local_folder(remotefolder)

        # Write the mailboxes
        mbnames.add(account.name, localfolder.getname(),
                    localrepos.getlocalroot())

        # Load status folder.
        statusfolder = statusrepos.getfolder(
            remotefolder.getvisiblename().replace(remoterepos.getsep(),
                                                  statusrepos.getsep()))
        statusfolder.openfiles()

        if localfolder.get_uidvalidity() == None:
            # This is a new folder, so delete the status cache to be
            # sure we don't have a conflict.
            # TODO: This does not work. We always return a value, need
            # to rework this...
            statusfolder.deletemessagelist()

        statusfolder.cachemessagelist()

        # Load local folder.
        ui.syncingfolder(remoterepos, remotefolder, localrepos, localfolder)

        # Retrieve messagelists, taking into account age-restriction
        # options
        maxage = localfolder.getmaxage()
        localstart = localfolder.getstartdate()
        remotestart = remotefolder.getstartdate()
        if (maxage != None) + (localstart != None) + (remotestart != None) > 1:
            six.reraise(
                OfflineImapError(
                    "You can set at most one of the "
                    "following: maxage, startdate (for the local folder), "
                    "startdate (for the remote folder)",
                    OfflineImapError.ERROR.REPO), None,
                exc_info()[2])
        if (maxage != None or localstart or remotestart) and quick:
            # IMAP quickchanged isn't compatible with options that
            # involve restricting the messagelist, since the "quick"
            # check can only retrieve a full list of UIDs in the folder.
            ui.warn("Quick syncs (-q) not supported in conjunction "
                    "with maxage or startdate; ignoring -q.")
        if maxage != None:
            cachemessagelists_upto_date(localfolder, remotefolder, maxage)
        elif localstart != None:
            cachemessagelists_startdate(remotefolder, localfolder, localstart)
            check_uid_validity(localfolder, remotefolder, statusfolder)
        elif remotestart != None:
            cachemessagelists_startdate(localfolder, remotefolder, remotestart)
            check_uid_validity(localfolder, remotefolder, statusfolder)
        else:
            localfolder.cachemessagelist()
            if quick:
                if (not localfolder.quickchanged(statusfolder)
                        and not remotefolder.quickchanged(statusfolder)):
                    ui.skippingfolder(remotefolder)
                    localrepos.restore_atime()
                    return
            check_uid_validity(localfolder, remotefolder, statusfolder)
            remotefolder.cachemessagelist()

        # Synchronize remote changes.
        if not localrepos.getconfboolean('readonly', False):
            ui.syncingmessages(remoterepos, remotefolder, localrepos,
                               localfolder)
            remotefolder.syncmessagesto(localfolder, statusfolder)
        else:
            ui.debug(
                '', "Not syncing to read-only repository '%s'" %
                localrepos.getname())

        # Synchronize local changes.
        if not remoterepos.getconfboolean('readonly', False):
            ui.syncingmessages(localrepos, localfolder, remoterepos,
                               remotefolder)
            localfolder.syncmessagesto(remotefolder, statusfolder)
        else:
            ui.debug(
                '', "Not syncing to read-only repository '%s'" %
                remoterepos.getname())

        statusfolder.save()
        localrepos.restore_atime()
    except (KeyboardInterrupt, SystemExit):
        raise
    except OfflineImapError as e:
        # bubble up severe Errors, skip folder otherwise
        if e.severity > OfflineImapError.ERROR.FOLDER:
            raise
        else:
            ui.error(e,
                     exc_info()[2],
                     msg="Aborting sync, folder '%s' "
                     "[acc: '%s']" % (localfolder, account))
    except Exception as e:
        ui.error(
            e,
            msg="ERROR in syncfolder for %s folder %s: %s" %
            (account, remotefolder.getvisiblename(), traceback.format_exc()))
    finally:
        for folder in ["statusfolder", "localfolder", "remotefolder"]:
            if folder in locals():
                locals()[folder].dropmessagelistcache()
        statusfolder.closefiles()
Ejemplo n.º 23
0
    def __init__(self, repos):
        """:repos: a IMAPRepository instance."""

        self.ui = getglobalui()
        self.repos = repos
        self.config = repos.getconfig()

        self.preauth_tunnel = repos.getpreauthtunnel()
        self.transport_tunnel = repos.gettransporttunnel()
        if self.preauth_tunnel and self.transport_tunnel:
            raise OfflineImapError(
                '%s: ' % repos + 'you must enable precisely one '
                'type of tunnel (preauth or transport), '
                'not both', OfflineImapError.ERROR.REPO)
        self.tunnel = \
            self.preauth_tunnel if self.preauth_tunnel \
            else self.transport_tunnel

        self.username = \
            None if self.preauth_tunnel else repos.getuser()
        self.user_identity = repos.get_remote_identity()
        self.authmechs = repos.get_auth_mechanisms()
        self.password = None
        self.passworderror = None
        self.goodpassword = None

        self.usessl = repos.getssl()
        self.useipv6 = repos.getipv6()
        if self.useipv6 is True:
            self.af = socket.AF_INET6
        elif self.useipv6 is False:
            self.af = socket.AF_INET
        else:
            self.af = socket.AF_UNSPEC
        self.hostname = None if self.transport_tunnel else repos.gethost()
        self.port = repos.getport()
        if self.port is None:
            self.port = 993 if self.usessl else 143
        self.sslclientcert = repos.getsslclientcert()
        self.sslclientkey = repos.getsslclientkey()
        self.sslcacertfile = repos.getsslcacertfile()
        if self.sslcacertfile is None:
            self.__verifycert = None  # Disable cert verification.
            # This way of working sucks hard...
        self.fingerprint = repos.get_ssl_fingerprint()
        self.tlslevel = repos.gettlslevel()
        self.sslversion = repos.getsslversion()
        self.starttls = repos.getstarttls()

        if self.tlslevel is not "tls_compat" and self.sslversion is None:
            raise Exception("When 'tls_version' is not 'tls_compat' "
                            "the 'ssl_version' must be set explicitly.")

        self.oauth2_refresh_token = repos.getoauth2_refresh_token()
        self.oauth2_access_token = repos.getoauth2_access_token()
        self.oauth2_client_id = repos.getoauth2_client_id()
        self.oauth2_client_secret = repos.getoauth2_client_secret()
        self.oauth2_request_url = repos.getoauth2_request_url()

        self.delim = None
        self.root = None
        self.maxconnections = repos.getmaxconnections()
        self.availableconnections = []
        self.assignedconnections = []
        self.lastowner = {}
        self.semaphore = BoundedSemaphore(self.maxconnections)
        self.connectionlock = Lock()
        self.reference = repos.getreference()
        self.idlefolders = repos.getidlefolders()
        self.gss_step = self.GSS_STATE_STEP
        self.gss_vc = None
        self.gssapi = False

        # In order to support proxy connection, we have to override the
        # default socket instance with our own socksified socket instance.
        # We add this option to bypass the GFW in China.
        self.proxied_socket = self._get_proxy('proxy', socket.socket)

        # Turns out that the GFW in China is no longer blocking imap.gmail.com
        # However accounts.google.com (for oauth2) definitey is.  Therefore
        # it is not strictly necessary to use a proxy for *both* IMAP *and*
        # oauth2, so a new option is added: authproxy.

        # Set proxy for use in authentication (only) if desired.
        # If not set, is same as proxy option (compatible with current configs)
        # To use a proxied_socket but not an authproxied_socket
        # set authproxy = '' in config
        self.authproxied_socket = self._get_proxy('authproxy',
                                                  self.proxied_socket)
Ejemplo n.º 24
0
    def __init__(self, repos):
        self.ui = getglobalui()
        self.repos = repos
        self.config = repos.getconfig()

        self.preauth_tunnel = repos.getpreauthtunnel()
        self.transport_tunnel = repos.gettransporttunnel()
        if self.preauth_tunnel and self.transport_tunnel:
            raise OfflineImapError(
                '%s: ' % repos + 'you must enable precisely one '
                'type of tunnel (preauth or transport), '
                'not both', OfflineImapError.ERROR.REPO)
        self.tunnel = \
            self.preauth_tunnel if self.preauth_tunnel \
            else self.transport_tunnel

        self.username = \
            None if self.preauth_tunnel else repos.getuser()
        self.user_identity = repos.get_remote_identity()
        self.authmechs = repos.get_auth_mechanisms()
        self.password = None
        self.passworderror = None
        self.goodpassword = None

        self.usessl = repos.getssl()
        self.hostname = \
            None if self.preauth_tunnel else repos.gethost()
        self.port = repos.getport()
        if self.port == None:
            self.port = 993 if self.usessl else 143
        self.sslclientcert = repos.getsslclientcert()
        self.sslclientkey = repos.getsslclientkey()
        self.sslcacertfile = repos.getsslcacertfile()
        if self.sslcacertfile is None:
            self.__verifycert = None  # disable cert verification
        self.fingerprint = repos.get_ssl_fingerprint()
        self.sslversion = repos.getsslversion()
        self.tlslevel = repos.gettlslevel()

        self.oauth2_refresh_token = repos.getoauth2_refresh_token()
        self.oauth2_access_token = repos.getoauth2_access_token()
        self.oauth2_client_id = repos.getoauth2_client_id()
        self.oauth2_client_secret = repos.getoauth2_client_secret()
        self.oauth2_request_url = repos.getoauth2_request_url()

        self.delim = None
        self.root = None
        self.maxconnections = repos.getmaxconnections()
        self.availableconnections = []
        self.assignedconnections = []
        self.lastowner = {}
        self.semaphore = BoundedSemaphore(self.maxconnections)
        self.connectionlock = Lock()
        self.reference = repos.getreference()
        self.idlefolders = repos.getidlefolders()
        self.gss_step = self.GSS_STATE_STEP
        self.gss_vc = None
        self.gssapi = False

        # In order to support proxy connection, we have to override the
        # default socket instance with our own socksified socket instance.
        # We add this option to bypass the GFW in China.
        _account_section = 'Account ' + self.repos.account.name
        if not self.config.has_option(_account_section, 'proxy'):
            self.proxied_socket = socket.socket
        else:
            proxy = self.config.get(_account_section, 'proxy')
            # Powered by PySocks.
            try:
                import socks
                proxy_type, host, port = proxy.split(":")
                port = int(port)
                socks.setdefaultproxy(getattr(socks, proxy_type), host, port)
                self.proxied_socket = socks.socksocket
            except ImportError:
                self.ui.warn("PySocks not installed, ignoring proxy option.")
                self.proxied_socket = socket.socket
            except (AttributeError, ValueError) as e:
                self.ui.warn("Bad proxy option %s for account %s: %s "
                             "Ignoring proxy option." %
                             (proxy, self.repos.account.name, e))
                self.proxied_socket = socket.socket
Ejemplo n.º 25
0
    def getpassword(self):
        """Return the IMAP password for this repository.

        It tries to get passwords in the following order:

        1. evaluate Repository 'remotepasseval'
        2. read password from Repository 'remotepass'
        3. read password from file specified in Repository 'remotepassfile'
        4. read password from ~/.netrc
        5. read password from /etc/netrc

        On success we return the password.
        If all strategies fail we return None."""

        # 1. Evaluate Repository 'remotepasseval'.
        passwd = self.getconf('remotepasseval', None)
        if passwd is not None:
            l_pass = self.localeval.eval(passwd)

            # We need a str password
            if isinstance(l_pass, bytes):
                return l_pass.decode(encoding='utf-8')
            elif isinstance(l_pass, str):
                return l_pass

            # If is not bytes or str, we have a problem
            raise OfflineImapError(
                "Could not get a right password format for"
                " repository %s. Type found: %s. "
                "Please, open a bug." % (self.name, type(l_pass)),
                OfflineImapError.ERROR.FOLDER)

        # 2. Read password from Repository 'remotepass'.
        password = self.getconf('remotepass', None)
        if password is not None:
            # Assume the configuration file to be UTF-8 encoded so we must not
            # encode this string again.
            return password
        # 3. Read password from file specified in Repository 'remotepassfile'.
        passfile = self.getconf('remotepassfile', None)
        if passfile is not None:
            file_desc = open(os.path.expanduser(passfile),
                             'r',
                             encoding='utf-8')
            password = file_desc.readline().strip()
            file_desc.close()
            return password.encode('UTF-8')
        # 4. Read password from ~/.netrc.
        try:
            netrcentry = netrc.netrc().authenticators(self.gethost())
        except IOError as inst:
            if inst.errno != errno.ENOENT:
                raise
        else:
            if netrcentry:
                user = self.getuser()
                if user is None or user == netrcentry[0]:
                    return netrcentry[2]
        # 5. Read password from /etc/netrc.
        try:
            netrcentry = netrc.netrc('/etc/netrc')\
                .authenticators(self.gethost())
        except IOError as inst:
            if inst.errno not in (errno.ENOENT, errno.EACCES):
                raise
        else:
            if netrcentry:
                user = self.getuser()
                if user is None or user == netrcentry[0]:
                    return netrcentry[2]
        # No strategy yielded a password!
        return None
Ejemplo n.º 26
0
    def __authn_helper(self, imapobj):
        """Authentication machinery for self.acquireconnection().

        Raises OfflineImapError() of type ERROR.REPO when
        there are either fatal problems or no authentications
        succeeded.

        If any authentication method succeeds, routine should exit:
        warnings for failed methods are to be produced in the
        respective except blocks."""

        # Stack stores pairs of (method name, exception)
        exc_stack = []
        tried_to_authn = False
        tried_tls = False
        # Authentication routines, hash keyed by method name
        # with value that is a tuple with
        # - authentication function,
        # - tryTLS flag,
        # - check IMAP capability flag.
        auth_methods = {
            "GSSAPI": (self.__authn_gssapi, False, True),
            "XOAUTH2": (self.__authn_xoauth2, True, True),
            "CRAM-MD5": (self.__authn_cram_md5, True, True),
            "PLAIN": (self.__authn_plain, True, True),
            "LOGIN": (self.__authn_login, True, False),
        }

        # GSSAPI is tried first by default: we will probably go TLS after it and
        # GSSAPI mustn't be tunneled over TLS.
        for m in self.authmechs:
            if m not in auth_methods:
                raise Exception("Bad authentication method %s, "
                                "please, file OfflineIMAP bug" % m)

            func, tryTLS, check_cap = auth_methods[m]

            # TLS must be initiated before checking capabilities:
            # they could have been changed after STARTTLS.
            if tryTLS and self.starttls and not tried_tls:
                tried_tls = True
                self.__start_tls(imapobj)

            if check_cap:
                cap = "AUTH=" + m
                if cap not in imapobj.capabilities:
                    continue

            tried_to_authn = True
            self.ui.debug('imap', 'Attempting ' '%s authentication' % m)
            try:
                if func(imapobj):
                    return
            except (imapobj.error, OfflineImapError) as e:
                self.ui.warn('%s authentication failed: %s' % (m, e))
                exc_stack.append((m, e))

        if len(exc_stack):
            msg = "\n\t".join(
                [": ".join((x[0], str(x[1]))) for x in exc_stack])
            raise OfflineImapError(
                "All authentication types "
                "failed:\n\t%s" % msg, OfflineImapError.ERROR.REPO)

        if not tried_to_authn:
            methods = ", ".join([
                x[5:] for x in
                [x for x in imapobj.capabilities if x[0:5] == "AUTH="]
            ])
            raise OfflineImapError(
                "Repository %s: no supported "
                "authentication mechanisms found; configured %s, "
                "server advertises %s" %
                (self.repos, ", ".join(self.authmechs), methods),
                OfflineImapError.ERROR.REPO)
Ejemplo n.º 27
0
    def _fetch_from_imap(self, uids, retry_num=1):
        """Fetches data from IMAP server.

        Arguments:
        - uids: message UIDS (OfflineIMAP3: First UID returned only)
        - retry_num: number of retries to make

        Returns: data obtained by this query."""

        imapobj = self.imapserver.acquireconnection()
        try:
            query = "(%s)" % (" ".join(self.imap_query))
            fails_left = retry_num  # Retry on dropped connection.
            while fails_left:
                try:
                    imapobj.select(self.getfullIMAPname(), readonly=True)
                    res_type, data = imapobj.uid('fetch', uids, query)
                    break
                except imapobj.abort as e:
                    fails_left -= 1
                    # self.ui.error() will show the original traceback.
                    if fails_left <= 0:
                        message = ("%s, while fetching msg %r in folder %r."
                                   " Max retry reached (%d)" %
                                   (e, uids, self.name, retry_num))
                        raise OfflineImapError(message,
                                               OfflineImapError.ERROR.MESSAGE)
                    self.ui.error("%s. While fetching msg %r in folder %r."
                                  " Query: %s Retrying (%d/%d)" %
                                  (e, uids, self.name, query,
                                   retry_num - fails_left, retry_num))
                    # Release dropped connection, and get a new one.
                    self.imapserver.releaseconnection(imapobj, True)
                    imapobj = self.imapserver.acquireconnection()
        finally:
            # The imapobj here might be different than the one created before
            # the ``try`` clause. So please avoid transforming this to a nice
            # ``with`` without taking this into account.
            self.imapserver.releaseconnection(imapobj)

        # Ensure to not consider unsolicited FETCH responses caused by flag
        # changes from concurrent connections.  These appear as strings in
        # 'data' (the BODY response appears as a tuple).  This should leave
        # exactly one response.
        if res_type == 'OK':
            data = [res for res in data if not isinstance(res, bytes)]

        # Could not fetch message.  Note: it is allowed by rfc3501 to return any
        # data for the UID FETCH command.
        if data == [None] or res_type != 'OK' or len(data) != 1:
            severity = OfflineImapError.ERROR.MESSAGE
            reason = "IMAP server '%s' failed to fetch messages UID '%s'. " \
                     "Server responded: %s %s" % (self.getrepository(), uids,
                                                  res_type, data)
            if data == [None] or len(data) < 1:
                # IMAP server did not find a message with this UID.
                reason = "IMAP server '%s' does not have a message " \
                         "with UID '%s'" % (self.getrepository(), uids)
            raise OfflineImapError(reason, severity)

        # JI: In offlineimap, this function returned a tuple of strings for each
        # fetched UID, offlineimap3 calls to the imap object return bytes and so
        # originally a fixed, utf-8 conversion was done and *only* the first
        # response (d[0]) was returned.  Note that this alters the behavior
        # between code bases.  However, it seems like a single UID is the intent
        # of this function so retaining the modfication here for now.
        #
        # TODO: Can we assume the server response containing the meta data is
        # always 'utf-8' encoded?  Assuming yes for now.
        #
        # Convert responses, d[0][0], into a 'utf-8' string (from bytes) and
        # Convert email, d[0][1], into a message object (from bytes)

        ndata0 = data[0][0].decode('utf-8')
        try:
            ndata1 = self.parser['8bit-RFC'].parsebytes(data[0][1])
        except:
            err = exc_info()
            response_type = type(data[0][1]).__name__
            msg_id = self._extract_message_id(data[0][1])[0].decode(
                'ascii', errors='surrogateescape')
            raise OfflineImapError(
                "Exception parsing message with ID ({}) from imaplib (response type: {}).\n {}: {}"
                .format(msg_id, response_type, err[0].__name__,
                        err[1]), OfflineImapError.ERROR.MESSAGE)
        if len(ndata1.defects) > 0:
            # We don't automatically apply fixes as to attempt to preserve the original message
            self.ui.warn("UID {} has defects: {}".format(uids, ndata1.defects))
            if any(
                    isinstance(defect, NoBoundaryInMultipartDefect)
                    for defect in ndata1.defects):
                # (Hopefully) Rare defect from a broken client where multipart boundary is
                # not properly quoted.  Attempt to solve by fixing the boundary and parsing
                self.ui.warn(" ... applying multipart boundary fix.")
                ndata1 = self.parser['8bit-RFC'].parsebytes(
                    self._quote_boundary_fix(data[0][1]))
            try:
                # See if the defects after fixes are preventing us from obtaining bytes
                _ = ndata1.as_bytes(policy=self.policy['8bit-RFC'])
            except UnicodeEncodeError as err:
                # Unknown issue which is causing failure of as_bytes()
                msg_id = self.getmessageheader(ndata1, "message-id")
                if msg_id is None:
                    msg_id = '<Unknown Message-ID>'
                raise OfflineImapError(
                    "UID {} ({}) has defects preventing it from being processed!\n  {}: {}"
                    .format(uids, msg_id,
                            type(err).__name__,
                            err), OfflineImapError.ERROR.MESSAGE)
        ndata = [ndata0, ndata1]

        return ndata