def test_01_imapsplit(self): """Test imaputil.imapsplit()""" res = imaputil.imapsplit(b'(\\HasNoChildren) "." "INBOX.Sent"') self.assertEqual(res, [b'(\\HasNoChildren)', b'"."', b'"INBOX.Sent"']) res = imaputil.imapsplit(b'"mo\\" o" sdfsdf') self.assertEqual(res, [b'"mo\\" o"', b'sdfsdf'])
def _messagelabels_aux(self, arg, uidlist, labels): """Common code to savemessagelabels and addmessagelabels""" labels = labels - self.ignorelabels uidlist = [uid for uid in uidlist if uid > 0] if len(uidlist) > 0: imapobj = self.imapserver.acquireconnection() try: labels_str = '(' + ' '.join( [imaputil.quote(lb) for lb in labels]) + ')' # Coalesce uid's into ranges uid_str = imaputil.uid_sequence(uidlist) result = self._store_to_imap(imapobj, uid_str, arg, labels_str) except imapobj.readonly: self.ui.labelstoreadonly(self, uidlist, labels) return None finally: self.imapserver.releaseconnection(imapobj) if result: retlabels = imaputil.flags2hash( imaputil.imapsplit(result)[1])['X-GM-LABELS'] retlabels = set([ imaputil.dequote(lb) for lb in imaputil.imapsplit(retlabels) ]) return retlabels return None
def savemessageflags(self, uid, flags): """Change a message's flags to `flags`. Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" imapobj = self.imapserver.acquireconnection() try: try: imapobj.select(self.getfullname()) except imapobj.readonly: self.ui.flagstoreadonly(self, [uid], flags) return result = imapobj.uid('store', '%d' % uid, 'FLAGS', imaputil.flagset2flagstring(flags)) assert result[0] == 'OK', 'Error with store: ' + '. '.join( result[1]) finally: self.imapserver.releaseconnection(imapobj) result = result[1][0] if not result: self.messagelist[uid]['flags'] = flags else: flags = imaputil.flags2hash(imaputil.imapsplit(result)[1])['FLAGS'] self.messagelist[uid]['flags'] = imaputil.flagstring2flagset(flags)
def cachemessagelist(self): if not self.synclabels: return super(GmailFolder, self).cachemessagelist() self.messagelist = {} self.ui.collectingdata(None, self) imapobj = self.imapserver.acquireconnection() try: msgsToFetch = self._msgs_to_fetch(imapobj) if not msgsToFetch: return # No messages to sync # Get the flags and UIDs for these. single-quotes prevent # imaplib2 from quoting the sequence. # # NB: msgsToFetch are sequential numbers, not UID's res_type, response = imapobj.fetch("'%s'" % msgsToFetch, '(FLAGS X-GM-LABELS UID)') if res_type != 'OK': raise OfflineImapError("FETCHING UIDs in folder [%s]%s failed. " % \ (self.getrepository(), self) + \ "Server responded '[%s] %s'" % \ (res_type, response), OfflineImapError.ERROR.FOLDER), \ None, exc_info()[2] finally: self.imapserver.releaseconnection(imapobj) for messagestr in response: # looks like: '1 (FLAGS (\\Seen Old) X-GM-LABELS (\\Inbox \\Favorites) UID 4807)' or None if no msg # Discard initial message number. if messagestr == None: continue messagestr = messagestr.split(' ', 1)[1] options = imaputil.flags2hash(messagestr) if not 'UID' in options: self.ui.warn('No UID in message with options %s' %\ str(options), minor = 1) else: uid = long(options['UID']) self.messagelist[uid] = self.msglist_item_initializer(uid) flags = imaputil.flagsimap2maildir(options['FLAGS']) m = re.search('\(([^\)]*)\)', options['X-GM-LABELS']) if m: labels = set([ imaputil.dequote(lb) for lb in imaputil.imapsplit(m.group(1)) ]) else: labels = set() labels = labels - self.ignorelabels rtime = imaplibutil.Internaldate2epoch(messagestr) self.messagelist[uid] = { 'uid': uid, 'flags': flags, 'labels': labels, 'time': rtime }
def cachemessagelist(self, min_date=None, min_uid=None): if not self.synclabels: return super(GmailFolder, self).cachemessagelist( min_date=min_date, min_uid=min_uid) self.dropmessagelistcache() self.ui.collectingdata(None, self) imapobj = self.imapserver.acquireconnection() try: msgsToFetch = self._msgs_to_fetch( imapobj, min_date=min_date, min_uid=min_uid) if not msgsToFetch: return # No messages to sync # Get the flags and UIDs for these. single-quotes prevent # imaplib2 from quoting the sequence. # # NB: msgsToFetch are sequential numbers, not UID's res_type, response = imapobj.fetch("'%s'"% msgsToFetch, '(FLAGS X-GM-LABELS UID)') if res_type != 'OK': six.reraise(OfflineImapError, OfflineImapError( "FETCHING UIDs in folder [%s]%s failed. "% (self.getrepository(), self) + "Server responded '[%s] %s'"% (res_type, response), OfflineImapError.ERROR.FOLDER), exc_info()[2]) finally: self.imapserver.releaseconnection(imapobj) for messagestr in response: # looks like: '1 (FLAGS (\\Seen Old) X-GM-LABELS (\\Inbox \\Favorites) UID 4807)' or None if no msg # Discard initial message number. if messagestr == None: continue messagestr = messagestr.split(' ', 1)[1] # e.g.: {'X-GM-LABELS': '("Webserver (RW.net)" "\\Inbox" GInbox)', 'FLAGS': '(\\Seen)', 'UID': '275440'} options = imaputil.flags2hash(messagestr) if not 'UID' in options: self.ui.warn('No UID in message with options %s' %\ str(options), minor = 1) else: uid = int(options['UID']) self.messagelist[uid] = self.msglist_item_initializer(uid) flags = imaputil.flagsimap2maildir(options['FLAGS']) # e.g.: '("Webserver (RW.net)" "\\Inbox" GInbox)' m = re.search('^[(](.*)[)]', options['X-GM-LABELS']) if m: labels = set([imaputil.dequote(lb) for lb in imaputil.imapsplit(m.group(1))]) else: labels = set() labels = labels - self.ignorelabels rtime = imaplibutil.Internaldate2epoch(messagestr) self.messagelist[uid] = {'uid': uid, 'flags': flags, 'labels': labels, 'time': rtime}
def getfolders(self): if self.folders != None: return self.folders retval = [] imapobj = self.imapserver.acquireconnection() # check whether to list all folders, or subscribed only listfunction = imapobj.list if self.getconfboolean('subscribedonly', False): listfunction = imapobj.lsub try: listresult = listfunction(directory=self.imapserver.reference)[1] finally: self.imapserver.releaseconnection(imapobj) for string in listresult: if string == None or \ (type(string) == types.StringType and string == ''): # Bug in imaplib: empty strings in results from # literals. continue flags, delim, name = imaputil.imapsplit(string) flaglist = [x.lower() for x in imaputil.flagsplit(flags)] if '\\noselect' in flaglist: continue foldername = imaputil.dequote(name) retval.append(self.getfoldertype()(self.imapserver, foldername, self)) # filter out the folder? if not self.folderfilter(foldername): self.ui.debug( 'imap', "Filtering out '%s'[%s] due to folderfilt" "er" % (foldername, self)) retval[-1].sync_this = False # Add all folderincludes if len(self.folderincludes): imapobj = self.imapserver.acquireconnection() try: for foldername in self.folderincludes: try: imapobj.select(foldername, readonly=True) except OfflineImapError, e: # couldn't select this folderinclude, so ignore folder. if e.severity > OfflineImapError.ERROR.FOLDER: raise self.ui.error(e, exc_info()[2], 'Invalid folderinclude:') continue retval.append(self.getfoldertype()(self.imapserver, foldername, self)) finally: self.imapserver.releaseconnection(imapobj) retval.sort(lambda x, y: self.foldersort(x.getvisiblename(), y.getvisiblename())) self.folders = retval return self.folders
def getfolders(self): if self.folders != None: return self.folders retval = [] imapobj = self.imapserver.acquireconnection() # check whether to list all folders, or subscribed only listfunction = imapobj.list if self.config.has_option(self.getsection(), 'subscribedonly'): if self.getconf('subscribedonly') == "yes": listfunction = imapobj.lsub try: listresult = listfunction(directory = self.imapserver.reference)[1] finally: self.imapserver.releaseconnection(imapobj) for string in listresult: if string == None or \ (type(string) == types.StringType and string == ''): # Bug in imaplib: empty strings in results from # literals. continue flags, delim, name = imaputil.imapsplit(string) flaglist = [x.lower() for x in imaputil.flagsplit(flags)] if '\\noselect' in flaglist: continue foldername = imaputil.dequote(name) if not self.folderfilter(foldername): self.ui.debug('imap',"Filtering out '%s' due to folderfilter" %\ foldername) continue retval.append(self.getfoldertype()(self.imapserver, foldername, self.nametrans(foldername), self.accountname, self)) if len(self.folderincludes): imapobj = self.imapserver.acquireconnection() try: for foldername in self.folderincludes: try: imapobj.select(foldername, readonly = 1) except OfflineImapError, e: # couldn't select this folderinclude, so ignore folder. if e.severity > OfflineImapError.ERROR.FOLDER: raise self.ui.error(e, exc_info()[2], 'Invalid folderinclude:') continue retval.append(self.getfoldertype()(self.imapserver, foldername, self.nametrans(foldername), self.accountname, self)) finally: self.imapserver.releaseconnection(imapobj) retval.sort(lambda x, y: self.foldersort(x.getvisiblename(), y.getvisiblename())) self.folders = retval return retval
def processmessagesflags(self, operation, uidlist, flags): # XXX: the imapobj.myrights(...) calls dies with an error # report from Gmail server stating that IMAP command # 'MYRIGHTS' is not implemented. So, this # `processmessagesflags` is just a copy from `IMAPFolder`, # with the references to `imapobj.myrights()` deleted This # shouldn't hurt, however, Gmail users always have full # control over all their mailboxes (apparently). if len(uidlist) > 101: # Hack for those IMAP ervers with a limited line length self.processmessagesflags(operation, uidlist[:100], flags) self.processmessagesflags(operation, uidlist[100:], flags) return imapobj = self.imapserver.acquireconnection() try: imapobj.select(self.getfullname()) r = imapobj.uid('store', imaputil.listjoin(uidlist), operation + 'FLAGS', imaputil.flagsmaildir2imap(flags)) assert r[0] == 'OK', 'Error with store: ' + '. '.join(r[1]) r = r[1] finally: self.imapserver.releaseconnection(imapobj) needupdate = copy(uidlist) for result in r: if result == None: # Compensate for servers that don't return anything from # STORE. continue attributehash = imaputil.flags2hash(imaputil.imapsplit(result)[1]) if not ('UID' in attributehash and 'FLAGS' in attributehash): # Compensate for servers that don't return a UID attribute. continue flags = attributehash['FLAGS'] uid = long(attributehash['UID']) self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags) try: needupdate.remove(uid) except ValueError: # Let it slide if it's not in the list pass for uid in needupdate: if operation == '+': for flag in flags: if not flag in self.messagelist[uid]['flags']: self.messagelist[uid]['flags'].append(flag) self.messagelist[uid]['flags'].sort() elif operation == '-': for flag in flags: if flag in self.messagelist[uid]['flags']: self.messagelist[uid]['flags'].remove(flag)
def processmessagesflags(self, operation, uidlist, flags): if len(uidlist) > 101: # Hack for those IMAP ervers with a limited line length self.processmessagesflags(operation, uidlist[:100], flags) self.processmessagesflags(operation, uidlist[100:], flags) return imapobj = self.imapserver.acquireconnection() try: try: imapobj.select(self.getfullname()) except imapobj.readonly: UIBase.getglobalui().flagstoreadonly(self, uidlist, flags) return r = imapobj.uid('store', imaputil.listjoin(uidlist), operation + 'FLAGS', imaputil.flagsmaildir2imap(flags)) assert r[0] == 'OK', 'Error with store: ' + '. '.join(r[1]) r = r[1] finally: self.imapserver.releaseconnection(imapobj) # Some IMAP servers do not always return a result. Therefore, # only update the ones that it talks about, and manually fix # the others. needupdate = copy(uidlist) for result in r: if result == None: # Compensate for servers that don't return anything from # STORE. continue attributehash = imaputil.flags2hash(imaputil.imapsplit(result)[1]) if not ('UID' in attributehash and 'FLAGS' in attributehash): # Compensate for servers that don't return a UID attribute. continue lflags = attributehash['FLAGS'] uid = long(attributehash['UID']) self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(lflags) try: needupdate.remove(uid) except ValueError: # Let it slide if it's not in the list pass for uid in needupdate: if operation == '+': for flag in flags: if not flag in self.messagelist[uid]['flags']: self.messagelist[uid]['flags'].append(flag) self.messagelist[uid]['flags'].sort() elif operation == '-': for flag in flags: if flag in self.messagelist[uid]['flags']: self.messagelist[uid]['flags'].remove(flag)
def getfolders(self): if self.folders != None: return self.folders retval = [] imapobj = self.imapserver.acquireconnection() # check whether to list all folders, or subscribed only listfunction = imapobj.list if self.config.has_option(self.getsection(), 'subscribedonly'): if self.getconf('subscribedonly') == "yes": listfunction = imapobj.lsub try: listresult = listfunction(directory=self.imapserver.reference)[1] finally: self.imapserver.releaseconnection(imapobj) for string in listresult: if string == None or \ (type(string) == types.StringType and string == ''): # Bug in imaplib: empty strings in results from # literals. continue flags, delim, name = imaputil.imapsplit(string) flaglist = [x.lower() for x in imaputil.flagsplit(flags)] if '\\noselect' in flaglist: continue foldername = imaputil.dequote(name) if not self.folderfilter(foldername): self.ui.debug('imap',"Filtering out '%s' due to folderfilter" %\ foldername) continue retval.append(self.getfoldertype()(self.imapserver, foldername, self.nametrans(foldername), self.accountname, self)) if len(self.folderincludes): imapobj = self.imapserver.acquireconnection() try: for foldername in self.folderincludes: try: imapobj.select(foldername, readonly=1) except ValueError: continue retval.append( self.getfoldertype()(self.imapserver, foldername, self.nametrans(foldername), self.accountname, self)) finally: self.imapserver.releaseconnection(imapobj) retval.sort(lambda x, y: self.foldersort(x.getvisiblename(), y.getvisiblename())) self.folders = retval return retval
def cachemessagelist(self): if not self.synclabels: return super(GmailFolder, self).cachemessagelist() self.ui.collectingdata(None, self) self.messagelist = {} imapobj = self.imapserver.acquireconnection() try: msgsToFetch = self._msgs_to_fetch(imapobj) if not msgsToFetch: return # No messages to sync # Get the flags and UIDs for these. single-quotes prevent # imaplib2 from quoting the sequence. # Note: msgsToFetch are sequential numbers, not UID's res_type, response = imapobj.fetch("'%s'" % msgsToFetch, '(FLAGS X-GM-LABELS UID)') if res_type != 'OK': raise OfflineImapError("FETCHING UIDs in folder [%s]%s failed. " "Server responded '[%s] %s'" % ( self.getrepository(), self, res_type, response), OfflineImapError.ERROR.FOLDER) finally: self.imapserver.releaseconnection(imapobj) for messagestr in response: # looks like: '1 (FLAGS (\\Seen Old) UID 4807)' or None if no msg # Discard initial message number. if messagestr == None: continue messagestr = messagestr.split(' ', 1)[1] options = imaputil.flags2hash(messagestr) if not 'UID' in options: self.ui.warn('No UID in message with options %s' %\ str(options), minor = 1) else: uid = long(options['UID']) flags = imaputil.flagsimap2maildir(options['FLAGS']) m = re.search('\(([^\)]*)\)', options['X-GM-LABELS']) if m: labels = set([imaputil.dequote(lb) for lb in imaputil.imapsplit(m.group(1))]) else: labels = set() labels = labels - self.ignorelabels rtime = imaplibutil.Internaldate2epoch(messagestr) self.messagelist[uid] = {'uid': uid, 'flags': flags, 'labels': labels, 'time': rtime}
def cachemessagelist(self): if not self.synclabels: return super(GmailFolder, self).cachemessagelist() self.ui.collectingdata(None, self) self.messagelist = {} imapobj = self.imapserver.acquireconnection() try: msgsToFetch = self._msgs_to_fetch(imapobj) if not msgsToFetch: return # No messages to sync # Get the flags and UIDs for these. single-quotes prevent # imaplib2 from quoting the sequence. data = self._fetch_from_imap(imapobj, "'%s'" % msgsToFetch, '(FLAGS X-GM-LABELS UID)') finally: self.imapserver.releaseconnection(imapobj) for messagestr in data: # looks like: '1 (FLAGS (\\Seen Old) UID 4807)' or None if no msg # Discard initial message number. if messagestr == None: continue messagestr = messagestr.split(' ', 1)[1] options = imaputil.flags2hash(messagestr) if not 'UID' in options: self.ui.warn('No UID in message with options %s' %\ str(options), minor = 1) else: uid = long(options['UID']) flags = imaputil.flagsimap2maildir(options['FLAGS']) m = re.search('\(([^\)]*)\)', options['X-GM-LABELS']) if m: labels = set([ imaputil.dequote(lb) for lb in imaputil.imapsplit(m.group(1)) ]) else: labels = set() labels = labels - self.ignorelabels rtime = imaplibutil.Internaldate2epoch(messagestr) self.messagelist[uid] = { 'uid': uid, 'flags': flags, 'labels': labels, 'time': rtime }
def getmessage(self, uid): """Retrieve message with UID from the IMAP server (incl body). Also gets Gmail labels and embeds them into the message. :returns: the message body or throws and OfflineImapError (probably severity MESSAGE) if e.g. no message with this UID could be found. """ imapobj = self.imapserver.acquireconnection() try: data = self._fetch_from_imap(imapobj, str(uid), 2) finally: self.imapserver.releaseconnection(imapobj) # data looks now e.g. #[('320 (X-GM-LABELS (...) UID 17061 BODY[] {2565}','msgbody....')] # we only asked for one message, and that msg is in data[0]. # msbody is in [0][1]. body = data[0][1].replace("\r\n", "\n") # Embed the labels into the message headers if self.synclabels: m = re.search('X-GM-LABELS\s*\(([^\)]*)\)', data[0][0]) if m: labels = set([ imaputil.dequote(lb) for lb in imaputil.imapsplit(m.group(1)) ]) else: labels = set() labels = labels - self.ignorelabels labels_str = imaputil.format_labels_string(self.labelsheader, sorted(labels)) # First remove old label headers that may be in the message content retrieved # from gmail Then add a labels header with current gmail labels. body = self.deletemessageheaders(body, self.labelsheader) body = self.addmessageheader(body, '\n', self.labelsheader, labels_str) if len(body) > 200: dbg_output = "%s...%s" % (str(body)[:150], str(body)[-50:]) else: dbg_output = body self.ui.debug( 'imap', "Returned object from fetching %d: '%s'" % (uid, dbg_output)) return body
def _messagelabels_aux(self, arg, uidlist, labels): """Common code to savemessagelabels and addmessagelabels""" labels = labels - self.ignorelabels uidlist = [uid for uid in uidlist if uid > 0] if len(uidlist) > 0: imapobj = self.imapserver.acquireconnection() try: labels_str = '(' + ' '.join([imaputil.quote(lb) for lb in labels]) + ')' # Coalesce uid's into ranges uid_str = imaputil.uid_sequence(uidlist) result = self._store_to_imap(imapobj, uid_str, arg, labels_str) except imapobj.readonly: self.ui.labelstoreadonly(self, uidlist, data) return None finally: self.imapserver.releaseconnection(imapobj) if result: retlabels = imaputil.flags2hash(imaputil.imapsplit(result)[1])['X-GM-LABELS'] retlabels = set([imaputil.dequote(lb) for lb in imaputil.imapsplit(retlabels)]) return retlabels return None
def getfolders(self): if self.folders != None: return self.folders retval = [] imapobj = self.imapserver.acquireconnection() # check whether to list all folders, or subscribed only listfunction = imapobj.list if self.config.has_option(self.getsection(), 'subscribedonly'): if self.getconf('subscribedonly') == "yes": listfunction = imapobj.lsub try: listresult = listfunction(directory = self.imapserver.reference)[1] finally: self.imapserver.releaseconnection(imapobj) for string in listresult: if string == None or \ (type(string) == types.StringType and string == ''): # Bug in imaplib: empty strings in results from # literals. continue flags, delim, name = imaputil.imapsplit(string) flaglist = [x.lower() for x in imaputil.flagsplit(flags)] if '\\noselect' in flaglist: continue foldername = imaputil.dequote(name) if not self.folderfilter(foldername): continue retval.append(self.getfoldertype()(self.imapserver, foldername, self.nametrans(foldername), self.accountname, self)) if len(self.folderincludes): imapobj = self.imapserver.acquireconnection() try: for foldername in self.folderincludes: try: imapobj.select(foldername, readonly = 1) except ValueError: continue retval.append(self.getfoldertype()(self.imapserver, foldername, self.nametrans(foldername), self.accountname, self)) finally: self.imapserver.releaseconnection(imapobj) retval.sort(lambda x, y: self.foldersort(x.getvisiblename(), y.getvisiblename())) self.folders = retval return retval
def getmessage(self, uid): """Retrieve message with UID from the IMAP server (incl body). Also gets Gmail labels and embeds them into the message. :returns: the message body or throws and OfflineImapError (probably severity MESSAGE) if e.g. no message with this UID could be found. """ data = self._fetch_from_imap(str(uid), self.retrycount) # data looks now e.g. # ['320 (X-GM-LABELS (...) UID 17061 BODY[] {2565}',<email.message.EmailMessage object>] # we only asked for one message, and that msg is in data[1]. msg = data[1] # Embed the labels into the message headers if self.synclabels: m = re.search('X-GM-LABELS\s*[(](.*)[)]', data[0]) if m: labels = set([ imaputil.dequote(lb) for lb in imaputil.imapsplit(m.group(1)) ]) else: labels = set() labels = labels - self.ignorelabels labels_str = imaputil.format_labels_string(self.labelsheader, sorted(labels)) # First remove old label headers that may be in the message body retrieved # from gmail Then add a labels header with current gmail labels. self.deletemessageheaders(msg, self.labelsheader) self.addmessageheader(msg, self.labelsheader, labels_str) if self.ui.is_debugging('imap'): # Optimization: don't create the debugging objects unless needed msg_s = msg.as_string(policy=self.policy['8bit-RFC']) if len(msg_s) > 200: dbg_output = "%s...%s" % (msg_s[:150], msg_s[-50:]) else: dbg_output = msg_s self.ui.debug( 'imap', "Returned object from fetching %d: '%s'" % (uid, dbg_output)) return msg
def processmessagesflags(self, operation, uidlist, flags): if len(uidlist) > 101: # Hack for those IMAP ervers with a limited line length self.processmessagesflags(operation, uidlist[:100], flags) self.processmessagesflags(operation, uidlist[100:], flags) return imapobj = self.imapserver.acquireconnection() try: try: imapobj.select(self.getfullname()) except imapobj.readonly: self.ui.flagstoreadonly(self, uidlist, flags) return r = imapobj.uid( "store", imaputil.uid_sequence(uidlist), operation + "FLAGS", imaputil.flagsmaildir2imap(flags) ) assert r[0] == "OK", "Error with store: " + ". ".join(r[1]) r = r[1] finally: self.imapserver.releaseconnection(imapobj) # Some IMAP servers do not always return a result. Therefore, # only update the ones that it talks about, and manually fix # the others. needupdate = list(uidlist) for result in r: if result == None: # Compensate for servers that don't return anything from # STORE. continue attributehash = imaputil.flags2hash(imaputil.imapsplit(result)[1]) if not ("UID" in attributehash and "FLAGS" in attributehash): # Compensate for servers that don't return a UID attribute. continue flagstr = attributehash["FLAGS"] uid = long(attributehash["UID"]) self.messagelist[uid]["flags"] = imaputil.flagsimap2maildir(flagstr) try: needupdate.remove(uid) except ValueError: # Let it slide if it's not in the list pass for uid in needupdate: if operation == "+": self.messagelist[uid]["flags"] |= flags elif operation == "-": self.messagelist[uid]["flags"] -= flags
def savemessageflags(self, uid, flags): imapobj = self.imapserver.acquireconnection() try: try: imapobj.select(self.getfullname()) except imapobj.readonly: self.ui.flagstoreadonly(self, [uid], flags) return result = imapobj.uid("store", "%d" % uid, "FLAGS", imaputil.flagsmaildir2imap(flags)) assert result[0] == "OK", "Error with store: " + ". ".join(result[1]) finally: self.imapserver.releaseconnection(imapobj) result = result[1][0] if not result: self.messagelist[uid]["flags"] = flags else: flags = imaputil.flags2hash(imaputil.imapsplit(result)[1])["FLAGS"] self.messagelist[uid]["flags"] = imaputil.flagsimap2maildir(flags)
def __processmessagesflags_real(self, operation, uidlist, flags): imapobj = self.imapserver.acquireconnection() try: try: imapobj.select(self.getfullname()) except imapobj.readonly: self.ui.flagstoreadonly(self, uidlist, flags) return response = imapobj.uid('store', imaputil.uid_sequence(uidlist), operation + 'FLAGS', imaputil.flagsmaildir2imap(flags)) if response[0] != 'OK': raise OfflineImapError( 'Error with store: %s' % '. '.join(response[1]), OfflineImapError.ERROR.MESSAGE) response = response[1] finally: self.imapserver.releaseconnection(imapobj) # Some IMAP servers do not always return a result. Therefore, # only update the ones that it talks about, and manually fix # the others. needupdate = list(uidlist) for result in response: if result is None: # Compensate for servers that don't return anything from # STORE. continue attributehash = imaputil.flags2hash(imaputil.imapsplit(result)[1]) if not ('UID' in attributehash and 'FLAGS' in attributehash): # Compensate for servers that don't return a UID attribute. continue flagstr = attributehash['FLAGS'] uid = int(attributehash['UID']) self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir( flagstr) try: needupdate.remove(uid) except ValueError: # Let it slide if it's not in the list. pass for uid in needupdate: if operation == '+': self.messagelist[uid]['flags'] |= flags elif operation == '-': self.messagelist[uid]['flags'] -= flags
def savemessageflags(self, uid, flags): imapobj = self.imapserver.acquireconnection() try: try: imapobj.select(self.getfullname()) except imapobj.readonly: self.ui.flagstoreadonly(self, [uid], flags) return result = imapobj.uid('store', '%d' % uid, 'FLAGS', imaputil.flagsmaildir2imap(flags)) assert result[0] == 'OK', 'Error with store: ' + '. '.join(result[1]) finally: self.imapserver.releaseconnection(imapobj) result = result[1][0] if not result: self.messagelist[uid]['flags'] = flags else: flags = imaputil.flags2hash(imaputil.imapsplit(result)[1])['FLAGS'] self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
def __processmessagesflags_real(self, operation, uidlist, flags): imapobj = self.imapserver.acquireconnection() try: try: imapobj.select(self.getfullname()) except imapobj.readonly: self.ui.flagstoreadonly(self, uidlist, flags) return response = imapobj.uid('store', imaputil.uid_sequence(uidlist), operation + 'FLAGS', imaputil.flagsmaildir2imap(flags)) if response[0] != 'OK': raise OfflineImapError( 'Error with store: %s'% '. '.join(response[1]), OfflineImapError.ERROR.MESSAGE) response = response[1] finally: self.imapserver.releaseconnection(imapobj) # Some IMAP servers do not always return a result. Therefore, # only update the ones that it talks about, and manually fix # the others. needupdate = list(uidlist) for result in response: if result is None: # Compensate for servers that don't return anything from # STORE. continue attributehash = imaputil.flags2hash(imaputil.imapsplit(result)[1]) if not ('UID' in attributehash and 'FLAGS' in attributehash): # Compensate for servers that don't return a UID attribute. continue flagstr = attributehash['FLAGS'] uid = int(attributehash['UID']) self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flagstr) try: needupdate.remove(uid) except ValueError: # Let it slide if it's not in the list. pass for uid in needupdate: if operation == '+': self.messagelist[uid]['flags'] |= flags elif operation == '-': self.messagelist[uid]['flags'] -= flags
def getmessage(self, uid): """Retrieve message with UID from the IMAP server (incl body). Also gets Gmail labels and embeds them into the message. :returns: the message body or throws and OfflineImapError (probably severity MESSAGE) if e.g. no message with this UID could be found. """ imapobj = self.imapserver.acquireconnection() try: data = self._fetch_from_imap(imapobj, str(uid), 2) finally: self.imapserver.releaseconnection(imapobj) # data looks now e.g. #[('320 (X-GM-LABELS (...) UID 17061 BODY[] {2565}','msgbody....')] # we only asked for one message, and that msg is in data[0]. # msbody is in [0][1]. body = data[0][1].replace("\r\n", "\n") # Embed the labels into the message headers if self.synclabels: m = re.search('X-GM-LABELS\s*\(([^\)]*)\)', data[0][0]) if m: labels = set([imaputil.dequote(lb) for lb in imaputil.imapsplit(m.group(1))]) else: labels = set() labels = labels - self.ignorelabels labels_str = imaputil.format_labels_string(self.labelsheader, sorted(labels)) # First remove old label headers that may be in the message content retrieved # from gmail Then add a labels header with current gmail labels. body = self.deletemessageheaders(body, self.labelsheader) body = self.addmessageheader(body, '\n', self.labelsheader, labels_str) if len(body)>200: dbg_output = "%s...%s"% (str(body)[:150], str(body)[-50:]) else: dbg_output = body self.ui.debug('imap', "Returned object from fetching %d: '%s'"% (uid, dbg_output)) return body
def savemessageflags(self, uid, flags): imapobj = self.imapserver.acquireconnection() try: try: imapobj.select(self.getfullname()) except imapobj.readonly: self.ui.flagstoreadonly(self, [uid], flags) return result = imapobj.uid('store', '%d' % uid, 'FLAGS', imaputil.flagsmaildir2imap(flags)) assert result[0] == 'OK', 'Error with store: ' + '. '.join(r[1]) finally: self.imapserver.releaseconnection(imapobj) result = result[1][0] if not result: self.messagelist[uid]['flags'] = flags else: flags = imaputil.flags2hash(imaputil.imapsplit(result)[1])['FLAGS'] self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
def cachemessagelist(self): if not self.synclabels: return super(GmailFolder, self).cachemessagelist() self.ui.collectingdata(None, self) self.messagelist = {} imapobj = self.imapserver.acquireconnection() try: msgsToFetch = self._msgs_to_fetch(imapobj) if not msgsToFetch: return # No messages to sync # Get the flags and UIDs for these. single-quotes prevent # imaplib2 from quoting the sequence. data = self._fetch_from_imap(imapobj, "'%s'" % msgsToFetch, '(FLAGS X-GM-LABELS UID)') finally: self.imapserver.releaseconnection(imapobj) for messagestr in data: # looks like: '1 (FLAGS (\\Seen Old) UID 4807)' or None if no msg # Discard initial message number. if messagestr == None: continue messagestr = messagestr.split(' ', 1)[1] options = imaputil.flags2hash(messagestr) if not 'UID' in options: self.ui.warn('No UID in message with options %s' %\ str(options), minor = 1) else: uid = long(options['UID']) flags = imaputil.flagsimap2maildir(options['FLAGS']) m = re.search('\(([^\)]*)\)', options['X-GM-LABELS']) if m: labels = set([imaputil.dequote(lb) for lb in imaputil.imapsplit(m.group(1))]) else: labels = set() labels = labels - self.ignorelabels rtime = imaplibutil.Internaldate2epoch(messagestr) self.messagelist[uid] = {'uid': uid, 'flags': flags, 'labels': labels, 'time': rtime}
def savemessageflags(self, uid, flags): """Change a message's flags to `flags`. Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" imapobj = self.imapserver.acquireconnection() try: result = self._store_to_imap(imapobj, str(uid), 'FLAGS', imaputil.flagsmaildir2imap(flags)) except imapobj.readonly: self.ui.flagstoreadonly(self, [uid], flags) return finally: self.imapserver.releaseconnection(imapobj) if not result: self.messagelist[uid]['flags'] = flags else: flags = imaputil.flags2hash(imaputil.imapsplit(result)[1])['FLAGS'] self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
def savemessageflags(self, uid, flags): """Change a message's flags to `flags`. Note that this function does not check against dryrun settings, so you need to ensure that it is never called in a dryrun mode.""" imapobj = self.imapserver.acquireconnection() try: try: imapobj.select(self.getfullname()) except imapobj.readonly: self.ui.flagstoreadonly(self, [uid], flags) return result = imapobj.uid('store', '%d' % uid, 'FLAGS', imaputil.flagsmaildir2imap(flags)) assert result[0] == 'OK', 'Error with store: ' + '. '.join(result[1]) finally: self.imapserver.releaseconnection(imapobj) result = result[1][0] if not result: self.messagelist[uid]['flags'] = flags else: flags = imaputil.flags2hash(imaputil.imapsplit(result)[1])['FLAGS'] self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
def getfolders(self): """Return a list of instances of OfflineIMAP representative folder.""" if self.folders != None: return self.folders retval = [] imapobj = self.imapserver.acquireconnection() # check whether to list all folders, or subscribed only listfunction = imapobj.list if self.getconfboolean('subscribedonly', False): listfunction = imapobj.lsub try: listresult = listfunction(directory=self.imapserver.reference)[1] finally: self.imapserver.releaseconnection(imapobj) for s in listresult: if s == None or \ (isinstance(s, basestring) and s == ''): # Bug in imaplib: empty strings in results from # literals. TODO: still relevant? continue flags, delim, name = imaputil.imapsplit(s) flaglist = [x.lower() for x in imaputil.flagsplit(flags)] if '\\noselect' in flaglist: continue foldername = imaputil.dequote(name) retval.append(self.getfoldertype()(self.imapserver, foldername, self)) # Add all folderincludes if len(self.folderincludes): imapobj = self.imapserver.acquireconnection() try: for foldername in self.folderincludes: try: imapobj.select(foldername, readonly=True) except OfflineImapError as e: # couldn't select this folderinclude, so ignore folder. if e.severity > OfflineImapError.ERROR.FOLDER: raise self.ui.error(e, exc_info()[2], 'Invalid folderinclude:') continue retval.append(self.getfoldertype()(self.imapserver, foldername, self)) finally: self.imapserver.releaseconnection(imapobj) if self.foldersort is None: # default sorting by case insensitive transposed name retval.sort(key=lambda x: str.lower(x.getvisiblename())) else: # do foldersort in a python3-compatible way # http://bytes.com/topic/python/answers/844614-python-3-sorting-comparison-function def cmp2key(mycmp): """Converts a cmp= function into a key= function We need to keep cmp functions for backward compatibility""" class K: def __init__(self, obj, *args): self.obj = obj def __cmp__(self, other): return mycmp(self.obj.getvisiblename(), other.obj.getvisiblename()) return K retval.sort(key=cmp2key(self.foldersort)) self.folders = retval return self.folders
def cachemessagelist(self, min_date=None, min_uid=None): if not self.synclabels: return super(GmailFolder, self).cachemessagelist(min_date=min_date, min_uid=min_uid) self.dropmessagelistcache() self.ui.collectingdata(None, self) imapobj = self.imapserver.acquireconnection() try: msgsToFetch = self._msgs_to_fetch(imapobj, min_date=min_date, min_uid=min_uid) if not msgsToFetch: return # No messages to sync # Get the flags and UIDs for these. single-quotes prevent # imaplib2 from quoting the sequence. # # NB: msgsToFetch are sequential numbers, not UID's res_type, response = imapobj.fetch("%s" % msgsToFetch, '(FLAGS X-GM-LABELS UID)') if res_type != 'OK': raise OfflineImapError( "FETCHING UIDs in folder [%s]%s failed. " % (self.getrepository(), self) + "Server responded '[%s] %s'" % (res_type, response), OfflineImapError.ERROR.FOLDER, exc_info()[2]) finally: self.imapserver.releaseconnection(imapobj) for messagestr in response: # looks like: '1 (FLAGS (\\Seen Old) X-GM-LABELS (\\Inbox \\Favorites) UID 4807)' or None if no msg # Discard initial message number. if messagestr is None: continue messagestr = messagestr.decode('utf-8').split(' ', 1)[1] # e.g.: {'X-GM-LABELS': '("Webserver (RW.net)" "\\Inbox" GInbox)', 'FLAGS': '(\\Seen)', 'UID': '275440'} options = imaputil.flags2hash(messagestr) if 'UID' not in options: self.ui.warn('No UID in message with options %s' % str(options), minor=1) else: uid = int(options['UID']) self.messagelist[uid] = self.msglist_item_initializer(uid) flags = imaputil.flagsimap2maildir(options['FLAGS']) # e.g.: '("Webserver (RW.net)" "\\Inbox" GInbox)' m = re.search('^[(](.*)[)]', options['X-GM-LABELS']) if m: labels = set([ imaputil.dequote(lb) for lb in imaputil.imapsplit(m.group(1)) ]) else: labels = set() labels = labels - self.ignorelabels rtime = imaplibutil.Internaldate2epoch(messagestr.encode()) self.messagelist[uid] = { 'uid': uid, 'flags': flags, 'labels': labels, 'time': rtime }
def acquireconnection(self): """Fetches a connection from the pool, making sure to create a new one if needed, to obey the maximum connection limits, etc. Opens a connection to the server and returns an appropriate object.""" self.semaphore.acquire() self.connectionlock.acquire() curThread = currentThread() imapobj = None if len(self.availableconnections): # One is available. # Try to find one that previously belonged to this thread # as an optimization. Start from the back since that's where # they're popped on. for i in range(len(self.availableconnections) - 1, -1, -1): tryobj = self.availableconnections[i] if self.lastowner[tryobj] == curThread.ident: imapobj = tryobj del (self.availableconnections[i]) break if not imapobj: imapobj = self.availableconnections[0] del (self.availableconnections[0]) self.assignedconnections.append(imapobj) self.lastowner[imapobj] = curThread.ident self.connectionlock.release() return imapobj self.connectionlock.release() # Release until need to modify data # Must be careful here that if we fail we should bail out gracefully # and release locks / threads so that the next attempt can try... success = False try: while success is not True: # Generate a new connection. if self.tunnel: self.ui.connecting(self.repos.getname(), 'tunnel', self.tunnel) imapobj = imaplibutil.IMAP4_Tunnel( self.tunnel, timeout=socket.getdefaulttimeout(), use_socket=self.proxied_socket, ) success = True elif self.usessl: self.ui.connecting(self.repos.getname(), self.hostname, self.port) self.ui.debug( 'imap', "%s: level '%s', version '%s'" % (self.repos.getname(), self.tlslevel, self.sslversion)) imapobj = imaplibutil.WrappedIMAP4_SSL( host=self.hostname, port=self.port, keyfile=self.sslclientkey, certfile=self.sslclientcert, ca_certs=self.sslcacertfile, cert_verify_cb=self.__verifycert, ssl_version=self.sslversion, timeout=socket.getdefaulttimeout(), fingerprint=self.fingerprint, use_socket=self.proxied_socket, tls_level=self.tlslevel, af=self.af, ) else: self.ui.connecting(self.repos.getname(), self.hostname, self.port) imapobj = imaplibutil.WrappedIMAP4( self.hostname, self.port, timeout=socket.getdefaulttimeout(), use_socket=self.proxied_socket, af=self.af, ) if not self.preauth_tunnel: try: self.__authn_helper(imapobj) self.goodpassword = self.password success = True except OfflineImapError as e: self.passworderror = str(e) raise # Enable compression if self.repos.getconfboolean('usecompression', 0): imapobj.enable_compression() # update capabilities after login, e.g. gmail serves different ones typ, dat = imapobj.capability() if dat != [None]: imapobj.capabilities = tuple(dat[-1].upper().split()) if self.delim == None: listres = imapobj.list(self.reference, '""')[1] if listres == [None] or listres == None: # Some buggy IMAP servers do not respond well to LIST "" "" # Work around them. listres = imapobj.list(self.reference, '"*"')[1] if listres == [None] or listres == None: # No Folders were returned. This occurs, e.g. if the # 'reference' prefix does not exist on the mail # server. Raise exception. err = "Server '%s' returned no folders in '%s'"% \ (self.repos.getname(), self.reference) self.ui.warn(err) raise Exception(err) self.delim, self.root = \ imaputil.imapsplit(listres[0])[1:] self.delim = imaputil.dequote(self.delim) self.root = imaputil.dequote(self.root) with self.connectionlock: self.assignedconnections.append(imapobj) self.lastowner[imapobj] = curThread.ident return imapobj except Exception as e: """If we are here then we did not succeed in getting a connection - we should clean up and then re-raise the error...""" self.semaphore.release() severity = OfflineImapError.ERROR.REPO if type(e) == gaierror: #DNS related errors. Abort Repo sync #TODO: special error msg for e.errno == 2 "Name or service not known"? reason = "Could not resolve name '%s' for repository "\ "'%s'. Make sure you have configured the ser"\ "ver name correctly and that you are online."%\ (self.hostname, self.repos) six.reraise(OfflineImapError, OfflineImapError(reason, severity), exc_info()[2]) elif isinstance(e, SSLError) and e.errno == errno.EPERM: # SSL unknown protocol error # happens e.g. when connecting via SSL to a non-SSL service if self.port != 993: reason = "Could not connect via SSL to host '%s' and non-s"\ "tandard ssl port %d configured. Make sure you connect"\ " to the correct port. Got: %s"% ( self.hostname, self.port, e) else: reason = "Unknown SSL protocol connecting to host '%s' for "\ "repository '%s'. OpenSSL responded:\n%s"\ % (self.hostname, self.repos, e) six.reraise(OfflineImapError, OfflineImapError(reason, severity), exc_info()[2]) elif isinstance(e, socket.error) and e.args[0] == errno.ECONNREFUSED: # "Connection refused", can be a non-existing port, or an unauthorized # webproxy (open WLAN?) reason = "Connection to host '%s:%d' for repository '%s' was "\ "refused. Make sure you have the right host and port "\ "configured and that you are actually able to access the "\ "network."% (self.hostname, self.port, self.repos) six.reraise(OfflineImapError, OfflineImapError(reason, severity), exc_info()[2]) # Could not acquire connection to the remote; # socket.error(last_error) raised if str(e)[:24] == "can't open socket; error": six.reraise( OfflineImapError, OfflineImapError( "Could not connect to remote server '%s' " "for repository '%s'. Remote does not answer." % (self.hostname, self.repos), OfflineImapError.ERROR.REPO), exc_info()[2]) else: # re-raise all other errors raise
def acquireconnection(self): """Fetches a connection from the pool, making sure to create a new one if needed, to obey the maximum connection limits, etc. Opens a connection to the server and returns an appropriate object.""" self.semaphore.acquire() self.connectionlock.acquire() curThread = currentThread() imapobj = None if len(self.availableconnections): # One is available. # Try to find one that previously belonged to this thread # as an optimization. Start from the back since that's where # they're popped on. imapobj = None for i in range(len(self.availableconnections) - 1, -1, -1): tryobj = self.availableconnections[i] if self.lastowner[tryobj] == curThread.ident: imapobj = tryobj del(self.availableconnections[i]) break if not imapobj: imapobj = self.availableconnections[0] del(self.availableconnections[0]) self.assignedconnections.append(imapobj) self.lastowner[imapobj] = curThread.ident self.connectionlock.release() return imapobj self.connectionlock.release() # Release until need to modify data """ Must be careful here that if we fail we should bail out gracefully and release locks / threads so that the next attempt can try... """ success = 0 try: while not success: # Generate a new connection. if self.tunnel: self.ui.connecting('tunnel', self.tunnel) imapobj = imaplibutil.IMAP4_Tunnel(self.tunnel, timeout=socket.getdefaulttimeout()) success = 1 elif self.usessl: self.ui.connecting(self.hostname, self.port) fingerprint = self.repos.get_ssl_fingerprint() imapobj = imaplibutil.WrappedIMAP4_SSL(self.hostname, self.port, self.sslclientkey, self.sslclientcert, self.sslcacertfile, self.verifycert, timeout=socket.getdefaulttimeout(), fingerprint=fingerprint ) else: self.ui.connecting(self.hostname, self.port) imapobj = imaplibutil.WrappedIMAP4(self.hostname, self.port, timeout=socket.getdefaulttimeout()) if not self.tunnel: try: # Try GSSAPI and continue if it fails if 'AUTH=GSSAPI' in imapobj.capabilities and have_gss: self.connectionlock.acquire() self.ui.debug('imap', 'Attempting GSSAPI authentication') try: imapobj.authenticate('GSSAPI', self.gssauth) except imapobj.error as val: self.gssapi = False self.ui.debug('imap', 'GSSAPI Authentication failed') else: self.gssapi = True kerberos.authGSSClientClean(self.gss_vc) self.gss_vc = None self.gss_step = self.GSS_STATE_STEP #if we do self.password = None then the next attempt cannot try... #self.password = None self.connectionlock.release() if not self.gssapi: if 'STARTTLS' in imapobj.capabilities and not\ self.usessl: self.ui.debug('imap', 'Using STARTTLS connection') imapobj.starttls() if 'AUTH=CRAM-MD5' in imapobj.capabilities: self.ui.debug('imap', 'Attempting CRAM-MD5 authentication') try: imapobj.authenticate('CRAM-MD5', self.md5handler) except imapobj.error as val: self.plainauth(imapobj) else: # Use plaintext login, unless # LOGINDISABLED (RFC2595) if 'LOGINDISABLED' in imapobj.capabilities: raise OfflineImapError("Plaintext login " "disabled by server. Need to use SSL?", OfflineImapError.ERROR.REPO) self.plainauth(imapobj) # Would bail by here if there was a failure. success = 1 self.goodpassword = self.password except imapobj.error as val: self.passworderror = str(val) raise # update capabilities after login, e.g. gmail serves different ones typ, dat = imapobj.capability() if dat != [None]: imapobj.capabilities = tuple(dat[-1].upper().split()) if self.delim == None: listres = imapobj.list(self.reference, '""')[1] if listres == [None] or listres == None: # Some buggy IMAP servers do not respond well to LIST "" "" # Work around them. listres = imapobj.list(self.reference, '"*"')[1] if listres == [None] or listres == None: # No Folders were returned. This occurs, e.g. if the # 'reference' prefix does not exist on the mail # server. Raise exception. err = "Server '%s' returned no folders in '%s'" % \ (self.repos.getname(), self.reference) self.ui.warn(err) raise Exception(err) self.delim, self.root = \ imaputil.imapsplit(listres[0])[1:] self.delim = imaputil.dequote(self.delim) self.root = imaputil.dequote(self.root) self.connectionlock.acquire() self.assignedconnections.append(imapobj) self.lastowner[imapobj] = curThread.ident self.connectionlock.release() return imapobj except Exception as e: """If we are here then we did not succeed in getting a connection - we should clean up and then re-raise the error...""" self.semaphore.release() if(self.connectionlock.locked()): self.connectionlock.release() severity = OfflineImapError.ERROR.REPO if type(e) == gaierror: #DNS related errors. Abort Repo sync #TODO: special error msg for e.errno == 2 "Name or service not known"? reason = "Could not resolve name '%s' for repository "\ "'%s'. Make sure you have configured the ser"\ "ver name correctly and that you are online."%\ (self.hostname, self.repos) raise OfflineImapError(reason, severity) elif isinstance(e, SSLError) and e.errno == 1: # SSL unknown protocol error # happens e.g. when connecting via SSL to a non-SSL service if self.port != 993: reason = "Could not connect via SSL to host '%s' and non-s"\ "tandard ssl port %d configured. Make sure you connect"\ " to the correct port." % (self.hostname, self.port) else: reason = "Unknown SSL protocol connecting to host '%s' for"\ "repository '%s'. OpenSSL responded:\n%s"\ % (self.hostname, self.repos, e) raise OfflineImapError(reason, severity) elif isinstance(e, socket.error) and e.args[0] == errno.ECONNREFUSED: # "Connection refused", can be a non-existing port, or an unauthorized # webproxy (open WLAN?) reason = "Connection to host '%s:%d' for repository '%s' was "\ "refused. Make sure you have the right host and port "\ "configured and that you are actually able to access the "\ "network." % (self.hostname, self.port, self.reposname) raise OfflineImapError(reason, severity) # Could not acquire connection to the remote; # socket.error(last_error) raised if str(e)[:24] == "can't open socket; error": raise OfflineImapError("Could not connect to remote server '%s' "\ "for repository '%s'. Remote does not answer." % (self.hostname, self.repos), OfflineImapError.ERROR.REPO) else: # re-raise all other errors raise
# Would bail by here if there was a failure. success = 1 self.goodpassword = self.password except imapobj.error, val: self.passworderror = str(val) raise #self.password = None if self.delim == None: listres = imapobj.list(self.reference, '""')[1] if listres == [None] or listres == None: # Some buggy IMAP servers do not respond well to LIST "" "" # Work around them. listres = imapobj.list(self.reference, '"*"')[1] self.delim, self.root = \ imaputil.imapsplit(listres[0])[1:] self.delim = imaputil.dequote(self.delim) self.root = imaputil.dequote(self.root) self.connectionlock.acquire() self.assignedconnections.append(imapobj) self.lastowner[imapobj] = thread.get_ident() self.connectionlock.release() return imapobj except: """If we are here then we did not succeed in getting a connection - we should clean up and then re-raise the error...""" self.semaphore.release() #Make sure that this can be retried the next time... self.passworderror = None
if self.delim == None: listres = imapobj.list(self.reference, '""')[1] if listres == [None] or listres == None: # Some buggy IMAP servers do not respond well to LIST "" "" # Work around them. listres = imapobj.list(self.reference, '"*"')[1] if listres == [None] or listres == None: # No Folders were returned. This occurs, e.g. if the # 'reference' prefix does not exist on the mail # server. Raise exception. err = "Server '%s' returned no folders in '%s'" % \ (self.repos.getname(), self.reference) self.ui.warn(err) raise Exception(err) self.delim, self.root = \ imaputil.imapsplit(listres[0])[1:] self.delim = imaputil.dequote(self.delim) self.root = imaputil.dequote(self.root) self.connectionlock.acquire() self.assignedconnections.append(imapobj) self.lastowner[imapobj] = thread.get_ident() self.connectionlock.release() return imapobj except: """If we are here then we did not succeed in getting a connection - we should clean up and then re-raise the error...""" self.semaphore.release() #Make sure that this can be retried the next time... self.passworderror = None
def getfolders(self): if self.folders != None: return self.folders retval = [] imapobj = self.imapserver.acquireconnection() # check whether to list all folders, or subscribed only listfunction = imapobj.list if self.getconfboolean('subscribedonly', False): listfunction = imapobj.lsub try: listresult = listfunction(directory = self.imapserver.reference)[1] finally: self.imapserver.releaseconnection(imapobj) for string in listresult: if string == None or \ (isinstance(string, basestring) and string == ''): # Bug in imaplib: empty strings in results from # literals. TODO: still relevant? continue flags, delim, name = imaputil.imapsplit(string) flaglist = [x.lower() for x in imaputil.flagsplit(flags)] if '\\noselect' in flaglist: continue foldername = imaputil.dequote(name) retval.append(self.getfoldertype()(self.imapserver, foldername, self)) # Add all folderincludes if len(self.folderincludes): imapobj = self.imapserver.acquireconnection() try: for foldername in self.folderincludes: try: imapobj.select(foldername, readonly = True) except OfflineImapError as e: # couldn't select this folderinclude, so ignore folder. if e.severity > OfflineImapError.ERROR.FOLDER: raise self.ui.error(e, exc_info()[2], 'Invalid folderinclude:') continue retval.append(self.getfoldertype()(self.imapserver, foldername, self)) finally: self.imapserver.releaseconnection(imapobj) if self.foldersort is None: # default sorting by case insensitive transposed name retval.sort(key=lambda x: str.lower(x.getvisiblename())) else: # do foldersort in a python3-compatible way # http://bytes.com/topic/python/answers/844614-python-3-sorting-comparison-function def cmp2key(mycmp): """Converts a cmp= function into a key= function We need to keep cmp functions for backward compatibility""" class K: def __init__(self, obj, *args): self.obj = obj def __cmp__(self, other): return mycmp(self.obj.getvisiblename(), other.obj.getvisiblename()) return K retval.sort(key=cmp2key(self.foldersort)) self.folders = retval return self.folders
def getfolders(self): """Return a list of instances of OfflineIMAP representative folder.""" if self.folders is not None: return self.folders retval = [] imapobj = self.imapserver.acquireconnection() # check whether to list all folders, or subscribed only listfunction = imapobj.list if self.getconfboolean('subscribedonly', False): listfunction = imapobj.lsub try: result, listresult = \ listfunction(directory=self.imapserver.reference, pattern='"*"') if result != 'OK': raise OfflineImapError("Could not list the folders for" " repository %s. Server responded: %s" % (self.name, str(listresult)), OfflineImapError.ERROR.FOLDER) finally: self.imapserver.releaseconnection(imapobj) for fldr in listresult: if fldr is None or (isinstance(fldr, str) and fldr == ''): # Bug in imaplib: empty strings in results from # literals. TODO: still relevant? continue try: flags, delim, name = imaputil.imapsplit(fldr) except ValueError: self.ui.error( "could not correctly parse server response; got: %s" % fldr) raise flaglist = [x.lower() for x in imaputil.flagsplit(flags)] if '\\noselect' in flaglist: continue retval.append(self.getfoldertype()(self.imapserver, name, self)) # Add all folderincludes if len(self.folderincludes): imapobj = self.imapserver.acquireconnection() try: for foldername in self.folderincludes: try: imapobj.select(imaputil.utf8_IMAP(foldername), readonly=True) except OfflineImapError as exc: # couldn't select this folderinclude, so ignore folder. if exc.severity > OfflineImapError.ERROR.FOLDER: raise self.ui.error(exc, exc_info()[2], 'Invalid folderinclude:') continue retval.append(self.getfoldertype()( self.imapserver, foldername, self, decode=False)) finally: self.imapserver.releaseconnection(imapobj) if self.foldersort is None: # default sorting by case insensitive transposed name retval.sort(key=lambda x: str.lower(x.getvisiblename())) else: # do foldersort in a python3-compatible way # http://bytes.com/topic/python/answers/ \ # 844614-python-3-sorting-comparison-function def cmp2key(mycmp): """Converts a cmp= function into a key= function We need to keep cmp functions for backward compatibility""" class K: """ Class to compare getvisiblename() between two objects. """ def __init__(self, obj, *args): self.obj = obj def __cmp__(self, other): return mycmp(self.obj.getvisiblename(), other.obj.getvisiblename()) def __lt__(self, other): return self.__cmp__(other) < 0 def __le__(self, other): return self.__cmp__(other) <= 0 def __gt__(self, other): return self.__cmp__(other) > 0 def __ge__(self, other): return self.__cmp__(other) >= 0 def __eq__(self, other): return self.__cmp__(other) == 0 def __ne__(self, other): return self.__cmp__(other) != 0 return K retval.sort(key=cmp2key(self.foldersort)) self.folders = retval return self.folders
def acquireconnection(self): """Fetches a connection from the pool, making sure to create a new one if needed, to obey the maximum connection limits, etc. Opens a connection to the server and returns an appropriate object.""" self.semaphore.acquire() self.connectionlock.acquire() curThread = currentThread() imapobj = None if len(self.availableconnections): # One is available. # Try to find one that previously belonged to this thread # as an optimization. Start from the back since that's where # they're popped on. imapobj = None for i in range(len(self.availableconnections) - 1, -1, -1): tryobj = self.availableconnections[i] if self.lastowner[tryobj] == curThread.ident: imapobj = tryobj del (self.availableconnections[i]) break if not imapobj: imapobj = self.availableconnections[0] del (self.availableconnections[0]) self.assignedconnections.append(imapobj) self.lastowner[imapobj] = curThread.ident self.connectionlock.release() return imapobj self.connectionlock.release() # Release until need to modify data """ Must be careful here that if we fail we should bail out gracefully and release locks / threads so that the next attempt can try... """ success = 0 try: while not success: # Generate a new connection. if self.tunnel: self.ui.connecting('tunnel', self.tunnel) imapobj = imaplibutil.IMAP4_Tunnel( self.tunnel, timeout=socket.getdefaulttimeout()) success = 1 elif self.usessl: self.ui.connecting(self.hostname, self.port) fingerprint = self.repos.get_ssl_fingerprint() imapobj = imaplibutil.WrappedIMAP4_SSL( self.hostname, self.port, self.sslclientkey, self.sslclientcert, self.sslcacertfile, self.verifycert, timeout=socket.getdefaulttimeout(), fingerprint=fingerprint) else: self.ui.connecting(self.hostname, self.port) imapobj = imaplibutil.WrappedIMAP4( self.hostname, self.port, timeout=socket.getdefaulttimeout()) if not self.tunnel: try: # Try GSSAPI and continue if it fails if 'AUTH=GSSAPI' in imapobj.capabilities and have_gss: self.connectionlock.acquire() self.ui.debug('imap', 'Attempting GSSAPI authentication') try: imapobj.authenticate('GSSAPI', self.gssauth) except imapobj.error as val: self.gssapi = False self.ui.debug('imap', 'GSSAPI Authentication failed') else: self.gssapi = True kerberos.authGSSClientClean(self.gss_vc) self.gss_vc = None self.gss_step = self.GSS_STATE_STEP #if we do self.password = None then the next attempt cannot try... #self.password = None self.connectionlock.release() if not self.gssapi: if 'STARTTLS' in imapobj.capabilities and not\ self.usessl: self.ui.debug('imap', 'Using STARTTLS connection') imapobj.starttls() if 'AUTH=CRAM-MD5' in imapobj.capabilities: self.ui.debug( 'imap', 'Attempting CRAM-MD5 authentication') try: imapobj.authenticate( 'CRAM-MD5', self.md5handler) except imapobj.error as val: self.plainauth(imapobj) else: # Use plaintext login, unless # LOGINDISABLED (RFC2595) if 'LOGINDISABLED' in imapobj.capabilities: raise OfflineImapError( "Plaintext login " "disabled by server. Need to use SSL?", OfflineImapError.ERROR.REPO) self.plainauth(imapobj) # Would bail by here if there was a failure. success = 1 self.goodpassword = self.password except imapobj.error as val: self.passworderror = str(val) raise # update capabilities after login, e.g. gmail serves different ones typ, dat = imapobj.capability() if dat != [None]: imapobj.capabilities = tuple(dat[-1].upper().split()) if self.delim == None: listres = imapobj.list(self.reference, '""')[1] if listres == [None] or listres == None: # Some buggy IMAP servers do not respond well to LIST "" "" # Work around them. listres = imapobj.list(self.reference, '"*"')[1] if listres == [None] or listres == None: # No Folders were returned. This occurs, e.g. if the # 'reference' prefix does not exist on the mail # server. Raise exception. err = "Server '%s' returned no folders in '%s'" % \ (self.repos.getname(), self.reference) self.ui.warn(err) raise Exception(err) self.delim, self.root = \ imaputil.imapsplit(listres[0])[1:] self.delim = imaputil.dequote(self.delim) self.root = imaputil.dequote(self.root) self.connectionlock.acquire() self.assignedconnections.append(imapobj) self.lastowner[imapobj] = curThread.ident self.connectionlock.release() return imapobj except Exception as e: """If we are here then we did not succeed in getting a connection - we should clean up and then re-raise the error...""" self.semaphore.release() if (self.connectionlock.locked()): self.connectionlock.release() severity = OfflineImapError.ERROR.REPO if type(e) == gaierror: #DNS related errors. Abort Repo sync #TODO: special error msg for e.errno == 2 "Name or service not known"? reason = "Could not resolve name '%s' for repository "\ "'%s'. Make sure you have configured the ser"\ "ver name correctly and that you are online."%\ (self.hostname, self.repos) raise OfflineImapError(reason, severity) elif isinstance(e, SSLError) and e.errno == 1: # SSL unknown protocol error # happens e.g. when connecting via SSL to a non-SSL service if self.port != 993: reason = "Could not connect via SSL to host '%s' and non-s"\ "tandard ssl port %d configured. Make sure you connect"\ " to the correct port." % (self.hostname, self.port) else: reason = "Unknown SSL protocol connecting to host '%s' for"\ "repository '%s'. OpenSSL responded:\n%s"\ % (self.hostname, self.repos, e) raise OfflineImapError(reason, severity) elif isinstance(e, socket.error) and e.args[0] == errno.ECONNREFUSED: # "Connection refused", can be a non-existing port, or an unauthorized # webproxy (open WLAN?) reason = "Connection to host '%s:%d' for repository '%s' was "\ "refused. Make sure you have the right host and port "\ "configured and that you are actually able to access the "\ "network." % (self.hostname, self.port, self.reposname) raise OfflineImapError(reason, severity) # Could not acquire connection to the remote; # socket.error(last_error) raised if str(e)[:24] == "can't open socket; error": raise OfflineImapError("Could not connect to remote server '%s' "\ "for repository '%s'. Remote does not answer." % (self.hostname, self.repos), OfflineImapError.ERROR.REPO) else: # re-raise all other errors raise
def acquireconnection(self): """Fetches a connection from the pool, making sure to create a new one if needed, to obey the maximum connection limits, etc. Opens a connection to the server and returns an appropriate object.""" self.semaphore.acquire() self.connectionlock.acquire() curThread = currentThread() imapobj = None if len(self.availableconnections): # One is available. # Try to find one that previously belonged to this thread # as an optimization. Start from the back since that's where # they're popped on. for i in range(len(self.availableconnections) - 1, -1, -1): tryobj = self.availableconnections[i] if self.lastowner[tryobj] == curThread.ident: imapobj = tryobj del(self.availableconnections[i]) break if not imapobj: imapobj = self.availableconnections[0] del(self.availableconnections[0]) self.assignedconnections.append(imapobj) self.lastowner[imapobj] = curThread.ident self.connectionlock.release() return imapobj self.connectionlock.release() # Release until need to modify data # Must be careful here that if we fail we should bail out gracefully # and release locks / threads so that the next attempt can try... success = False try: while success is not True: # Generate a new connection. if self.tunnel: self.ui.connecting( self.repos.getname(), 'tunnel', self.tunnel) imapobj = imaplibutil.IMAP4_Tunnel( self.tunnel, timeout=socket.getdefaulttimeout(), use_socket=self.proxied_socket, ) success = True elif self.usessl: self.ui.connecting( self.repos.getname(), self.hostname, self.port) self.ui.debug('imap', "%s: level '%s', version '%s'"% (self.repos.getname(), self.tlslevel, self.sslversion)) imapobj = imaplibutil.WrappedIMAP4_SSL( host=self.hostname, port=self.port, keyfile=self.sslclientkey, certfile=self.sslclientcert, ca_certs=self.sslcacertfile, cert_verify_cb=self.__verifycert, ssl_version=self.sslversion, timeout=socket.getdefaulttimeout(), fingerprint=self.fingerprint, use_socket=self.proxied_socket, tls_level=self.tlslevel, af=self.af, ) else: self.ui.connecting( self.repos.getname(), self.hostname, self.port) imapobj = imaplibutil.WrappedIMAP4( self.hostname, self.port, timeout=socket.getdefaulttimeout(), use_socket=self.proxied_socket, af=self.af, ) if not self.preauth_tunnel: try: self.__authn_helper(imapobj) self.goodpassword = self.password success = True except OfflineImapError as e: self.passworderror = str(e) raise # Enable compression if self.repos.getconfboolean('usecompression', 0): imapobj.enable_compression() # update capabilities after login, e.g. gmail serves different ones typ, dat = imapobj.capability() if dat != [None]: imapobj.capabilities = tuple(dat[-1].upper().split()) if self.delim == None: listres = imapobj.list(self.reference, '""')[1] if listres == [None] or listres == None: # Some buggy IMAP servers do not respond well to LIST "" "" # Work around them. listres = imapobj.list(self.reference, '"*"')[1] if listres == [None] or listres == None: # No Folders were returned. This occurs, e.g. if the # 'reference' prefix does not exist on the mail # server. Raise exception. err = "Server '%s' returned no folders in '%s'"% \ (self.repos.getname(), self.reference) self.ui.warn(err) raise Exception(err) self.delim, self.root = \ imaputil.imapsplit(listres[0])[1:] self.delim = imaputil.dequote(self.delim) self.root = imaputil.dequote(self.root) with self.connectionlock: self.assignedconnections.append(imapobj) self.lastowner[imapobj] = curThread.ident return imapobj except Exception as e: """If we are here then we did not succeed in getting a connection - we should clean up and then re-raise the error...""" self.semaphore.release() severity = OfflineImapError.ERROR.REPO if type(e) == gaierror: #DNS related errors. Abort Repo sync #TODO: special error msg for e.errno == 2 "Name or service not known"? reason = "Could not resolve name '%s' for repository "\ "'%s'. Make sure you have configured the ser"\ "ver name correctly and that you are online."%\ (self.hostname, self.repos) six.reraise(OfflineImapError, OfflineImapError(reason, severity), exc_info()[2]) elif isinstance(e, SSLError) and e.errno == errno.EPERM: # SSL unknown protocol error # happens e.g. when connecting via SSL to a non-SSL service if self.port != 993: reason = "Could not connect via SSL to host '%s' and non-s"\ "tandard ssl port %d configured. Make sure you connect"\ " to the correct port. Got: %s"% ( self.hostname, self.port, e) else: reason = "Unknown SSL protocol connecting to host '%s' for "\ "repository '%s'. OpenSSL responded:\n%s"\ % (self.hostname, self.repos, e) six.reraise(OfflineImapError, OfflineImapError(reason, severity), exc_info()[2]) elif isinstance(e, socket.error) and e.args[0] == errno.ECONNREFUSED: # "Connection refused", can be a non-existing port, or an unauthorized # webproxy (open WLAN?) reason = "Connection to host '%s:%d' for repository '%s' was "\ "refused. Make sure you have the right host and port "\ "configured and that you are actually able to access the "\ "network."% (self.hostname, self.port, self.repos) six.reraise(OfflineImapError, OfflineImapError(reason, severity), exc_info()[2]) # Could not acquire connection to the remote; # socket.error(last_error) raised if str(e)[:24] == "can't open socket; error": six.reraise(OfflineImapError, OfflineImapError( "Could not connect to remote server '%s' " "for repository '%s'. Remote does not answer."% (self.hostname, self.repos), OfflineImapError.ERROR.REPO), exc_info()[2]) else: # re-raise all other errors raise