def savemessage(self, uid, content, flags, rtime):
"""Save the message on the Server
This backend always assigns a new uid, so the uid arg is ignored.
This function will update the self.messagelist dict to contain
the new message after sucessfully saving it.
:param rtime: A timestamp to be used as the mail date
:returns: the UID of the new message as assigned by the server. If the
message is saved, but it's UID can not be found, it will
return 0. If the message can't be written (folder is
read-only for example) it will return -1."""
self.ui.debug('imap', 'savemessage: called')
# already have it, just save modified flags
if uid > 0 and self.uidexists(uid):
self.savemessageflags(uid, flags)
return uid
retry_left = 2 # succeeded in APPENDING?
imapobj = self.imapserver.acquireconnection()
try:
while retry_left:
# UIDPLUS extension provides us with an APPENDUID response.
use_uidplus = 'UIDPLUS' in imapobj.capabilities
# get the date of the message, so we can pass it to the server.
date = self.getmessageinternaldate(content, rtime)
content = re.sub("(?<!\r)\n", "\r\n", content)
if not use_uidplus:
# insert a random unique header that we can fetch later
(headername,
headervalue) = self.generate_randomheader(content)
self.ui.debug('imap', 'savemessage: header is: %s: %s' %\
(headername, headervalue))
content = self.savemessage_addheader(
content, headername, headervalue)
if len(content) > 200:
dbg_output = "%s...%s" % (content[:150], content[-50:])
else:
dbg_output = content
self.ui.debug(
'imap', "savemessage: date: %s, content: '%s'" %
(date, dbg_output))
try:
# Select folder for append and make the box READ-WRITE
imapobj.select(self.getfullname())
except imapobj.readonly:
# readonly exception. Return original uid to notify that
# we did not save the message. (see savemessage in Base.py)
self.ui.msgtoreadonly(self, uid, content, flags)
return uid
#Do the APPEND
try:
(typ,
dat) = imapobj.append(self.getfullname(),
imaputil.flagsmaildir2imap(flags),
date, content)
retry_left = 0 # Mark as success
except imapobj.abort, e:
# connection has been reset, release connection and retry.
retry_left -= 1
self.imapserver.releaseconnection(imapobj, True)
imapobj = self.imapserver.acquireconnection()
if not retry_left:
raise OfflineImapError(
"Saving msg in folder '%s', "
"repository '%s' failed (abort). Server reponded: %s\n"
"Message content was: %s" %
(self, self.getrepository(), str(e), dbg_output),
OfflineImapError.ERROR.MESSAGE)
self.ui.error(e, exc_info()[2])
except imapobj.error, e: # APPEND failed
# If the server responds with 'BAD', append()
# raise()s directly. So we catch that too.
# drop conn, it might be bad.
self.imapserver.releaseconnection(imapobj, True)
imapobj = None
raise OfflineImapError(
"Saving msg folder '%s', repo '%s'"
"failed (error). Server reponded: %s\nMessage content was: "
"%s" %
(self, self.getrepository(), str(e), dbg_output),
OfflineImapError.ERROR.MESSAGE)
def getfolders(self):
"""Return a list of instances of OfflineIMAP representative folder."""
if self.folders is not None:
return self.folders
retval = []
imapobj = self.imapserver.acquireconnection()
# check whether to list all folders, or subscribed only
listfunction = imapobj.list
if self.getconfboolean('subscribedonly', False):
listfunction = imapobj.lsub
try:
result, listresult = \
listfunction(directory=self.imapserver.reference, pattern='"*"')
if result != 'OK':
raise OfflineImapError(
"Could not list the folders for"
" repository %s. Server responded: %s" %
(self.name, str(listresult)),
OfflineImapError.ERROR.FOLDER)
finally:
self.imapserver.releaseconnection(imapobj)
for fldr in listresult:
if fldr is None or (isinstance(fldr, str) and fldr == ''):
# Bug in imaplib: empty strings in results from
# literals. TODO: still relevant?
continue
try:
flags, delim, name = imaputil.imapsplit(fldr)
except ValueError:
self.ui.error(
"could not correctly parse server response; got: %s" %
fldr)
raise
flaglist = [x.lower() for x in imaputil.flagsplit(flags)]
if '\\noselect' in flaglist:
continue
retval.append(self.getfoldertype()(self.imapserver, name, self))
# Add all folderincludes
if len(self.folderincludes):
imapobj = self.imapserver.acquireconnection()
try:
for foldername in self.folderincludes:
try:
imapobj.select(imaputil.utf8_IMAP(foldername),
readonly=True)
except OfflineImapError as exc:
# couldn't select this folderinclude, so ignore folder.
if exc.severity > OfflineImapError.ERROR.FOLDER:
raise
self.ui.error(exc,
exc_info()[2], 'Invalid folderinclude:')
continue
retval.append(self.getfoldertype()(self.imapserver,
foldername,
self,
decode=False))
finally:
self.imapserver.releaseconnection(imapobj)
if self.foldersort is None:
# default sorting by case insensitive transposed name
retval.sort(key=lambda x: str.lower(x.getvisiblename()))
else:
# do foldersort in a python3-compatible way
# http://bytes.com/topic/python/answers/ \
# 844614-python-3-sorting-comparison-function
def cmp2key(mycmp):
"""Converts a cmp= function into a key= function
We need to keep cmp functions for backward compatibility"""
class K:
"""
Class to compare getvisiblename() between two objects.
"""
def __init__(self, obj, *args):
self.obj = obj
def __cmp__(self, other):
return mycmp(self.obj.getvisiblename(),
other.obj.getvisiblename())
def __lt__(self, other):
return self.__cmp__(other) < 0
def __le__(self, other):
return self.__cmp__(other) <= 0
def __gt__(self, other):
return self.__cmp__(other) > 0
def __ge__(self, other):
return self.__cmp__(other) >= 0
def __eq__(self, other):
return self.__cmp__(other) == 0
def __ne__(self, other):
return self.__cmp__(other) != 0
return K
retval.sort(key=cmp2key(self.foldersort))
self.folders = retval
return self.folders
def acquireconnection(self):
"""Fetches a connection from the pool, making sure to create a new one
if needed, to obey the maximum connection limits, etc.
Opens a connection to the server and returns an appropriate
object."""
self.semaphore.acquire()
self.connectionlock.acquire()
curThread = currentThread()
imapobj = None
if len(self.availableconnections): # One is available.
# Try to find one that previously belonged to this thread
# as an optimization. Start from the back since that's where
# they're popped on.
imapobj = None
for i in range(len(self.availableconnections) - 1, -1, -1):
tryobj = self.availableconnections[i]
if self.lastowner[tryobj] == curThread.ident:
imapobj = tryobj
del (self.availableconnections[i])
break
if not imapobj:
imapobj = self.availableconnections[0]
del (self.availableconnections[0])
self.assignedconnections.append(imapobj)
self.lastowner[imapobj] = curThread.ident
self.connectionlock.release()
return imapobj
self.connectionlock.release() # Release until need to modify data
# Must be careful here that if we fail we should bail out gracefully
# and release locks / threads so that the next attempt can try...
success = 0
try:
while not success:
# Generate a new connection.
if self.tunnel:
self.ui.connecting('tunnel', self.tunnel)
imapobj = imaplibutil.IMAP4_Tunnel(
self.tunnel,
timeout=socket.getdefaulttimeout(),
use_socket=self.proxied_socket,
)
success = 1
elif self.usessl:
self.ui.connecting(self.hostname, self.port)
imapobj = imaplibutil.WrappedIMAP4_SSL(
self.hostname,
self.port,
self.sslclientkey,
self.sslclientcert,
self.sslcacertfile,
self.__verifycert,
self.sslversion,
timeout=socket.getdefaulttimeout(),
fingerprint=self.fingerprint,
use_socket=self.proxied_socket,
tls_level=self.tlslevel,
)
else:
self.ui.connecting(self.hostname, self.port)
imapobj = imaplibutil.WrappedIMAP4(
self.hostname,
self.port,
timeout=socket.getdefaulttimeout(),
use_socket=self.proxied_socket,
)
if not self.preauth_tunnel:
try:
self.__authn_helper(imapobj)
self.goodpassword = self.password
success = 1
except OfflineImapError as e:
self.passworderror = str(e)
raise
# Enable compression
if self.repos.getconfboolean('usecompression', 0):
imapobj.enable_compression()
# update capabilities after login, e.g. gmail serves different ones
typ, dat = imapobj.capability()
if dat != [None]:
imapobj.capabilities = tuple(dat[-1].upper().split())
if self.delim == None:
listres = imapobj.list(self.reference, '""')[1]
if listres == [None] or listres == None:
# Some buggy IMAP servers do not respond well to LIST "" ""
# Work around them.
listres = imapobj.list(self.reference, '"*"')[1]
if listres == [None] or listres == None:
# No Folders were returned. This occurs, e.g. if the
# 'reference' prefix does not exist on the mail
# server. Raise exception.
err = "Server '%s' returned no folders in '%s'"% \
(self.repos.getname(), self.reference)
self.ui.warn(err)
raise Exception(err)
self.delim, self.root = \
imaputil.imapsplit(listres[0])[1:]
self.delim = imaputil.dequote(self.delim)
self.root = imaputil.dequote(self.root)
with self.connectionlock:
self.assignedconnections.append(imapobj)
self.lastowner[imapobj] = curThread.ident
return imapobj
except Exception as e:
"""If we are here then we did not succeed in getting a
connection - we should clean up and then re-raise the
error..."""
self.semaphore.release()
severity = OfflineImapError.ERROR.REPO
if type(e) == gaierror:
#DNS related errors. Abort Repo sync
#TODO: special error msg for e.errno == 2 "Name or service not known"?
reason = "Could not resolve name '%s' for repository "\
"'%s'. Make sure you have configured the ser"\
"ver name correctly and that you are online."%\
(self.hostname, self.repos)
raise OfflineImapError(reason, severity), None, exc_info()[2]
elif isinstance(e, SSLError) and e.errno == errno.EPERM:
# SSL unknown protocol error
# happens e.g. when connecting via SSL to a non-SSL service
if self.port != 993:
reason = "Could not connect via SSL to host '%s' and non-s"\
"tandard ssl port %d configured. Make sure you connect"\
" to the correct port."% (self.hostname, self.port)
else:
reason = "Unknown SSL protocol connecting to host '%s' for "\
"repository '%s'. OpenSSL responded:\n%s"\
% (self.hostname, self.repos, e)
raise OfflineImapError(reason, severity), None, exc_info()[2]
elif isinstance(e,
socket.error) and e.args[0] == errno.ECONNREFUSED:
# "Connection refused", can be a non-existing port, or an unauthorized
# webproxy (open WLAN?)
reason = "Connection to host '%s:%d' for repository '%s' was "\
"refused. Make sure you have the right host and port "\
"configured and that you are actually able to access the "\
"network."% (self.hostname, self.port, self.repos)
raise OfflineImapError(reason, severity), None, exc_info()[2]
# Could not acquire connection to the remote;
# socket.error(last_error) raised
if str(e)[:24] == "can't open socket; error":
raise OfflineImapError("Could not connect to remote server '%s' "\
"for repository '%s'. Remote does not answer."
% (self.hostname, self.repos),
OfflineImapError.ERROR.REPO), None, exc_info()[2]
else:
# re-raise all other errors
raise
def __xoauth2handler(self, response):
now = datetime.datetime.now()
if self.oauth2_access_token_expires_at \
and self.oauth2_access_token_expires_at < now:
self.oauth2_access_token = None
self.ui.debug('imap',
'xoauth2handler: oauth2_access_token expired')
if self.oauth2_access_token is None:
if self.oauth2_request_url is None:
raise OfflineImapError(
"No remote oauth2_request_url for "
"repository '%s' specified." % self,
OfflineImapError.ERROR.REPO)
# Generate new access token.
params = {}
params['client_id'] = self.oauth2_client_id
params['client_secret'] = self.oauth2_client_secret
params['refresh_token'] = self.oauth2_refresh_token
params['grant_type'] = 'refresh_token'
self.ui.debug('imap',
'xoauth2handler: url "%s"' % self.oauth2_request_url)
self.ui.debug('imap', 'xoauth2handler: params "%s"' % params)
original_socket = socket.socket
socket.socket = self.authproxied_socket
try:
response = urllib.request.urlopen(
self.oauth2_request_url,
urllib.parse.urlencode(params).encode('utf-8')).read()
except Exception as e:
try:
msg = "%s (configuration is: %s)" % (e, str(params))
except Exception as eparams:
msg = "%s [cannot display configuration: %s]" % (e,
eparams)
self.ui.error(msg)
raise
finally:
socket.socket = original_socket
resp = json.loads(response)
self.ui.debug('imap', 'xoauth2handler: response "%s"' % resp)
if 'error' in resp:
raise OfflineImapError("xoauth2handler got: %s" % resp,
OfflineImapError.ERROR.REPO)
self.oauth2_access_token = resp['access_token']
if 'expires_in' in resp:
self.oauth2_access_token_expires_at = now + datetime.timedelta(
seconds=resp['expires_in'] / 2)
self.ui.debug(
'imap', 'xoauth2handler: access_token "%s expires %s"' %
(self.oauth2_access_token, self.oauth2_access_token_expires_at))
auth_string = 'user=%s\1auth=Bearer %s\1\1' % (
self.username, self.oauth2_access_token)
# auth_string = base64.b64encode(auth_string)
self.ui.debug('imap', 'xoauth2handler: returning "%s"' % auth_string)
return auth_string
def getmaxage(self):
if self.config.getdefault("Account %s" % self.accountname, "maxage",
None):
raise OfflineImapError(
"maxage is not supported on IMAP-IMAP sync",
OfflineImapError.ERROR.REPO), None, exc_info()[2]
def cachemessagelist(self, min_date=None, min_uid=None):
if not self.synclabels:
return super(GmailFolder, self).cachemessagelist(min_date=min_date,
min_uid=min_uid)
self.dropmessagelistcache()
self.ui.collectingdata(None, self)
imapobj = self.imapserver.acquireconnection()
try:
msgsToFetch = self._msgs_to_fetch(imapobj,
min_date=min_date,
min_uid=min_uid)
if not msgsToFetch:
return # No messages to sync
# Get the flags and UIDs for these. single-quotes prevent
# imaplib2 from quoting the sequence.
#
# NB: msgsToFetch are sequential numbers, not UID's
res_type, response = imapobj.fetch("'%s'" % msgsToFetch,
'(FLAGS X-GM-LABELS UID)')
if res_type != 'OK':
six.reraise(
OfflineImapError,
OfflineImapError(
"FETCHING UIDs in folder [%s]%s failed. " %
(self.getrepository(), self) +
"Server responded '[%s] %s'" % (res_type, response),
OfflineImapError.ERROR.FOLDER),
exc_info()[2])
finally:
self.imapserver.releaseconnection(imapobj)
for messagestr in response:
# looks like: '1 (FLAGS (\\Seen Old) X-GM-LABELS (\\Inbox \\Favorites) UID 4807)' or None if no msg
# Discard initial message number.
if messagestr == None:
continue
messagestr = messagestr.split(' ', 1)[1]
options = imaputil.flags2hash(messagestr)
if not 'UID' in options:
self.ui.warn('No UID in message with options %s' %\
str(options),
minor = 1)
else:
uid = int(options['UID'])
self.messagelist[uid] = self.msglist_item_initializer(uid)
flags = imaputil.flagsimap2maildir(options['FLAGS'])
m = re.search('\(([^\)]*)\)', options['X-GM-LABELS'])
if m:
labels = set([
imaputil.dequote(lb)
for lb in imaputil.imapsplit(m.group(1))
])
else:
labels = set()
labels = labels - self.ignorelabels
rtime = imaplibutil.Internaldate2epoch(messagestr)
self.messagelist[uid] = {
'uid': uid,
'flags': flags,
'labels': labels,
'time': rtime
}
def cachemessagelist(self):
maxage = self.config.getdefaultint("Account %s" % self.accountname,
"maxage", -1)
maxsize = self.config.getdefaultint("Account %s" % self.accountname,
"maxsize", -1)
self.messagelist = {}
imapobj = self.imapserver.acquireconnection()
try:
res_type, imapdata = imapobj.select(self.getfullname(), True, True)
if imapdata == [None] or imapdata[0] == '0':
# Empty folder, no need to populate message list
return
# By default examine all UIDs in this folder
msgsToFetch = '1:*'
if (maxage != -1) | (maxsize != -1):
search_cond = "("
if (maxage != -1):
#find out what the oldest message is that we should look at
oldest_struct = time.gmtime(time.time() -
(60 * 60 * 24 * maxage))
if oldest_struct[0] < 1900:
raise OfflineImapError(
"maxage setting led to year %d. "
"Abort syncing." % oldest_struct[0],
OfflineImapError.ERROR.REPO)
search_cond += "SINCE %02d-%s-%d" % (
oldest_struct[2], MonthNames[oldest_struct[1]],
oldest_struct[0])
if (maxsize != -1):
if (maxage != -1): # There are two conditions, add space
search_cond += " "
search_cond += "SMALLER %d" % maxsize
search_cond += ")"
res_type, res_data = imapobj.search(None, search_cond)
if res_type != 'OK':
raise OfflineImapError(
"SEARCH in folder [%s]%s failed. "
"Search string was '%s'. Server responded '[%s] %s'" %
(self.getrepository(), self, search_cond, res_type,
res_data), OfflineImapError.ERROR.FOLDER)
# Result UIDs are seperated by space, coalesce into ranges
msgsToFetch = imaputil.uid_sequence(res_data[0].split())
if not msgsToFetch:
return # No messages to sync
# Get the flags and UIDs for these. single-quotes prevent
# imaplib2 from quoting the sequence.
res_type, response = imapobj.fetch("'%s'" % msgsToFetch,
'(FLAGS UID)')
if res_type != 'OK':
raise OfflineImapError(
"FETCHING UIDs in folder [%s]%s failed. "
"Server responded '[%s] %s'" %
(self.getrepository(), self, res_type, response),
OfflineImapError.ERROR.FOLDER)
finally:
self.imapserver.releaseconnection(imapobj)
for messagestr in response:
# looks like: '1 (FLAGS (\\Seen Old) UID 4807)' or None if no msg
# Discard initial message number.
if messagestr == None:
continue
messagestr = messagestr.split(' ', 1)[1]
options = imaputil.flags2hash(messagestr)
if not 'UID' in options:
self.ui.warn('No UID in message with options %s' %\
str(options),
minor = 1)
else:
uid = long(options['UID'])
flags = imaputil.flagsimap2maildir(options['FLAGS'])
rtime = imaplibutil.Internaldate2epoch(messagestr)
self.messagelist[uid] = {
'uid': uid,
'flags': flags,
'time': rtime
}
def __savemessage_fetchheaders(self, imapobj, headername, headervalue):
""" We fetch all new mail headers and search for the right
X-OfflineImap line by hand. The response from the server has form:
(
'OK',
[
(
'185 (RFC822.HEADER {1789}',
'... mail headers ...'
),
' UID 2444)',
(
'186 (RFC822.HEADER {1789}',
'... 2nd mail headers ...'
),
' UID 2445)'
]
)
We need to locate the UID just after mail headers containing our
X-OfflineIMAP line.
Returns UID when found, 0 when not found."""
self.ui.debug('imap', '__savemessage_fetchheaders called for %s: %s'% \
(headername, headervalue))
# run "fetch X:* rfc822.header"
# since we stored the mail we are looking for just recently, it would
# not be optimal to fetch all messages. So we'll find highest message
# UID in our local messagelist and search from there (exactly from
# UID+1). That works because UIDs are guaranteed to be unique and
# ascending.
if self.getmessagelist():
start = 1 + max(self.getmessagelist().keys())
else:
# Folder was empty - start from 1
start = 1
# Imaplib quotes all parameters of a string type. That must not happen
# with the range X:*. So we use bytearray to stop imaplib from getting
# in our way
result = imapobj.uid('FETCH', bytearray('%d:*' % start),
'rfc822.header')
if result[0] != 'OK':
raise OfflineImapError(
'Error fetching mail headers: %s' % '. '.join(result[1]),
OfflineImapError.ERROR.MESSAGE)
result = result[1]
found = 0
for item in result:
if found == 0 and type(item) == type(()):
# Walk just tuples
if re.search("(?:^|\\r|\\n)%s:\s*%s(?:\\r|\\n)" %
(headername, headervalue),
item[1],
flags=re.IGNORECASE):
found = 1
elif found == 1:
if type(item) == type(""):
uid = re.search("UID\s+(\d+)", item, flags=re.IGNORECASE)
if uid:
return int(uid.group(1))
else:
self.ui.warn(
"Can't parse FETCH response, can't find UID: %s",
result.__repr__())
else:
self.ui.warn(
"Can't parse FETCH response, we awaited string: %s",
result.__repr__())
return 0
def _fetch_from_imap(self, uids, retry_num=1):
"""Fetches data from IMAP server.
Arguments:
- uids: message UIDS (OfflineIMAP3: First UID returned only)
- retry_num: number of retries to make
Returns: data obtained by this query."""
imapobj = self.imapserver.acquireconnection()
try:
query = "(%s)" % (" ".join(self.imap_query))
fails_left = retry_num # Retry on dropped connection.
while fails_left:
try:
imapobj.select(self.getfullIMAPname(), readonly=True)
res_type, data = imapobj.uid('fetch', uids, query)
break
except imapobj.abort as e:
fails_left -= 1
# self.ui.error() will show the original traceback.
if fails_left <= 0:
message = ("%s, while fetching msg %r in folder %r."
" Max retry reached (%d)" %
(e, uids, self.name, retry_num))
raise OfflineImapError(message,
OfflineImapError.ERROR.MESSAGE)
self.ui.error("%s. While fetching msg %r in folder %r."
" Query: %s Retrying (%d/%d)" %
(e, uids, self.name, query,
retry_num - fails_left, retry_num))
# Release dropped connection, and get a new one.
self.imapserver.releaseconnection(imapobj, True)
imapobj = self.imapserver.acquireconnection()
finally:
# The imapobj here might be different than the one created before
# the ``try`` clause. So please avoid transforming this to a nice
# ``with`` without taking this into account.
self.imapserver.releaseconnection(imapobj)
# Ensure to not consider unsolicited FETCH responses caused by flag
# changes from concurrent connections. These appear as strings in
# 'data' (the BODY response appears as a tuple). This should leave
# exactly one response.
if res_type == 'OK':
data = [res for res in data if not isinstance(res, bytes)]
# Could not fetch message. Note: it is allowed by rfc3501 to return any
# data for the UID FETCH command.
if data == [None] or res_type != 'OK' or len(data) != 1:
severity = OfflineImapError.ERROR.MESSAGE
reason = "IMAP server '%s' failed to fetch messages UID '%s'. " \
"Server responded: %s %s" % (self.getrepository(), uids,
res_type, data)
if data == [None] or len(data) < 1:
# IMAP server did not find a message with this UID.
reason = "IMAP server '%s' does not have a message " \
"with UID '%s'" % (self.getrepository(), uids)
raise OfflineImapError(reason, severity)
# JI: In offlineimap, this function returned a tuple of strings for each
# fetched UID, offlineimap3 calls to the imap object return bytes and so
# originally a fixed, utf-8 conversion was done and *only* the first
# response (d[0]) was returned. Note that this alters the behavior
# between code bases. However, it seems like a single UID is the intent
# of this function so retaining the modfication here for now.
#
# TODO: Can we assume the server response containing the meta data is
# always 'utf-8' encoded? Assuming yes for now.
#
# Convert responses, d[0][0], into a 'utf-8' string (from bytes) and
# Convert email, d[0][1], into a message object (from bytes)
ndata0 = data[0][0].decode('utf-8')
ndata1 = self.parser['8bit-RFC'].parsebytes(data[0][1])
ndata = [ndata0, ndata1]
return ndata
def acquireconnection(self):
"""Fetches a connection from the pool, making sure to create a new one
if needed, to obey the maximum connection limits, etc.
Opens a connection to the server and returns an appropriate
object."""
self.semaphore.acquire()
self.connectionlock.acquire()
imapobj = None
if len(self.availableconnections): # One is available.
# Try to find one that previously belonged to this thread
# as an optimization. Start from the back since that's where
# they're popped on.
imapobj = None
for i in range(len(self.availableconnections) - 1, -1, -1):
tryobj = self.availableconnections[i]
if self.lastowner[tryobj] == get_ident():
imapobj = tryobj
del (self.availableconnections[i])
break
if not imapobj:
imapobj = self.availableconnections[0]
del (self.availableconnections[0])
self.assignedconnections.append(imapobj)
self.lastowner[imapobj] = get_ident()
self.connectionlock.release()
return imapobj
self.connectionlock.release() # Release until need to modify data
""" Must be careful here that if we fail we should bail out gracefully
and release locks / threads so that the next attempt can try...
"""
success = 0
try:
while not success:
# Generate a new connection.
if self.tunnel:
self.ui.connecting('tunnel', self.tunnel)
imapobj = imaplibutil.IMAP4_Tunnel(
self.tunnel, timeout=socket.getdefaulttimeout())
success = 1
elif self.usessl:
self.ui.connecting(self.hostname, self.port)
fingerprint = self.repos.get_ssl_fingerprint()
imapobj = imaplibutil.WrappedIMAP4_SSL(
self.hostname,
self.port,
self.sslclientkey,
self.sslclientcert,
self.sslcacertfile,
self.verifycert,
timeout=socket.getdefaulttimeout(),
fingerprint=fingerprint)
else:
self.ui.connecting(self.hostname, self.port)
imapobj = imaplibutil.WrappedIMAP4(
self.hostname,
self.port,
timeout=socket.getdefaulttimeout())
if not self.tunnel:
try:
# Try GSSAPI and continue if it fails
if 'AUTH=GSSAPI' in imapobj.capabilities and have_gss:
self.connectionlock.acquire()
self.ui.debug('imap',
'Attempting GSSAPI authentication')
try:
imapobj.authenticate('GSSAPI', self.gssauth)
except imapobj.error, val:
self.gssapi = False
self.ui.debug('imap',
'GSSAPI Authentication failed')
else:
self.gssapi = True
kerberos.authGSSClientClean(self.gss_vc)
self.gss_vc = None
self.gss_step = self.GSS_STATE_STEP
#if we do self.password = None then the next attempt cannot try...
#self.password = None
self.connectionlock.release()
if not self.gssapi:
if 'STARTTLS' in imapobj.capabilities and not\
self.usessl:
self.ui.debug('imap',
'Using STARTTLS connection')
imapobj.starttls()
if 'AUTH=CRAM-MD5' in imapobj.capabilities:
self.ui.debug(
'imap',
'Attempting CRAM-MD5 authentication')
try:
imapobj.authenticate(
'CRAM-MD5', self.md5handler)
except imapobj.error, val:
self.plainauth(imapobj)
else:
# Use plaintext login, unless
# LOGINDISABLED (RFC2595)
if 'LOGINDISABLED' in imapobj.capabilities:
raise OfflineImapError(
"Plaintext login "
"disabled by server. Need to use SSL?",
OfflineImapError.ERROR.REPO)
self.plainauth(imapobj)
# Would bail by here if there was a failure.
success = 1
self.goodpassword = self.password
except imapobj.error, val:
self.passworderror = str(val)
raise
def savemessage(self, uid, msg, flags, rtime):
"""Save the message on the Server
This backend always assigns a new uid, so the uid arg is ignored.
This function will update the self.messagelist dict to contain
the new message after sucessfully saving it.
See folder/Base for details. Note that savemessage() does not
check against dryrun settings, so you need to ensure that
savemessage is never called in a dryrun mode.
:param uid: Message UID
:param msg: Message Object
:param flags: Message flags
:param rtime: A timestamp to be used as the mail date
:returns: the UID of the new message as assigned by the server. If the
message is saved, but it's UID can not be found, it will
return 0. If the message can't be written (folder is
read-only for example) it will return -1."""
self.ui.savemessage('imap', uid, flags, self)
# Already have it, just save modified flags.
if uid > 0 and self.uidexists(uid):
self.savemessageflags(uid, flags)
return uid
# Filter user requested headers before uploading to the IMAP server
self.deletemessageheaders(msg, self.filterheaders)
# Should just be able to set the policy, to use CRLF in msg output
output_policy = self.policy['8bit-RFC']
# Get the date of the message, so we can pass it to the server.
date = self.__getmessageinternaldate(msg, rtime)
# Message-ID is handy for debugging messages.
msg_id = self.getmessageheader(msg, "message-id")
if not msg_id:
msg_id = '[unknown message-id]'
retry_left = 2 # succeeded in APPENDING?
imapobj = self.imapserver.acquireconnection()
# NB: in the finally clause for this try we will release
# NB: the acquired imapobj, so don't do that twice unless
# NB: you will put another connection to imapobj. If you
# NB: really do need to release connection manually, set
# NB: imapobj to None.
try:
while retry_left:
# XXX: we can mangle message only once, out of the loop
# UIDPLUS extension provides us with an APPENDUID response.
use_uidplus = 'UIDPLUS' in imapobj.capabilities
if not use_uidplus:
# Insert a random unique header that we can fetch later.
(headername,
headervalue) = self.__generate_randomheader(msg)
self.ui.debug(
'imap', 'savemessage: header is: %s: %s' %
(headername, headervalue))
self.addmessageheader(msg, headername, headervalue)
if self.ui.is_debugging('imap'):
# Optimization: don't create the debugging objects unless needed
msg_s = msg.as_string(policy=output_policy)
if len(msg_s) > 200:
dbg_output = "%s...%s" % (msg_s[:150], msg_s[-50:])
else:
dbg_output = msg_s
self.ui.debug(
'imap', "savemessage: date: %s, content: '%s'" %
(date, dbg_output))
try:
# Select folder for append and make the box READ-WRITE.
imapobj.select(self.getfullIMAPname())
except imapobj.readonly:
# readonly exception. Return original uid to notify that
# we did not save the message. (see savemessage in Base.py)
self.ui.msgtoreadonly(self, uid)
return uid
# Do the APPEND.
try:
(typ,
dat) = imapobj.append(self.getfullIMAPname(),
imaputil.flagsmaildir2imap(flags),
date,
msg.as_bytes(policy=output_policy))
# This should only catch 'NO' responses since append()
# will raise an exception for 'BAD' responses:
if typ != 'OK':
# For example, Groupwise IMAP server
# can return something like:
#
# NO APPEND The 1500 MB storage limit \
# has been exceeded.
#
# In this case, we should immediately abort
# the repository sync and continue
# with the next account.
err_msg = \
"Saving msg (%s) in folder '%s', " \
"repository '%s' failed (abort). " \
"Server responded: %s %s\n" % \
(msg_id, self, self.getrepository(), typ, dat)
raise OfflineImapError(err_msg,
OfflineImapError.ERROR.REPO)
retry_left = 0 # Mark as success.
except imapobj.abort as e:
# Connection has been reset, release connection and retry.
retry_left -= 1
self.imapserver.releaseconnection(imapobj, True)
imapobj = self.imapserver.acquireconnection()
if not retry_left:
raise OfflineImapError(
"Saving msg (%s) in folder '%s', "
"repository '%s' failed (abort). "
"Server responded: %s\n"
"Message content was: %s" %
(msg_id, self, self.getrepository(), str(e),
dbg_output), OfflineImapError.ERROR.MESSAGE,
exc_info()[2])
# XXX: is this still needed?
self.ui.error(e, exc_info()[2])
except imapobj.error as e: # APPEND failed
# If the server responds with 'BAD', append()
# raise()s directly. So we catch that too.
# drop conn, it might be bad.
self.imapserver.releaseconnection(imapobj, True)
imapobj = None
raise OfflineImapError(
"Saving msg (%s) folder '%s', repo '%s'"
"failed (error). Server responded: %s\n"
"Message content was: %s" %
(msg_id, self, self.getrepository(), str(e),
dbg_output), OfflineImapError.ERROR.MESSAGE,
exc_info()[2])
# Checkpoint. Let it write out stuff, etc. Eg searches for
# just uploaded messages won't work if we don't do this.
(typ, dat) = imapobj.check()
assert (typ == 'OK')
# Get the new UID, do we use UIDPLUS?
if use_uidplus:
# Get new UID from the APPENDUID response, it could look
# like OK [APPENDUID 38505 3955] APPEND completed with
# 38505 bein folder UIDvalidity and 3955 the new UID.
# note: we would want to use .response() here but that
# often seems to return [None], even though we have
# data. TODO
resp = imapobj._get_untagged_response('APPENDUID')
if resp == [None] or resp is None:
self.ui.warn(
"Server supports UIDPLUS but got no APPENDUID "
"appending a message. Got: %s." % str(resp))
return 0
try:
# Convert the UID from [b'4 1532'] to ['4 1532']
s_uid = [x.decode('utf-8') for x in resp]
# Now, read the UID field
uid = int(s_uid[-1].split(' ')[1])
except ValueError:
uid = 0 # Definetly not what we should have.
except Exception:
raise OfflineImapError(
"Unexpected response: %s" % str(resp),
OfflineImapError.ERROR.MESSAGE)
if uid == 0:
self.ui.warn("savemessage: Server supports UIDPLUS, but"
" we got no usable UID back. APPENDUID "
"reponse was '%s'" % str(resp))
else:
try:
# We don't use UIDPLUS.
uid = self.__savemessage_searchforheader(
imapobj, headername, headervalue)
# See docs for savemessage in Base.py for explanation
# of this and other return values.
if uid == 0:
self.ui.debug(
'imap', 'savemessage: attempt to get new UID '
'UID failed. Search headers manually.')
uid = self.__savemessage_fetchheaders(
imapobj, headername, headervalue)
self.ui.warn("savemessage: Searching mails for new "
"Message-ID failed. "
"Could not determine new UID on %s." %
self.getname())
# Something wrong happened while trying to get the UID. Explain
# the error might be about the 'get UID' process not necesseraly
# the APPEND.
except Exception:
self.ui.warn(
"%s: could not determine the UID while we got "
"no error while appending the "
"email with '%s: %s'" %
(self.getname(), headername, headervalue))
raise
finally:
if imapobj:
self.imapserver.releaseconnection(imapobj)
if uid: # Avoid UID FETCH 0 crash happening later on.
self.messagelist[uid] = self.msglist_item_initializer(uid)
self.messagelist[uid]['flags'] = flags
self.ui.debug('imap', 'savemessage: returning new UID %d' % uid)
return uid
def savemessage(self, uid, content, flags, rtime):
"""Save the message on the Server
This backend always assigns a new uid, so the uid arg is ignored.
This function will update the self.messagelist dict to contain
the new message after sucessfully saving it.
See folder/Base for details. Note that savemessage() does not
check against dryrun settings, so you need to ensure that
savemessage is never called in a dryrun mode.
:param rtime: A timestamp to be used as the mail date
:returns: the UID of the new message as assigned by the server. If the
message is saved, but it's UID can not be found, it will
return 0. If the message can't be written (folder is
read-only for example) it will return -1."""
self.ui.savemessage('imap', uid, flags, self)
# already have it, just save modified flags
if uid > 0 and self.uidexists(uid):
self.savemessageflags(uid, flags)
return uid
retry_left = 2 # succeeded in APPENDING?
imapobj = self.imapserver.acquireconnection()
try:
while retry_left:
# UIDPLUS extension provides us with an APPENDUID response.
use_uidplus = 'UIDPLUS' in imapobj.capabilities
# get the date of the message, so we can pass it to the server.
date = self.getmessageinternaldate(content, rtime)
content = re.sub("(?<!\r)\n", "\r\n", content)
if not use_uidplus:
# insert a random unique header that we can fetch later
(headername,
headervalue) = self.generate_randomheader(content)
self.ui.debug('imap', 'savemessage: header is: %s: %s' %\
(headername, headervalue))
content = self.savemessage_addheader(
content, headername, headervalue)
if len(content) > 200:
dbg_output = "%s...%s" % (content[:150], content[-50:])
else:
dbg_output = content
self.ui.debug(
'imap', "savemessage: date: %s, content: '%s'" %
(date, dbg_output))
try:
# Select folder for append and make the box READ-WRITE
imapobj.select(self.getfullname())
except imapobj.readonly:
# readonly exception. Return original uid to notify that
# we did not save the message. (see savemessage in Base.py)
self.ui.msgtoreadonly(self, uid, content, flags)
return uid
#Do the APPEND
try:
(typ,
dat) = imapobj.append(self.getfullname(),
imaputil.flagsmaildir2imap(flags),
date, content)
retry_left = 0 # Mark as success
except imapobj.abort as e:
# connection has been reset, release connection and retry.
retry_left -= 1
self.imapserver.releaseconnection(imapobj, True)
imapobj = self.imapserver.acquireconnection()
if not retry_left:
raise OfflineImapError(
"Saving msg in folder '%s', "
"repository '%s' failed (abort). Server reponded: %s\n"
"Message content was: %s" %
(self, self.getrepository(), str(e), dbg_output),
OfflineImapError.ERROR.MESSAGE)
self.ui.error(e, exc_info()[2])
except imapobj.error as e: # APPEND failed
# If the server responds with 'BAD', append()
# raise()s directly. So we catch that too.
# drop conn, it might be bad.
self.imapserver.releaseconnection(imapobj, True)
imapobj = None
raise OfflineImapError(
"Saving msg folder '%s', repo '%s'"
"failed (error). Server reponded: %s\nMessage content was: "
"%s" %
(self, self.getrepository(), str(e), dbg_output),
OfflineImapError.ERROR.MESSAGE)
# Checkpoint. Let it write out stuff, etc. Eg searches for
# just uploaded messages won't work if we don't do this.
(typ, dat) = imapobj.check()
assert (typ == 'OK')
# get the new UID. Test for APPENDUID response even if the
# server claims to not support it, as e.g. Gmail does :-(
if use_uidplus or imapobj._get_untagged_response(
'APPENDUID', True):
# get new UID from the APPENDUID response, it could look
# like OK [APPENDUID 38505 3955] APPEND completed with
# 38505 bein folder UIDvalidity and 3955 the new UID.
# note: we would want to use .response() here but that
# often seems to return [None], even though we have
# data. TODO
resp = imapobj._get_untagged_response('APPENDUID')
if resp == [None]:
self.ui.warn(
"Server supports UIDPLUS but got no APPENDUID "
"appending a message.")
return 0
uid = long(resp[-1].split(' ')[1])
if uid == 0:
self.ui.warn(
"savemessage: Server supports UIDPLUS, but"
" we got no usable uid back. APPENDUID reponse was "
"'%s'" % str(resp))
else:
# we don't support UIDPLUS
uid = self.savemessage_searchforheader(imapobj, headername,
headervalue)
# See docs for savemessage in Base.py for explanation
# of this and other return values
if uid == 0:
self.ui.debug(
'imap', 'savemessage: attempt to get new UID '
'UID failed. Search headers manually.')
uid = self.savemessage_fetchheaders(
imapobj, headername, headervalue)
self.ui.warn(
'imap', "savemessage: Searching mails for new "
"Message-ID failed. Could not determine new UID.")
finally:
self.imapserver.releaseconnection(imapobj)
if uid: # avoid UID FETCH 0 crash happening later on
self.messagelist[uid] = {'uid': uid, 'flags': flags}
self.ui.debug('imap', 'savemessage: returning new UID %d' % uid)
return uid
def savemessagelabels(self, uid, labels, ignorelabels=None):
"""Change a message's labels to `labels`.
Note that this function does not check against dryrun settings,
so you need to ensure that it is never called in a dryrun mode."""
if ignorelabels is None:
ignorelabels = set()
filename = self.messagelist[uid]['filename']
filepath = os.path.join(self.getfullname(), filename)
file = open(filepath, 'rt')
content = file.read()
file.close()
oldlabels = set()
for hstr in self.getmessageheaderlist(content, self.labelsheader):
oldlabels.update(
imaputil.labels_from_header(self.labelsheader, hstr))
labels = labels - ignorelabels
ignoredlabels = oldlabels & ignorelabels
oldlabels = oldlabels - ignorelabels
# Nothing to change.
if labels == oldlabels:
return
# Change labels into content.
labels_str = imaputil.format_labels_string(
self.labelsheader, sorted(labels | ignoredlabels))
# First remove old labels header, and then add the new one.
content = self.deletemessageheaders(content, self.labelsheader)
content = self.addmessageheader(content, '\n', self.labelsheader,
labels_str)
mtime = int(os.stat(filepath).st_mtime)
# Write file with new labels to a unique file in tmp.
messagename = self.new_message_filename(uid, set())
tmpname = self.save_to_tmp_file(messagename, content)
tmppath = os.path.join(self.getfullname(), tmpname)
# Move to actual location.
try:
os.rename(tmppath, filepath)
except OSError as e:
raise OfflineImapError(
"Can't rename file '%s' to '%s': %s" %
(tmppath, filepath, e[1]), OfflineImapError.ERROR.FOLDER,
exc_info()[2])
# If utime_from_header=true, we don't want to change the mtime.
if self._utime_from_header and mtime:
os.utime(filepath, (mtime, mtime))
# save the new mtime and labels
self.messagelist[uid]['mtime'] = int(os.stat(filepath).st_mtime)
self.messagelist[uid]['labels'] = labels
def __savemessage_fetchheaders(self, imapobj, headername, headervalue):
""" We fetch all new mail headers and search for the right
X-OfflineImap line by hand. The response from the server has form:
(
'OK',
[
(
'185 (RFC822.HEADER {1789}',
'... mail headers ...'
),
' UID 2444)',
(
'186 (RFC822.HEADER {1789}',
'... 2nd mail headers ...'
),
' UID 2445)'
]
)
We need to locate the UID just after mail headers containing our
X-OfflineIMAP line.
Returns UID when found, 0 when not found."""
self.ui.debug(
'imap', '__savemessage_fetchheaders called for %s: %s' %
(headername, headervalue))
# Run "fetch X:* rfc822.header".
# Since we stored the mail we are looking for just recently, it would
# not be optimal to fetch all messages. So we'll find highest message
# UID in our local messagelist and search from there (exactly from
# UID+1). That works because UIDs are guaranteed to be unique and
# ascending.
if self.getmessagelist():
start = 1 + max(self.getmessagelist().keys())
else:
# Folder was empty - start from 1.
start = 1
result = imapobj.uid('FETCH', '%d:*' % start, 'rfc822.header')
if result[0] != 'OK':
msg = 'Error fetching mail headers: %s' % '. '.join(result[1])
raise OfflineImapError(msg, OfflineImapError.ERROR.MESSAGE)
# result is like:
# [
# ('185 (RFC822.HEADER {1789}', '... mail headers ...'),
# ' UID 2444)',
# ('186 (RFC822.HEADER {1789}', '... 2nd mail headers ...'),
# ' UID 2445)'
# ]
result = result[1]
found = None
# item is like:
# ('185 (RFC822.HEADER {1789}', '... mail headers ...'), ' UID 2444)'
for item in result:
if found is None and type(item) == tuple:
# Decode the value
item = [x.decode('utf-8') for x in item]
# Walk just tuples.
if re.search("(?:^|\\r|\\n)%s:\s*%s(?:\\r|\\n)" %
(headername, headervalue),
item[1],
flags=re.IGNORECASE):
found = item[0]
elif found is not None:
if isinstance(item, bytes):
item = item.decode('utf-8')
uid = re.search("UID\s+(\d+)", item, flags=re.IGNORECASE)
if uid:
return int(uid.group(1))
else:
# This parsing is for Davmail.
# https://github.com/OfflineIMAP/offlineimap/issues/479
# item is like:
# ')'
# and item[0] stored in "found" is like:
# '1694 (UID 1694 RFC822.HEADER {1294}'
uid = re.search("\d+\s+\(UID\s+(\d+)",
found,
flags=re.IGNORECASE)
if uid:
return int(uid.group(1))
self.ui.warn("Can't parse FETCH response, "
"can't find UID in %s" % item)
self.ui.debug('imap', "Got: %s" % repr(result))
else:
self.ui.warn("Can't parse FETCH response, "
"we awaited string: %s" % repr(item))
return 0
def __sync(self):
"""Synchronize the account once, then return.
Assumes that `self.remoterepos`, `self.localrepos`, and
`self.statusrepos` has already been populated, so it should only
be called from the :meth:`syncrunner` function."""
folderthreads = []
hook_env = {
'OIMAP_ACCOUNT_NAME': self.getname(),
}
self.callhook('presynchook', hook_env)
if self.utf_8_support and self.remoterepos.getdecodefoldernames():
raise OfflineImapError(
"Configuration mismatch in account " +
"'%s'. " % self.getname() +
"\nAccount setting 'utf8foldernames' and repository " +
"setting 'decodefoldernames'\nmay not be used at the " +
"same time. This account has not been synchronized.\n" +
"Please check the configuration and documentation.",
OfflineImapError.ERROR.REPO)
quickconfig = self.getconfint('quick', 0)
if quickconfig < 0:
quick = True
elif quickconfig > 0:
if self.quicknum == 0 or self.quicknum > quickconfig:
self.quicknum = 1
quick = False
else:
self.quicknum = self.quicknum + 1
quick = True
else:
quick = False
try:
startedThread = False
remoterepos = self.remoterepos
localrepos = self.localrepos
statusrepos = self.statusrepos
# Init repos with list of folders, so we have them (and the
# folder delimiter etc).
remoterepos.getfolders()
localrepos.getfolders()
remoterepos.sync_folder_structure(localrepos, statusrepos)
# Replicate the folderstructure between REMOTE to LOCAL.
if not localrepos.getconfboolean('readonly', False):
self.ui.syncfolders(remoterepos, localrepos)
# Iterate through all folders on the remote repo and sync.
for remotefolder in remoterepos.getfolders():
# Check for CTRL-C or SIGTERM.
if Account.abort_NOW_signal.is_set():
break
if not remotefolder.sync_this:
self.ui.debug(
'', "Not syncing filtered folder '%s'"
"[%s]" % (remotefolder.getname(), remoterepos))
continue # Ignore filtered folder.
# The remote folder names must not have the local sep char in
# their names since this would cause troubles while converting
# the name back (from local to remote).
sep = localrepos.getsep()
if (sep != os.path.sep and sep != remoterepos.getsep()
and sep in remotefolder.getname()):
self.ui.warn(
'', "Ignoring folder '%s' due to unsupported "
"'%s' character serving as local separator." %
(remotefolder.getname(), localrepos.getsep()))
continue # Ignore unsupported folder name.
localfolder = self.get_local_folder(remotefolder)
if not localfolder.sync_this:
self.ui.debug(
'', "Not syncing filtered folder '%s'"
"[%s]" %
(localfolder.getname(), localfolder.repository))
continue # Ignore filtered folder.
if not globals.options.singlethreading:
thread = InstanceLimitedThread(
limitNamespace="%s%s" %
(FOLDER_NAMESPACE, self.remoterepos.getname()),
target=syncfolder,
name="Folder %s [acc: %s]" %
(remotefolder.getexplainedname(), self),
args=(self, remotefolder, quick))
thread.start()
folderthreads.append(thread)
else:
syncfolder(self, remotefolder, quick)
startedThread = True
# Wait for all threads to finish.
for thr in folderthreads:
thr.join()
if startedThread is True:
mbnames.writeIntermediateFile(
self.name) # Write out mailbox names.
else:
msg = "Account {}: no folder to sync (folderfilter issue?)".format(
self)
raise OfflineImapError(msg, OfflineImapError.ERROR.REPO)
localrepos.forgetfolders()
remoterepos.forgetfolders()
except:
# Error while syncing. Drop all connections that we have, they
# might be bogus by now (e.g. after suspend).
localrepos.dropconnections()
remoterepos.dropconnections()
raise
else:
# Sync went fine. Hold or drop depending on config.
localrepos.holdordropconnections()
remoterepos.holdordropconnections()
self.callhook('postsynchook', hook_env)
def _fetch_from_imap(self, uids, retry_num=1):
"""Fetches data from IMAP server.
Arguments:
- uids: message UIDS
- retry_num: number of retries to make
Returns: data obtained by this query."""
imapobj = self.imapserver.acquireconnection()
try:
query = "(%s)" % (" ".join(self.imap_query))
fails_left = retry_num # Retry on dropped connection.
while fails_left:
try:
imapobj.select(self.getfullIMAPname(), readonly=True)
res_type, data = imapobj.uid('fetch', uids, query)
break
except imapobj.abort as e:
fails_left -= 1
# self.ui.error() will show the original traceback.
if fails_left <= 0:
message = ("%s, while fetching msg %r in folder %r."
" Max retry reached (%d)" %
(e, uids, self.name, retry_num))
raise OfflineImapError(message,
OfflineImapError.ERROR.MESSAGE)
self.ui.error("%s. While fetching msg %r in folder %r."
" Query: %s Retrying (%d/%d)" %
(e, uids, self.name, query,
retry_num - fails_left, retry_num))
# Release dropped connection, and get a new one.
self.imapserver.releaseconnection(imapobj, True)
imapobj = self.imapserver.acquireconnection()
finally:
# The imapobj here might be different than the one created before
# the ``try`` clause. So please avoid transforming this to a nice
# ``with`` without taking this into account.
self.imapserver.releaseconnection(imapobj)
# Ensure to not consider unsolicited FETCH responses caused by flag
# changes from concurrent connections. These appear as strings in
# 'data' (the BODY response appears as a tuple). This should leave
# exactly one response.
if res_type == 'OK':
data = [res for res in data if not isinstance(res, bytes)]
# Could not fetch message. Note: it is allowed by rfc3501 to return any
# data for the UID FETCH command.
if data == [None] or res_type != 'OK' or len(data) != 1:
severity = OfflineImapError.ERROR.MESSAGE
reason = "IMAP server '%s' failed to fetch messages UID '%s'. " \
"Server responded: %s %s" % (self.getrepository(), uids,
res_type, data)
if data == [None] or len(data) < 1:
# IMAP server did not find a message with this UID.
reason = "IMAP server '%s' does not have a message " \
"with UID '%s'" % (self.getrepository(), uids)
raise OfflineImapError(reason, severity)
# Convert bytes to str
ndata0 = data[0][0].decode('utf-8')
ndata1 = data[0][1].decode('utf-8', errors='replace')
ndata = [ndata0, ndata1]
return ndata
def syncfolder(account, remotefolder, quick):
"""Synchronizes given remote folder for the specified account.
Filtered folders on the remote side will not invoke this function.
When called in concurrently for the same localfolder, syncs are
serialized."""
def acquire_mutex():
account_name = account.getname()
localfolder_name = localfolder.getfullname()
with SYNC_MUTEXES_LOCK:
if SYNC_MUTEXES.get(account_name) is None:
SYNC_MUTEXES[account_name] = {}
# The localfolder full name is good to uniquely identify the sync
# transaction.
if SYNC_MUTEXES[account_name].get(localfolder_name) is None:
#XXX: This lock could be an external file lock so we can remove
# the lock at the account level.
SYNC_MUTEXES[account_name][localfolder_name] = Lock()
# Acquire the lock.
SYNC_MUTEXES[account_name][localfolder_name].acquire()
def release_mutex():
SYNC_MUTEXES[account.getname()][localfolder.getfullname()].release()
def check_uid_validity():
# If either the local or the status folder has messages and
# there is a UID validity problem, warn and abort. If there are
# no messages, UW IMAPd loses UIDVALIDITY. But we don't really
# need it if both local folders are empty. So, in that case,
# just save it off.
if localfolder.getmessagecount() > 0 or statusfolder.getmessagecount(
) > 0:
if not localfolder.check_uidvalidity():
ui.validityproblem(localfolder)
localfolder.repository.restore_atime()
return
if not remotefolder.check_uidvalidity():
ui.validityproblem(remotefolder)
localrepos.restore_atime()
return
else:
# Both folders empty, just save new UIDVALIDITY.
localfolder.save_uidvalidity()
remotefolder.save_uidvalidity()
def cachemessagelists_upto_date(date):
"""Returns messages with uid > min(uids of messages newer than date)."""
remotefolder.cachemessagelist(
min_date=time.gmtime(time.mktime(date) + 24 * 60 * 60))
uids = remotefolder.getmessageuidlist()
localfolder.dropmessagelistcache()
if len(uids) > 0:
# Reload the remote message list from min_uid. This avoid issues for
# old messages, which has been added from local on any previous run
# (IOW, message is older than maxage _and_ has high enough UID).
remotefolder.dropmessagelistcache()
remotefolder.cachemessagelist(min_uid=min(uids))
localfolder.cachemessagelist(min_uid=min(uids))
else:
# Remote folder UIDs list is empty for the given range. We still
# might have valid local UIDs for this range (e.g.: new local
# emails).
localfolder.cachemessagelist(min_date=date)
uids = localfolder.getmessageuidlist()
# Take care to only consider positive uids. Negative UIDs might be
# present due to new emails.
uids = [uid for uid in uids if uid > 0]
if len(uids) > 0:
# Update the remote cache list for this new min(uids).
remotefolder.dropmessagelistcache()
remotefolder.cachemessagelist(min_uid=min(uids))
def cachemessagelists_startdate(new, partial, date):
"""Retrieve messagelists when startdate has been set for
the folder 'partial'.
Idea: suppose you want to clone the messages after date in one
account (partial) to a new one (new). If new is empty, then copy
messages in partial newer than date to new, and keep track of the
min uid. On subsequent syncs, sync all the messages in new against
those after that min uid in partial. This is a partial replacement
for maxage in the IMAP-IMAP sync case, where maxage doesn't work:
the UIDs of the messages in localfolder might not be in the same
order as those of corresponding messages in remotefolder, so if L in
local corresponds to R in remote, the ranges [L, ...] and [R, ...]
might not correspond. But, if we're cloning a folder into a new one,
[min_uid, ...] does correspond to [1, ...].
This is just for IMAP-IMAP. For Maildir-IMAP, use maxage instead."""
new.cachemessagelist()
min_uid = partial.retrieve_min_uid()
if min_uid == None: # min_uid file didn't exist
if len(new.getmessageuidlist()) > 0:
raise OfflineImapError(
"To use startdate on Repository %s, "
"Repository %s must be empty" %
(partial.repository.name, new.repository.name),
OfflineImapError.ERROR.MESSAGE)
else:
partial.cachemessagelist(min_date=date)
# messagelist.keys() instead of getuidmessagelist() because in
# the UID mapped case we want the actual local UIDs, not their
# remote counterparts.
positive_uids = [
uid for uid in list(partial.messagelist.keys()) if uid > 0
]
if len(positive_uids) > 0:
min_uid = min(positive_uids)
else:
min_uid = 1
partial.save_min_uid(min_uid)
else:
partial.cachemessagelist(min_uid=min_uid)
remoterepos = account.remoterepos
localrepos = account.localrepos
statusrepos = account.statusrepos
ui = getglobalui()
ui.registerthread(account)
try:
# Load local folder.
localfolder = account.get_local_folder(remotefolder)
# Acquire the mutex to start syncing.
acquire_mutex()
# Add the folder to the mbnames mailboxes.
mbnames.add(account.name, localrepos.getlocalroot(),
localfolder.getname())
# Load status folder.
statusfolder = statusrepos.getfolder(
remotefolder.getvisiblename().replace(remoterepos.getsep(),
statusrepos.getsep()))
statusfolder.openfiles()
statusfolder.cachemessagelist()
# Load local folder.
ui.syncingfolder(remoterepos, remotefolder, localrepos, localfolder)
# Retrieve messagelists, taking into account age-restriction
# options.
maxage = localfolder.getmaxage()
localstart = localfolder.getstartdate()
remotestart = remotefolder.getstartdate()
if (maxage != None) + (localstart != None) + (remotestart != None) > 1:
six.reraise(
OfflineImapError,
OfflineImapError(
"You can set at most one of the "
"following: maxage, startdate (for the local "
"folder), startdate (for the remote folder)",
OfflineImapError.ERROR.REPO),
exc_info()[2])
if (maxage != None or localstart or remotestart) and quick:
# IMAP quickchanged isn't compatible with options that
# involve restricting the messagelist, since the "quick"
# check can only retrieve a full list of UIDs in the folder.
ui.warn("Quick syncs (-q) not supported in conjunction "
"with maxage or startdate; ignoring -q.")
if maxage != None:
cachemessagelists_upto_date(maxage)
check_uid_validity()
elif localstart != None:
cachemessagelists_startdate(remotefolder, localfolder, localstart)
check_uid_validity()
elif remotestart != None:
cachemessagelists_startdate(localfolder, remotefolder, remotestart)
check_uid_validity()
else:
localfolder.cachemessagelist()
if quick:
if (not localfolder.quickchanged(statusfolder)
and not remotefolder.quickchanged(statusfolder)):
ui.skippingfolder(remotefolder)
localrepos.restore_atime()
return
check_uid_validity()
remotefolder.cachemessagelist()
# Synchronize remote changes.
if not localrepos.getconfboolean('readonly', False):
ui.syncingmessages(remoterepos, remotefolder, localrepos,
localfolder)
remotefolder.syncmessagesto(localfolder, statusfolder)
else:
ui.debug(
'', "Not syncing to read-only repository '%s'" %
localrepos.getname())
# Synchronize local changes.
if not remoterepos.getconfboolean('readonly', False):
ui.syncingmessages(localrepos, localfolder, remoterepos,
remotefolder)
localfolder.syncmessagesto(remotefolder, statusfolder)
else:
ui.debug(
'', "Not syncing to read-only repository '%s'" %
remoterepos.getname())
statusfolder.save()
localrepos.restore_atime()
except (KeyboardInterrupt, SystemExit):
raise
except OfflineImapError as e:
# Bubble up severe Errors, skip folder otherwise.
if e.severity > OfflineImapError.ERROR.FOLDER:
raise
else:
ui.error(e,
exc_info()[2],
msg="Aborting sync, folder '%s' "
"[acc: '%s']" % (localfolder, account))
except Exception as e:
ui.error(
e,
msg="ERROR in syncfolder for %s folder %s: %s" %
(account, remotefolder.getvisiblename(), traceback.format_exc()))
finally:
for folder in ["statusfolder", "localfolder", "remotefolder"]:
if folder in locals():
locals()[folder].dropmessagelistcache()
statusfolder.closefiles()
# Release the mutex of this sync transaction.
release_mutex()
def savemessage(self, uid, content, flags, rtime):
"""Save the message on the Server
This backend always assigns a new uid, so the uid arg is ignored.
This function will update the self.messagelist dict to contain
the new message after sucessfully saving it.
See folder/Base for details. Note that savemessage() does not
check against dryrun settings, so you need to ensure that
savemessage is never called in a dryrun mode.
:param rtime: A timestamp to be used as the mail date
:returns: the UID of the new message as assigned by the server. If the
message is saved, but it's UID can not be found, it will
return 0. If the message can't be written (folder is
read-only for example) it will return -1."""
self.ui.savemessage('imap', uid, flags, self)
# already have it, just save modified flags
if uid > 0 and self.uidexists(uid):
self.savemessageflags(uid, flags)
return uid
content = self.deletemessageheaders(content, self.filterheaders)
# Use proper CRLF all over the message
content = re.sub("(?<!\r)\n", CRLF, content)
# get the date of the message, so we can pass it to the server.
date = self.__getmessageinternaldate(content, rtime)
# Message-ID is handy for debugging messages
msg_id = self.getmessageheader(content, "message-id")
if not msg_id:
msg_id = '[unknown message-id]'
retry_left = 2 # succeeded in APPENDING?
imapobj = self.imapserver.acquireconnection()
# NB: in the finally clause for this try we will release
# NB: the acquired imapobj, so don't do that twice unless
# NB: you will put another connection to imapobj. If you
# NB: really do need to release connection manually, set
# NB: imapobj to None.
try:
while retry_left:
# XXX: we can mangle message only once, out of the loop
# UIDPLUS extension provides us with an APPENDUID response.
use_uidplus = 'UIDPLUS' in imapobj.capabilities
if not use_uidplus:
# insert a random unique header that we can fetch later
(headername,
headervalue) = self.__generate_randomheader(content)
self.ui.debug(
'imap', 'savemessage: header is: %s: %s' %
(headername, headervalue))
content = self.addmessageheader(content, CRLF, headername,
headervalue)
if len(content) > 200:
dbg_output = "%s...%s" % (content[:150], content[-50:])
else:
dbg_output = content
self.ui.debug(
'imap', "savemessage: date: %s, content: '%s'" %
(date, dbg_output))
try:
# Select folder for append and make the box READ-WRITE
imapobj.select(self.getfullname())
except imapobj.readonly:
# readonly exception. Return original uid to notify that
# we did not save the message. (see savemessage in Base.py)
self.ui.msgtoreadonly(self, uid, content, flags)
return uid
#Do the APPEND
try:
(typ,
dat) = imapobj.append(self.getfullname(),
imaputil.flagsmaildir2imap(flags),
date, content)
# This should only catch 'NO' responses since append()
# will raise an exception for 'BAD' responses:
if typ != 'OK':
# For example, Groupwise IMAP server can return something like:
#
# NO APPEND The 1500 MB storage limit has been exceeded.
#
# In this case, we should immediately abort the repository sync
# and continue with the next account.
msg = \
"Saving msg (%s) in folder '%s', repository '%s' failed (abort). " \
"Server responded: %s %s\n"% \
(msg_id, self, self.getrepository(), typ, dat)
raise OfflineImapError(msg,
OfflineImapError.ERROR.REPO)
retry_left = 0 # Mark as success
except imapobj.abort as e:
# connection has been reset, release connection and retry.
retry_left -= 1
self.imapserver.releaseconnection(imapobj, True)
imapobj = self.imapserver.acquireconnection()
if not retry_left:
raise OfflineImapError("Saving msg (%s) in folder '%s', "
"repository '%s' failed (abort). Server responded: %s\n"
"Message content was: %s"%
(msg_id, self, self.getrepository(), str(e), dbg_output),
OfflineImapError.ERROR.MESSAGE), \
None, exc_info()[2]
# XXX: is this still needed?
self.ui.error(e, exc_info()[2])
except imapobj.error as e: # APPEND failed
# If the server responds with 'BAD', append()
# raise()s directly. So we catch that too.
# drop conn, it might be bad.
self.imapserver.releaseconnection(imapobj, True)
imapobj = None
raise OfflineImapError(
"Saving msg (%s) folder '%s', repo '%s'"
"failed (error). Server responded: %s\nMessage content was: "
"%s" % (msg_id, self, self.getrepository(), str(e),
dbg_output),
OfflineImapError.ERROR.MESSAGE), None, exc_info()[2]
# Checkpoint. Let it write out stuff, etc. Eg searches for
# just uploaded messages won't work if we don't do this.
(typ, dat) = imapobj.check()
assert (typ == 'OK')
# get the new UID, do we use UIDPLUS?
if use_uidplus:
# get new UID from the APPENDUID response, it could look
# like OK [APPENDUID 38505 3955] APPEND completed with
# 38505 bein folder UIDvalidity and 3955 the new UID.
# note: we would want to use .response() here but that
# often seems to return [None], even though we have
# data. TODO
resp = imapobj._get_untagged_response('APPENDUID')
if resp == [None] or resp is None:
self.ui.warn(
"Server supports UIDPLUS but got no APPENDUID "
"appending a message.")
return 0
uid = long(resp[-1].split(' ')[1])
if uid == 0:
self.ui.warn(
"savemessage: Server supports UIDPLUS, but"
" we got no usable uid back. APPENDUID reponse was "
"'%s'" % str(resp))
else:
# we don't support UIDPLUS
uid = self.__savemessage_searchforheader(
imapobj, headername, headervalue)
# See docs for savemessage in Base.py for explanation
# of this and other return values
if uid == 0:
self.ui.debug(
'imap', 'savemessage: attempt to get new UID '
'UID failed. Search headers manually.')
uid = self.__savemessage_fetchheaders(
imapobj, headername, headervalue)
self.ui.warn(
'imap', "savemessage: Searching mails for new "
"Message-ID failed. Could not determine new UID.")
finally:
if imapobj: self.imapserver.releaseconnection(imapobj)
if uid: # avoid UID FETCH 0 crash happening later on
self.messagelist[uid] = self.msglist_item_initializer(uid)
self.messagelist[uid]['flags'] = flags
self.ui.debug('imap', 'savemessage: returning new UID %d' % uid)
return uid
self.semaphore.release()
#Make sure that this can be retried the next time...
self.passworderror = None
if (self.connectionlock.locked()):
self.connectionlock.release()
severity = OfflineImapError.ERROR.REPO
if type(e) == gaierror:
#DNS related errors. Abort Repo sync
#TODO: special error msg for e.errno == 2 "Name or service not known"?
reason = "Could not resolve name '%s' for repository "\
"'%s'. Make sure you have configured the ser"\
"ver name correctly and that you are online."%\
(self.hostname, self.reposname)
raise OfflineImapError(reason, severity)
elif SSLError and isinstance(e, SSLError) and e.errno == 1:
# SSL unknown protocol error
# happens e.g. when connecting via SSL to a non-SSL service
if self.port != 443:
reason = "Could not connect via SSL to host '%s' and non-s"\
"tandard ssl port %d configured. Make sure you connect"\
" to the correct port." % (self.hostname, self.port)
else:
reason = "Unknown SSL protocol connecting to host '%s' for"\
"repository '%s'. OpenSSL responded:\n%s"\
% (self.hostname, self.reposname, e)
raise OfflineImapError(reason, severity)
elif isinstance(e,
class IMAPRepository(BaseRepository):
def __init__(self, reposname, account):
"""Initialize an IMAPRepository object."""
BaseRepository.__init__(self, reposname, account)
# self.ui is being set by the BaseRepository
self._host = None
self.imapserver = imapserver.IMAPServer(self)
self.folders = None
def startkeepalive(self):
keepalivetime = self.getkeepalive()
if not keepalivetime: return
self.kaevent = Event()
self.kathread = ExitNotifyThread(target=self.imapserver.keepalive,
name="Keep alive " + self.getname(),
args=(keepalivetime, self.kaevent))
self.kathread.setDaemon(1)
self.kathread.start()
def stopkeepalive(self):
if not hasattr(self, 'kaevent'):
# Keepalive is not active.
return
self.kaevent.set()
del self.kathread
del self.kaevent
def holdordropconnections(self):
if not self.getholdconnectionopen():
self.dropconnections()
def dropconnections(self):
self.imapserver.close()
def getholdconnectionopen(self):
if self.getidlefolders():
return 1
return self.getconfboolean("holdconnectionopen", 0)
def getkeepalive(self):
num = self.getconfint("keepalive", 0)
if num == 0 and self.getidlefolders():
return 29 * 60
else:
return num
def getsep(self):
return self.imapserver.delim
def gethost(self):
"""Return the configured hostname to connect to
:returns: hostname as string or throws Exception"""
if self._host: # use cached value if possible
return self._host
# 1) check for remotehosteval setting
if self.config.has_option(self.getsection(), 'remotehosteval'):
host = self.getconf('remotehosteval')
try:
host = self.localeval.eval(host)
except Exception, e:
raise OfflineImapError("remotehosteval option for repository "\
"'%s' failed:\n%s" % (self, e),
OfflineImapError.ERROR.REPO)
if host:
self._host = host
return self._host
# 2) check for plain remotehost setting
host = self.getconf('remotehost', None)
if host != None:
self._host = host
return self._host
# no success
raise OfflineImapError("No remote host for repository "\
"'%s' specified." % self,
OfflineImapError.ERROR.REPO)
def _msgs_to_fetch(self, imapobj):
"""
Determines sequence numbers of messages to be fetched.
Message sequence numbers (MSNs) are more easily compacted
into ranges which makes transactions slightly faster.
Arguments:
- imapobj: instance of IMAPlib
Returns: range(s) for messages or None if no messages
are to be fetched.
"""
res_type, imapdata = imapobj.select(self.getfullname(), True, True)
if imapdata == [None] or imapdata[0] == '0':
# Empty folder, no need to populate message list
return None
# By default examine all messages in this folder
msgsToFetch = '1:*'
maxage = self.config.getdefaultint("Account %s" % self.accountname,
"maxage", -1)
maxsize = self.config.getdefaultint("Account %s" % self.accountname,
"maxsize", -1)
# Build search condition
if (maxage != -1) | (maxsize != -1):
search_cond = "("
if (maxage != -1):
#find out what the oldest message is that we should look at
oldest_struct = time.gmtime(time.time() -
(60 * 60 * 24 * maxage))
if oldest_struct[0] < 1900:
raise OfflineImapError(
"maxage setting led to year %d. "
"Abort syncing." % oldest_struct[0],
OfflineImapError.ERROR.REPO)
search_cond += "SINCE %02d-%s-%d" % (
oldest_struct[2], MonthNames[oldest_struct[1]],
oldest_struct[0])
if (maxsize != -1):
if (maxage != -1): # There are two conditions, add space
search_cond += " "
search_cond += "SMALLER %d" % maxsize
search_cond += ")"
res_type, res_data = imapobj.search(None, search_cond)
if res_type != 'OK':
raise OfflineImapError(
"SEARCH in folder [%s]%s failed. "
"Search string was '%s'. Server responded '[%s] %s'" %
(self.getrepository(), self, search_cond, res_type,
res_data), OfflineImapError.ERROR.FOLDER)
# Resulting MSN are separated by space, coalesce into ranges
msgsToFetch = imaputil.uid_sequence(res_data[0].split())
return msgsToFetch
def syncfolder(account, remotefolder, quick):
"""Synchronizes given remote folder for the specified account.
Filtered folders on the remote side will not invoke this function."""
def check_uid_validity(localfolder, remotefolder, statusfolder):
# If either the local or the status folder has messages and
# there is a UID validity problem, warn and abort. If there are
# no messages, UW IMAPd loses UIDVALIDITY. But we don't really
# need it if both local folders are empty. So, in that case,
# just save it off.
if localfolder.getmessagecount() > 0 or statusfolder.getmessagecount(
) > 0:
if not localfolder.check_uidvalidity():
ui.validityproblem(localfolder)
localfolder.repository.restore_atime()
return
if not remotefolder.check_uidvalidity():
ui.validityproblem(remotefolder)
localrepos.restore_atime()
return
else:
# Both folders empty, just save new UIDVALIDITY
localfolder.save_uidvalidity()
remotefolder.save_uidvalidity()
def save_min_uid(folder, min_uid):
uidfile = folder.get_min_uid_file()
fd = open(uidfile, 'wt')
fd.write(str(min_uid) + "\n")
fd.close()
def cachemessagelists_upto_date(localfolder, remotefolder, date):
""" Returns messages with uid > min(uids of messages newer than date)."""
localfolder.cachemessagelist(min_date=date)
check_uid_validity(localfolder, remotefolder, statusfolder)
# local messagelist had date restriction applied already. Restrict
# sync to messages with UIDs >= min_uid from this list.
#
# local messagelist might contain new messages (with uid's < 0).
positive_uids = [
uid for uid in localfolder.getmessageuidlist() if uid > 0
]
if len(positive_uids) > 0:
remotefolder.cachemessagelist(min_uid=min(positive_uids))
else:
# No messages with UID > 0 in range in localfolder.
# date restriction was applied with respect to local dates but
# remote folder timezone might be different from local, so be
# safe and make sure the range isn't bigger than in local.
remotefolder.cachemessagelist(
min_date=time.gmtime(time.mktime(date) + 24 * 60 * 60))
def cachemessagelists_startdate(new, partial, date):
""" Retrieve messagelists when startdate has been set for
the folder 'partial'.
Idea: suppose you want to clone the messages after date in one
account (partial) to a new one (new). If new is empty, then copy
messages in partial newer than date to new, and keep track of the
min uid. On subsequent syncs, sync all the messages in new against
those after that min uid in partial. This is a partial replacement
for maxage in the IMAP-IMAP sync case, where maxage doesn't work:
the UIDs of the messages in localfolder might not be in the same
order as those of corresponding messages in remotefolder, so if L in
local corresponds to R in remote, the ranges [L, ...] and [R, ...]
might not correspond. But, if we're cloning a folder into a new one,
[min_uid, ...] does correspond to [1, ...].
This is just for IMAP-IMAP. For Maildir-IMAP, use maxage instead.
"""
new.cachemessagelist()
min_uid = partial.retrieve_min_uid()
if min_uid == None: # min_uid file didn't exist
if len(new.getmessageuidlist()) > 0:
raise OfflineImapError(
"To use startdate on Repository %s, "
"Repository %s must be empty" %
(partial.repository.name, new.repository.name),
OfflineImapError.ERROR.MESSAGE)
else:
partial.cachemessagelist(min_date=date)
# messagelist.keys() instead of getuidmessagelist() because in
# the UID mapped case we want the actual local UIDs, not their
# remote counterparts
positive_uids = [
uid for uid in list(partial.messagelist.keys()) if uid > 0
]
if len(positive_uids) > 0:
min_uid = min(positive_uids)
else:
min_uid = 1
save_min_uid(partial, min_uid)
else:
partial.cachemessagelist(min_uid=min_uid)
remoterepos = account.remoterepos
localrepos = account.localrepos
statusrepos = account.statusrepos
ui = getglobalui()
ui.registerthread(account)
try:
# Load local folder.
localfolder = account.get_local_folder(remotefolder)
# Write the mailboxes
mbnames.add(account.name, localfolder.getname(),
localrepos.getlocalroot())
# Load status folder.
statusfolder = statusrepos.getfolder(
remotefolder.getvisiblename().replace(remoterepos.getsep(),
statusrepos.getsep()))
statusfolder.openfiles()
if localfolder.get_uidvalidity() == None:
# This is a new folder, so delete the status cache to be
# sure we don't have a conflict.
# TODO: This does not work. We always return a value, need
# to rework this...
statusfolder.deletemessagelist()
statusfolder.cachemessagelist()
# Load local folder.
ui.syncingfolder(remoterepos, remotefolder, localrepos, localfolder)
# Retrieve messagelists, taking into account age-restriction
# options
maxage = localfolder.getmaxage()
localstart = localfolder.getstartdate()
remotestart = remotefolder.getstartdate()
if (maxage != None) + (localstart != None) + (remotestart != None) > 1:
six.reraise(
OfflineImapError(
"You can set at most one of the "
"following: maxage, startdate (for the local folder), "
"startdate (for the remote folder)",
OfflineImapError.ERROR.REPO), None,
exc_info()[2])
if (maxage != None or localstart or remotestart) and quick:
# IMAP quickchanged isn't compatible with options that
# involve restricting the messagelist, since the "quick"
# check can only retrieve a full list of UIDs in the folder.
ui.warn("Quick syncs (-q) not supported in conjunction "
"with maxage or startdate; ignoring -q.")
if maxage != None:
cachemessagelists_upto_date(localfolder, remotefolder, maxage)
elif localstart != None:
cachemessagelists_startdate(remotefolder, localfolder, localstart)
check_uid_validity(localfolder, remotefolder, statusfolder)
elif remotestart != None:
cachemessagelists_startdate(localfolder, remotefolder, remotestart)
check_uid_validity(localfolder, remotefolder, statusfolder)
else:
localfolder.cachemessagelist()
if quick:
if (not localfolder.quickchanged(statusfolder)
and not remotefolder.quickchanged(statusfolder)):
ui.skippingfolder(remotefolder)
localrepos.restore_atime()
return
check_uid_validity(localfolder, remotefolder, statusfolder)
remotefolder.cachemessagelist()
# Synchronize remote changes.
if not localrepos.getconfboolean('readonly', False):
ui.syncingmessages(remoterepos, remotefolder, localrepos,
localfolder)
remotefolder.syncmessagesto(localfolder, statusfolder)
else:
ui.debug(
'', "Not syncing to read-only repository '%s'" %
localrepos.getname())
# Synchronize local changes.
if not remoterepos.getconfboolean('readonly', False):
ui.syncingmessages(localrepos, localfolder, remoterepos,
remotefolder)
localfolder.syncmessagesto(remotefolder, statusfolder)
else:
ui.debug(
'', "Not syncing to read-only repository '%s'" %
remoterepos.getname())
statusfolder.save()
localrepos.restore_atime()
except (KeyboardInterrupt, SystemExit):
raise
except OfflineImapError as e:
# bubble up severe Errors, skip folder otherwise
if e.severity > OfflineImapError.ERROR.FOLDER:
raise
else:
ui.error(e,
exc_info()[2],
msg="Aborting sync, folder '%s' "
"[acc: '%s']" % (localfolder, account))
except Exception as e:
ui.error(
e,
msg="ERROR in syncfolder for %s folder %s: %s" %
(account, remotefolder.getvisiblename(), traceback.format_exc()))
finally:
for folder in ["statusfolder", "localfolder", "remotefolder"]:
if folder in locals():
locals()[folder].dropmessagelistcache()
statusfolder.closefiles()
def __init__(self, repos):
""":repos: a IMAPRepository instance."""
self.ui = getglobalui()
self.repos = repos
self.config = repos.getconfig()
self.preauth_tunnel = repos.getpreauthtunnel()
self.transport_tunnel = repos.gettransporttunnel()
if self.preauth_tunnel and self.transport_tunnel:
raise OfflineImapError(
'%s: ' % repos + 'you must enable precisely one '
'type of tunnel (preauth or transport), '
'not both', OfflineImapError.ERROR.REPO)
self.tunnel = \
self.preauth_tunnel if self.preauth_tunnel \
else self.transport_tunnel
self.username = \
None if self.preauth_tunnel else repos.getuser()
self.user_identity = repos.get_remote_identity()
self.authmechs = repos.get_auth_mechanisms()
self.password = None
self.passworderror = None
self.goodpassword = None
self.usessl = repos.getssl()
self.useipv6 = repos.getipv6()
if self.useipv6 is True:
self.af = socket.AF_INET6
elif self.useipv6 is False:
self.af = socket.AF_INET
else:
self.af = socket.AF_UNSPEC
self.hostname = None if self.transport_tunnel else repos.gethost()
self.port = repos.getport()
if self.port is None:
self.port = 993 if self.usessl else 143
self.sslclientcert = repos.getsslclientcert()
self.sslclientkey = repos.getsslclientkey()
self.sslcacertfile = repos.getsslcacertfile()
if self.sslcacertfile is None:
self.__verifycert = None # Disable cert verification.
# This way of working sucks hard...
self.fingerprint = repos.get_ssl_fingerprint()
self.tlslevel = repos.gettlslevel()
self.sslversion = repos.getsslversion()
self.starttls = repos.getstarttls()
if self.tlslevel is not "tls_compat" and self.sslversion is None:
raise Exception("When 'tls_version' is not 'tls_compat' "
"the 'ssl_version' must be set explicitly.")
self.oauth2_refresh_token = repos.getoauth2_refresh_token()
self.oauth2_access_token = repos.getoauth2_access_token()
self.oauth2_client_id = repos.getoauth2_client_id()
self.oauth2_client_secret = repos.getoauth2_client_secret()
self.oauth2_request_url = repos.getoauth2_request_url()
self.delim = None
self.root = None
self.maxconnections = repos.getmaxconnections()
self.availableconnections = []
self.assignedconnections = []
self.lastowner = {}
self.semaphore = BoundedSemaphore(self.maxconnections)
self.connectionlock = Lock()
self.reference = repos.getreference()
self.idlefolders = repos.getidlefolders()
self.gss_step = self.GSS_STATE_STEP
self.gss_vc = None
self.gssapi = False
# In order to support proxy connection, we have to override the
# default socket instance with our own socksified socket instance.
# We add this option to bypass the GFW in China.
self.proxied_socket = self._get_proxy('proxy', socket.socket)
# Turns out that the GFW in China is no longer blocking imap.gmail.com
# However accounts.google.com (for oauth2) definitey is. Therefore
# it is not strictly necessary to use a proxy for *both* IMAP *and*
# oauth2, so a new option is added: authproxy.
# Set proxy for use in authentication (only) if desired.
# If not set, is same as proxy option (compatible with current configs)
# To use a proxied_socket but not an authproxied_socket
# set authproxy = '' in config
self.authproxied_socket = self._get_proxy('authproxy',
self.proxied_socket)
def __init__(self, repos):
self.ui = getglobalui()
self.repos = repos
self.config = repos.getconfig()
self.preauth_tunnel = repos.getpreauthtunnel()
self.transport_tunnel = repos.gettransporttunnel()
if self.preauth_tunnel and self.transport_tunnel:
raise OfflineImapError(
'%s: ' % repos + 'you must enable precisely one '
'type of tunnel (preauth or transport), '
'not both', OfflineImapError.ERROR.REPO)
self.tunnel = \
self.preauth_tunnel if self.preauth_tunnel \
else self.transport_tunnel
self.username = \
None if self.preauth_tunnel else repos.getuser()
self.user_identity = repos.get_remote_identity()
self.authmechs = repos.get_auth_mechanisms()
self.password = None
self.passworderror = None
self.goodpassword = None
self.usessl = repos.getssl()
self.hostname = \
None if self.preauth_tunnel else repos.gethost()
self.port = repos.getport()
if self.port == None:
self.port = 993 if self.usessl else 143
self.sslclientcert = repos.getsslclientcert()
self.sslclientkey = repos.getsslclientkey()
self.sslcacertfile = repos.getsslcacertfile()
if self.sslcacertfile is None:
self.__verifycert = None # disable cert verification
self.fingerprint = repos.get_ssl_fingerprint()
self.sslversion = repos.getsslversion()
self.tlslevel = repos.gettlslevel()
self.oauth2_refresh_token = repos.getoauth2_refresh_token()
self.oauth2_access_token = repos.getoauth2_access_token()
self.oauth2_client_id = repos.getoauth2_client_id()
self.oauth2_client_secret = repos.getoauth2_client_secret()
self.oauth2_request_url = repos.getoauth2_request_url()
self.delim = None
self.root = None
self.maxconnections = repos.getmaxconnections()
self.availableconnections = []
self.assignedconnections = []
self.lastowner = {}
self.semaphore = BoundedSemaphore(self.maxconnections)
self.connectionlock = Lock()
self.reference = repos.getreference()
self.idlefolders = repos.getidlefolders()
self.gss_step = self.GSS_STATE_STEP
self.gss_vc = None
self.gssapi = False
# In order to support proxy connection, we have to override the
# default socket instance with our own socksified socket instance.
# We add this option to bypass the GFW in China.
_account_section = 'Account ' + self.repos.account.name
if not self.config.has_option(_account_section, 'proxy'):
self.proxied_socket = socket.socket
else:
proxy = self.config.get(_account_section, 'proxy')
# Powered by PySocks.
try:
import socks
proxy_type, host, port = proxy.split(":")
port = int(port)
socks.setdefaultproxy(getattr(socks, proxy_type), host, port)
self.proxied_socket = socks.socksocket
except ImportError:
self.ui.warn("PySocks not installed, ignoring proxy option.")
self.proxied_socket = socket.socket
except (AttributeError, ValueError) as e:
self.ui.warn("Bad proxy option %s for account %s: %s "
"Ignoring proxy option." %
(proxy, self.repos.account.name, e))
self.proxied_socket = socket.socket
def getpassword(self):
"""Return the IMAP password for this repository.
It tries to get passwords in the following order:
1. evaluate Repository 'remotepasseval'
2. read password from Repository 'remotepass'
3. read password from file specified in Repository 'remotepassfile'
4. read password from ~/.netrc
5. read password from /etc/netrc
On success we return the password.
If all strategies fail we return None."""
# 1. Evaluate Repository 'remotepasseval'.
passwd = self.getconf('remotepasseval', None)
if passwd is not None:
l_pass = self.localeval.eval(passwd)
# We need a str password
if isinstance(l_pass, bytes):
return l_pass.decode(encoding='utf-8')
elif isinstance(l_pass, str):
return l_pass
# If is not bytes or str, we have a problem
raise OfflineImapError(
"Could not get a right password format for"
" repository %s. Type found: %s. "
"Please, open a bug." % (self.name, type(l_pass)),
OfflineImapError.ERROR.FOLDER)
# 2. Read password from Repository 'remotepass'.
password = self.getconf('remotepass', None)
if password is not None:
# Assume the configuration file to be UTF-8 encoded so we must not
# encode this string again.
return password
# 3. Read password from file specified in Repository 'remotepassfile'.
passfile = self.getconf('remotepassfile', None)
if passfile is not None:
file_desc = open(os.path.expanduser(passfile),
'r',
encoding='utf-8')
password = file_desc.readline().strip()
file_desc.close()
return password.encode('UTF-8')
# 4. Read password from ~/.netrc.
try:
netrcentry = netrc.netrc().authenticators(self.gethost())
except IOError as inst:
if inst.errno != errno.ENOENT:
raise
else:
if netrcentry:
user = self.getuser()
if user is None or user == netrcentry[0]:
return netrcentry[2]
# 5. Read password from /etc/netrc.
try:
netrcentry = netrc.netrc('/etc/netrc')\
.authenticators(self.gethost())
except IOError as inst:
if inst.errno not in (errno.ENOENT, errno.EACCES):
raise
else:
if netrcentry:
user = self.getuser()
if user is None or user == netrcentry[0]:
return netrcentry[2]
# No strategy yielded a password!
return None
def __authn_helper(self, imapobj):
"""Authentication machinery for self.acquireconnection().
Raises OfflineImapError() of type ERROR.REPO when
there are either fatal problems or no authentications
succeeded.
If any authentication method succeeds, routine should exit:
warnings for failed methods are to be produced in the
respective except blocks."""
# Stack stores pairs of (method name, exception)
exc_stack = []
tried_to_authn = False
tried_tls = False
# Authentication routines, hash keyed by method name
# with value that is a tuple with
# - authentication function,
# - tryTLS flag,
# - check IMAP capability flag.
auth_methods = {
"GSSAPI": (self.__authn_gssapi, False, True),
"XOAUTH2": (self.__authn_xoauth2, True, True),
"CRAM-MD5": (self.__authn_cram_md5, True, True),
"PLAIN": (self.__authn_plain, True, True),
"LOGIN": (self.__authn_login, True, False),
}
# GSSAPI is tried first by default: we will probably go TLS after it and
# GSSAPI mustn't be tunneled over TLS.
for m in self.authmechs:
if m not in auth_methods:
raise Exception("Bad authentication method %s, "
"please, file OfflineIMAP bug" % m)
func, tryTLS, check_cap = auth_methods[m]
# TLS must be initiated before checking capabilities:
# they could have been changed after STARTTLS.
if tryTLS and self.starttls and not tried_tls:
tried_tls = True
self.__start_tls(imapobj)
if check_cap:
cap = "AUTH=" + m
if cap not in imapobj.capabilities:
continue
tried_to_authn = True
self.ui.debug('imap', 'Attempting ' '%s authentication' % m)
try:
if func(imapobj):
return
except (imapobj.error, OfflineImapError) as e:
self.ui.warn('%s authentication failed: %s' % (m, e))
exc_stack.append((m, e))
if len(exc_stack):
msg = "\n\t".join(
[": ".join((x[0], str(x[1]))) for x in exc_stack])
raise OfflineImapError(
"All authentication types "
"failed:\n\t%s" % msg, OfflineImapError.ERROR.REPO)
if not tried_to_authn:
methods = ", ".join([
x[5:] for x in
[x for x in imapobj.capabilities if x[0:5] == "AUTH="]
])
raise OfflineImapError(
"Repository %s: no supported "
"authentication mechanisms found; configured %s, "
"server advertises %s" %
(self.repos, ", ".join(self.authmechs), methods),
OfflineImapError.ERROR.REPO)
def _fetch_from_imap(self, uids, retry_num=1):
"""Fetches data from IMAP server.
Arguments:
- uids: message UIDS (OfflineIMAP3: First UID returned only)
- retry_num: number of retries to make
Returns: data obtained by this query."""
imapobj = self.imapserver.acquireconnection()
try:
query = "(%s)" % (" ".join(self.imap_query))
fails_left = retry_num # Retry on dropped connection.
while fails_left:
try:
imapobj.select(self.getfullIMAPname(), readonly=True)
res_type, data = imapobj.uid('fetch', uids, query)
break
except imapobj.abort as e:
fails_left -= 1
# self.ui.error() will show the original traceback.
if fails_left <= 0:
message = ("%s, while fetching msg %r in folder %r."
" Max retry reached (%d)" %
(e, uids, self.name, retry_num))
raise OfflineImapError(message,
OfflineImapError.ERROR.MESSAGE)
self.ui.error("%s. While fetching msg %r in folder %r."
" Query: %s Retrying (%d/%d)" %
(e, uids, self.name, query,
retry_num - fails_left, retry_num))
# Release dropped connection, and get a new one.
self.imapserver.releaseconnection(imapobj, True)
imapobj = self.imapserver.acquireconnection()
finally:
# The imapobj here might be different than the one created before
# the ``try`` clause. So please avoid transforming this to a nice
# ``with`` without taking this into account.
self.imapserver.releaseconnection(imapobj)
# Ensure to not consider unsolicited FETCH responses caused by flag
# changes from concurrent connections. These appear as strings in
# 'data' (the BODY response appears as a tuple). This should leave
# exactly one response.
if res_type == 'OK':
data = [res for res in data if not isinstance(res, bytes)]
# Could not fetch message. Note: it is allowed by rfc3501 to return any
# data for the UID FETCH command.
if data == [None] or res_type != 'OK' or len(data) != 1:
severity = OfflineImapError.ERROR.MESSAGE
reason = "IMAP server '%s' failed to fetch messages UID '%s'. " \
"Server responded: %s %s" % (self.getrepository(), uids,
res_type, data)
if data == [None] or len(data) < 1:
# IMAP server did not find a message with this UID.
reason = "IMAP server '%s' does not have a message " \
"with UID '%s'" % (self.getrepository(), uids)
raise OfflineImapError(reason, severity)
# JI: In offlineimap, this function returned a tuple of strings for each
# fetched UID, offlineimap3 calls to the imap object return bytes and so
# originally a fixed, utf-8 conversion was done and *only* the first
# response (d[0]) was returned. Note that this alters the behavior
# between code bases. However, it seems like a single UID is the intent
# of this function so retaining the modfication here for now.
#
# TODO: Can we assume the server response containing the meta data is
# always 'utf-8' encoded? Assuming yes for now.
#
# Convert responses, d[0][0], into a 'utf-8' string (from bytes) and
# Convert email, d[0][1], into a message object (from bytes)
ndata0 = data[0][0].decode('utf-8')
try:
ndata1 = self.parser['8bit-RFC'].parsebytes(data[0][1])
except:
err = exc_info()
response_type = type(data[0][1]).__name__
msg_id = self._extract_message_id(data[0][1])[0].decode(
'ascii', errors='surrogateescape')
raise OfflineImapError(
"Exception parsing message with ID ({}) from imaplib (response type: {}).\n {}: {}"
.format(msg_id, response_type, err[0].__name__,
err[1]), OfflineImapError.ERROR.MESSAGE)
if len(ndata1.defects) > 0:
# We don't automatically apply fixes as to attempt to preserve the original message
self.ui.warn("UID {} has defects: {}".format(uids, ndata1.defects))
if any(
isinstance(defect, NoBoundaryInMultipartDefect)
for defect in ndata1.defects):
# (Hopefully) Rare defect from a broken client where multipart boundary is
# not properly quoted. Attempt to solve by fixing the boundary and parsing
self.ui.warn(" ... applying multipart boundary fix.")
ndata1 = self.parser['8bit-RFC'].parsebytes(
self._quote_boundary_fix(data[0][1]))
try:
# See if the defects after fixes are preventing us from obtaining bytes
_ = ndata1.as_bytes(policy=self.policy['8bit-RFC'])
except UnicodeEncodeError as err:
# Unknown issue which is causing failure of as_bytes()
msg_id = self.getmessageheader(ndata1, "message-id")
if msg_id is None:
msg_id = '<Unknown Message-ID>'
raise OfflineImapError(
"UID {} ({}) has defects preventing it from being processed!\n {}: {}"
.format(uids, msg_id,
type(err).__name__,
err), OfflineImapError.ERROR.MESSAGE)
ndata = [ndata0, ndata1]
return ndata