def syncfoldersto(self, dest, copyfolders):
"""Syncs the folders in this repository to those in dest.
It does NOT sync the contents of those folders.
For every time dest.makefolder() is called, also call makefolder()
on each folder in copyfolders."""
src = self
srcfolders = src.getfolders()
destfolders = dest.getfolders()
# Create hashes with the names, but convert the source folders
# to the dest folder's sep.
srchash = {}
for folder in srcfolders:
srchash[folder.getvisiblename().replace(src.getsep(), dest.getsep())] = \
folder
desthash = {}
for folder in destfolders:
desthash[folder.getvisiblename()] = folder
#
# Find new folders.
#
for key in srchash.keys():
if not key in desthash:
try:
dest.makefolder(key)
for copyfolder in copyfolders:
copyfolder.makefolder(
key.replace(dest.getsep(), copyfolder.getsep()))
except:
UIBase.getglobalui().warn("ERROR Attempting to make folder " \
+ key + ":" +str(sys.exc_info()[1]))
def copymessageto(self, uid, applyto, register = 1):
# Sometimes, it could be the case that if a sync takes awhile,
# a message might be deleted from the maildir before it can be
# synced to the status cache. This is only a problem with
# self.getmessage(). So, don't call self.getmessage unless
# really needed.
if register:
UIBase.getglobalui().registerthread(self.getaccountname())
UIBase.getglobalui().copyingmessage(uid, self, applyto)
message = ''
# If any of the destinations actually stores the message body,
# load it up.
for object in applyto:
if object.storesmessages():
message = self.getmessage(uid)
break
flags = self.getmessageflags(uid)
rtime = self.getmessagetime(uid)
for object in applyto:
newuid = object.savemessage(uid, message, flags, rtime)
if newuid > 0 and newuid != uid:
# Change the local uid.
self.savemessage(newuid, message, flags, rtime)
self.deletemessage(uid)
uid = newuid
def syncmessagesto_neguid_msg(self, uid, dest, applyto, register = 1):
if register:
UIBase.getglobalui().registerthread(self.getaccountname())
UIBase.getglobalui().copyingmessage(uid, self, applyto)
successobject = None
successuid = None
message = self.getmessage(uid)
flags = self.getmessageflags(uid)
rtime = self.getmessagetime(uid)
for tryappend in applyto:
successuid = tryappend.savemessage(uid, message, flags, rtime)
if successuid >= 0:
successobject = tryappend
break
# Did we succeed?
if successobject != None:
if successuid: # Only if IMAP actually assigned a UID
# Copy the message to the other remote servers.
for appendserver in \
[x for x in applyto if x != successobject]:
appendserver.savemessage(successuid, message, flags, rtime)
# Copy to its new name on the local server and delete
# the one without a UID.
self.savemessage(successuid, message, flags, rtime)
self.deletemessage(uid) # It'll be re-downloaded.
else:
# Did not find any server to take this message. Ignore.
pass
def syncmessagesto_neguid_msg(self, uid, dest, applyto, register=1):
if register:
UIBase.getglobalui().registerthread(self.getaccountname())
UIBase.getglobalui().copyingmessage(uid, self, applyto)
successobject = None
successuid = None
message = self.getmessage(uid)
flags = self.getmessageflags(uid)
rtime = self.getmessagetime(uid)
for tryappend in applyto:
successuid = tryappend.savemessage(uid, message, flags, rtime)
if successuid >= 0:
successobject = tryappend
break
# Did we succeed?
if successobject != None:
if successuid: # Only if IMAP actually assigned a UID
# Copy the message to the other remote servers.
for appendserver in \
[x for x in applyto if x != successobject]:
appendserver.savemessage(successuid, message, flags, rtime)
# Copy to its new name on the local server and delete
# the one without a UID.
self.savemessage(successuid, message, flags, rtime)
self.deletemessage(uid) # It'll be re-downloaded.
else:
# Did not find any server to take this message. Ignore.
pass
def syncfoldersto(self, dest, copyfolders):
"""Syncs the folders in this repository to those in dest.
It does NOT sync the contents of those folders.
For every time dest.makefolder() is called, also call makefolder()
on each folder in copyfolders."""
src = self
srcfolders = src.getfolders()
destfolders = dest.getfolders()
# Create hashes with the names, but convert the source folders
# to the dest folder's sep.
srchash = {}
for folder in srcfolders:
srchash[folder.getvisiblename().replace(src.getsep(), dest.getsep())] = \
folder
desthash = {}
for folder in destfolders:
desthash[folder.getvisiblename()] = folder
#
# Find new folders.
#
for key in srchash.keys():
if not key in desthash:
try:
dest.makefolder(key)
for copyfolder in copyfolders:
copyfolder.makefolder(key.replace(dest.getsep(), copyfolder.getsep()))
except:
UIBase.getglobalui().warn("ERROR Attempting to make folder " \
+ key + ":" +str(sys.exc_info()[1]))
def processmessagesflags(self, operation, uidlist, flags):
if len(uidlist) > 101:
# Hack for those IMAP ervers with a limited line length
self.processmessagesflags(operation, uidlist[:100], flags)
self.processmessagesflags(operation, uidlist[100:], flags)
return
imapobj = self.imapserver.acquireconnection()
try:
try:
imapobj.select(self.getfullname())
except imapobj.readonly:
UIBase.getglobalui().flagstoreadonly(self, uidlist, flags)
return
r = imapobj.uid('store',
imaputil.listjoin(uidlist),
operation + 'FLAGS',
imaputil.flagsmaildir2imap(flags))
assert r[0] == 'OK', 'Error with store: ' + '. '.join(r[1])
r = r[1]
finally:
self.imapserver.releaseconnection(imapobj)
# Some IMAP servers do not always return a result. Therefore,
# only update the ones that it talks about, and manually fix
# the others.
needupdate = copy(uidlist)
for result in r:
if result == None:
# Compensate for servers that don't return anything from
# STORE.
continue
attributehash = imaputil.flags2hash(imaputil.imapsplit(result)[1])
if not ('UID' in attributehash and 'FLAGS' in attributehash):
# Compensate for servers that don't return a UID attribute.
continue
lflags = attributehash['FLAGS']
uid = long(attributehash['UID'])
self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(lflags)
try:
needupdate.remove(uid)
except ValueError: # Let it slide if it's not in the list
pass
for uid in needupdate:
if operation == '+':
for flag in flags:
if not flag in self.messagelist[uid]['flags']:
self.messagelist[uid]['flags'].append(flag)
self.messagelist[uid]['flags'].sort()
elif operation == '-':
for flag in flags:
if flag in self.messagelist[uid]['flags']:
self.messagelist[uid]['flags'].remove(flag)
def syncmessagesto_delete(self, dest, applyto):
"""Pass 3 of folder synchronization.
Look for message present in dest but not in self.
If any, delete them."""
deletelist = []
for uid in dest.getmessageuidlist():
if uid < 0:
continue
if not self.uidexists(uid):
deletelist.append(uid)
if len(deletelist):
UIBase.getglobalui().deletingmessages(deletelist, applyto)
for object in applyto:
object.deletemessages(deletelist)
def syncmessagesto_delete(self, dest, applyto):
"""Pass 3 of folder synchronization.
Look for message present in dest but not in self.
If any, delete them."""
deletelist = []
self_messagelist = self.getmessagelist()
for uid in dest.getmessagelist().keys():
if uid < 0:
continue
if not uid in self_messagelist:
deletelist.append(uid)
if len(deletelist):
UIBase.getglobalui().deletingmessages(deletelist, applyto)
for object in applyto:
object.deletemessages(deletelist)
def savemessage(self, uid, content, flags, rtime):
# This function only ever saves to tmp/,
# but it calls savemessageflags() to actually save to cur/ or new/.
ui = UIBase.getglobalui()
ui.debug('perscon', 'savemessage: called to write with flags %s and content %s' % \
(repr(flags), "<?>"))
if uid < 0:
# We cannot assign a new uid.
return uid
if uid in self.messagelist:
# We already have it.
self.savemessageflags(uid, flags)
return uid
msguid = "IMAP.%s.%s.%s" % (self.accountname, self.name, uid)
atts = {}
flags.sort ()
msg = EmailJSON.convert_mail_to_dict(content, self.name, msguid, flags, rtime, atts)
msg = EmailJSON.tojson(msg)
try:
r = self.repository.rpc("doc/%s" % msguid, data=msg)
except urllib2.HTTPError as e:
print e.read ()
print msg
os._exit(1)
self.messagelist[uid] = {'uid': uid, 'flags': flags }
ui.debug('perscon', 'savemessage: returning uid %d' % uid)
return uid
def savemessage(self, uid, content, flags, rtime):
# This function only ever saves to tmp/,
# but it calls savemessageflags() to actually save to cur/ or new/.
ui = UIBase.getglobalui()
ui.debug('perscon', 'savemessage: called to write with flags %s and content %s' % \
(repr(flags), "<?>"))
if uid < 0:
# We cannot assign a new uid.
return uid
if uid in self.messagelist:
# We already have it.
self.savemessageflags(uid, flags)
return uid
msguid = "IMAP.%s.%s.%s" % (self.accountname, self.name, uid)
atts = {}
flags.sort()
msg = EmailJSON.convert_mail_to_dict(content, self.name, msguid, flags,
rtime, atts)
msg = EmailJSON.tojson(msg)
try:
r = self.repository.rpc("doc/%s" % msguid, data=msg)
except urllib2.HTTPError as e:
print e.read()
print msg
os._exit(1)
self.messagelist[uid] = {'uid': uid, 'flags': flags}
ui.debug('perscon', 'savemessage: returning uid %d' % uid)
return uid
def syncitall(accounts, config, siglisteners):
currentThread().setExitMessage('SYNC_WITH_TIMER_TERMINATE')
ui = UIBase.getglobalui()
threads = threadutil.threadlist()
mbnames.init(config, accounts)
for accountname in accounts:
syncaccount(threads, config, accountname, siglisteners)
# Wait for the threads to finish.
threads.reset()
def md5handler(self, response):
ui = UIBase.getglobalui()
challenge = response.strip()
ui.debug('imap', 'md5handler: got challenge %s' % challenge)
passwd = self.repos.getpassword()
retval = self.username + ' ' + hmac.new(passwd, challenge).hexdigest()
ui.debug('imap', 'md5handler: returning %s' % retval)
return retval
def savemessageflags(self, uid, flags):
imapobj = self.imapserver.acquireconnection()
try:
try:
imapobj.select(self.getfullname())
except imapobj.readonly:
UIBase.getglobalui().flagstoreadonly(self, [uid], flags)
return
result = imapobj.uid("store", "%d" % uid, "FLAGS", imaputil.flagsmaildir2imap(flags))
assert result[0] == "OK", "Error with store: " + ". ".join(r[1])
finally:
self.imapserver.releaseconnection(imapobj)
result = result[1][0]
if not result:
self.messagelist[uid]["flags"] = flags
else:
flags = imaputil.flags2hash(imaputil.imapsplit(result)[1])["FLAGS"]
self.messagelist[uid]["flags"] = imaputil.flagsimap2maildir(flags)
def syncitall(accounts, config):
currentThread().setExitMessage('SYNC_WITH_TIMER_TERMINATE')
ui = UIBase.getglobalui()
threads = threadutil.threadlist()
mbnames.init(config, accounts)
for accountname in accounts:
syncaccount(threads, config, accountname)
# Wait for the threads to finish.
threads.reset()
def md5handler(self, response):
ui = UIBase.getglobalui()
challenge = response.strip()
ui.debug('imap', 'md5handler: got challenge %s' % challenge)
passwd = self.getpassword()
retval = self.username + ' ' + hmac.new(passwd, challenge).hexdigest()
ui.debug('imap', 'md5handler: returning %s' % retval)
return retval
def syncmessagesto_flags(self, dest, applyto):
"""Pass 4 of folder synchronization.
Look for any flag matching issues -- set dest message to have the
same flags that we have."""
# As an optimization over previous versions, we store up which flags
# are being used for an add or a delete. For each flag, we store
# a list of uids to which it should be added. Then, we can call
# addmessagesflags() to apply them in bulk, rather than one
# call per message as before. This should result in some significant
# performance improvements.
addflaglist = {}
delflaglist = {}
for uid in self.getmessagelist().keys():
if uid < 0: # Ignore messages missed by pass 1
continue
selfflags = self.getmessageflags(uid)
destflags = dest.getmessageflags(uid)
addflags = [x for x in selfflags if x not in destflags]
for flag in addflags:
if not flag in addflaglist:
addflaglist[flag] = []
addflaglist[flag].append(uid)
delflags = [x for x in destflags if x not in selfflags]
for flag in delflags:
if not flag in delflaglist:
delflaglist[flag] = []
delflaglist[flag].append(uid)
for object in applyto:
for flag in addflaglist.keys():
UIBase.getglobalui().addingflags(addflaglist[flag], flag,
[object])
object.addmessagesflags(addflaglist[flag], [flag])
for flag in delflaglist.keys():
UIBase.getglobalui().deletingflags(delflaglist[flag], flag,
[object])
object.deletemessagesflags(delflaglist[flag], [flag])
def __init__(self, config, name):
self.config = config
self.name = name
self.metadatadir = config.getmetadatadir()
self.localeval = config.getlocaleval()
self.ui = UIBase.getglobalui()
self.refreshperiod = self.getconffloat('autorefresh', 0.0)
self.quicknum = 0
if self.refreshperiod == 0.0:
self.refreshperiod = None
def __init__(self, name, sep, repository, accountname, config):
self.name = name
self.config = config
self.sep = sep
self.messagelist = None
self.repository = repository
self.accountname = accountname
self.ui = UIBase.getglobalui()
self.debug("name=%s sep=%s acct=%s" % (name, sep, accountname))
BaseFolder.__init__(self)
def __init__(self, config, name):
self.config = config
self.name = name
self.metadatadir = config.getmetadatadir()
self.localeval = config.getlocaleval()
self.ui = UIBase.getglobalui()
self.refreshperiod = self.getconffloat('autorefresh', 0.0)
self.quicknum = 0
if self.refreshperiod == 0.0:
self.refreshperiod = None
def __init__(self, name, sep, repository, accountname, config):
self.name = name
self.config = config
self.sep = sep
self.messagelist = None
self.repository = repository
self.accountname = accountname
self.ui = UIBase.getglobalui()
self.debug("name=%s sep=%s acct=%s" % (name, sep, accountname))
BaseFolder.__init__(self)
def savemessageflags(self, uid, flags):
imapobj = self.imapserver.acquireconnection()
try:
try:
imapobj.select(self.getfullname())
except imapobj.readonly:
UIBase.getglobalui().flagstoreadonly(self, [uid], flags)
return
result = imapobj.uid('store', '%d' % uid, 'FLAGS',
imaputil.flagsmaildir2imap(flags))
assert result[0] == 'OK', 'Error with store: ' + '. '.join(r[1])
finally:
self.imapserver.releaseconnection(imapobj)
result = result[1][0]
if not result:
self.messagelist[uid]['flags'] = flags
else:
flags = imaputil.flags2hash(imaputil.imapsplit(result)[1])['FLAGS']
self.messagelist[uid]['flags'] = imaputil.flagsimap2maildir(flags)
def syncmessagesto_flags(self, dest, applyto):
"""Pass 4 of folder synchronization.
Look for any flag matching issues -- set dest message to have the
same flags that we have."""
# As an optimization over previous versions, we store up which flags
# are being used for an add or a delete. For each flag, we store
# a list of uids to which it should be added. Then, we can call
# addmessagesflags() to apply them in bulk, rather than one
# call per message as before. This should result in some significant
# performance improvements.
addflaglist = {}
delflaglist = {}
for uid in self.getmessagelist().keys():
if uid < 0: # Ignore messages missed by pass 1
continue
selfflags = self.getmessageflags(uid)
destflags = dest.getmessageflags(uid)
addflags = [x for x in selfflags if x not in destflags]
for flag in addflags:
if not flag in addflaglist:
addflaglist[flag] = []
addflaglist[flag].append(uid)
delflags = [x for x in destflags if x not in selfflags]
for flag in delflags:
if not flag in delflaglist:
delflaglist[flag] = []
delflaglist[flag].append(uid)
for object in applyto:
for flag in addflaglist.keys():
UIBase.getglobalui().addingflags(addflaglist[flag], flag, [object])
object.addmessagesflags(addflaglist[flag], [flag])
for flag in delflaglist.keys():
UIBase.getglobalui().deletingflags(delflaglist[flag], flag, [object])
object.deletemessagesflags(delflaglist[flag], [flag])
def __init__(self, root, name, sep, repository, accountname, config):
self.name = name
self.config = config
self.dofsync = config.getdefaultboolean("general", "fsync", True)
self.root = root
self.sep = sep
self.ui = UIBase.getglobalui()
self.messagelist = None
self.repository = repository
self.accountname = accountname
BaseFolder.__init__(self)
def deletemessages_noconvert(self, uidlist):
# Weed out ones not in self.messagelist
uidlist = [uid for uid in uidlist if uid in self.messagelist]
if not len(uidlist):
return
self.addmessagesflags_noconvert(uidlist, ['T'])
imapobj = self.imapserver.acquireconnection()
try:
try:
imapobj.select(self.getfullname())
except imapobj.readonly:
UIBase.getglobalui().deletereadonly(self, uidlist)
return
if self.expunge:
assert(imapobj.expunge()[0] == 'OK')
finally:
self.imapserver.releaseconnection(imapobj)
for uid in uidlist:
del self.messagelist[uid]
def getmessage(self, uid):
ui = UIBase.getglobalui()
imapobj = self.imapserver.acquireconnection()
try:
imapobj.select(self.getfullname(), readonly=1)
initialresult = imapobj.uid("fetch", "%d" % uid, "(BODY.PEEK[])")
ui.debug("imap", "Returned object from fetching %d: %s" % (uid, str(initialresult)))
return initialresult[1][0][1].replace("\r\n", "\n")
finally:
self.imapserver.releaseconnection(imapobj)
def getmessage(self, uid):
ui = UIBase.getglobalui()
imapobj = self.imapserver.acquireconnection()
try:
imapobj.select(self.getfullname(), readonly = 1)
initialresult = imapobj.uid('fetch', '%d' % uid, '(BODY.PEEK[])')
ui.debug('imap', 'Returned object from fetching %d: %s' % \
(uid, str(initialresult)))
return initialresult[1][0][1].replace("\r\n", "\n")
finally:
self.imapserver.releaseconnection(imapobj)
def syncmessagesto(self, dest, applyto=None):
"""Syncs messages in this folder to the destination.
If applyto is specified, it should be a list of folders (don't forget
to include dest!) to which all write actions should be applied.
It defaults to [dest] if not specified. It is important that
the UID generator be listed first in applyto; that is, the other
applyto ones should be the ones that "copy" the main action."""
if applyto == None:
applyto = [dest]
try:
self.syncmessagesto_neguid(dest, applyto)
except:
UIBase.getglobalui().warn("ERROR attempting to handle negative uids " \
+ "for account " + self.getaccountname() + ":" + str(sys.exc_info()[1]))
#all threads launched here are in try / except clauses when they copy anyway...
self.syncmessagesto_copy(dest, applyto)
try:
self.syncmessagesto_delete(dest, applyto)
except:
UIBase.getglobalui().warn("ERROR attempting to delete messages " \
+ "for account " + self.getaccountname() + ":" + str(sys.exc_info()[1]))
# Now, the message lists should be identical wrt the uids present.
# (except for potential negative uids that couldn't be placed
# anywhere)
try:
self.syncmessagesto_flags(dest, applyto)
except:
UIBase.getglobalui().warn("ERROR attempting to sync flags " \
+ "for account " + self.getaccountname() + ":" + str(sys.exc_info()[1]))
def syncmessagesto(self, dest, applyto = None):
"""Syncs messages in this folder to the destination.
If applyto is specified, it should be a list of folders (don't forget
to include dest!) to which all write actions should be applied.
It defaults to [dest] if not specified. It is important that
the UID generator be listed first in applyto; that is, the other
applyto ones should be the ones that "copy" the main action."""
if applyto == None:
applyto = [dest]
try:
self.syncmessagesto_neguid(dest, applyto)
except:
UIBase.getglobalui().warn("ERROR attempting to handle negative uids " \
+ "for account " + self.getaccountname() + ":" + str(sys.exc_info()[1]))
#all threads launched here are in try / except clauses when they copy anyway...
self.syncmessagesto_copy(dest, applyto)
try:
self.syncmessagesto_delete(dest, applyto)
except:
UIBase.getglobalui().warn("ERROR attempting to delete messages " \
+ "for account " + self.getaccountname() + ":" + str(sys.exc_info()[1]))
# Now, the message lists should be identical wrt the uids present.
# (except for potential negative uids that couldn't be placed
# anywhere)
try:
self.syncmessagesto_flags(dest, applyto)
except:
UIBase.getglobalui().warn("ERROR attempting to sync flags " \
+ "for account " + self.getaccountname() + ":" + str(sys.exc_info()[1]))
def copymessageto(self, uid, applyto, register=1):
# Sometimes, it could be the case that if a sync takes awhile,
# a message might be deleted from the maildir before it can be
# synced to the status cache. This is only a problem with
# self.getmessage(). So, don't call self.getmessage unless
# really needed.
try:
if register:
UIBase.getglobalui().registerthread(self.getaccountname())
UIBase.getglobalui().copyingmessage(uid, self, applyto)
message = ''
# If any of the destinations actually stores the message body,
# load it up.
for object in applyto:
if object.storesmessages():
message = self.getmessage(uid)
break
flags = self.getmessageflags(uid)
rtime = self.getmessagetime(uid)
for object in applyto:
newuid = object.savemessage(uid, message, flags, rtime)
if newuid > 0 and newuid != uid:
# Change the local uid.
self.savemessage(newuid, message, flags, rtime)
self.deletemessage(uid)
uid = newuid
except:
UIBase.getglobalui().warn("ERROR attempting to copy message " + str(uid) \
+ " for account " + self.getaccountname() + ":" + str(sys.exc_info()[1]))
def __init__(self, reposname, account):
"""Initialize a PersConRepository object. Takes a URL
to the server holding all the mail."""
BaseRepository.__init__(self, reposname, account)
self.ui = UIBase.getglobalui()
self.localurl = self.getconf('localurl')
self.folders = None
self.debug("PersConRepository initialized, sep is " + repr(self.getsep())
+ " localurl=" + repr(self.localurl))
Perscon_utils.init_url (self.localurl)
# test the Personal Container connection
self.rpc("ping")
def getpassword(self):
if self.goodpassword != None:
return self.goodpassword
if self.password != None and self.passworderror == None:
return self.password
self.password = UIBase.getglobalui().getpass(self.reposname,
self.config,
self.passworderror)
self.passworderror = None
return self.password
def __init__(self, reposname, account):
"""Initialize a PersConRepository object. Takes a URL
to the server holding all the mail."""
BaseRepository.__init__(self, reposname, account)
self.ui = UIBase.getglobalui()
self.localurl = self.getconf('localurl')
self.folders = None
self.debug("PersConRepository initialized, sep is " +
repr(self.getsep()) + " localurl=" + repr(self.localurl))
Perscon_utils.init_url(self.localurl)
# test the Personal Container connection
self.rpc("ping")
def getpassword(self):
if self.goodpassword != None:
return self.goodpassword
if self.password != None and self.passworderror == None:
return self.password
self.password = UIBase.getglobalui().getpass(self.reposname,
self.config,
self.passworderror)
self.passworderror = None
return self.password
def gssauth(self, response):
data = base64.b64encode(response)
try:
if self.gss_step == self.GSS_STATE_STEP:
if not self.gss_vc:
rc, self.gss_vc = kerberos.authGSSClientInit('imap@' +
self.hostname)
response = kerberos.authGSSClientResponse(self.gss_vc)
rc = kerberos.authGSSClientStep(self.gss_vc, data)
if rc != kerberos.AUTH_GSS_CONTINUE:
self.gss_step = self.GSS_STATE_WRAP
elif self.gss_step == self.GSS_STATE_WRAP:
rc = kerberos.authGSSClientUnwrap(self.gss_vc, data)
response = kerberos.authGSSClientResponse(self.gss_vc)
rc = kerberos.authGSSClientWrap(self.gss_vc, response,
self.username)
response = kerberos.authGSSClientResponse(self.gss_vc)
except kerberos.GSSError, err:
# Kerberos errored out on us, respond with None to cancel the
# authentication
UIBase.getglobalui().debug('imap',
'%s: %s' % (err[0][0], err[1][0]))
return None
def gssauth(self, response):
data = base64.b64encode(response)
try:
if self.gss_step == self.GSS_STATE_STEP:
if not self.gss_vc:
rc, self.gss_vc = kerberos.authGSSClientInit('imap@' +
self.hostname)
response = kerberos.authGSSClientResponse(self.gss_vc)
rc = kerberos.authGSSClientStep(self.gss_vc, data)
if rc != kerberos.AUTH_GSS_CONTINUE:
self.gss_step = self.GSS_STATE_WRAP
elif self.gss_step == self.GSS_STATE_WRAP:
rc = kerberos.authGSSClientUnwrap(self.gss_vc, data)
response = kerberos.authGSSClientResponse(self.gss_vc)
rc = kerberos.authGSSClientWrap(self.gss_vc, response,
self.username)
response = kerberos.authGSSClientResponse(self.gss_vc)
except kerberos.GSSError, err:
# Kerberos errored out on us, respond with None to cancel the
# authentication
UIBase.getglobalui().debug('imap',
'%s: %s' % (err[0][0], err[1][0]))
return None
def __init__(self, reposname, account):
"""Initialize a MaildirRepository object. Takes a path name
to the directory holding all the Maildir directories."""
BaseRepository.__init__(self, reposname, account)
self.root = self.getlocalroot()
self.folders = None
self.ui = UIBase.getglobalui()
self.debug("MaildirRepository initialized, sep is " + repr(self.getsep()))
self.folder_atimes = []
# Create the top-level folder if it doesn't exist
if not os.path.isdir(self.root):
os.mkdir(self.root, 0700)
def keepalive(self, timeout, event):
"""Sends a NOOP to each connection recorded. It will wait a maximum
of timeout seconds between doing this, and will continue to do so
until the Event object as passed is true. This method is expected
to be invoked in a separate thread, which should be join()'d after
the event is set."""
ui = UIBase.getglobalui()
ui.debug('imap', 'keepalive thread started')
while 1:
ui.debug('imap', 'keepalive: top of loop')
time.sleep(timeout)
ui.debug('imap', 'keepalive: after wait')
if event.isSet():
ui.debug('imap', 'keepalive: event is set; exiting')
return
ui.debug('imap', 'keepalive: acquiring connectionlock')
self.connectionlock.acquire()
numconnections = len(self.assignedconnections) + \
len(self.availableconnections)
self.connectionlock.release()
ui.debug('imap', 'keepalive: connectionlock released')
threads = []
imapobjs = []
for i in range(numconnections):
ui.debug(
'imap', 'keepalive: processing connection %d of %d' %
(i, numconnections))
imapobj = self.acquireconnection()
ui.debug('imap', 'keepalive: connection %d acquired' % i)
imapobjs.append(imapobj)
thr = threadutil.ExitNotifyThread(target=imapobj.noop)
thr.setDaemon(1)
thr.start()
threads.append(thr)
ui.debug('imap', 'keepalive: thread started')
ui.debug('imap', 'keepalive: joining threads')
for thr in threads:
# Make sure all the commands have completed.
thr.join()
ui.debug('imap', 'keepalive: releasing connections')
for imapobj in imapobjs:
self.releaseconnection(imapobj)
ui.debug('imap', 'keepalive: bottom of loop')
def cachemessagelist(self):
imapobj = self.imapserver.acquireconnection()
self.messagelist = {}
try:
# Primes untagged_responses
imapobj.select(self.getfullname(), readonly = 1, force = 1)
try:
# Some mail servers do not return an EXISTS response if
# the folder is empty.
maxmsgid = long(imapobj.untagged_responses['EXISTS'][0])
except KeyError:
return
if maxmsgid < 1:
# No messages; return
return
# Now, get the flags and UIDs for these.
# We could conceivably get rid of maxmsgid and just say
# '1:*' here.
response = imapobj.fetch('1:%d' % maxmsgid, '(FLAGS UID INTERNALDATE)')[1]
finally:
self.imapserver.releaseconnection(imapobj)
for messagestr in response:
# Discard the message number.
messagestr = string.split(messagestr, maxsplit = 1)[1]
options = imaputil.flags2hash(messagestr)
if not options.has_key('UID'):
UIBase.getglobalui().warn('No UID in message with options %s' %\
str(options),
minor = 1)
else:
uid = long(options['UID'])
flags = imaputil.flagsimap2maildir(options['FLAGS'])
rtime = imaplibutil.Internaldate2epoch(messagestr)
self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime}
def savemessage_searchforheader(self, imapobj, headername, headervalue):
if imapobj.untagged_responses.has_key('APPENDUID'):
return long(imapobj.untagged_responses['APPENDUID'][-1].split(' ')[1])
ui = UIBase.getglobalui()
ui.debug('imap', 'savemessage_searchforheader called for %s: %s' % \
(headername, headervalue))
# Now find the UID it got.
headervalue = imapobj._quote(headervalue)
try:
matchinguids = imapobj.uid('search', 'HEADER', headername, headervalue)[1][0]
except imapobj.error, err:
# IMAP server doesn't implement search or had a problem.
ui.debug('imap', "savemessage_searchforheader: got IMAP error '%s' while attempting to UID SEARCH for message with header %s" % (err, headername))
return 0
def __init__(self, reposname, account):
"""Initialize a MaildirRepository object. Takes a path name
to the directory holding all the Maildir directories."""
BaseRepository.__init__(self, reposname, account)
self.root = self.getlocalroot()
self.folders = None
self.ui = UIBase.getglobalui()
self.debug("MaildirRepository initialized, sep is " +
repr(self.getsep()))
self.folder_atimes = []
# Create the top-level folder if it doesn't exist
if not os.path.isdir(self.root):
os.mkdir(self.root, 0700)
def keepalive(self, timeout, event):
"""Sends a NOOP to each connection recorded. It will wait a maximum
of timeout seconds between doing this, and will continue to do so
until the Event object as passed is true. This method is expected
to be invoked in a separate thread, which should be join()'d after
the event is set."""
ui = UIBase.getglobalui()
ui.debug('imap', 'keepalive thread started')
while 1:
ui.debug('imap', 'keepalive: top of loop')
time.sleep(timeout)
ui.debug('imap', 'keepalive: after wait')
if event.isSet():
ui.debug('imap', 'keepalive: event is set; exiting')
return
ui.debug('imap', 'keepalive: acquiring connectionlock')
self.connectionlock.acquire()
numconnections = len(self.assignedconnections) + \
len(self.availableconnections)
self.connectionlock.release()
ui.debug('imap', 'keepalive: connectionlock released')
threads = []
imapobjs = []
for i in range(numconnections):
ui.debug('imap', 'keepalive: processing connection %d of %d' % (i, numconnections))
imapobj = self.acquireconnection()
ui.debug('imap', 'keepalive: connection %d acquired' % i)
imapobjs.append(imapobj)
thr = threadutil.ExitNotifyThread(target = imapobj.noop)
thr.setDaemon(1)
thr.start()
threads.append(thr)
ui.debug('imap', 'keepalive: thread started')
ui.debug('imap', 'keepalive: joining threads')
for thr in threads:
# Make sure all the commands have completed.
thr.join()
ui.debug('imap', 'keepalive: releasing connections')
for imapobj in imapobjs:
self.releaseconnection(imapobj)
ui.debug('imap', 'keepalive: bottom of loop')
def savemessage_addheader(self, content, headername, headervalue):
ui = UIBase.getglobalui()
ui.debug("imap", "savemessage_addheader: called to add %s: %s" % (headername, headervalue))
insertionpoint = content.find("\r\n")
ui.debug("imap", "savemessage_addheader: insertionpoint = %d" % insertionpoint)
leader = content[0:insertionpoint]
ui.debug("imap", "savemessage_addheader: leader = %s" % repr(leader))
if insertionpoint == 0 or insertionpoint == -1:
newline = ""
insertionpoint = 0
else:
newline = "\r\n"
newline += "%s: %s" % (headername, headervalue)
ui.debug("imap", "savemessage_addheader: newline = " + repr(newline))
trailer = content[insertionpoint:]
ui.debug("imap", "savemessage_addheader: trailer = " + repr(trailer))
return leader + newline + trailer
def threadexited(thread):
"""Called when a thread exits."""
ui = UIBase.getglobalui()
if thread.getExitCause() == 'EXCEPTION':
if isinstance(thread.getExitException(), SystemExit):
# Bring a SystemExit into the main thread.
# Do not send it back to UI layer right now.
# Maybe later send it to ui.terminate?
raise SystemExit
ui.threadException(thread) # Expected to terminate
sys.exit(100) # Just in case...
os._exit(100)
elif thread.getExitMessage() == 'SYNC_WITH_TIMER_TERMINATE':
ui.terminate()
# Just in case...
sys.exit(100)
os._exit(100)
else:
ui.threadExited(thread)
def threadexited(thread):
"""Called when a thread exits."""
ui = UIBase.getglobalui()
if thread.getExitCause() == 'EXCEPTION':
if isinstance(thread.getExitException(), SystemExit):
# Bring a SystemExit into the main thread.
# Do not send it back to UI layer right now.
# Maybe later send it to ui.terminate?
raise SystemExit
ui.threadException(thread) # Expected to terminate
sys.exit(100) # Just in case...
os._exit(100)
elif thread.getExitMessage() == 'SYNC_WITH_TIMER_TERMINATE':
ui.terminate()
# Just in case...
sys.exit(100)
os._exit(100)
else:
ui.threadExited(thread)
def savemessage_addheader(self, content, headername, headervalue):
ui = UIBase.getglobalui()
ui.debug('imap',
'savemessage_addheader: called to add %s: %s' % (headername,
headervalue))
insertionpoint = content.find("\r\n")
ui.debug('imap', 'savemessage_addheader: insertionpoint = %d' % insertionpoint)
leader = content[0:insertionpoint]
ui.debug('imap', 'savemessage_addheader: leader = %s' % repr(leader))
if insertionpoint == 0 or insertionpoint == -1:
newline = ''
insertionpoint = 0
else:
newline = "\r\n"
newline += "%s: %s" % (headername, headervalue)
ui.debug('imap', 'savemessage_addheader: newline = ' + repr(newline))
trailer = content[insertionpoint:]
ui.debug('imap', 'savemessage_addheader: trailer = ' + repr(trailer))
return leader + newline + trailer
def savemessage_addheader(self, content, headername, headervalue):
raise NotImplemented
ui = UIBase.getglobalui()
ui.debug('imap',
'savemessage_addheader: called to add %s: %s' % (headername,
headervalue))
insertionpoint = content.find("\r\n")
ui.debug('imap', 'savemessage_addheader: insertionpoint = %d' % insertionpoint)
leader = content[0:insertionpoint]
ui.debug('imap', 'savemessage_addheader: leader = %s' % repr(leader))
if insertionpoint == 0 or insertionpoint == -1:
newline = ''
insertionpoint = 0
else:
newline = "\r\n"
newline += "%s: %s" % (headername, headervalue)
ui.debug('imap', 'savemessage_addheader: newline = ' + repr(newline))
trailer = content[insertionpoint:]
ui.debug('imap', 'savemessage_addheader: trailer = ' + repr(trailer))
return leader + newline + trailer
def startup(versionno):
assert versionno == version.versionstr, "Revision of main program (%s) does not match that of library (%s). Please double-check your PYTHONPATH and installation locations." % (
versionno, version.versionstr)
options = {}
if '--help' in sys.argv[1:]:
sys.stdout.write(version.getcmdhelp() + "\n")
sys.exit(0)
for optlist in getopt(sys.argv[1:], 'P:1oqa:c:d:l:u:hk:f:')[0]:
options[optlist[0]] = optlist[1]
if options.has_key('-h'):
sys.stdout.write(version.getcmdhelp())
sys.stdout.write("\n")
sys.exit(0)
configfilename = os.path.expanduser("~/.offlineimaprc")
if options.has_key('-c'):
configfilename = options['-c']
if options.has_key('-P'):
if not options.has_key('-1'):
sys.stderr.write("FATAL: profile mode REQUIRES -1\n")
sys.exit(100)
profiledir = options['-P']
os.mkdir(profiledir)
threadutil.setprofiledir(profiledir)
sys.stderr.write(
"WARNING: profile mode engaged;\nPotentially large data will be created in "
+ profiledir + "\n")
config = CustomConfigParser()
if not os.path.exists(configfilename):
sys.stderr.write(" *** Config file %s does not exist; aborting!\n" %
configfilename)
sys.exit(1)
config.read(configfilename)
# override config values with option '-k'
for option in options.keys():
if option == '-k':
(key, value) = options['-k'].split('=', 1)
if ':' in key:
(secname, key) = key.split(':', 1)
section = secname.replace("_", " ")
else:
section = "general"
config.set(section, key, value)
ui = offlineimap.ui.detector.findUI(config, options.get('-u'))
UIBase.setglobalui(ui)
if options.has_key('-l'):
ui.setlogfd(open(options['-l'], 'wt'))
ui.init_banner()
if options.has_key('-d'):
for debugtype in options['-d'].split(','):
ui.add_debug(debugtype.strip())
if debugtype == 'imap':
imaplib.Debug = 5
if debugtype == 'thread':
threading._VERBOSE = 1
if options.has_key('-o'):
# FIXME: maybe need a better
for section in accounts.getaccountlist(config):
config.remove_option('Account ' + section, "autorefresh")
if options.has_key('-q'):
for section in accounts.getaccountlist(config):
config.set('Account ' + section, "quick", '-1')
if options.has_key('-f'):
foldernames = options['-f'].replace(" ", "").split(",")
folderfilter = "lambda f: f in %s" % foldernames
folderincludes = "[]"
for accountname in accounts.getaccountlist(config):
account_section = 'Account ' + accountname
remote_repo_section = 'Repository ' + \
config.get(account_section, 'remoterepository')
local_repo_section = 'Repository ' + \
config.get(account_section, 'localrepository')
for section in [remote_repo_section, local_repo_section]:
config.set(section, "folderfilter", folderfilter)
config.set(section, "folderincludes", folderincludes)
lock(config, ui)
def sigterm_handler(signum, frame):
# die immediately
ui.terminate(errormsg="terminating...")
signal.signal(signal.SIGTERM, sigterm_handler)
try:
pidfd = open(config.getmetadatadir() + "/pid", "w")
pidfd.write(str(os.getpid()) + "\n")
pidfd.close()
except:
pass
try:
if options.has_key('-l'):
sys.stderr = ui.logfile
socktimeout = config.getdefaultint("general", "socktimeout", 0)
if socktimeout > 0:
socket.setdefaulttimeout(socktimeout)
activeaccounts = config.get("general", "accounts")
if options.has_key('-a'):
activeaccounts = options['-a']
activeaccounts = activeaccounts.replace(" ", "")
activeaccounts = activeaccounts.split(",")
allaccounts = accounts.AccountHashGenerator(config)
syncaccounts = []
for account in activeaccounts:
if account not in allaccounts:
if len(allaccounts) == 0:
errormsg = 'The account "%s" does not exist because no accounts are defined!' % account
else:
errormsg = 'The account "%s" does not exist. Valid accounts are:' % account
for name in allaccounts.keys():
errormsg += '\n%s' % name
ui.terminate(1,
errortitle='Unknown Account "%s"' % account,
errormsg=errormsg)
if account not in syncaccounts:
syncaccounts.append(account)
server = None
remoterepos = None
localrepos = None
if options.has_key('-1'):
threadutil.initInstanceLimit("ACCOUNTLIMIT", 1)
else:
threadutil.initInstanceLimit(
"ACCOUNTLIMIT",
config.getdefaultint("general", "maxsyncaccounts", 1))
for reposname in config.getsectionlist('Repository'):
for instancename in [
"FOLDER_" + reposname, "MSGCOPY_" + reposname
]:
if options.has_key('-1'):
threadutil.initInstanceLimit(instancename, 1)
else:
threadutil.initInstanceLimit(
instancename,
config.getdefaultint('Repository ' + reposname,
"maxconnections", 1))
siglisteners = []
def sig_handler(signum, frame):
if signum == signal.SIGUSR1:
# tell each account to do a full sync asap
signum = (1, )
elif signum == signal.SIGHUP:
# tell each account to die asap
signum = (2, )
elif signum == signal.SIGUSR2:
# tell each account to do a full sync asap, then die
signum = (1, 2)
# one listener per account thread (up to maxsyncaccounts)
for listener in siglisteners:
for sig in signum:
listener.put_nowait(sig)
signal.signal(signal.SIGHUP, sig_handler)
signal.signal(signal.SIGUSR1, sig_handler)
signal.signal(signal.SIGUSR2, sig_handler)
threadutil.initexitnotify()
t = ExitNotifyThread(target=syncmaster.syncitall,
name='Sync Runner',
kwargs={
'accounts': syncaccounts,
'config': config,
'siglisteners': siglisteners
})
t.setDaemon(1)
t.start()
except:
ui.mainException()
try:
threadutil.exitnotifymonitorloop(threadutil.threadexited)
except SystemExit:
raise
except:
ui.mainException() # Also expected to terminate.
def acquireconnection(self):
"""Fetches a connection from the pool, making sure to create a new one
if needed, to obey the maximum connection limits, etc.
Opens a connection to the server and returns an appropriate
object."""
self.semaphore.acquire()
self.connectionlock.acquire()
imapobj = None
if len(self.availableconnections): # One is available.
# Try to find one that previously belonged to this thread
# as an optimization. Start from the back since that's where
# they're popped on.
threadid = thread.get_ident()
imapobj = None
for i in range(len(self.availableconnections) - 1, -1, -1):
tryobj = self.availableconnections[i]
if self.lastowner[tryobj] == threadid:
imapobj = tryobj
del(self.availableconnections[i])
break
if not imapobj:
imapobj = self.availableconnections[0]
del(self.availableconnections[0])
self.assignedconnections.append(imapobj)
self.lastowner[imapobj] = thread.get_ident()
self.connectionlock.release()
return imapobj
self.connectionlock.release() # Release until need to modify data
""" Must be careful here that if we fail we should bail out gracefully
and release locks / threads so that the next attempt can try...
"""
success = 0
try:
while not success:
# Generate a new connection.
if self.tunnel:
UIBase.getglobalui().connecting('tunnel', self.tunnel)
imapobj = UsefulIMAP4_Tunnel(self.tunnel)
success = 1
elif self.usessl:
UIBase.getglobalui().connecting(self.hostname, self.port)
imapobj = UsefulIMAP4_SSL(self.hostname, self.port,
self.sslclientkey, self.sslclientcert)
else:
UIBase.getglobalui().connecting(self.hostname, self.port)
imapobj = UsefulIMAP4(self.hostname, self.port)
imapobj.mustquote = imaplibutil.mustquote
if not self.tunnel:
try:
# Try GSSAPI and continue if it fails
if 'AUTH=GSSAPI' in imapobj.capabilities and have_gss:
UIBase.getglobalui().debug('imap',
'Attempting GSSAPI authentication')
try:
imapobj.authenticate('GSSAPI', self.gssauth)
except imapobj.error, val:
self.gssapi = False
UIBase.getglobalui().debug('imap',
'GSSAPI Authentication failed')
else:
self.gssapi = True
#if we do self.password = None then the next attempt cannot try...
#self.password = None
if not self.gssapi:
if 'AUTH=CRAM-MD5' in imapobj.capabilities:
UIBase.getglobalui().debug('imap',
'Attempting CRAM-MD5 authentication')
try:
imapobj.authenticate('CRAM-MD5', self.md5handler)
except imapobj.error, val:
self.plainauth(imapobj)
else:
self.plainauth(imapobj)
# Would bail by here if there was a failure.
success = 1
self.goodpassword = self.password
except imapobj.error, val:
self.passworderror = str(val)
raise
def plainauth(self, imapobj):
UIBase.getglobalui().debug('imap',
'Attempting plain authentication')
imapobj.login(self.username, self.getpassword())
def debug(*args):
msg = []
for arg in args:
msg.append(str(arg))
UIBase.getglobalui().debug('imap', " ".join(msg))
def __init__(self, srcusername, srcpasswd, dstusername, dstpasswd):
try:
import fcntl
hasfcntl = 1
except:
hasfcntl = 0
lockfd = None
def lock(config, ui):
if not hasfcntl:
return
lockfd = open(self.configuration.getmetadatadir() + "/lock", "w")
try:
fcntl.flock(lockfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError:
ui.locked()
ui.terminate(1)
self.configuration = CustomConfigParser()
self.configuration.add_section('general')
self.configuration.set('general','accounts', dstusername)
self.configuration.add_section('Account ' + dstusername)
self.configuration.set('Account ' + dstusername, 'localrepository', dstusername+'_local')
self.configuration.set('Account ' + dstusername, 'remoterepository', dstusername+'_remote')
self.configuration.add_section('Repository ' + dstusername + '_local')
self.configuration.add_section('Repository ' + dstusername + '_remote')
self.configuration.set('Repository ' + dstusername + '_local', 'type', 'IMAP')
self.configuration.set('Repository ' + dstusername + '_local', 'remotehost', newserveraddr)
self.configuration.set('Repository ' + dstusername + '_local', 'remoteuser', dstusername)
self.configuration.set('Repository ' + dstusername + '_local', 'remotepass', dstpasswd)
self.configuration.set('Repository ' + dstusername + '_remote', 'type', 'IMAP')
self.configuration.set('Repository ' + dstusername + '_remote', 'remotehost', server)
self.configuration.set('Repository ' + dstusername + '_remote', 'remoteuser', srcusername)
self.configuration.set('Repository ' + dstusername + '_remote', 'remotepass', srcpasswd)
self.monothread = 0
# Setup a interface
ui = offlineimap.ui.detector.findUI(self.configuration, 'TTY.TTYUI')
UIBase.setglobalui(ui)
# ui.add_debug('imap')
# ui.add_debug('thread')
# imaplib.Debug = 5
# threading._VERBOSE = 1
lock(self.configuration, ui)
def sigterm_handler(signum, frame):
# die immediately
ui.terminate(errormsg="terminating...")
signal.signal(signal.SIGTERM,sigterm_handler)
try:
pidfd = open(config.getmetadatadir() + "/pid", "w")
pidfd.write(str(os.getpid()) + "\n")
pidfd.close()
except:
pass
try:
activeaccounts = self.configuration.get("general", "accounts")
activeaccounts = activeaccounts.replace(" ", "")
activeaccounts = activeaccounts.split(",")
allaccounts = accounts.AccountHashGenerator(self.configuration)
if self.monothread:
threadutil.initInstanceLimit("ACCOUNTLIMIT", 1)
else:
threadutil.initInstanceLimit("ACCOUNTLIMIT", self.configuration.getdefaultint("general", "maxsyncaccounts", 1))
for reposname in self.configuration.getsectionlist('Repository'):
for instancename in ["FOLDER_" + reposname, "MSGCOPY_" + reposname]:
if self.monothread:
threadutil.initInstanceLimit(instancename, 1)
else:
threadutil.initInstanceLimit(instancename, self.configuration.getdefaultint('Repository ' + reposname, "maxconnections", 1))
syncaccounts = []
for account in activeaccounts:
if account not in syncaccounts:
syncaccounts.append(account)
siglisteners = []
def sig_handler(signum, frame):
if signum == signal.SIGUSR1:
# tell each account to do a full sync asap
signum = (1,)
elif signum == signal.SIGHUP:
# tell each account to die asap
signum = (2,)
elif signum == signal.SIGUSR2:
# tell each account to do a full sync asap, then die
signum = (1, 2)
# one listener per account thread (up to maxsyncaccounts)
for listener in siglisteners:
for sig in signum:
listener.put_nowait(sig)
signal.signal(signal.SIGHUP,sig_handler)
signal.signal(signal.SIGUSR1,sig_handler)
signal.signal(signal.SIGUSR2,sig_handler)
threadutil.initexitnotify()
t = ExitNotifyThread(target=syncmaster.syncitall,
name='Sync Runner',
kwargs = {'accounts': syncaccounts,
'config': self.configuration,
'siglisteners': siglisteners})
t.setDaemon(1)
t.start()
except:
ui.mainException()
try:
threadutil.exitnotifymonitorloop(threadutil.threadexited)
except SystemExit:
raise
except:
ui.mainException() # Also expected to terminate.
rmtree(self.configuration.getmetadatadir())
def __init__(self, srcusername, srcpasswd, dstusername, dstpasswd):
try:
import fcntl
hasfcntl = 1
except:
hasfcntl = 0
lockfd = None
def lock(config, ui):
if not hasfcntl:
return
lockfd = open(self.configuration.getmetadatadir() + "/lock", "w")
try:
fcntl.flock(lockfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError:
ui.locked()
ui.terminate(1)
self.configuration = CustomConfigParser()
self.configuration.add_section('general')
self.configuration.set('general', 'accounts', dstusername)
self.configuration.add_section('Account ' + dstusername)
self.configuration.set('Account ' + dstusername, 'localrepository',
dstusername + '_local')
self.configuration.set('Account ' + dstusername, 'remoterepository',
dstusername + '_remote')
self.configuration.add_section('Repository ' + dstusername + '_local')
self.configuration.add_section('Repository ' + dstusername + '_remote')
self.configuration.set('Repository ' + dstusername + '_local', 'type',
'IMAP')
self.configuration.set('Repository ' + dstusername + '_local',
'remotehost', newserveraddr)
self.configuration.set('Repository ' + dstusername + '_local',
'remoteuser', dstusername)
self.configuration.set('Repository ' + dstusername + '_local',
'remotepass', dstpasswd)
self.configuration.set('Repository ' + dstusername + '_remote', 'type',
'IMAP')
self.configuration.set('Repository ' + dstusername + '_remote',
'remotehost', server)
self.configuration.set('Repository ' + dstusername + '_remote',
'remoteuser', srcusername)
self.configuration.set('Repository ' + dstusername + '_remote',
'remotepass', srcpasswd)
self.monothread = 0
# Setup a interface
ui = offlineimap.ui.detector.findUI(self.configuration, 'TTY.TTYUI')
UIBase.setglobalui(ui)
# ui.add_debug('imap')
# ui.add_debug('thread')
# imaplib.Debug = 5
# threading._VERBOSE = 1
lock(self.configuration, ui)
def sigterm_handler(signum, frame):
# die immediately
ui.terminate(errormsg="terminating...")
signal.signal(signal.SIGTERM, sigterm_handler)
try:
pidfd = open(config.getmetadatadir() + "/pid", "w")
pidfd.write(str(os.getpid()) + "\n")
pidfd.close()
except:
pass
try:
activeaccounts = self.configuration.get("general", "accounts")
activeaccounts = activeaccounts.replace(" ", "")
activeaccounts = activeaccounts.split(",")
allaccounts = accounts.AccountHashGenerator(self.configuration)
if self.monothread:
threadutil.initInstanceLimit("ACCOUNTLIMIT", 1)
else:
threadutil.initInstanceLimit(
"ACCOUNTLIMIT",
self.configuration.getdefaultint("general",
"maxsyncaccounts", 1))
for reposname in self.configuration.getsectionlist('Repository'):
for instancename in [
"FOLDER_" + reposname, "MSGCOPY_" + reposname
]:
if self.monothread:
threadutil.initInstanceLimit(instancename, 1)
else:
threadutil.initInstanceLimit(
instancename,
self.configuration.getdefaultint(
'Repository ' + reposname, "maxconnections",
1))
syncaccounts = []
for account in activeaccounts:
if account not in syncaccounts:
syncaccounts.append(account)
siglisteners = []
def sig_handler(signum, frame):
if signum == signal.SIGUSR1:
# tell each account to do a full sync asap
signum = (1, )
elif signum == signal.SIGHUP:
# tell each account to die asap
signum = (2, )
elif signum == signal.SIGUSR2:
# tell each account to do a full sync asap, then die
signum = (1, 2)
# one listener per account thread (up to maxsyncaccounts)
for listener in siglisteners:
for sig in signum:
listener.put_nowait(sig)
signal.signal(signal.SIGHUP, sig_handler)
signal.signal(signal.SIGUSR1, sig_handler)
signal.signal(signal.SIGUSR2, sig_handler)
threadutil.initexitnotify()
t = ExitNotifyThread(target=syncmaster.syncitall,
name='Sync Runner',
kwargs={
'accounts': syncaccounts,
'config': self.configuration,
'siglisteners': siglisteners
})
t.setDaemon(1)
t.start()
except:
ui.mainException()
try:
threadutil.exitnotifymonitorloop(threadutil.threadexited)
except SystemExit:
raise
except:
ui.mainException() # Also expected to terminate.
rmtree(self.configuration.getmetadatadir())
def cachemessagelist(self):
imapobj = self.imapserver.acquireconnection()
self.messagelist = {}
try:
# Primes untagged_responses
imapobj.select(self.getfullname(), readonly = 1, force = 1)
maxage = self.config.getdefaultint("Account " + self.accountname, "maxage", -1)
maxsize = self.config.getdefaultint("Account " + self.accountname, "maxsize", -1)
if (maxage != -1) | (maxsize != -1):
try:
search_condition = "(";
if(maxage != -1):
#find out what the oldest message is that we should look at
oldest_time_struct = time.gmtime(time.time() - (60*60*24*maxage))
#format this manually - otherwise locales could cause problems
monthnames_standard = ["Jan", "Feb", "Mar", "Apr", "May", \
"June", "July", "Aug", "Sep", "Oct", "Nov", "Dec"]
our_monthname = monthnames_standard[oldest_time_struct[1]-1]
daystr = "%(day)02d" % {'day' : oldest_time_struct[2]}
date_search_str = "SINCE " + daystr + "-" + our_monthname \
+ "-" + str(oldest_time_struct[0])
search_condition += date_search_str
if(maxsize != -1):
if(maxage != 1): #There are two conditions - add a space
search_condition += " "
search_condition += "SMALLER " + self.config.getdefault("Account " + self.accountname, "maxsize", -1)
search_condition += ")"
searchresult = imapobj.search(None, search_condition)
#result would come back seperated by space - to change into a fetch
#statement we need to change space to comma
messagesToFetch = searchresult[1][0].replace(" ", ",")
except KeyError:
return
if len(messagesToFetch) < 1:
# No messages; return
return
else:
try:
# Some mail servers do not return an EXISTS response if
# the folder is empty.
maxmsgid = long(imapobj.untagged_responses['EXISTS'][0])
messagesToFetch = '1:%d' % maxmsgid;
except KeyError:
return
if maxmsgid < 1:
#no messages; return
return
# Now, get the flags and UIDs for these.
# We could conceivably get rid of maxmsgid and just say
# '1:*' here.
response = imapobj.fetch(messagesToFetch, '(FLAGS UID)')[1]
finally:
self.imapserver.releaseconnection(imapobj)
for messagestr in response:
# Discard the message number.
messagestr = string.split(messagestr, maxsplit = 1)[1]
options = imaputil.flags2hash(messagestr)
if not options.has_key('UID'):
UIBase.getglobalui().warn('No UID in message with options %s' %\
str(options),
minor = 1)
else:
uid = long(options['UID'])
flags = imaputil.flagsimap2maildir(options['FLAGS'])
rtime = imaplibutil.Internaldate2epoch(messagestr)
self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime}
def savemessage(self, uid, content, flags, rtime):
imapobj = self.imapserver.acquireconnection()
ui = UIBase.getglobalui()
ui.debug('imap', 'savemessage: called')
try:
try:
imapobj.select(self.getfullname()) # Needed for search
except imapobj.readonly:
ui.msgtoreadonly(self, uid, content, flags)
# Return indicating message taken, but no UID assigned.
# Fudge it.
return 0
# This backend always assigns a new uid, so the uid arg is ignored.
# In order to get the new uid, we need to save off the message ID.
message = rfc822.Message(StringIO(content))
datetuple_msg = rfc822.parsedate(message.getheader('Date'))
# Will be None if missing or not in a valid format.
# If time isn't known
if rtime == None and datetuple_msg == None:
datetuple = time.localtime()
elif rtime == None:
datetuple = datetuple_msg
else:
datetuple = time.localtime(rtime)
try:
if datetuple[0] < 1981:
raise ValueError
# Check for invalid date
datetuple_check = time.localtime(time.mktime(datetuple))
if datetuple[:2] != datetuple_check[:2]:
raise ValueError
# This could raise a value error if it's not a valid format.
date = imaplib.Time2Internaldate(datetuple)
except (ValueError, OverflowError):
# Argh, sometimes it's a valid format but year is 0102
# or something. Argh. It seems that Time2Internaldate
# will rause a ValueError if the year is 0102 but not 1902,
# but some IMAP servers nonetheless choke on 1902.
date = imaplib.Time2Internaldate(time.localtime())
ui.debug('imap', 'savemessage: using date ' + str(date))
content = re.sub("(?<!\r)\n", "\r\n", content)
ui.debug('imap', 'savemessage: initial content is: ' + repr(content))
(headername, headervalue) = self.savemessage_getnewheader(content)
ui.debug('imap', 'savemessage: new headers are: %s: %s' % \
(headername, headervalue))
content = self.savemessage_addheader(content, headername,
headervalue)
ui.debug('imap', 'savemessage: new content is: ' + repr(content))
ui.debug('imap', 'savemessage: new content length is ' + \
str(len(content)))
assert(imapobj.append(self.getfullname(),
imaputil.flagsmaildir2imap(flags),
date, content)[0] == 'OK')
# Checkpoint. Let it write out the messages, etc.
assert(imapobj.check()[0] == 'OK')
# Keep trying until we get the UID.
ui.debug('imap', 'savemessage: first attempt to get new UID')
uid = self.savemessage_searchforheader(imapobj, headername,
headervalue)
# See docs for savemessage in Base.py for explanation of this and other return values
if uid <= 0:
ui.debug('imap', 'savemessage: first attempt to get new UID failed. Going to run a NOOP and try again.')
assert(imapobj.noop()[0] == 'OK')
uid = self.savemessage_searchforheader(imapobj, headername,
headervalue)
finally:
self.imapserver.releaseconnection(imapobj)
if uid: # avoid UID FETCH 0 crash happening later on
self.messagelist[uid] = {'uid': uid, 'flags': flags}
ui.debug('imap', 'savemessage: returning %d' % uid)
return uid
def new_mesg(self, s, secs=None):
if secs is None:
secs = time.time()
tm = time.strftime('%M:%S', time.localtime(secs))
UIBase.getglobalui().debug('imap', ' %s.%02d %s' % (tm, (secs*100)%100, s))
def savemessage(self, uid, content, flags, rtime):
# This function only ever saves to tmp/,
# but it calls savemessageflags() to actually save to cur/ or new/.
ui = UIBase.getglobalui()
ui.debug('maildir', 'savemessage: called to write for %s with flags %s uid %s rtime %s in %s and content %s' % \
(self.accountname, repr(flags), repr(uid), repr(rtime), repr(self.getvisiblename()), repr(content)))
if uid < 0:
# We cannot assign a new uid.
return uid
if uid in self.messagelist:
# We already have it.
self.savemessageflags(uid, flags)
return uid
# Otherwise, save the message in tmp/ and then call savemessageflags()
# to give it a permanent home.
tmpdir = os.path.join(self.getfullname(), 'tmp')
messagename = None
attempts = 0
while 1:
if attempts > 15:
raise IOError, "Couldn't write to file %s" % messagename
timeval, timeseq = gettimeseq()
messagename = '%d_%d.%d.%s,U=%d,FMD5=%s' % \
(timeval,
timeseq,
os.getpid(),
socket.gethostname(),
uid,
md5(self.getvisiblename()).hexdigest())
if os.path.exists(os.path.join(tmpdir, messagename)):
time.sleep(2)
attempts += 1
else:
break
tmpmessagename = messagename.split(',')[0]
ui.debug('maildir',
'savemessage: using temporary name %s' % tmpmessagename)
file = open(os.path.join(tmpdir, tmpmessagename), "wt")
file.write(content)
# Make sure the data hits the disk
file.flush()
if self.dofsync:
os.fsync(file.fileno())
file.close()
if rtime != None:
os.utime(os.path.join(tmpdir, tmpmessagename), (rtime, rtime))
ui.debug('maildir', 'savemessage: moving from %s to %s' % \
(tmpmessagename, os.path.join(tmpdir, messagename)))
if tmpmessagename != messagename: # then rename it
os.rename(os.path.join(tmpdir, tmpmessagename),
os.path.join(tmpdir, messagename))
if self.dofsync:
try:
# fsync the directory (safer semantics in Linux)
fd = os.open(tmpdir, os.O_RDONLY)
os.fsync(fd)
os.close(fd)
except:
pass
self.messagelist[uid] = {
'uid': uid,
'flags': [],
'filename': os.path.join(tmpdir, messagename)
}
username = self.config.getdefault("Account " + self.accountname,
"username", "")
on_savemessage(self.accountname, username,
os.path.join(tmpdir, messagename), flags,
self.getvisiblename())
self.savemessageflags(uid, flags)
ui.debug('maildir', 'savemessage: returning uid %d' % uid)
return uid
