def maildir_source(self):
from mailbox import Maildir
import codecs
def get_encoding(enc, default='latin1'):
"""
Return *enc* if *enc* is a valid encoding,
*default* otherwise.
"""
if not enc:
return default
try:
codecs.lookup(enc)
return enc
except LookupError:
return default
m = Maildir(self._infile, create=False)
if self._folder:
m = m.get_folder(self._folder)
for _, msg in m.iteritems():
for part in self.get_plaintext_parts(msg):
enc = get_encoding(part.get_content_charset())
content_bytes = part.get_payload(decode=True)
content = content_bytes.decode(encoding=enc, errors='ignore')
yield from self.filter_text(content)
def maildir_source(self):
from mailbox import Maildir
import codecs
def get_encoding(enc, default='latin1'):
"""Return *enc* if *enc* is a valid encoding,
*default* otherwise.
"""
if not enc:
return default
try:
codecs.lookup(enc)
return enc
except LookupError:
return default
m = Maildir(self._infile, create=False)
if self._folder:
m = m.get_folder(self._folder)
for _, msg in m.iteritems():
for part in self.get_plaintext_parts(msg):
enc = get_encoding(part.get_content_charset())
content_bytes = part.get_payload(decode=True)
content = content_bytes.decode(encoding=enc, errors='ignore')
yield from self.filter_text(content)
class moveMail:
def __init__(self, oldBoxDir, newBoxDir, daystoremove):
from sys import stdout
from mailbox import Maildir, MaildirMessage, NoSuchMailboxError
from os.path import expanduser
from time import time, strftime, gmtime
self.logfile = strftime(
expanduser("~/") + "mail-remove-log-%Y%m%d%H%M", gmtime())
# DeltaT is epoch now - 60(s)*m*d*nm where n is number of months - here set for 4 months
self.deltaT = int(round(time() - (60 * 60 * 24 * int(daystoremove))))
self.newBox = Maildir(newBoxDir, create=True, factory=MaildirMessage)
try:
self.oldBox = Maildir(oldBoxDir,
create=False,
factory=MaildirMessage)
except NoSuchMailboxError:
print("[E] Invalid mail dir. Aborting")
return (1)
def moveByLabel(self):
from email.utils import parsedate_tz, mktime_tz
from mailbox import NoSuchMailboxError
for folder in self.oldBox.list_folders():
_c_moved = 0
_c_rej = 0
_c_total = self.oldBox.get_folder(folder).__len__()
print("\n[I] Folder " + folder + "", end="")
for key, msg in self.oldBox.get_folder(folder).iteritems():
_date = msg['Date']
if _date:
if (mktime_tz(parsedate_tz(_date)) - self.deltaT) < 0:
if _c_moved == 0:
#To detect if no thing is moved, so this can be a new folder
try:
self.newBox.get_folder(folder)
except NoSuchMailboxError:
print("[I]\tCreating in new: %s" % folder)
self.newBox.add_folder(folder)
# Mooooooooooooo'ving!
self.newBox.get_folder(folder).add(msg)
self.oldBox.get_folder(folder).remove(key)
_c_moved += 1
print(
"\r[I]\tStats: Not moved (Bad Mail): %d/%d // Moved: %d/%d"
% (_c_rej, _c_total, _c_moved, _c_total),
end="")
else:
_c_rej += 1
if _c_moved >= _c_total:
print("\n[W]\tRemoving folder %s" % folder, end="")
print("")
class moveMail:
def __init__(self, oldBoxDir, newBoxDir, daystoremove):
from sys import stdout
from mailbox import Maildir,MaildirMessage,NoSuchMailboxError
from os.path import expanduser
from time import time, strftime, gmtime
self.logfile=strftime(expanduser("~/")+"mail-remove-log-%Y%m%d%H%M", gmtime())
# DeltaT is epoch now - 60(s)*m*d*nm where n is number of months - here set for 4 months
self.deltaT=int(round(time()-(60*60*24*int(daystoremove))))
self.newBox = Maildir(newBoxDir, create=True, factory=MaildirMessage)
try:
self.oldBox = Maildir(oldBoxDir, create=False, factory=MaildirMessage)
except NoSuchMailboxError:
print ("[E] Invalid mail dir. Aborting")
return(1)
def moveByLabel (self):
from email.utils import parsedate_tz,mktime_tz
from mailbox import NoSuchMailboxError
for folder in self.oldBox.list_folders():
_c_moved=0
_c_rej=0
_c_total=self.oldBox.get_folder(folder).__len__()
print("\n[I] Folder " + folder + "", end="")
for key, msg in self.oldBox.get_folder(folder).iteritems():
_date=msg['Date']
if _date:
if (mktime_tz(parsedate_tz(_date)) - self.deltaT) < 0:
if _c_moved == 0:
#To detect if no thing is moved, so this can be a new folder
try:
self.newBox.get_folder(folder)
except NoSuchMailboxError:
print("[I]\tCreating in new: %s" % folder)
self.newBox.add_folder(folder)
# Mooooooooooooo'ving!
self.newBox.get_folder(folder).add(msg)
self.oldBox.get_folder(folder).remove(key)
_c_moved += 1
print("\r[I]\tStats: Not moved (Bad Mail): %d/%d // Moved: %d/%d" % (_c_rej,_c_total,_c_moved,_c_total), end="")
else:
_c_rej += 1
if _c_moved >= _c_total:
print("\n[W]\tRemoving folder %s" % folder, end="")
print("")
def _load_mails_as_is(mail_paths, store):
deferreds = []
for path in mail_paths:
if isfile(path):
mbox_mails = mbox(path, factory=None)
yield add_mail_folder(store, mbox_mails, 'INBOX', deferreds)
else:
maildir = Maildir(path, factory=None)
yield add_mail_folder(store, maildir, 'INBOX', deferreds)
for mail_folder_name in maildir.list_folders():
mail_folder = maildir.get_folder(mail_folder_name)
yield add_mail_folder(store, mail_folder, mail_folder_name, deferreds)
yield defer.gatherResults(deferreds, consumeErrors=True)
def load_mails(args, mail_paths):
leap_session, soledad = args
account = leap_session.account
deferreds = []
for path in mail_paths:
maildir = Maildir(path, factory=None)
add_mail_folder(account, maildir, 'INBOX', deferreds)
for mail_folder_name in maildir.list_folders():
mail_folder = maildir.get_folder(mail_folder_name)
add_mail_folder(account, mail_folder, mail_folder_name, deferreds)
yield defer.DeferredList(deferreds)
defer.returnValue(args)
def _load_mails_as_is(mail_paths, store):
deferreds = []
for path in mail_paths:
if isfile(path):
mbox_mails = mbox(path, factory=None)
yield add_mail_folder(store, mbox_mails, 'INBOX', deferreds)
else:
maildir = Maildir(path, factory=None)
yield add_mail_folder(store, maildir, 'INBOX', deferreds)
for mail_folder_name in maildir.list_folders():
mail_folder = maildir.get_folder(mail_folder_name)
yield add_mail_folder(store, mail_folder, mail_folder_name, deferreds)
yield defer.gatherResults(deferreds, consumeErrors=True)
def load_mails(args, mail_paths):
leap_session, soledad = args
account = leap_session.account
deferreds = []
for path in mail_paths:
maildir = Maildir(path, factory=None)
add_mail_folder(account, maildir, 'INBOX', deferreds)
add_mail_folder(account, maildir, 'DRAFTS', deferreds)
for mail_folder_name in maildir.list_folders():
mail_folder = maildir.get_folder(mail_folder_name)
add_mail_folder(account, mail_folder, mail_folder_name, deferreds)
yield defer.DeferredList(deferreds)
defer.returnValue(args)
def load_mails(args, mail_paths):
leap_session, soledad = args
store = leap_session.mail_store
deferreds = []
for path in mail_paths:
maildir = Maildir(path, factory=None)
yield add_mail_folder(store, maildir, 'INBOX', deferreds)
for mail_folder_name in maildir.list_folders():
mail_folder = maildir.get_folder(mail_folder_name)
yield add_mail_folder(store, mail_folder, mail_folder_name, deferreds)
yield defer.gatherResults(deferreds, consumeErrors=True)
defer.returnValue(args)
def load_mails(args, mail_paths):
leap_session, soledad = args
store = leap_session.mail_store
deferreds = []
for path in mail_paths:
maildir = Maildir(path, factory=None)
yield add_mail_folder(store, maildir, 'INBOX', deferreds)
for mail_folder_name in maildir.list_folders():
mail_folder = maildir.get_folder(mail_folder_name)
yield add_mail_folder(store, mail_folder, mail_folder_name,
deferreds)
yield defer.gatherResults(deferreds, consumeErrors=True)
defer.returnValue(args)
workers.append(worker)
worker.start()
# spawn log and email writing workers
logger = LogWorker(logQueue)
logger.start()
mboxer = MboxWorker(errorQueue, logQueue, luser)
mboxer.start()
# load workQueue
mail = {}
maildir = Maildir(baseMailDir + domain + "/" + luser, factory=None)
folders = maildir.list_folders()
folders.append('') # empty string means INBOX.
for folder in folders:
mail[folder] = maildir.get_folder(folder)
for folder in mail:
for eid, msg in mail[folder].iteritems():
#print domain, user, folder, msg.get_flags(), eid, len(str(msg))
# Process emails here.
email = (domain,USER,folder,eid,msg.get_flags(),str(msg))
workQueue.put(email)
while not workQueue.empty():
if debug:
print "workQueue: " + folder + " " + str(workQueue.qsize())
time.sleep(60)
logQueue.put("elapsed time:" + str(datetime.datetime.now() - started))
# kill ALL the threads!