def add_file(go, path):
if not go:
print(f"Would add {path}")
return
# Open the mailbox
md = Maildir(DEST.joinpath(DEST_FOLDER), create=False)
# Determine the UID for the next message
uid_re = re.compile(",U=([0-9]+),")
uid = 1 + max(
chain([0],
map(lambda key: int(uid_re.search(key).groups()[0]), md.keys())))
print(f"Next UID: {uid}")
# Determine the time when the message was originally delivered
msg_bytes = path.read_bytes()
msg = MaildirMessage(msg_bytes)
msg_time = time.mktime(parsedate(msg["Date"]))
# Create a Maildir filename in the format used by OfflineIMAP
key = (f"{int(msg_time)}_0.{os.getpid()}.{socket.gethostname()},U={uid},"
f"FMD5={md5(DEST_FOLDER.encode()).hexdigest()}:2,S")
# Complete path
dest = DEST.joinpath(DEST_FOLDER, "cur", key)
assert not dest.exists() and dest.parent.exists()
# Write the file
print(f"Write {key}")
dest.write_bytes(msg_bytes)
# Update the utime
os.utime(dest, (msg_time, msg_time))
def read_mailboxes(self, mailboxes):
"""Return list of mailboxes.
Raise exception on invalid mailbox.
"""
mboxes = []
state = []
unread = []
if mailboxes:
for mdir in mailboxes:
try:
mbox = Maildir(mdir, create=False)
mbox.keys()
mboxes.append(mbox)
state.append('')
unread.append(0)
except NoSuchMailboxError:
raise MailstatusException(
"invalid path: {path}".format(path=mdir))
except FileNotFoundError:
raise MailstatusException(
"invalid maildir: {path}".format(path=mdir))
self.mboxes = mboxes
self.mbox_state = state
self.unread = unread
def maildir_source(self):
from mailbox import Maildir
import codecs
def get_encoding(enc, default='latin1'):
"""
Return *enc* if *enc* is a valid encoding,
*default* otherwise.
"""
if not enc:
return default
try:
codecs.lookup(enc)
return enc
except LookupError:
return default
m = Maildir(self._infile, create=False)
if self._folder:
m = m.get_folder(self._folder)
for _, msg in m.iteritems():
for part in self.get_plaintext_parts(msg):
enc = get_encoding(part.get_content_charset())
content_bytes = part.get_payload(decode=True)
content = content_bytes.decode(encoding=enc, errors='ignore')
yield from self.filter_text(content)
def maildir_source(self):
from mailbox import Maildir
import codecs
def get_encoding(enc, default='latin1'):
"""Return *enc* if *enc* is a valid encoding,
*default* otherwise.
"""
if not enc:
return default
try:
codecs.lookup(enc)
return enc
except LookupError:
return default
m = Maildir(self._infile, create=False)
if self._folder:
m = m.get_folder(self._folder)
for _, msg in m.iteritems():
for part in self.get_plaintext_parts(msg):
enc = get_encoding(part.get_content_charset())
content_bytes = part.get_payload(decode=True)
content = content_bytes.decode(encoding=enc, errors='ignore')
yield from self.filter_text(content)
def create(self, path, mails, compression='xz', server=None):
path = Path.cwd() / path
with TemporaryDirectory(prefix="mailarchive-") as tmpdir:
with tmp_chdir(tmpdir), tmp_umask(0o077):
basedir = Path(path.name.split('.')[0])
maildir = Maildir(basedir, create=True)
self.mailindex = MailIndex(server=server)
last_folder = None
for folder, msgbytes in mails:
if folder != last_folder:
mailfolder = maildir.add_folder(folder)
last_folder = folder
sha256 = hashlib.sha256(msgbytes).hexdigest()
key = mailfolder.add(msgbytes)
msg = mailfolder.get_message(key)
idx_item = {
"Date": msg.get("Date"),
"From": msg.get("From"),
"MessageId": msg.get("Message-Id"),
"Subject": msg.get("Subject"),
"To": msg.get("To"),
"checksum": {
"sha256": sha256
},
"folder": folder,
"key": key,
}
self.mailindex.append(idx_item)
with TemporaryFile(dir=tmpdir) as tmpf:
self.mailindex.write(tmpf)
tmpf.seek(0)
self.add_metadata(".mailindex.yaml", tmpf)
super().create(path, compression, [basedir])
return self
def test_maildir(self):
tmpdir = mkdtemp('archweb')
mdir = tmpdir + '/maildir'
maildir = Maildir(mdir)
msg = Message()
msg['subject'] = 'John Doe'
msg['to'] = 'John Doe <*****@*****.**>'
maildir.add(msg)
# Invalid
call_command('donor_import', mdir)
self.assertEqual(len(Donor.objects.all()), 0)
# Valid
msg = Message()
msg['subject'] = 'Receipt [$25.00] By: David Doe [[email protected]]'
msg['to'] = 'John Doe <*****@*****.**>'
maildir.add(msg)
call_command('donor_import', mdir)
self.assertEqual(len(Donor.objects.all()), 1)
# Re-running should result in no new donor
call_command('donor_import', mdir)
self.assertEqual(len(Donor.objects.all()), 1)
rmtree(tmpdir)
def import_email(email, import_path, format, **kwargs):
from caliopen.core.user import User
from caliopen.core.contact import Contact, ContactLookup
from caliopen.core.mail import MailMessage
from caliopen.smtp.agent import DeliveryAgent
AVATAR_DIR = '../../../caliopen.ng/src/assets/images/avatars'
if format == 'maildir':
emails = Maildir(import_path, factory=message_from_file)
mode = 'maildir'
else:
if os.path.isdir(import_path):
mode = 'mbox_directory'
emails = {}
files = [
f for f in listdir(import_path)
if os.path.isfile(os.path.join(import_path, f))
]
for f in files:
with open('%s/%s' % (import_path, f)) as fh:
emails[f] = message_from_file(fh)
else:
mode = 'mbox'
emails = mbox(import_path)
print email
user = User.get(email)
agent = DeliveryAgent()
mailfrom = ''
rcpts = [email]
log.info("Processing mode %s" % mode)
msgs = []
for key, mail in emails.iteritems():
# Create contact for user
log.info('Processing mail %s' % key)
msgs.append(MailMessage(mail))
msgs = sorted(msgs, key=lambda msg: msg.date)
for msg in msgs:
for type, addresses in msg.recipients.iteritems():
if not addresses:
continue
for alias, _address in addresses:
lookup = ContactLookup.get(user, alias)
if not lookup:
log.info('Creating contact %s' % alias)
infos = {'mail': alias}
name, domain = alias.split('@')
if os.path.isfile('%s/%s.png' % (AVATAR_DIR, name)):
infos.update({'avatar': '%s.png' % name})
Contact.create(user, infos)
res = agent.process(mailfrom, rcpts, msg.mail.as_string())
log.info('Process result %r' % res)
def clear_mailbox(cls, directory):
"""
Method clears mailbox by removing symlinks.
Args:
directory (str): Path to directory for clearing the mailbox
"""
mailbox = Maildir(directory)
mailbox.clear()
def import_email(email, import_path, format, **kwargs):
from caliop.helpers.log import log
from caliop.core.user import User
from caliop.core.contact import Contact, ContactLookup
from caliop.core.mail import MailMessage
from caliop.smtp.agent import DeliveryAgent
AVATAR_DIR = '../../caliop.ng/src/assets/images/avatars'
if format == 'maildir':
emails = Maildir(import_path, factory=message_from_file)
mode = 'maildir'
else:
if os.path.isdir(import_path):
mode = 'mbox_directory'
emails = {}
files = [f for f in listdir(import_path) if
os.path.isfile(os.path.join(import_path, f))]
for f in files:
with open('%s/%s' % (import_path, f)) as fh:
emails[f] = message_from_file(fh)
else:
mode = 'mbox'
emails = mbox(import_path)
user = User.get(email)
agent = DeliveryAgent()
mailfrom = ''
rcpts = [email]
log.info("Processing mode %s" % mode)
msgs = []
for key, mail in emails.iteritems():
# Create contact for user
log.info('Processing mail %s' % key)
msgs.append(MailMessage(mail))
msgs = sorted(msgs, key=lambda msg: msg.date)
for msg in msgs:
for type, addresses in msg.recipients.iteritems():
if not addresses:
continue
for alias, _address in addresses:
lookup = ContactLookup.get(user, alias)
if not lookup:
log.info('Creating contact %s' % alias)
infos = {'mail': alias}
name, domain = alias.split('@')
if os.path.isfile('%s/%s.png' % (AVATAR_DIR, name)):
infos.update({'avatar': '%s.png' % name})
Contact.create(user, infos)
res = agent.process(mailfrom, rcpts, msg.mail.as_string())
log.info('Process result %r' % res)
def _maildir_delivery(self, username, domain, data):
try:
path = os.path.join(os.path.expanduser(MAILDIR_DELIVERY_USER), domain, username)
mdir = Maildir(path)
mdirid = mdir.add(data)
print "Maildir Delivered %s@%s %s/%s" % (username, domain, path, mdirid)
except:
# XXX log errors with maildir but continue
print "Maildir delivery error %s@%s " % (username, domain)
def send_email(self, message):
"""Sends an email directly (no queuing).
No retries are done, the caller should handle MailDeliveryException in order to ensure that
the mail is never lost.
:param message: the email.message.Message to send. The envelope sender will be extracted from the
``Return-Path`` or ``From`` headers. The envelope recipients will be
extracted from the combined list of ``To``, ``CC`` and ``BCC`` headers.
:return: the Message-ID of the message that was just sent, if successfully sent, otherwise raises
MailDeliveryException and logs root cause.
"""
smtp_from = message['Return-Path'] or message['From']
assert smtp_from, "The Return-Path or From header is required for any outbound email"
# The email's "Envelope From" (Return-Path), and all recipient addresses must only contain ASCII characters.
from_rfc2822 = extract_rfc2822_addresses(smtp_from)
assert len(
from_rfc2822
) == 1, "Malformed 'Return-Path' or 'From' address - it may only contain plain ASCII characters"
smtp_from = from_rfc2822[0]
email_to = message['To']
email_cc = message['Cc']
email_bcc = message['Bcc']
smtp_to_list = filter(
None,
flatten(
map(extract_rfc2822_addresses,
[email_to, email_cc, email_bcc])))
assert smtp_to_list, "At least one valid recipient address should be specified for outgoing emails (To/Cc/Bcc)"
try:
message_id = message['Message-Id']
# Add email in Maildir if smtp_host contains maildir.
if self.smtp_host.startswith('maildir:/'):
from mailbox import Maildir
maildir_path = self.smtp_host[8:]
mdir = Maildir(maildir_path, factory=None, create=True)
mdir.add(message.as_string(True))
return message_id
try:
self.conn.sendmail(smtp_from, smtp_to_list,
message.as_string())
finally:
try:
# Close Connection of SMTP Server
self.conn.quit()
except Exception:
# ignored, just a consequence of the previous exception
pass
except Exception, e:
msg = "Mail delivery failed via SMTP server '%s'.\n%s: %s" % (
unicode(self.smtp_host), e.__class__.__name__, unicode(e))
raise ZatoException(self.cid, msg)
def __init__(self, mailDirPath: Path, mailBoxesPath: Path):
super().__init__()
self.clearMailDirPath = mailDirPath
self.clearMailDirPath.mkdir(parents=True, exist_ok=True)
self.clearMailDir = Maildir(str(self.clearMailDirPath))
self.mailBoxesPath = mailBoxesPath
self.mailBoxes = {}
# Signals
self.sNewMessage = AsyncSignal(str)
def _email_send(smtp_from, smtp_to_list, message, openobject_id=None, ssl=False, debug=False):
"""Low-level method to send directly a Message through the configured smtp server.
:param smtp_from: RFC-822 envelope FROM (not displayed to recipient)
:param smtp_to_list: RFC-822 envelope RCPT_TOs (not displayed to recipient)
:param message: an email.message.Message to send
:param debug: True if messages should be output to stderr before being sent,
and smtplib.SMTP put into debug mode.
:return: True if the mail was delivered successfully to the smtp,
else False (+ exception logged)
"""
if openobject_id:
message['Message-Id'] = "<%s-openobject-%s@%s>" % (time.time(), openobject_id, socket.gethostname())
try:
smtp_server = config['smtp_server']
if smtp_server.startswith('maildir:/'):
from mailbox import Maildir
maildir_path = smtp_server[8:]
mdir = Maildir(maildir_path,factory=None, create = True)
mdir.add(msg.as_string(True))
return True
oldstderr = smtplib.stderr
if not ssl: ssl = config.get('smtp_ssl', False)
s = smtplib.SMTP()
try:
# in case of debug, the messages are printed to stderr.
if debug:
smtplib.stderr = WriteToLogger()
s.set_debuglevel(int(bool(debug))) # 0 or 1
s.connect(smtp_server, config['smtp_port'])
if ssl:
s.ehlo()
s.starttls()
s.ehlo()
if config['smtp_user'] or config['smtp_password']:
s.login(config['smtp_user'], config['smtp_password'])
s.sendmail(smtp_from, smtp_to_list, message.as_string())
finally:
try:
s.quit()
if debug:
smtplib.stderr = oldstderr
except:
# ignored, just a consequence of the previous exception
pass
except Exception, e:
_logger.error('could not deliver email', exc_info=True)
return False
def _maildir_delivery(self, username, domain, data):
try:
path = os.path.join(os.path.expanduser(MAILDIR_DELIVERY_USER),
domain, username)
mdir = Maildir(path)
mdirid = mdir.add(data)
print "Maildir Delivered %s@%s %s/%s" % (username, domain, path,
mdirid)
except:
# XXX log errors with maildir but continue
print "Maildir delivery error %s@%s " % (username, domain)
def main(args=sys.argv):
parser = option_parser()
opts, args = parser.parse_args(args)
if len(args) > 1:
if len(args) < 4:
print ('You must specify the from address, to address and body text'
' on the command line')
return 1
msg = compose_mail(args[1], args[2], args[3], subject=opts.subject,
attachment=opts.attachment)
from_, to = args[1:3]
eto = [extract_email_address(x.strip()) for x in to.split(',')]
efrom = extract_email_address(from_)
else:
msg = sys.stdin.read()
from email import message_from_string
from email.utils import getaddresses
eml = message_from_string(msg)
tos = eml.get_all('to', [])
ccs = eml.get_all('cc', []) + eml.get_all('bcc', [])
all_tos = []
for x in tos + ccs:
all_tos.extend(y.strip() for y in x.split(','))
eto = list(map(extract_email_address, all_tos))
if not eto:
raise ValueError('Email from STDIN does not specify any recipients')
efrom = getaddresses(eml.get_all('from', []))
if not efrom:
raise ValueError('Email from STDIN does not specify a sender')
efrom = efrom[0][1]
outbox = None
if opts.outbox is not None:
outbox = os.path.abspath(os.path.expanduser(opts.outbox))
from mailbox import Maildir
outbox = Maildir(opts.outbox, factory=None)
if opts.fork:
if os.fork() != 0:
return 0
try:
sendmail(msg, efrom, eto, localhost=opts.localhost, verbose=opts.verbose,
timeout=opts.timeout, relay=opts.relay, username=opts.username,
password=opts.password, port=opts.port,
encryption=opts.encryption_method)
except:
if outbox is not None:
outbox.add(msg)
outbox.close()
print 'Delivery failed. Message saved to', opts.outbox
raise
return 0
def send_email(self, message):
"""Sends an email directly (no queuing).
No retries are done, the caller should handle MailDeliveryException in order to ensure that
the mail is never lost.
:param message: the email.message.Message to send. The envelope sender will be extracted from the
``Return-Path`` or ``From`` headers. The envelope recipients will be
extracted from the combined list of ``To``, ``CC`` and ``BCC`` headers.
:return: the Message-ID of the message that was just sent, if successfully sent, otherwise raises
MailDeliveryException and logs root cause.
"""
smtp_from = message['Return-Path'] or message['From']
assert smtp_from, "The Return-Path or From header is required for any outbound email"
# The email's "Envelope From" (Return-Path), and all recipient addresses must only contain ASCII characters.
from_rfc2822 = extract_rfc2822_addresses(smtp_from)
assert len(from_rfc2822) == 1, "Malformed 'Return-Path' or 'From' address - it may only contain plain ASCII characters"
smtp_from = from_rfc2822[0]
email_to = message['To']
email_cc = message['Cc']
email_bcc = message['Bcc']
smtp_to_list = filter(None, flatten(map(extract_rfc2822_addresses,[email_to, email_cc, email_bcc])))
assert smtp_to_list, "At least one valid recipient address should be specified for outgoing emails (To/Cc/Bcc)"
try:
message_id = message['Message-Id']
# Add email in Maildir if smtp_host contains maildir.
if self.smtp_host.startswith('maildir:/'):
from mailbox import Maildir
maildir_path = self.smtp_host[8:]
mdir = Maildir(maildir_path, factory=None, create = True)
mdir.add(message.as_string(True))
return message_id
try:
self.conn.sendmail(smtp_from, smtp_to_list, message.as_string())
finally:
try:
# Close Connection of SMTP Server
self.conn.quit()
except Exception:
# ignored, just a consequence of the previous exception
pass
except Exception, e:
msg = "Mail delivery failed via SMTP server '%s'.\n%s: %s" % (unicode(self.smtp_host),
e.__class__.__name__,
unicode(e))
raise ZatoException(self.cid, msg)
def tickle_iterator( path ):
thetime = datetime.now()
for box, subdirs, _ in walk( path, topdown=True ):
# remove maildir meta directories from top-level
subdirs.remove('cur')
subdirs.remove('tmp')
subdirs.remove('new')
src_mbox = Maildir(box, factory=None)
src_mbox.lock()
for key in src_mbox.iterkeys():
path = '/'.join([box, src_mbox._toc[key]])
try:
tickle_time = None
start_time = thetime
if src_mbox[key]['X-Tickler']:
# already tickling (probably past due). get tickler time from tickler header.
try:
tickle_time = parse_time( src_mbox[key]['X-Tickler'] )
except Exception, e:
print e
else:
# compute tickle time from mailbox name
start_time = datetime.fromtimestamp(stat(path).st_ctime)
tickle_time = parse_time(basename( box ).replace('-', ' '), start_time=start_time)
due_in = thetime - tickle_time
yield {
'src' : src_mbox,
'key' : key,
'boxname' : basename( box ),
'path' : path,
'due_in' : format_timedelta( due_in ),
'due' : due_in > timedelta(seconds=1),
'start_time' : start_time,
'tickle_time' : tickle_time,
}
except DateTimeParseException:
# support for free-form todo box names
yield {
'src' : src_mbox,
'key' : key,
'boxname' : basename( box ),
'path' : path,
'due_in' : None,
'start_time' : None,
'tickle_time' : None,
'due' : False,
}
def _load_mails_as_is(mail_paths, store):
deferreds = []
for path in mail_paths:
if isfile(path):
mbox_mails = mbox(path, factory=None)
yield add_mail_folder(store, mbox_mails, 'INBOX', deferreds)
else:
maildir = Maildir(path, factory=None)
yield add_mail_folder(store, maildir, 'INBOX', deferreds)
for mail_folder_name in maildir.list_folders():
mail_folder = maildir.get_folder(mail_folder_name)
yield add_mail_folder(store, mail_folder, mail_folder_name, deferreds)
yield defer.gatherResults(deferreds, consumeErrors=True)
def load_mails(args, mail_paths):
leap_session, soledad = args
account = leap_session.account
deferreds = []
for path in mail_paths:
maildir = Maildir(path, factory=None)
add_mail_folder(account, maildir, 'INBOX', deferreds)
for mail_folder_name in maildir.list_folders():
mail_folder = maildir.get_folder(mail_folder_name)
add_mail_folder(account, mail_folder, mail_folder_name, deferreds)
yield defer.DeferredList(deferreds)
defer.returnValue(args)
def _load_mails_as_is(mail_paths, store):
deferreds = []
for path in mail_paths:
if isfile(path):
mbox_mails = mbox(path, factory=None)
yield add_mail_folder(store, mbox_mails, 'INBOX', deferreds)
else:
maildir = Maildir(path, factory=None)
yield add_mail_folder(store, maildir, 'INBOX', deferreds)
for mail_folder_name in maildir.list_folders():
mail_folder = maildir.get_folder(mail_folder_name)
yield add_mail_folder(store, mail_folder, mail_folder_name, deferreds)
yield defer.gatherResults(deferreds, consumeErrors=True)
def archive_message(mlist, message):
"""See `IArchiver`.
This archiver saves messages into a maildir.
"""
archive_dir = os.path.join(config.ARCHIVE_DIR, 'prototype')
try:
os.makedirs(archive_dir, 0o775)
except OSError as error:
# If this already exists, then we're fine
if error.errno != errno.EEXIST:
raise
# Maildir will throw an error if the directories are partially created
# (for instance the toplevel exists but cur, new, or tmp do not)
# therefore we don't create the toplevel as we did above.
list_dir = os.path.join(archive_dir, mlist.fqdn_listname)
mailbox = Maildir(list_dir, create=True, factory=None)
lock_file = os.path.join(
config.LOCK_DIR, '{0}-maildir.lock'.format(mlist.fqdn_listname))
# Lock the maildir as Maildir.add() is not threadsafe. Don't use the
# context manager because it's not an error if we can't acquire the
# archiver lock. We'll just log the problem and continue.
#
# XXX 2012-03-14 BAW: When we extend the chain/pipeline architecture
# to other runners, e.g. the archive runner, it would be better to let
# any TimeOutError propagate up. That would cause the message to be
# re-queued and tried again later, rather than being discarded as
# happens now below.
lock = Lock(lock_file)
try:
lock.lock(timeout=timedelta(seconds=1))
# Add the message to the maildir. The return value could be used
# to construct the file path if necessary. E.g.
#
# os.path.join(archive_dir, mlist.fqdn_listname, 'new',
# message_key)
mailbox.add(message)
except TimeOutError:
# Log the error and go on.
log.error('Unable to acquire prototype archiver lock for {0}, '
'discarding: {1}'.format(
mlist.fqdn_listname,
message.get('message-id', 'n/a')))
finally:
lock.unlock(unconditionally=True)
# Can we get return the URL of the archived message?
return None
def archive_message(mlist, message):
"""See `IArchiver`.
This archiver saves messages into a maildir.
"""
archive_dir = os.path.join(config.ARCHIVE_DIR, 'prototype')
try:
os.makedirs(archive_dir, 0o775)
except OSError as error:
# If this already exists, then we're fine
if error.errno != errno.EEXIST:
raise
# Maildir will throw an error if the directories are partially created
# (for instance the toplevel exists but cur, new, or tmp do not)
# therefore we don't create the toplevel as we did above.
list_dir = os.path.join(archive_dir, mlist.fqdn_listname)
mailbox = Maildir(list_dir, create=True, factory=None)
lock_file = os.path.join(
config.LOCK_DIR, '{0}-maildir.lock'.format(mlist.fqdn_listname))
# Lock the maildir as Maildir.add() is not threadsafe. Don't use the
# context manager because it's not an error if we can't acquire the
# archiver lock. We'll just log the problem and continue.
#
# XXX 2012-03-14 BAW: When we extend the chain/pipeline architecture
# to other runners, e.g. the archive runner, it would be better to let
# any TimeOutError propagate up. That would cause the message to be
# re-queued and tried again later, rather than being discarded as
# happens now below.
lock = Lock(lock_file)
try:
lock.lock(timeout=timedelta(seconds=1))
# Add the message to the maildir. The return value could be used
# to construct the file path if necessary. E.g.
#
# os.path.join(archive_dir, mlist.fqdn_listname, 'new',
# message_key)
mailbox.add(message)
except TimeOutError:
# Log the error and go on.
log.error('Unable to acquire prototype archiver lock for {0}, '
'discarding: {1}'.format(
mlist.fqdn_listname,
message.get('message-id', 'n/a')))
finally:
lock.unlock(unconditionally=True)
# Can we get return the URL of the archived message?
return None
def main() -> None:
args = parse_args()
global debug_enabled
debug_enabled = args.debug
mailbox = Maildir(
dirname=args.maildir_path,
factory=message_factory,
create=False,
)
message_filter = MessageFilter.create(args)
debug("filter: {}".format(message_filter))
matches = 0
for message in mailbox:
debug(
"message: {}".format((
message.get("From"),
message.get("Reply-To"),
message.get("To"),
message.get("Subject"),
), ), )
if match(message, message_filter):
matches += 1
debug("matched: {}".format(matches))
if args.count is None or args.count == matches:
exit(0)
exit(1)
def load_mails(args, mail_paths):
leap_session, soledad = args
store = leap_session.mail_store
deferreds = []
for path in mail_paths:
maildir = Maildir(path, factory=None)
yield add_mail_folder(store, maildir, 'INBOX', deferreds)
for mail_folder_name in maildir.list_folders():
mail_folder = maildir.get_folder(mail_folder_name)
yield add_mail_folder(store, mail_folder, mail_folder_name, deferreds)
yield defer.gatherResults(deferreds, consumeErrors=True)
defer.returnValue(args)
def enqueue_message(msg, queue_path, sender, recipients, return_msg=False, in_progress=False, **queue_args):
msg_bytes = serialize_message_with_queue_data(msg, sender=sender, recipients=recipients, **queue_args)
create_maildir_directories(queue_path)
mailbox = Maildir(queue_path)
sub_dir = 'cur' if in_progress else 'new'
return inject_message_into_maildir(msg_bytes, mailbox, sub_dir=sub_dir, return_msg=return_msg)
def get_message(self, key):
print 'MaildirMailbox.get_message'
""" Overloading for context data"""
message = Maildir.get_message(self, key)
# TODO status
setContext(message, key, 'read', 'add')
return message
async def run_tests():
with tempfile.TemporaryDirectory() as maildir:
# create temporary mailbox
for d in ['cur', 'new', 'tmp']:
maildirdir = os.path.join(maildir, d)
os.mkdir(maildirdir)
mailbox = Maildir(maildir)
# start smtp server
handler = MessageHandler(maildir)
controller = Controller(handler)
controller.start()
# run tests
tests = filter(
lambda f: f.startswith('test_'),
globals().keys(),
)
for test in tests:
print(test + '...', end=' ')
try:
await globals()[test](mailbox)
except AssertionError:
fail()
continue
succeed()
def cmd_dl(self, argv):
''' Collect messages from a POP3 server and deliver to a Maildir.
Usage: {cmd} [{{ssl,tcp}}:]{{netrc_account|[user@]host[!sni_name][:port]}} maildir
'''
pop_target = argv.pop(0)
maildir_path = argv.pop(0)
assert len(argv) == 0
if not isdirpath(maildir_path):
raise ValueError("maildir %s: not a directory" % (maildir_path,))
M = Maildir(maildir_path)
with POP3(pop_target) as pop3:
msg_uid_map = dict(pop3.client_uidl())
print(
f'{len(msg_uid_map)} message',
('' if len(msg_uid_map) == 1 else 's'),
('.' if len(msg_uid_map) == 0 else ':'),
sep=''
)
with ResultSet() as deleRs:
with ResultSet() as retrRs:
for msg_n in msg_uid_map.keys():
retrRs.add(pop3.dl_bg(msg_n, M, deleRs))
pop3.flush()
retrRs.wait()
# now the deleRs are all queued
pop3.flush()
if deleRs:
print("wait for DELEs...")
deleRs.wait()
def test_mailbox(self):
with SMTP(*self.address) as client:
client.sendmail(
'*****@*****.**', ['*****@*****.**'], """\
From: Anne Person <*****@*****.**>
To: Bart Person <*****@*****.**>
Subject: A test
Message-ID: <ant>
Hi Bart, this is Anne.
""")
client.sendmail(
'*****@*****.**', ['*****@*****.**'], """\
From: Cate Person <*****@*****.**>
To: Dave Person <*****@*****.**>
Subject: A test
Message-ID: <bee>
Hi Dave, this is Cate.
""")
client.sendmail(
'*****@*****.**', ['*****@*****.**'], """\
From: Elle Person <*****@*****.**>
To: Fred Person <*****@*****.**>
Subject: A test
Message-ID: <cat>
Hi Fred, this is Elle.
""")
# Check the messages in the mailbox.
mailbox = Maildir(self.maildir_path)
messages = sorted(mailbox, key=itemgetter('message-id'))
self.assertEqual(
list(message['message-id'] for message in messages),
['<ant>', '<bee>', '<cat>'])
def load_mails(args, mail_paths):
leap_session, soledad = args
account = leap_session.account
deferreds = []
for path in mail_paths:
maildir = Maildir(path, factory=None)
add_mail_folder(account, maildir, 'INBOX', deferreds)
add_mail_folder(account, maildir, 'DRAFTS', deferreds)
for mail_folder_name in maildir.list_folders():
mail_folder = maildir.get_folder(mail_folder_name)
add_mail_folder(account, mail_folder, mail_folder_name, deferreds)
yield defer.DeferredList(deferreds)
defer.returnValue(args)
def __init__(self, oldBoxDir, newBoxDir, daystoremove):
from sys import stdout
from mailbox import Maildir,MaildirMessage,NoSuchMailboxError
from os.path import expanduser
from time import time, strftime, gmtime
self.logfile=strftime(expanduser("~/")+"mail-remove-log-%Y%m%d%H%M", gmtime())
# DeltaT is epoch now - 60(s)*m*d*nm where n is number of months - here set for 4 months
self.deltaT=int(round(time()-(60*60*24*int(daystoremove))))
self.newBox = Maildir(newBoxDir, create=True, factory=MaildirMessage)
try:
self.oldBox = Maildir(oldBoxDir, create=False, factory=MaildirMessage)
except NoSuchMailboxError:
print ("[E] Invalid mail dir. Aborting")
return(1)
def _convert(self, item: Item, maildir: mailbox.Maildir) -> bool:
if item.type is Entry.folder:
newmaildir = maildir.add_folder(item.name)
newmaildir.lock()
source = pmlib.manager.get_source(item.data.type)
if source is None:
pmlib.log.warn(
item.name,
"Mailbox format is not yet implemented: {0:s}".format(
item.data.type.name))
newmaildir.unlock()
return True
item.report.target_format = self.target
check = source.read(item, newmaildir)
newmaildir.flush()
newmaildir.unlock()
return check
else:
pmlib.log.inform("TRAY", item.full_name)
newmaildir = maildir.add_folder(item.name)
newmaildir.lock()
# first convert folder
for _item in sorted(item.children, key=sort_items):
if _item.type is Entry.folder:
check = self._convert(_item, newmaildir)
if check is False:
return False
# then trays
for _item in sorted(item.children, key=sort_items):
if _item.type is not Entry.folder:
check = self._convert(_item, newmaildir)
if check is False:
return False
newmaildir.flush()
newmaildir.unlock()
maildir.flush()
return True
def load_mails(args, mail_paths):
leap_session, soledad = args
store = leap_session.mail_store
deferreds = []
for path in mail_paths:
maildir = Maildir(path, factory=None)
yield add_mail_folder(store, maildir, 'INBOX', deferreds)
for mail_folder_name in maildir.list_folders():
mail_folder = maildir.get_folder(mail_folder_name)
yield add_mail_folder(store, mail_folder, mail_folder_name,
deferreds)
yield defer.gatherResults(deferreds, consumeErrors=True)
defer.returnValue(args)
def open(self, basedir=None):
"""Maildirの利用を開始する"""
if not basedir: basedir = os.path.expanduser(user_mailbox)
mdir = Maildir(basedir, None)
self.mdir = mdir
self.cur = ''
self.fld = mdir
self.basedir = basedir
def __init__(self, oldBoxDir, newBoxDir, daystoremove):
from sys import stdout
from mailbox import Maildir, MaildirMessage, NoSuchMailboxError
from os.path import expanduser
from time import time, strftime, gmtime
self.logfile = strftime(
expanduser("~/") + "mail-remove-log-%Y%m%d%H%M", gmtime())
# DeltaT is epoch now - 60(s)*m*d*nm where n is number of months - here set for 4 months
self.deltaT = int(round(time() - (60 * 60 * 24 * int(daystoremove))))
self.newBox = Maildir(newBoxDir, create=True, factory=MaildirMessage)
try:
self.oldBox = Maildir(oldBoxDir,
create=False,
factory=MaildirMessage)
except NoSuchMailboxError:
print("[E] Invalid mail dir. Aborting")
return (1)
def populateMailbox(self, username):
# Here will the mail directory of the user be stored
pathShort = "/var/vmail/" + self.domain + "/" + username
pathLong = pathShort + "/Maildir"
# Create those directories
os.mkdir(pathShort)
mailbox = Maildir(pathLong)
# And now populate that directory
numOfMails = random.randint(1, 6666)
percentageRead = random.randint(0, 100)
for i in xrange(1, numOfMails):
message = MaildirMessage(message=str(i))
message.set_subdir("cur")
readSeed = random.randint(0, 100)
if percentageRead <= readSeed:
message.set_flags("S")
mailbox.add(message)
self.chmodMailbox(pathShort)
async def handle_DATA(self, server, session, envelope):
d = Maildir(MAILDIR_PATH)
message = email.message_from_bytes(envelope.original_content)
for r in envelope.rcpt_tos:
message['X-Rcpt-To'] = r
d.add(message)
try:
conn = smtplib.SMTP(RELAY_HOST, RELAY_PORT)
conn.ehlo()
conn.starttls()
conn.login(RELAY_USER, RELAY_PASS)
refused = conn.sendmail(envelope.mail_from, envelope.rcpt_tos,
envelope.original_content)
if refused:
print(refused, flush=True)
print("To: %r" % (envelope.rcpt_tos, ), flush=True)
return '250 OK'
except smtplib.SMTPException as exn:
traceback.print_exc()
return '550 SMTPException: %s' % exn
def inject_email(email, import_path, format, host, **kwargs):
"""Import emails for an user."""
if format == 'maildir':
emails = Maildir(import_path, factory=message_from_file)
mode = 'maildir'
else:
if os.path.isdir(import_path):
mode = 'mbox_directory'
emails = {}
files = [
f for f in os.listdir(import_path)
if os.path.isfile(os.path.join(import_path, f))
]
for f in files:
try:
log.debug('Importing mail from file {}'.format(f))
with open('%s/%s' % (import_path, f)) as fh:
emails[f] = message_from_file(fh)
except Exception as exc:
log.error('Error importing email {}'.format(exc))
else:
mode = 'mbox'
emails = mbox(import_path)
log.info("Processing mode %s" % mode)
if ':' in host:
host, port = host.split(':', 2)
port = int(port)
else:
port = 25
mails = emails.values()
total_msgs = len(mails)
log.info('Will inject mail for {} into {}:{}'.format([email], host, port))
smtp = smtplib.SMTP(host=host, port=port)
for i, mail in enumerate(mails, 1):
# Create contact for user
log.info('Injecting mail {}/{}'.format(i, total_msgs))
smtp.sendmail("*****@*****.**", [email], str(mail))
def test_maildir(self):
tmpdir = mkdtemp('archweb')
mdir = tmpdir + '/maildir'
maildir = Maildir(mdir)
msg = Message()
msg['subject'] = 'John Doe'
msg['to'] = 'John Doe <*****@*****.**>'
maildir.add(msg)
# Invalid
call_command('donor_import', mdir)
self.assertEqual(len(Donor.objects.all()), 0)
# Valid
msg = Message()
msg['subject'] = 'Receipt [$25.00] By: David Doe [[email protected]]'
msg['to'] = 'John Doe <*****@*****.**>'
maildir.add(msg)
call_command('donor_import', mdir)
self.assertEqual(len(Donor.objects.all()), 1)
# Re-running should result in no new donor
call_command('donor_import', mdir)
self.assertEqual(len(Donor.objects.all()), 1)
rmtree(tmpdir)
def add_maildir(self, maildir_path):
""" Load up a maildir add compute hash for each mail their contain. """
maildir = Maildir(maildir_path, create=False)
# Collate folders by hash.
print("Processing {} mails in {}".format(len(maildir), maildir._path))
for mail_id, message in maildir.iteritems():
mail_file = os.path.join(maildir._path, maildir._lookup(mail_id))
try:
mail_hash, header_text = self.compute_hash(
mail_file, message, self.use_message_id)
except InsufficientHeadersError as e:
print("WARNING: ignoring problematic {}: {}".format(
mail_file, e.args[0]))
else:
if self.mail_count > 0 and self.mail_count % 100 == 0:
print(".")
# print("Hash is {} for mail {!r}.".format(mail_hash, mail_id))
if mail_hash not in self.mails:
self.mails[mail_hash] = []
self.mails[mail_hash].append((mail_file, message))
self.mail_count += 1
def add_maildir(self, maildir_path):
""" Load up a maildir add compute hash for each mail their contain. """
maildir = Maildir(maildir_path, create=False)
# Collate folders by hash.
print("Processing {} mails in {}".format(len(maildir), maildir._path))
for mail_id, message in maildir.iteritems():
mail_file = os.path.join(maildir._path, maildir._lookup(mail_id))
try:
mail_hash, header_text = self.compute_hash(
mail_file, message, self.use_message_id)
except InsufficientHeadersError as e:
print("WARNING: ignoring problematic {}: {}".format(
mail_file, e.args[0]))
else:
if self.mail_count > 0 and self.mail_count % 100 == 0:
print(".")
# print("Hash is {} for mail {!r}.".format(mail_hash, mail_id))
if mail_hash not in self.mails:
self.mails[mail_hash] = []
self.mails[mail_hash].append((mail_file, message))
self.mail_count += 1
def add_maildir(self, maildir_path):
""" Load up a maildir and compute hash for each mail found. """
maildir_path = self.canonical_path(maildir_path)
logger.info("Opening maildir at {} ...".format(maildir_path))
# Maildir parser requires a string, not a unicode, as path.
maildir = Maildir(str(maildir_path), factory=None, create=False)
# Group folders by hash.
logger.info("{} mails found.".format(len(maildir)))
if self.conf.progress:
bar = ProgressBar(widgets=[Percentage(), Bar()],
max_value=len(maildir),
redirect_stderr=True,
redirect_stdout=True)
else:
def bar(x):
return x
for mail_id in bar(maildir.iterkeys()):
self.stats['mail_found'] += 1
mail_path = self.canonical_path(
os.path.join(maildir._path, maildir._lookup(mail_id)))
mail = Mail(mail_path, self.conf)
try:
mail_hash = mail.hash_key
except (InsufficientHeadersError, MissingMessageID) as expt:
logger.warning("Rejecting {}: {}".format(
mail_path, expt.args[0]))
self.stats['mail_rejected'] += 1
else:
logger.debug("Hash is {} for mail {!r}.".format(
mail_hash, mail_id))
# Use a set to deduplicate entries pointing to the same file.
self.mails.setdefault(mail_hash, set()).add(mail_path)
self.stats['mail_kept'] += 1
def test_mailbox_reset(self):
with SMTP(*self.address) as client:
client.sendmail(
'*****@*****.**', ['*****@*****.**'], """\
From: Anne Person <*****@*****.**>
To: Bart Person <*****@*****.**>
Subject: A test
Message-ID: <ant>
Hi Bart, this is Anne.
""")
self.handler.reset()
mailbox = Maildir(self.maildir_path)
self.assertEqual(list(mailbox), [])
class MaildirAdapter(Mapping):
def __init__(self, maildir_path):
self.mbox = Maildir(maildir_path, create=False)
def __getitem__(self, key):
actual_key = unquote(key)
msg = self.mbox[actual_key]
return MessageAdapter(msg)
def __iter__(self):
return iter(map(quote, self.mbox.keys()))
def __len__(self):
return len(self.mbox)
def test_email_works(tmpdir):
"""
This is a basic test of the email server working OK.
"""
email_server, maildir = generate_email_server(tmpdir)
email_server.start()
time.sleep(3.0)
# send a test email
email_sent = actions.send_email(
"*****@*****.**",
"Hello",
"this is a test",
["*****@*****.**"],
"127.0.0.1",
None,
None,
"salt",
port=9487,
)
assert email_sent is True
# check if it was received
mailbox = Maildir(maildir)
for _, message in mailbox.items():
if message["Subject"] == "Hello":
assert message["From"] == "*****@*****.**"
assert message["To"] == "*****@*****.**"
assert "this is a test" in message.as_string()
email_server.stop()
def add_maildir(self, maildir_path):
""" Load up a maildir and compute hash for each mail found. """
maildir_path = self.canonical_path(maildir_path)
logger.info("Opening maildir at {} ...".format(maildir_path))
# Maildir parser requires a string, not a unicode, as path.
maildir = Maildir(str(maildir_path), factory=None, create=False)
# Group folders by hash.
logger.info("{} mails found.".format(len(maildir)))
if self.conf.progress:
bar = ProgressBar(widgets=[Percentage(), Bar()],
max_value=len(maildir), redirect_stderr=True,
redirect_stdout=True)
else:
def bar(x):
return x
for mail_id in bar(maildir.iterkeys()):
self.stats['mail_found'] += 1
mail_path = self.canonical_path(os.path.join(
maildir._path, maildir._lookup(mail_id)))
mail = Mail(mail_path, self.conf)
try:
mail_hash = mail.hash_key
except (InsufficientHeadersError, MissingMessageID) as expt:
logger.warning(
"Rejecting {}: {}".format(mail_path, expt.args[0]))
self.stats['mail_rejected'] += 1
else:
logger.debug(
"Hash is {} for mail {!r}.".format(mail_hash, mail_id))
# Use a set to deduplicate entries pointing to the same file.
self.mails.setdefault(mail_hash, set()).add(mail_path)
self.stats['mail_kept'] += 1
def duplicates_run(opts, maildir_paths):
mails_by_hash = {}
mail_count = 0
check_maildirs_valid(maildir_paths)
for maildir_path in maildir_paths:
maildir = Maildir(maildir_path, factory=None)
mail_count += collate_folder_by_hash(mails_by_hash, maildir,
opts.message_id)
duplicates, sizes_too_dissimilar, diff_too_big, removed, sets = \
find_duplicates(mails_by_hash, opts)
report_results(duplicates, sizes_too_dissimilar, diff_too_big, removed,
sets, mail_count)
def store_mail(tf, mail, maildir, metadata):
"""Store the given mail inside the maildir with its metadata"""
# Retrieve the message
msg = MaildirMessage(tf.extractfile(mail['name']))
msg.set_flags(get_msgflags(metadata[mail['name']]))
folder = path.dirname(mail['name'])
# Find the mailbox in which to store the mail
md = None
if re.match('Inbox(\![0-9]+)?$', folder):
md = Maildir(maildir)
elif re.match('Inbox/', folder):
# Nested folders under Inbox, put them under a folder named
# 'INBOX'.
folder = folder.replace('/', '.').replace('Inbox', 'INBOX')
md = Maildir(path.join(maildir, '.' + folder), factory=None)
elif re.match('Sent(\ Items.*)?', folder):
md = Maildir(path.join(maildir, '.' + 'Sent'), factory=None)
else:
md = Maildir(path.join(maildir, '.' + folder), factory=None)
# Store the mail
md.add(msg)
def import_email(email, import_path, format, contact_probability,
**kwargs):
"""Import emails for an user."""
from caliopen_main.user.core import User, UserIdentity
from caliopen_main.contact.core import Contact, ContactLookup
from caliopen_main.message.parsers.mail import MailMessage
from caliopen_main.contact.parameters import NewContact, NewEmail
from caliopen_nats.delivery import UserMailDelivery
from caliopen_main.message.core import RawMessage
from caliopen_storage.config import Configuration
max_size = int(Configuration("global").get("object_store.db_size_limit"))
if 'to' in kwargs and kwargs['to']:
dest_email = kwargs['to']
else:
dest_email = email
if format == 'maildir':
if dest_email != email:
raise Exception('Cannot change To email using maildir format')
emails = Maildir(import_path, factory=message_from_file)
mode = 'maildir'
else:
if os.path.isdir(import_path):
mode = 'mbox_directory'
emails = {}
files = [f for f in os.listdir(import_path) if
os.path.isfile(os.path.join(import_path, f))]
for f in files:
try:
log.debug('Importing mail from file {}'.format(f))
with open('%s/%s' % (import_path, f)) as fh:
data = fh.read()
data = re.sub('^To: (.*)', 'To: %s' % dest_email,
data, flags=re.MULTILINE)
emails[f] = message_from_string(data)
except Exception as exc:
log.error('Error importing email {}'.format(exc))
else:
mode = 'mbox'
emails = mbox(import_path)
user = User.by_local_identifier(dest_email, 'email')
log.info("Processing mode %s" % mode)
for key, data in emails.iteritems():
# Prevent creating message too large to fit in db.
# (should use inject cmd for large messages)
size = len(data.as_string())
if size > max_size:
log.warn("Message too large to fit into db. \
Please, use 'inject' cmd for importing large emails.")
continue
raw = RawMessage.create(data.as_string())
log.debug('Created raw message {}'.format(raw.raw_msg_id))
message = MailMessage(data.as_string())
dice = random()
if dice <= contact_probability:
for participant in message.participants:
try:
ContactLookup.get(user, participant.address)
except NotFound:
log.info('Creating contact %s' % participant.address)
name, domain = participant.address.split('@')
contact_param = NewContact()
contact_param.family_name = name
if participant.address:
e_mail = NewEmail()
e_mail.address = participant.address
contact_param.emails = [e_mail]
Contact.create(user, contact_param)
log.info('No contact associated to raw {} '.format(raw.raw_msg_id))
processor = UserMailDelivery(user,
user.local_identities[0]) # assume one local identity
try:
obj_message = processor.process_raw(raw.raw_msg_id)
except Exception as exc:
log.exception(exc)
else:
log.info('Created message {}'.format(obj_message.message_id))
if ret != 0:
log.critical("child returned with error, aborting!")
sys.exit(ret)
continue
try:
change_to_user(user)
maildir_path = path + "/.maildir/"
if not os.path.exists(maildir_path):
log.warn("%s doesn't have a maildir" % user)
sys.exit(0)
maildir = Maildir(maildir_path, factory=None)
spam_open = maildir.add_folder('SA.Spam.' + FOLDER_OPEN)
spam_learend = maildir.add_folder('SA.Spam.' + FOLDER_LEARNED)
ham_open = maildir.add_folder('SA.Ham.' + FOLDER_OPEN)
ham_learned = maildir.add_folder('SA.Ham.' + FOLDER_LEARNED)
jobs = (
('spam', ('spamassassin', '-r', '-D' ), spam_open, spam_learend),
('ham', ('sa-learn', '--ham', '-D' ), ham_open, ham_learned)
)
for desc, cmd, mbox_todo, mbox_done in jobs:
log.info ("User %s: processing %s" % (user, desc))
# spawn email workers
workers = []
for i in range(procs):
worker = EmailWorker(config, workQueue, logQueue, errorQueue)
workers.append(worker)
worker.start()
# spawn log and email writing workers
logger = LogWorker(logQueue)
logger.start()
mboxer = MboxWorker(errorQueue, logQueue, luser)
mboxer.start()
# load workQueue
mail = {}
maildir = Maildir(baseMailDir + domain + "/" + luser, factory=None)
folders = maildir.list_folders()
folders.append('') # empty string means INBOX.
for folder in folders:
mail[folder] = maildir.get_folder(folder)
for folder in mail:
for eid, msg in mail[folder].iteritems():
#print domain, user, folder, msg.get_flags(), eid, len(str(msg))
# Process emails here.
email = (domain,USER,folder,eid,msg.get_flags(),str(msg))
workQueue.put(email)
while not workQueue.empty():
if debug:
print "workQueue: " + folder + " " + str(workQueue.qsize())
time.sleep(60)
def send_email(
self,
cr,
uid,
message,
mail_server_id=None,
smtp_server=None,
smtp_port=None,
smtp_user=None,
smtp_password=None,
smtp_encryption=None,
smtp_debug=False,
context=None,
):
"""Sends an email directly (no queuing).
No retries are done, the caller should handle MailDeliveryException in order to ensure that
the mail is never lost.
If the mail_server_id is provided, sends using this mail server, ignoring other smtp_* arguments.
If mail_server_id is None and smtp_server is None, use the default mail server (highest priority).
If mail_server_id is None and smtp_server is not None, use the provided smtp_* arguments.
If both mail_server_id and smtp_server are None, look for an 'smtp_server' value in server config,
and fails if not found.
:param message: the email.message.Message to send. The envelope sender will be extracted from the
``Return-Path`` or ``From`` headers. The envelope recipients will be
extracted from the combined list of ``To``, ``CC`` and ``BCC`` headers.
:param mail_server_id: optional id of ir.mail_server to use for sending. overrides other smtp_* arguments.
:param smtp_server: optional hostname of SMTP server to use
:param smtp_encryption: optional TLS mode, one of 'none', 'starttls' or 'ssl' (see ir.mail_server fields for explanation)
:param smtp_port: optional SMTP port, if mail_server_id is not passed
:param smtp_user: optional SMTP user, if mail_server_id is not passed
:param smtp_password: optional SMTP password to use, if mail_server_id is not passed
:param smtp_debug: optional SMTP debug flag, if mail_server_id is not passed
:return: the Message-ID of the message that was just sent, if successfully sent, otherwise raises
MailDeliveryException and logs root cause.
"""
smtp_from = message["Return-Path"] or message["From"]
assert smtp_from, "The Return-Path or From header is required for any outbound email"
# The email's "Envelope From" (Return-Path), and all recipient addresses must only contain ASCII characters.
from_rfc2822 = extract_rfc2822_addresses(smtp_from)
assert from_rfc2822, (
"Malformed 'Return-Path' or 'From' address: %r - " "It should contain one valid plain ASCII email"
) % smtp_from
# use last extracted email, to support rarities like 'Support@MyComp <*****@*****.**>'
smtp_from = from_rfc2822[-1]
email_to = message["To"]
email_cc = message["Cc"]
email_bcc = message["Bcc"]
smtp_to_list = filter(None, tools.flatten(map(extract_rfc2822_addresses, [email_to, email_cc, email_bcc])))
assert smtp_to_list, self.NO_VALID_RECIPIENT
# Do not actually send emails in testing mode!
if getattr(threading.currentThread(), "testing", False):
_logger.log(logging.INFO, "skip sending email in test mode")
return message["Message-Id"]
# Get SMTP Server Details from Mail Server
mail_server = None
if mail_server_id:
mail_server = self.browse(cr, SUPERUSER_ID, mail_server_id)
elif not smtp_server:
mail_server_ids = self.search(cr, SUPERUSER_ID, [], order="sequence", limit=1)
if mail_server_ids:
mail_server = self.browse(cr, SUPERUSER_ID, mail_server_ids[0])
if mail_server:
smtp_server = mail_server.smtp_host
smtp_user = mail_server.smtp_user
smtp_password = mail_server.smtp_pass
smtp_port = mail_server.smtp_port
smtp_encryption = mail_server.smtp_encryption
smtp_debug = smtp_debug or mail_server.smtp_debug
else:
# we were passed an explicit smtp_server or nothing at all
smtp_server = smtp_server or tools.config.get("smtp_server")
smtp_port = tools.config.get("smtp_port", 25) if smtp_port is None else smtp_port
smtp_user = smtp_user or tools.config.get("smtp_user")
smtp_password = smtp_password or tools.config.get("smtp_password")
if smtp_encryption is None and tools.config.get("smtp_ssl"):
smtp_encryption = "starttls" # STARTTLS is the new meaning of the smtp_ssl flag as of v7.0
if not smtp_server:
raise osv.except_osv(
_("Missing SMTP Server"),
_("Please define at least one SMTP server, or provide the SMTP parameters explicitly."),
)
try:
message_id = message["Message-Id"]
# Add email in Maildir if smtp_server contains maildir.
if smtp_server.startswith("maildir:/"):
from mailbox import Maildir
maildir_path = smtp_server[8:]
mdir = Maildir(maildir_path, factory=None, create=True)
mdir.add(message.as_string(True))
return message_id
try:
smtp = self.connect(smtp_server, smtp_port, smtp_user, smtp_password, smtp_encryption or False, smtp_debug)
# smtp.sendmail(smtp_from, smtp_to_list, message.as_string())
"""
johnw 02/18/2014 enhance the mail sending logic
to handle the "login and from address not mathcing" issue,
for example, 163.com will raise exception under this case
if from and login user is different, then to try to send using smtp_user again
"""
try:
smtp.sendmail(smtp_from, smtp_to_list, message.as_string())
except Exception, e:
if smtp_from != smtp_user:
smtp.sendmail(smtp_user, smtp_to_list, message.as_string())
else:
raise e
finally:
try:
# Close Connection of SMTP Server
smtp.quit()
except Exception:
# ignored, just a consequence of the previous exception
pass
except Exception, e:
msg = _("Mail delivery failed via SMTP server '%s'.\n%s: %s") % (
tools.ustr(smtp_server),
e.__class__.__name__,
tools.ustr(e),
)
_logger.exception(msg)
raise MailDeliveryException(_("Mail delivery failed"), msg)
class moveMail:
def __init__(self, oldBoxDir, newBoxDir, daystoremove):
from sys import stdout
from mailbox import Maildir,MaildirMessage,NoSuchMailboxError
from os.path import expanduser
from time import time, strftime, gmtime
self.logfile=strftime(expanduser("~/")+"mail-remove-log-%Y%m%d%H%M", gmtime())
# DeltaT is epoch now - 60(s)*m*d*nm where n is number of months - here set for 4 months
self.deltaT=int(round(time()-(60*60*24*int(daystoremove))))
self.newBox = Maildir(newBoxDir, create=True, factory=MaildirMessage)
try:
self.oldBox = Maildir(oldBoxDir, create=False, factory=MaildirMessage)
except NoSuchMailboxError:
print ("[E] Invalid mail dir. Aborting")
return(1)
def moveByLabel (self):
from email.utils import parsedate_tz,mktime_tz
from mailbox import NoSuchMailboxError
for folder in self.oldBox.list_folders():
_c_moved=0
_c_rej=0
_c_total=self.oldBox.get_folder(folder).__len__()
print("\n[I] Folder " + folder + "", end="")
for key, msg in self.oldBox.get_folder(folder).iteritems():
_date=msg['Date']
if _date:
if (mktime_tz(parsedate_tz(_date)) - self.deltaT) < 0:
if _c_moved == 0:
#To detect if no thing is moved, so this can be a new folder
try:
self.newBox.get_folder(folder)
except NoSuchMailboxError:
print("[I]\tCreating in new: %s" % folder)
self.newBox.add_folder(folder)
# Mooooooooooooo'ving!
self.newBox.get_folder(folder).add(msg)
self.oldBox.get_folder(folder).remove(key)
_c_moved += 1
print("\r[I]\tStats: Not moved (Bad Mail): %d/%d // Moved: %d/%d" % (_c_rej,_c_total,_c_moved,_c_total), end="")
else:
_c_rej += 1
if _c_moved >= _c_total:
print("\n[W]\tRemoving folder %s" % folder, end="")
print("")
def process(gen, output):
box = Maildir(output)
box.clear()
for filename in gen:
os.symlink(filename, join(box._path, "cur", basename(filename)))
def send_email(
self,
cr,
uid,
message,
mail_server_id=None,
smtp_server=None,
smtp_port=None,
smtp_user=None,
smtp_password=None,
smtp_encryption=None,
smtp_debug=False,
context=None,
):
"""Sends an email directly (no queuing).
No retries are done, the caller should handle MailDeliveryException in order to ensure that
the mail is never lost.
If the mail_server_id is provided, sends using this mail server, ignoring other smtp_* arguments.
If mail_server_id is None and smtp_server is None, use the default mail server (highest priority).
If mail_server_id is None and smtp_server is not None, use the provided smtp_* arguments.
If both mail_server_id and smtp_server are None, look for an 'smtp_server' value in server config,
and fails if not found.
:param message: the email.message.Message to send. The envelope sender will be extracted from the
``Return-Path`` (if present), or will be set to the default bounce address.
The envelope recipients will be extracted from the combined list of ``To``,
``CC`` and ``BCC`` headers.
:param mail_server_id: optional id of ir.mail_server to use for sending. overrides other smtp_* arguments.
:param smtp_server: optional hostname of SMTP server to use
:param smtp_encryption: optional TLS mode, one of 'none', 'starttls' or 'ssl' (see ir.mail_server fields for explanation)
:param smtp_port: optional SMTP port, if mail_server_id is not passed
:param smtp_user: optional SMTP user, if mail_server_id is not passed
:param smtp_password: optional SMTP password to use, if mail_server_id is not passed
:param smtp_debug: optional SMTP debug flag, if mail_server_id is not passed
:return: the Message-ID of the message that was just sent, if successfully sent, otherwise raises
MailDeliveryException and logs root cause.
"""
# Use the default bounce address **only if** no Return-Path was
# provided by caller. Caller may be using Variable Envelope Return
# Path (VERP) to detect no-longer valid email addresses.
smtp_from = message["Return-Path"]
if not smtp_from:
smtp_from = self._get_default_bounce_address(cr, uid, context=context)
if not smtp_from:
smtp_from = message["From"]
assert smtp_from, "The Return-Path or From header is required for any outbound email"
# The email's "Envelope From" (Return-Path), and all recipient addresses must only contain ASCII characters.
from_rfc2822 = extract_rfc2822_addresses(smtp_from)
assert from_rfc2822, (
"Malformed 'Return-Path' or 'From' address: %r - " "It should contain one valid plain ASCII email"
) % smtp_from
# use last extracted email, to support rarities like 'Support@MyComp <*****@*****.**>'
smtp_from = from_rfc2822[-1]
email_to = message["To"]
email_cc = message["Cc"]
email_bcc = message["Bcc"]
smtp_to_list = filter(None, tools.flatten(map(extract_rfc2822_addresses, [email_to, email_cc, email_bcc])))
assert smtp_to_list, self.NO_VALID_RECIPIENT
x_forge_to = message["X-Forge-To"]
if x_forge_to:
# `To:` header forged, e.g. for posting on mail.groups, to avoid confusion
del message["X-Forge-To"]
del message["To"] # avoid multiple To: headers!
message["To"] = x_forge_to
# Do not actually send emails in testing mode!
if getattr(threading.currentThread(), "testing", False):
_test_logger.info("skip sending email in test mode")
return message["Message-Id"]
# Get SMTP Server Details from Mail Server
mail_server = None
if mail_server_id:
mail_server = self.browse(cr, SUPERUSER_ID, mail_server_id)
elif not smtp_server:
mail_server_ids = self.search(cr, SUPERUSER_ID, [], order="sequence", limit=1)
if mail_server_ids:
mail_server = self.browse(cr, SUPERUSER_ID, mail_server_ids[0])
if mail_server:
smtp_server = mail_server.smtp_host
smtp_user = mail_server.smtp_user
smtp_password = mail_server.smtp_pass
smtp_port = mail_server.smtp_port
smtp_encryption = mail_server.smtp_encryption
smtp_debug = smtp_debug or mail_server.smtp_debug
else:
# we were passed an explicit smtp_server or nothing at all
smtp_server = smtp_server or tools.config.get("smtp_server")
smtp_port = tools.config.get("smtp_port", 25) if smtp_port is None else smtp_port
smtp_user = smtp_user or tools.config.get("smtp_user")
smtp_password = smtp_password or tools.config.get("smtp_password")
if smtp_encryption is None and tools.config.get("smtp_ssl"):
smtp_encryption = "starttls" # STARTTLS is the new meaning of the smtp_ssl flag as of v7.0
if not smtp_server:
raise UserError(
_("Missing SMTP Server")
+ "\n"
+ _("Please define at least one SMTP server, or provide the SMTP parameters explicitly.")
)
try:
message_id = message["Message-Id"]
# Add email in Maildir if smtp_server contains maildir.
if smtp_server.startswith("maildir:/"):
from mailbox import Maildir
maildir_path = smtp_server[8:]
mdir = Maildir(maildir_path, factory=None, create=True)
mdir.add(message.as_string(True))
return message_id
smtp = None
try:
smtp = self.connect(
smtp_server, smtp_port, smtp_user, smtp_password, smtp_encryption or False, smtp_debug
)
smtp.sendmail(smtp_from, smtp_to_list, message.as_string())
finally:
if smtp is not None:
smtp.quit()
except Exception, e:
msg = _("Mail delivery failed via SMTP server '%s'.\n%s: %s") % (
tools.ustr(smtp_server),
e.__class__.__name__,
tools.ustr(e),
)
_logger.info(msg)
raise MailDeliveryException(_("Mail Delivery Failed"), msg)
def empty_dir(directory):
box = Maildir(directory)
box.clear()
def send_email(self, cr, uid, message, mail_server_id=None, smtp_server=None, smtp_port=None,
smtp_user=None, smtp_password=None, smtp_encryption=None, smtp_debug=False,
context=None):
"""Sends an email directly (no queuing).
No retries are done, the caller should handle MailDeliveryException in order to ensure that
the mail is never lost.
If the mail_server_id is provided, sends using this mail server, ignoring other smtp_* arguments.
If mail_server_id is None and smtp_server is None, use the default mail server (highest priority).
If mail_server_id is None and smtp_server is not None, use the provided smtp_* arguments.
If both mail_server_id and smtp_server are None, look for an 'smtp_server' value in server config,
and fails if not found.
:param message: the email.message.Message to send. The envelope sender will be extracted from the
``Return-Path`` or ``From`` headers. The envelope recipients will be
extracted from the combined list of ``To``, ``CC`` and ``BCC`` headers.
:param mail_server_id: optional id of ir.mail_server to use for sending. overrides other smtp_* arguments.
:param smtp_server: optional hostname of SMTP server to use
:param smtp_encryption: optional TLS mode, one of 'none', 'starttls' or 'ssl' (see ir.mail_server fields for explanation)
:param smtp_port: optional SMTP port, if mail_server_id is not passed
:param smtp_user: optional SMTP user, if mail_server_id is not passed
:param smtp_password: optional SMTP password to use, if mail_server_id is not passed
:param smtp_debug: optional SMTP debug flag, if mail_server_id is not passed
:return: the Message-ID of the message that was just sent, if successfully sent, otherwise raises
MailDeliveryException and logs root cause.
"""
smtp_from = message['Return-Path'] or message['From']
assert smtp_from, "The Return-Path or From header is required for any outbound email"
# The email's "Envelope From" (Return-Path), and all recipient addresses must only contain ASCII characters.
from_rfc2822 = extract_rfc2822_addresses(smtp_from)
assert len(from_rfc2822) == 1, "Malformed 'Return-Path' or 'From' address - it may only contain plain ASCII characters"
smtp_from = from_rfc2822[0]
email_to = message['To']
email_cc = message['Cc']
email_bcc = message['Bcc']
smtp_to_list = filter(None, tools.flatten(map(extract_rfc2822_addresses,[email_to, email_cc, email_bcc])))
assert smtp_to_list, "At least one valid recipient address should be specified for outgoing emails (To/Cc/Bcc)"
# Do not actually send emails in testing mode!
if getattr(threading.currentThread(), 'testing', False):
_logger.log(logging.TEST, "skip sending email in test mode")
return message['Message-Id']
# Get SMTP Server Details from Mail Server
mail_server = None
if mail_server_id:
mail_server = self.browse(cr, SUPERUSER_ID, mail_server_id)
elif not smtp_server:
mail_server_ids = self.search(cr, SUPERUSER_ID, [], order='sequence', limit=1)
if mail_server_ids:
mail_server = self.browse(cr, SUPERUSER_ID, mail_server_ids[0])
if mail_server:
smtp_server = mail_server.smtp_host
smtp_user = mail_server.smtp_user
smtp_password = mail_server.smtp_pass
smtp_port = mail_server.smtp_port
smtp_encryption = mail_server.smtp_encryption
smtp_debug = smtp_debug or mail_server.smtp_debug
else:
# we were passed an explicit smtp_server or nothing at all
smtp_server = smtp_server or tools.config.get('smtp_server')
smtp_port = tools.config.get('smtp_port', 25) if smtp_port is None else smtp_port
smtp_user = smtp_user or tools.config.get('smtp_user')
smtp_password = smtp_password or tools.config.get('smtp_password')
if smtp_encryption is None and tools.config.get('smtp_ssl'):
smtp_encryption = 'starttls' # STARTTLS is the new meaning of the smtp_ssl flag as of v7.0
#update by frank
#smtp_user = smtp_from
pid = self.pool.get('res.users').browse(cr, uid, uid, context=context).partner_id._id
smtp_user = self.pool.get('res.partner').browse(cr, uid, pid, context=context).email
smtp_password = self.pool.get('res.users').browse(cr, uid, uid, context=context).x_mailpass
#/update
if not smtp_server:
raise osv.except_osv(
_("Missing SMTP Server"),
_("Please define at least one SMTP server, or provide the SMTP parameters explicitly."))
try:
message_id = message['Message-Id']
# Add email in Maildir if smtp_server contains maildir.
if smtp_server.startswith('maildir:/'):
from mailbox import Maildir
maildir_path = smtp_server[8:]
mdir = Maildir(maildir_path, factory=None, create = True)
mdir.add(message.as_string(True))
return message_id
try:
smtp = self.connect(smtp_server, smtp_port, smtp_user, smtp_password, smtp_encryption or False, smtp_debug)
#smtp = self.connect(smtp_server, smtp_port, smtp_from, smtp_password, smtp_encryption or False, smtp_debug)
smtp.sendmail(smtp_from, smtp_to_list, message.as_string())
finally:
try:
# Close Connection of SMTP Server
smtp.quit()
except Exception:
# ignored, just a consequence of the previous exception
pass
except Exception, e:
msg = _("Mail delivery failed via SMTP server '%s'.\n%s: %s") % (tools.ustr(smtp_server),
e.__class__.__name__,
tools.ustr(e))
_logger.exception(msg)
raise MailDeliveryException(_("Mail delivery failed"), msg)
def import_email(email, import_path, format, **kwargs):
from caliopen.base.user.core import User
from caliopen.base.user.core import Contact, ContactLookup
from caliopen.base.message.format.mail import MailMessage
from caliopen.base.user.parameters import NewContact, NewEmail
from caliopen.smtp.agent import DeliveryAgent
AVATAR_DIR = "../../../caliopen.ng/src/assets/images/avatars"
if format == "maildir":
emails = Maildir(import_path, factory=message_from_file)
mode = "maildir"
else:
if os.path.isdir(import_path):
mode = "mbox_directory"
emails = {}
files = [f for f in listdir(import_path) if os.path.isfile(os.path.join(import_path, f))]
for f in files:
with open("%s/%s" % (import_path, f)) as fh:
emails[f] = message_from_file(fh)
else:
mode = "mbox"
emails = mbox(import_path)
user = User.by_name(email)
agent = DeliveryAgent()
mailfrom = ""
rcpts = [email]
log.info("Processing mode %s" % mode)
msgs = []
for key, mail in emails.iteritems():
# Create contact for user
log.info("Processing mail %s" % key)
msgs.append(MailMessage(mail))
msgs = sorted(msgs, key=lambda msg: msg.date)
for msg in msgs:
for type, addresses in msg.recipients.iteritems():
if not addresses:
continue
for alias, _address in addresses:
try:
ContactLookup.get(user, alias)
except NotFound:
log.info("Creating contact %s" % alias)
infos = {"mail": alias}
name, domain = alias.split("@")
if os.path.isfile("%s/%s.png" % (AVATAR_DIR, name)):
infos.update({"avatar": "%s.png" % name})
contact = NewContact()
contact.family_name = name
email = NewEmail()
email.address = alias
Contact.create(user, contact, emails=[email])
res = agent.process(mailfrom, rcpts, msg.mail.as_string())
log.info("Process result %r" % res)