def _build_fetch_response(self, message, parts, by_uid=True):
response = ('%d FETCH (UID %s' % (message.id, message.uid)).encode() if by_uid \
else ('%d FETCH (' % message.id).encode()
for part in parts:
if part.startswith('(') or part.endswith(')'):
part = part.strip('()')
if not response.endswith(b' ') and not response.endswith(b'('):
response += b' '
if part == 'UID' and not by_uid:
response += ('UID %s' % message.uid).encode()
if part == 'BODY[]' or part == 'BODY.PEEK[]' or part == 'RFC822':
response += ('%s {%s}\r\n' % (part, len(
message.as_bytes()))).encode() + message.as_bytes()
if part == 'BODY.PEEK[HEADER.FIELDS':
fetch_header = FETCH_HEADERS_RE.match(' '.join(parts))
if fetch_header:
headers = fetch_header.group('headers')
message_headers = Message(policy=Compat32(linesep='\r\n'))
for hk in headers.split():
message_headers[hk] = message.email.get(hk, '')
response += ('BODY[HEADER.FIELDS (%s)] {%d}\r\n' %
(headers, len(message_headers.as_bytes()))
).encode() + message_headers.as_bytes()
if part == 'FLAGS':
response += ('FLAGS (%s)' % ' '.join(message.flags)).encode()
response = response.strip(b' ')
response += b')'
return response
def test_import(self):
tmpmail = mkdtemp('archweb') + "/mail"
msg = Message()
msg['subject'] = 'John Doe'
msg['to'] = 'John Doe <*****@*****.**>'
with open(tmpmail, 'wb') as fp:
fp.write(msg.as_bytes())
# Invalid
with self.assertRaises(SystemExit):
call_command('donor_import', tmpmail)
self.assertEqual(len(Donor.objects.all()), 0)
# Valid
msg = Message()
msg['subject'] = 'Receipt [$25.00] By: David Doe [[email protected]]'
msg['to'] = 'John Doe <*****@*****.**>'
with open(tmpmail, 'wb') as fp:
fp.write(msg.as_bytes())
call_command('donor_import', tmpmail)
self.assertEqual(len(Donor.objects.all()), 1)
# Re-running should result in no new donor
call_command('donor_import', tmpmail)
self.assertEqual(len(Donor.objects.all()), 1)
def to_bytes(msg: Message):
"""replace Message.as_bytes() method by trying different policies"""
try:
return msg.as_bytes()
except UnicodeEncodeError:
LOG.warning("as_bytes fails with default policy, try SMTP policy")
try:
return msg.as_bytes(policy=email.policy.SMTP)
except UnicodeEncodeError:
LOG.warning("as_bytes fails with SMTP policy, try SMTPUTF8 policy")
return msg.as_bytes(policy=email.policy.SMTPUTF8)
def send_sms(data):
text, coding, parts_count = detect_coding(data['text'])
result = {
'sent_text': data['text'],
'parts_count': parts_count,
'mobiles': {}
}
for mobile in data['mobiles']:
# generate message_id
message_id = str(uuid.uuid4())
result['mobiles'][mobile] = {}
result['mobiles'][mobile]['message_id'] = message_id
result['mobiles'][mobile]['dlr_status'] = os.path.join(
os.path.dirname(request.url),
os.path.basename(current_app.config['SENT']), message_id)
if not validate_mobile(mobile):
current_app.logger.info(
'Message from [%s] to [%s] have invalid mobile number' %
(auth.username(), mobile))
result['mobiles'][mobile][
'response'] = 'Failed: invalid mobile number'
continue
if access_mobile(mobile):
lock_file = os.path.join(current_app.config['OUTGOING'],
message_id + '.LOCK')
m = Message()
m.add_header('From', auth.username())
m.add_header('To', mobile)
m.add_header('Alphabet', coding)
if data.get('queue'):
result.update({'queue': data['queue']})
m.add_header('Queue', result.get('queue'))
m.set_payload(text)
with open(lock_file, write_mode) as fp:
if use_python3:
fp.write(m.as_bytes())
else:
fp.write(m.as_string())
msg_file = lock_file.split('.LOCK')[0]
os.rename(lock_file, msg_file)
os.chmod(msg_file, 0o660)
current_app.logger.info(
'Message from [%s] to [%s] placed to the spooler as [%s]' %
(auth.username(), mobile, msg_file))
result['mobiles'][mobile]['response'] = 'Ok'
else:
current_app.logger.info(
'Message from [%s] to [%s] have forbidden mobile number' %
(auth.username(), mobile))
result['mobiles'][mobile][
'response'] = 'Failed: forbidden mobile number'
return result
def _build_fetch_response(self, message, parts, by_uid=True):
response = ('%d FETCH (UID %s' % (message.id, message.uid)).encode() if by_uid \
else ('%d FETCH (' % message.id).encode()
for part in parts:
if part.startswith('(') or part.endswith(')'):
part = part.strip('()')
if not response.endswith(b' ') and not response.endswith(b'('):
response += b' '
if part == 'UID' and not by_uid:
response += ('UID %s' % message.uid).encode()
if part == 'BODY[]' or part == 'BODY.PEEK[]' or part == 'RFC822':
response += ('%s {%s}\r\n' % (part, len(message.as_bytes()))).encode() + message.as_bytes()
if part == 'BODY.PEEK[HEADER.FIELDS':
fetch_header = FETCH_HEADERS_RE.match(' '.join(parts))
if fetch_header:
headers = fetch_header.group('headers')
message_headers = Message(policy=Compat32(linesep='\r\n'))
for hk in headers.split():
message_headers[hk] = message.email.get(hk, '')
response += ('BODY[HEADER.FIELDS (%s)] {%d}\r\n' %
(headers, len(message_headers.as_bytes()))).encode() + message_headers.as_bytes()
if part == 'FLAGS':
response += ('FLAGS (%s)' % ' '.join(message.flags)).encode()
response = response.strip(b' ')
response += b')'
return response
def prepare_pgp_message(orig_msg: Message, pgp_fingerprint: str):
msg = MIMEMultipart("encrypted", protocol="application/pgp-encrypted")
# copy all headers from original message except all standard MIME headers
for i in reversed(range(len(orig_msg._headers))):
header_name = orig_msg._headers[i][0].lower()
if header_name.lower() not in _MIME_HEADERS:
msg[header_name] = orig_msg._headers[i][1]
# Delete unnecessary headers in orig_msg except to save space
delete_all_headers_except(
orig_msg,
_MIME_HEADERS,
)
first = MIMEApplication(_subtype="pgp-encrypted",
_encoder=encoders.encode_7or8bit,
_data="")
first.set_payload("Version: 1")
msg.attach(first)
second = MIMEApplication("octet-stream", _encoder=encoders.encode_7or8bit)
second.add_header("Content-Disposition", "inline")
# encrypt original message
encrypted_data = pgp_utils.encrypt_file(BytesIO(orig_msg.as_bytes()),
pgp_fingerprint)
second.set_payload(encrypted_data)
msg.attach(second)
return msg
def handle_email_sent_to_ourself(alias, mailbox, msg: Message, user):
# store the refused email
random_name = str(uuid.uuid4())
full_report_path = f"refused-emails/cycle-{random_name}.eml"
s3.upload_email_from_bytesio(full_report_path, BytesIO(msg.as_bytes()),
random_name)
refused_email = RefusedEmail.create(path=None,
full_report_path=full_report_path,
user_id=alias.user_id)
db.session.commit()
LOG.d("Create refused email %s", refused_email)
# link available for 6 days as it gets deleted in 7 days
refused_email_url = refused_email.get_url(expires_in=518400)
send_email_with_rate_control(
user,
ALERT_SEND_EMAIL_CYCLE,
mailbox.email,
f"Warning: email sent from {mailbox.email} to {alias.email} forms a cycle",
render(
"transactional/cycle-email.txt",
name=user.name or "",
alias=alias,
mailbox=mailbox,
refused_email_url=refused_email_url,
),
render(
"transactional/cycle-email.html",
name=user.name or "",
alias=alias,
mailbox=mailbox,
refused_email_url=refused_email_url,
),
)
def sendmail(msg: Message, dump_dir: Optional[str] = None) -> int:
# dump mail
msg_id = msg.get("Message-Id")
if dump_dir:
with open(f"{dump_dir}/{msg_id}.eml", "w") as dump:
dump.write(msg.as_string())
# send mail
p = subprocess.Popen(["/usr/sbin/sendmail", "-t", "-oi"],
stdin=subprocess.PIPE)
p.communicate(msg.as_bytes())
return p.returncode
def to_bytes(msg: Message):
"""replace Message.as_bytes() method by trying different policies"""
try:
return msg.as_bytes()
except UnicodeEncodeError:
LOG.warning("as_bytes fails with default policy, try SMTP policy")
try:
return msg.as_bytes(policy=email.policy.SMTP)
except UnicodeEncodeError:
LOG.warning("as_bytes fails with SMTP policy, try SMTPUTF8 policy")
try:
return msg.as_bytes(policy=email.policy.SMTPUTF8)
except UnicodeEncodeError:
LOG.warning(
"as_bytes fails with SMTPUTF8 policy, try converting to string"
)
msg_string = msg.as_string()
try:
return msg_string.encode()
except UnicodeEncodeError as e:
LOG.w("can't encode msg, err:%s", e)
return msg_string.encode(errors="replace")
def _prepare_payload(self):
"""Serializes a batch request body."""
main_message = Message()
main_message.add_header("Content-Type", "multipart/mixed")
main_message.set_boundary(self._current_boundary)
for _ in self.context.get_next_query():
part_message = Message()
part_message.add_header("Content-Type", "application/http")
part_message.add_header("Content-Transfer-Encoding", "binary")
request = self.context.build_request()
part_message.set_payload(self._serialize_request(request))
main_message.attach(part_message)
return main_message.as_bytes()
def write_patch_file(filename, commit_info, diff):
"""Write patch file"""
if not diff:
gbp.log.debug("I won't generate empty diff %s" % filename)
return None
try:
with open(filename, 'wb') as patch:
msg = Message()
charset = Charset('utf-8')
charset.body_encoding = None
charset.header_encoding = QP
# Write headers
name = commit_info['author']['name']
email = commit_info['author']['email']
# Git compat: put name in quotes if special characters found
if re.search(r'[,.@()\[\]\\\:;]', name):
name = '"%s"' % name
from_header = Header(header_name='from')
try:
from_header.append(name, 'us-ascii')
except UnicodeDecodeError:
from_header.append(name, charset)
from_header.append('<%s>' % email)
msg['From'] = from_header
date = commit_info['author'].datetime
datestr = date.strftime('%a, %-d %b %Y %H:%M:%S %z')
msg['Date'] = Header(datestr, 'us-ascii', 'date')
subject_header = Header(header_name='subject')
try:
subject_header.append(commit_info['subject'], 'us-ascii')
except UnicodeDecodeError:
subject_header.append(commit_info['subject'], charset)
msg['Subject'] = subject_header
# Write message body
if commit_info['body']:
# Strip extra linefeeds
body = commit_info['body'].rstrip() + '\n'
try:
msg.set_payload(body.encode('us-ascii'))
except (UnicodeEncodeError):
msg.set_payload(body, charset)
policy = Compat32(max_line_length=77)
patch.write(msg.as_bytes(unixfrom=False, policy=policy))
# Write diff
patch.write(b'---\n')
patch.write(diff)
except IOError as err:
raise GbpError('Unable to create patch file: %s' % err)
return filename
def write_patch_file(filename, commit_info, diff):
"""Write patch file"""
if not diff:
gbp.log.debug("I won't generate empty diff %s" % filename)
return None
try:
with open(filename, 'wb') as patch:
msg = Message()
charset = Charset('utf-8')
charset.body_encoding = None
charset.header_encoding = QP
# Write headers
name = commit_info['author']['name']
email = commit_info['author']['email']
# Git compat: put name in quotes if special characters found
if re.search("[,.@()\[\]\\\:;]", name):
name = '"%s"' % name
from_header = Header(header_name='from')
try:
from_header.append(name, 'us-ascii')
except UnicodeDecodeError:
from_header.append(name, charset)
from_header.append('<%s>' % email)
msg['From'] = from_header
date = commit_info['author'].datetime
datestr = date.strftime('%a, %-d %b %Y %H:%M:%S %z')
msg['Date'] = Header(datestr, 'us-ascii', 'date')
subject_header = Header(header_name='subject')
try:
subject_header.append(commit_info['subject'], 'us-ascii')
except UnicodeDecodeError:
subject_header.append(commit_info['subject'], charset)
msg['Subject'] = subject_header
# Write message body
if commit_info['body']:
# Strip extra linefeeds
body = commit_info['body'].rstrip() + '\n'
try:
msg.set_payload(body.encode('us-ascii'))
except (UnicodeEncodeError):
msg.set_payload(body, charset)
policy = Compat32(max_line_length=77)
patch.write(msg.as_bytes(unixfrom=False, policy=policy))
# Write diff
patch.write(b'---\n')
patch.write(diff)
except IOError as err:
raise GbpError('Unable to create patch file: %s' % err)
return filename
def send_sms(data):
text, coding, parts_count = detect_coding(data['text'])
result = {
'sent_text': data['text'],
'parts_count': parts_count,
'mobiles': {}
}
for mobile in data['mobiles']:
# generate message_id
message_id = str(uuid.uuid4())
result['mobiles'][mobile] = {}
result['mobiles'][mobile]['message_id'] = message_id
result['mobiles'][mobile]['dlr_status'] = os.path.join(os.path.dirname(request.url),
os.path.basename(current_app.config['SENT']), message_id)
if not validate_mobile(mobile):
current_app.logger.info('Message from [%s] to [%s] have invalid mobile number' % (auth.username(), mobile))
result['mobiles'][mobile]['response'] = 'Failed: invalid mobile number'
continue
if access_mobile(mobile):
lock_file = os.path.join(current_app.config['OUTGOING'], message_id + '.LOCK')
m = Message()
m.add_header('From', auth.username())
m.add_header('To', mobile)
m.add_header('Alphabet', coding)
if data.get('queue'):
result.update({'queue' : data['queue']})
m.add_header('Queue', result.get('queue'))
m.set_payload(text)
with open(lock_file, write_mode) as fp:
if use_python3:
fp.write(m.as_bytes())
else:
fp.write(m.as_string())
msg_file = lock_file.split('.LOCK')[0]
os.rename(lock_file, msg_file)
os.chmod(msg_file, 0o660)
current_app.logger.info('Message from [%s] to [%s] placed to the spooler as [%s]' % (auth.username(), mobile, msg_file))
result['mobiles'][mobile]['response'] = 'Ok'
else:
current_app.logger.info('Message from [%s] to [%s] have forbidden mobile number' % (auth.username(), mobile))
result['mobiles'][mobile]['response'] = 'Failed: forbidden mobile number'
return result
def save_email_for_debugging(msg: Message, file_name_prefix=None) -> str:
"""Save email for debugging to temporary location
Return the file path
"""
if TEMP_DIR:
file_name = str(uuid.uuid4()) + ".eml"
if file_name_prefix:
file_name = file_name_prefix + file_name
with open(os.path.join(TEMP_DIR, file_name), "wb") as f:
f.write(msg.as_bytes())
LOG.d("email saved to %s", file_name)
return file_name
return ""
def prepare_multipart_body(self):
# type: () -> None
"""Will prepare the body of this request according to the multipart information.
This call assumes the on_request policies have been applied already in their
correct context (sync/async)
Does nothing if "set_multipart_mixed" was never called.
"""
if not self.multipart_mixed_info:
return
requests = self.multipart_mixed_info[0] # type: List[HttpRequest]
boundary = self.multipart_mixed_info[2] # type: Optional[str]
# Update the main request with the body
main_message = Message()
main_message.add_header("Content-Type", "multipart/mixed")
if boundary:
main_message.set_boundary(boundary)
for i, req in enumerate(requests):
part_message = Message()
part_message.add_header("Content-Type", "application/http")
part_message.add_header("Content-Transfer-Encoding", "binary")
part_message.add_header("Content-ID", str(i))
part_message.set_payload(req.serialize())
main_message.attach(part_message)
try:
from email.policy import HTTP
full_message = main_message.as_bytes(policy=HTTP)
eol = b"\r\n"
except ImportError: # Python 2.7
# Right now we decide to not support Python 2.7 on serialization, since
# it doesn't serialize a valid HTTP request (and our main scenario Storage refuses it)
raise NotImplementedError(
"Multipart request are not supported on Python 2.7"
)
# full_message = main_message.as_string()
# eol = b'\n'
_, _, body = full_message.split(eol, 2)
self.set_bytes_body(body)
self.headers["Content-Type"] = (
"multipart/mixed; boundary=" + main_message.get_boundary()
)
def add_dkim_signature(msg: Message, email_domain: str):
delete_header(msg, "DKIM-Signature")
# Specify headers in "byte" form
# Generate message signature
sig = dkim.sign(
msg.as_bytes(),
DKIM_SELECTOR,
email_domain.encode(),
DKIM_PRIVATE_KEY.encode(),
include_headers=DKIM_HEADERS,
)
sig = sig.decode()
# remove linebreaks from sig
sig = sig.replace("\n", " ").replace("\r", "")
msg["DKIM-Signature"] = sig[len("DKIM-Signature: "):]
def process_message(email_msg: Message, checksum: str, m_uid: str):
"""
Process an entire message object.
Split attachment files from rawmsg, create db entries for
each rawmsg, message meta and attachment.
"""
# We need to parse attachments first.
# They are extracted and removed from messages.
_attachments = [process_attachment(part) for part in email_msg.walk()]
attachments = list(filter(None, _attachments))
has_attachments = len(attachments) > 0
# Parse metadata
from_ = email_msg.get('From')
to = email_msg.get('To')
subject = email_msg.get('Subject')
date = email.utils.parsedate_to_datetime(
email_msg.get('Date')) if email_msg.get('Date') else None
with db.atomic():
rmsg = RawMsg.create(email_blob=email_msg.as_bytes(),
original_checksum=checksum)
if has_attachments:
for file_checksum, filename, content_type in attachments:
log.debug(file_checksum, filename, content_type)
att = Attachment.create(
file_checksum=file_checksum,
original_filename=filename,
content_type=content_type,
)
rmsg.attachments.add(att)
mmeta = MsgMeta.create(
rawmsg=rmsg,
imap_uid=m_uid,
from_=from_,
to=to,
subject=subject,
date=date,
has_attachments=has_attachments,
)
log.debug(m_uid, from_, to, subject)
def add_dkim_signature(msg: Message, email_domain: str):
if RSPAMD_SIGN_DKIM:
LOG.d("DKIM signature will be added by rspamd")
msg[headers.SL_WANT_SIGNING] = "yes"
return
for dkim_headers in headers.DKIM_HEADERS:
try:
add_dkim_signature_with_header(msg, email_domain, dkim_headers)
return
except dkim.DKIMException:
LOG.w("DKIM fail with %s", dkim_headers, exc_info=True)
# try with another headers
continue
# To investigate why some emails can't be DKIM signed. todo: remove
if TEMP_DIR:
file_name = str(uuid.uuid4()) + ".eml"
with open(os.path.join(TEMP_DIR, file_name), "wb") as f:
f.write(msg.as_bytes())
LOG.w("email saved to %s", file_name)
raise Exception("Cannot create DKIM signature")
def copy(msg: Message) -> Message:
"""return a copy of message"""
return email.message_from_bytes(msg.as_bytes())
def handle_spam(
contact: Contact,
alias: Alias,
msg: Message,
user: User,
mailbox_email: str,
email_log: EmailLog,
):
# Store the report & original email
orig_msg = get_orig_message_from_spamassassin_report(msg)
# generate a name for the email
random_name = str(uuid.uuid4())
full_report_path = f"spams/full-{random_name}.eml"
s3.upload_email_from_bytesio(full_report_path, BytesIO(msg.as_bytes()),
random_name)
file_path = None
if orig_msg:
file_path = f"spams/{random_name}.eml"
s3.upload_email_from_bytesio(file_path, BytesIO(orig_msg.as_bytes()),
random_name)
refused_email = RefusedEmail.create(path=file_path,
full_report_path=full_report_path,
user_id=user.id)
db.session.flush()
email_log.refused_email_id = refused_email.id
db.session.commit()
LOG.d("Create spam email %s", refused_email)
refused_email_url = (URL + f"/dashboard/refused_email?highlight_id=" +
str(email_log.id))
disable_alias_link = f"{URL}/dashboard/unsubscribe/{alias.id}"
# inform user
LOG.d(
"Inform user %s about spam email sent by %s to alias %s",
user,
contact.website_email,
alias.email,
)
send_email_with_rate_control(
user,
ALERT_SPAM_EMAIL,
mailbox_email,
f"Email from {contact.website_email} to {alias.email} is detected as spam",
render(
"transactional/spam-email.txt",
name=user.name,
alias=alias,
website_email=contact.website_email,
disable_alias_link=disable_alias_link,
refused_email_url=refused_email_url,
),
render(
"transactional/spam-email.html",
name=user.name,
alias=alias,
website_email=contact.website_email,
disable_alias_link=disable_alias_link,
refused_email_url=refused_email_url,
),
)
def handle_bounce(contact: Contact, alias: Alias, msg: Message, user: User):
address = alias.email
email_log: EmailLog = EmailLog.create(contact_id=contact.id,
bounced=True,
user_id=contact.user_id)
db.session.commit()
nb_bounced = EmailLog.filter_by(contact_id=contact.id,
bounced=True).count()
disable_alias_link = f"{URL}/dashboard/unsubscribe/{alias.id}"
# <<< Store the bounced email >>>
# generate a name for the email
random_name = str(uuid.uuid4())
full_report_path = f"refused-emails/full-{random_name}.eml"
s3.upload_email_from_bytesio(full_report_path, BytesIO(msg.as_bytes()),
random_name)
orig_msg = get_orig_message_from_bounce(msg)
if not orig_msg:
LOG.error(
"Cannot parse original message from bounce message %s %s %s %s",
alias,
user,
contact,
full_report_path,
)
return
file_path = f"refused-emails/{random_name}.eml"
s3.upload_email_from_bytesio(file_path, BytesIO(orig_msg.as_bytes()),
random_name)
# <<< END Store the bounced email >>>
mailbox_id = int(orig_msg[_MAILBOX_ID_HEADER])
mailbox = Mailbox.get(mailbox_id)
if not mailbox or mailbox.user_id != user.id:
LOG.error(
"Tampered message mailbox_id %s, %s, %s, %s %s",
mailbox_id,
user,
alias,
contact,
full_report_path,
)
return
refused_email = RefusedEmail.create(path=file_path,
full_report_path=full_report_path,
user_id=user.id)
db.session.flush()
email_log.refused_email_id = refused_email.id
email_log.bounced_mailbox_id = mailbox.id
db.session.commit()
LOG.d("Create refused email %s", refused_email)
refused_email_url = (URL + f"/dashboard/refused_email?highlight_id=" +
str(email_log.id))
# inform user if this is the first bounced email
if nb_bounced == 1:
LOG.d(
"Inform user %s about bounced email sent by %s to alias %s",
user,
contact.website_email,
address,
)
send_email_with_rate_control(
user,
ALERT_BOUNCE_EMAIL,
# use user mail here as only user is authenticated to see the refused email
user.email,
f"Email from {contact.website_email} to {address} cannot be delivered to your inbox",
render(
"transactional/bounced-email.txt",
name=user.name,
alias=alias,
website_email=contact.website_email,
disable_alias_link=disable_alias_link,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
render(
"transactional/bounced-email.html",
name=user.name,
alias=alias,
website_email=contact.website_email,
disable_alias_link=disable_alias_link,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
# cannot include bounce email as it can contain spammy text
# bounced_email=msg,
)
# disable the alias the second time email is bounced
elif nb_bounced >= 2:
LOG.d(
"Bounce happens again with alias %s from %s. Disable alias now ",
address,
contact.website_email,
)
alias.enabled = False
db.session.commit()
send_email_with_rate_control(
user,
ALERT_BOUNCE_EMAIL,
# use user mail here as only user is authenticated to see the refused email
user.email,
f"Alias {address} has been disabled due to second undelivered email from {contact.website_email}",
render(
"transactional/automatic-disable-alias.txt",
name=user.name,
alias=alias,
website_email=contact.website_email,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
render(
"transactional/automatic-disable-alias.html",
name=user.name,
alias=alias,
website_email=contact.website_email,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
# cannot include bounce email as it can contain spammy text
# bounced_email=msg,
)
def handle_reply(envelope, smtp: SMTP, msg: Message,
rcpt_to: str) -> (bool, str):
"""
return whether an email has been delivered and
the smtp status ("250 Message accepted", "550 Non-existent email address", etc)
"""
reply_email = rcpt_to.lower().strip()
# reply_email must end with EMAIL_DOMAIN
if not reply_email.endswith(EMAIL_DOMAIN):
LOG.warning(f"Reply email {reply_email} has wrong domain")
return False, "550 SL E2"
contact = Contact.get_by(reply_email=reply_email)
if not contact:
LOG.warning(f"No such forward-email with {reply_email} as reply-email")
return False, "550 SL E4"
alias = contact.alias
address: str = contact.alias.email
alias_domain = address[address.find("@") + 1:]
# alias must end with one of the ALIAS_DOMAINS or custom-domain
if not email_belongs_to_alias_domains(alias.email):
if not CustomDomain.get_by(domain=alias_domain):
return False, "550 SL E5"
user = alias.user
mail_from = envelope.mail_from.lower().strip()
# bounce email initiated by Postfix
# can happen in case emails cannot be delivered to user-email
# in this case Postfix will try to send a bounce report to original sender, which is
# the "reply email"
if mail_from == "<>":
LOG.warning(
"Bounce when sending to alias %s from %s, user %s",
alias,
contact,
user,
)
handle_bounce(contact, alias, msg, user)
return False, "550 SL E6"
mailbox = Mailbox.get_by(email=mail_from, user_id=user.id)
if not mailbox or mailbox not in alias.mailboxes:
# only mailbox can send email to the reply-email
handle_unknown_mailbox(envelope, msg, reply_email, user, alias)
return False, "550 SL E7"
if ENFORCE_SPF and mailbox.force_spf:
ip = msg[_IP_HEADER]
if not spf_pass(ip, envelope, mailbox, user, alias,
contact.website_email, msg):
# cannot use 4** here as sender will retry. 5** because that generates bounce report
return True, "250 SL E11"
delete_header(msg, _IP_HEADER)
delete_header(msg, "DKIM-Signature")
delete_header(msg, "Received")
# make the email comes from alias
from_header = alias.email
# add alias name from alias
if alias.name:
LOG.d("Put alias name in from header")
from_header = formataddr((alias.name, alias.email))
elif alias.custom_domain:
LOG.d("Put domain default alias name in from header")
# add alias name from domain
if alias.custom_domain.name:
from_header = formataddr((alias.custom_domain.name, alias.email))
add_or_replace_header(msg, "From", from_header)
# some email providers like ProtonMail adds automatically the Reply-To field
# make sure to delete it
delete_header(msg, "Reply-To")
# remove sender header if present as this could reveal user real email
delete_header(msg, "Sender")
delete_header(msg, "X-Sender")
replace_header_when_reply(msg, alias, "To")
replace_header_when_reply(msg, alias, "Cc")
# Received-SPF is injected by postfix-policyd-spf-python can reveal user original email
delete_header(msg, "Received-SPF")
LOG.d(
"send email from %s to %s, mail_options:%s,rcpt_options:%s",
alias.email,
contact.website_email,
envelope.mail_options,
envelope.rcpt_options,
)
if alias_domain in ALIAS_DOMAINS:
add_dkim_signature(msg, alias_domain)
# add DKIM-Signature for custom-domain alias
else:
custom_domain: CustomDomain = CustomDomain.get_by(domain=alias_domain)
if custom_domain.dkim_verified:
add_dkim_signature(msg, alias_domain)
smtp.sendmail(
alias.email,
contact.website_email,
msg.as_bytes(),
envelope.mail_options,
envelope.rcpt_options,
)
EmailLog.create(contact_id=contact.id,
is_reply=True,
user_id=contact.user_id)
db.session.commit()
return True, "250 Message accepted for delivery"
def forward_email_to_mailbox(
alias,
msg: Message,
email_log: EmailLog,
contact: Contact,
envelope,
smtp: SMTP,
mailbox,
user,
) -> (bool, str):
LOG.d("Forward %s -> %s -> %s", contact, alias, mailbox)
spam_check = True
is_spam, spam_status = get_spam_info(msg)
if is_spam:
LOG.warning("Email detected as spam. Alias: %s, from: %s", alias,
contact)
email_log.is_spam = True
email_log.spam_status = spam_status
handle_spam(contact, alias, msg, user, mailbox.email, email_log)
return False, "550 SL E1"
# create PGP email if needed
if mailbox.pgp_finger_print and user.is_premium():
LOG.d("Encrypt message using mailbox %s", mailbox)
msg = prepare_pgp_message(msg, mailbox.pgp_finger_print)
# add custom header
add_or_replace_header(msg, "X-SimpleLogin-Type", "Forward")
# remove reply-to & sender header if present
delete_header(msg, "Reply-To")
delete_header(msg, "Sender")
delete_header(msg, _IP_HEADER)
add_or_replace_header(msg, _MAILBOX_ID_HEADER, str(mailbox.id))
# change the from header so the sender comes from @SL
# so it can pass DMARC check
# replace the email part in from: header
contact_from_header = msg["From"]
new_from_header = contact.new_addr()
add_or_replace_header(msg, "From", new_from_header)
LOG.d("new_from_header:%s, old header %s", new_from_header,
contact_from_header)
# replace CC & To emails by reply-email for all emails that are not alias
replace_header_when_forward(msg, alias, "Cc")
replace_header_when_forward(msg, alias, "To")
# append alias into the TO header if it's not present in To or CC
if should_append_alias(msg, alias.email):
LOG.d("append alias %s to TO header %s", alias, msg["To"])
if msg["To"]:
to_header = msg["To"] + "," + alias.email
else:
to_header = alias.email
add_or_replace_header(msg, "To", to_header.strip())
# add List-Unsubscribe header
if UNSUBSCRIBER:
unsubscribe_link = f"mailto:{UNSUBSCRIBER}?subject={alias.id}="
add_or_replace_header(msg, "List-Unsubscribe", f"<{unsubscribe_link}>")
else:
unsubscribe_link = f"{URL}/dashboard/unsubscribe/{alias.id}"
add_or_replace_header(msg, "List-Unsubscribe", f"<{unsubscribe_link}>")
add_or_replace_header(msg, "List-Unsubscribe-Post",
"List-Unsubscribe=One-Click")
add_dkim_signature(msg, EMAIL_DOMAIN)
LOG.d(
"Forward mail from %s to %s, mail_options %s, rcpt_options %s ",
contact.website_email,
mailbox.email,
envelope.mail_options,
envelope.rcpt_options,
)
# smtp.send_message has UnicodeEncodeErroremail issue
# encode message raw directly instead
smtp.sendmail(
contact.reply_email,
mailbox.email,
msg.as_bytes(),
envelope.mail_options,
envelope.rcpt_options,
)
db.session.commit()
return True, "250 Message accepted for delivery"
async def forward_email_to_mailbox(
alias,
msg: Message,
email_log: EmailLog,
contact: Contact,
envelope,
smtp: SMTP,
mailbox,
user,
) -> (bool, str):
LOG.d("Forward %s -> %s -> %s", contact, alias, mailbox)
# sanity check: make sure mailbox is not actually an alias
if get_email_domain_part(alias.email) == get_email_domain_part(
mailbox.email):
LOG.exception(
"Mailbox has the same domain as alias. %s -> %s -> %s",
contact,
alias,
mailbox,
)
return False, "550 SL E14"
# Spam check
spam_status = ""
is_spam = False
if SPAMASSASSIN_HOST:
start = time.time()
spam_score = await get_spam_score(msg)
LOG.d(
"%s -> %s - spam score %s in %s seconds",
contact,
alias,
spam_score,
time.time() - start,
)
email_log.spam_score = spam_score
db.session.commit()
if (user.max_spam_score and spam_score > user.max_spam_score) or (
not user.max_spam_score and spam_score > MAX_SPAM_SCORE):
is_spam = True
spam_status = "Spam detected by SpamAssassin server"
else:
is_spam, spam_status = get_spam_info(msg,
max_score=user.max_spam_score)
if is_spam:
LOG.warning("Email detected as spam. Alias: %s, from: %s", alias,
contact)
email_log.is_spam = True
email_log.spam_status = spam_status
db.session.commit()
handle_spam(contact, alias, msg, user, mailbox, email_log)
return False, "550 SL E1 Email detected as spam"
# create PGP email if needed
if mailbox.pgp_finger_print and user.is_premium(
) and not alias.disable_pgp:
LOG.d("Encrypt message using mailbox %s", mailbox)
try:
msg = prepare_pgp_message(msg, mailbox.pgp_finger_print)
except PGPException:
LOG.exception("Cannot encrypt message %s -> %s. %s %s", contact,
alias, mailbox, user)
# so the client can retry later
return False, "421 SL E12 Retry later"
# add custom header
add_or_replace_header(msg, _DIRECTION, "Forward")
# remove reply-to & sender header if present
delete_header(msg, "Reply-To")
delete_header(msg, "Sender")
delete_header(msg, _IP_HEADER)
add_or_replace_header(msg, _MAILBOX_ID_HEADER, str(mailbox.id))
add_or_replace_header(msg, _EMAIL_LOG_ID_HEADER, str(email_log.id))
add_or_replace_header(msg, _MESSAGE_ID,
make_msgid(str(email_log.id), EMAIL_DOMAIN))
# change the from header so the sender comes from @SL
# so it can pass DMARC check
# replace the email part in from: header
contact_from_header = msg["From"]
new_from_header = contact.new_addr()
add_or_replace_header(msg, "From", new_from_header)
LOG.d("new_from_header:%s, old header %s", new_from_header,
contact_from_header)
# replace CC & To emails by reply-email for all emails that are not alias
replace_header_when_forward(msg, alias, "Cc")
replace_header_when_forward(msg, alias, "To")
# append alias into the TO header if it's not present in To or CC
if should_append_alias(msg, alias.email):
LOG.d("append alias %s to TO header %s", alias, msg["To"])
if msg["To"]:
to_header = msg["To"] + "," + alias.email
else:
to_header = alias.email
add_or_replace_header(msg, "To", to_header.strip())
# add List-Unsubscribe header
if UNSUBSCRIBER:
unsubscribe_link = f"mailto:{UNSUBSCRIBER}?subject={alias.id}="
add_or_replace_header(msg, "List-Unsubscribe", f"<{unsubscribe_link}>")
else:
unsubscribe_link = f"{URL}/dashboard/unsubscribe/{alias.id}"
add_or_replace_header(msg, "List-Unsubscribe", f"<{unsubscribe_link}>")
add_or_replace_header(msg, "List-Unsubscribe-Post",
"List-Unsubscribe=One-Click")
add_dkim_signature(msg, EMAIL_DOMAIN)
LOG.d(
"Forward mail from %s to %s, mail_options %s, rcpt_options %s ",
contact.website_email,
mailbox.email,
envelope.mail_options,
envelope.rcpt_options,
)
# smtp.send_message has UnicodeEncodeErroremail issue
# encode message raw directly instead
smtp.sendmail(
contact.reply_email,
mailbox.email,
msg.as_bytes(),
envelope.mail_options,
envelope.rcpt_options,
)
db.session.commit()
return True, "250 Message accepted for delivery"
def handle_bounce(contact: Contact, alias: Alias, msg: Message, user: User):
disable_alias_link = f"{URL}/dashboard/unsubscribe/{alias.id}"
# Store the bounced email
# generate a name for the email
random_name = str(uuid.uuid4())
full_report_path = f"refused-emails/full-{random_name}.eml"
s3.upload_email_from_bytesio(full_report_path, BytesIO(msg.as_bytes()),
random_name)
file_path = None
mailbox = None
email_log: EmailLog = None
orig_msg = get_orig_message_from_bounce(msg)
if not orig_msg:
# Some MTA does not return the original message in bounce message
# nothing we can do here
LOG.warning(
"Cannot parse original message from bounce message %s %s %s %s",
alias,
user,
contact,
full_report_path,
)
else:
file_path = f"refused-emails/{random_name}.eml"
s3.upload_email_from_bytesio(file_path, BytesIO(orig_msg.as_bytes()),
random_name)
try:
mailbox_id = int(orig_msg[_MAILBOX_ID_HEADER])
except TypeError:
LOG.exception(
"cannot parse mailbox from original message header %s",
orig_msg[_MAILBOX_ID_HEADER],
)
else:
mailbox = Mailbox.get(mailbox_id)
if not mailbox or mailbox.user_id != user.id:
LOG.exception(
"Tampered message mailbox_id %s, %s, %s, %s %s",
mailbox_id,
user,
alias,
contact,
full_report_path,
)
# cannot use this tampered mailbox, reset it
mailbox = None
# try to get the original email_log
try:
email_log_id = int(orig_msg[_EMAIL_LOG_ID_HEADER])
except TypeError:
LOG.exception(
"cannot parse email log from original message header %s",
orig_msg[_EMAIL_LOG_ID_HEADER],
)
else:
email_log = EmailLog.get(email_log_id)
refused_email = RefusedEmail.create(path=file_path,
full_report_path=full_report_path,
user_id=user.id)
db.session.flush()
LOG.d("Create refused email %s", refused_email)
if not mailbox:
LOG.debug("Try to get mailbox from bounce report")
try:
mailbox_id = int(get_header_from_bounce(msg, _MAILBOX_ID_HEADER))
except Exception:
LOG.exception("cannot get mailbox-id from bounce report, %s",
refused_email)
else:
mailbox = Mailbox.get(mailbox_id)
if not mailbox or mailbox.user_id != user.id:
LOG.exception(
"Tampered message mailbox_id %s, %s, %s, %s %s",
mailbox_id,
user,
alias,
contact,
full_report_path,
)
mailbox = None
if not email_log:
LOG.d("Try to get email log from bounce report")
try:
email_log_id = int(
get_header_from_bounce(msg, _EMAIL_LOG_ID_HEADER))
except Exception:
LOG.exception("cannot get email log id from bounce report, %s",
refused_email)
else:
email_log = EmailLog.get(email_log_id)
# use the default mailbox as the last option
if not mailbox:
LOG.warning("Use %s default mailbox %s", alias, refused_email)
mailbox = alias.mailbox
# create a new email log as the last option
if not email_log:
LOG.warning("cannot get the original email_log, create a new one")
email_log: EmailLog = EmailLog.create(contact_id=contact.id,
user_id=contact.user_id)
email_log.bounced = True
email_log.refused_email_id = refused_email.id
email_log.bounced_mailbox_id = mailbox.id
db.session.commit()
refused_email_url = (URL + f"/dashboard/refused_email?highlight_id=" +
str(email_log.id))
nb_bounced = EmailLog.filter_by(contact_id=contact.id,
bounced=True).count()
if nb_bounced >= 2 and alias.cannot_be_disabled:
LOG.warning("%s cannot be disabled", alias)
# inform user if this is the first bounced email
if nb_bounced == 1 or (nb_bounced >= 2 and alias.cannot_be_disabled):
LOG.d(
"Inform user %s about bounced email sent by %s to alias %s",
user,
contact.website_email,
alias,
)
send_email_with_rate_control(
user,
ALERT_BOUNCE_EMAIL,
user.email,
f"Email from {contact.website_email} to {alias.email} cannot be delivered to your inbox",
render(
"transactional/bounced-email.txt",
name=user.name,
alias=alias,
website_email=contact.website_email,
disable_alias_link=disable_alias_link,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
render(
"transactional/bounced-email.html",
name=user.name,
alias=alias,
website_email=contact.website_email,
disable_alias_link=disable_alias_link,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
)
# disable the alias the second time email is bounced
elif nb_bounced >= 2:
LOG.d(
"Bounce happens again with alias %s from %s. Disable alias now ",
alias,
contact.website_email,
)
alias.enabled = False
db.session.commit()
send_email_with_rate_control(
user,
ALERT_BOUNCE_EMAIL,
user.email,
f"Alias {alias.email} has been disabled due to second undelivered email from {contact.website_email}",
render(
"transactional/automatic-disable-alias.txt",
name=user.name,
alias=alias,
website_email=contact.website_email,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
render(
"transactional/automatic-disable-alias.html",
name=user.name,
alias=alias,
website_email=contact.website_email,
refused_email_url=refused_email_url,
mailbox_email=mailbox.email,
),
)
def prepare_multipart_body(self, content_index=0):
# type: (int) -> int
"""Will prepare the body of this request according to the multipart information.
This call assumes the on_request policies have been applied already in their
correct context (sync/async)
Does nothing if "set_multipart_mixed" was never called.
:param int content_index: The current index of parts within the batch message.
:returns: The updated index after all parts in this request have been added.
:rtype: int
"""
if not self.multipart_mixed_info:
return 0
requests = self.multipart_mixed_info[0] # type: List[HttpRequest]
boundary = self.multipart_mixed_info[2] # type: Optional[str]
# Update the main request with the body
main_message = Message()
main_message.add_header("Content-Type", "multipart/mixed")
if boundary:
main_message.set_boundary(boundary)
for req in requests:
part_message = Message()
if req.multipart_mixed_info:
content_index = req.prepare_multipart_body(
content_index=content_index)
part_message.add_header("Content-Type",
req.headers['Content-Type'])
payload = req.serialize()
# We need to remove the ~HTTP/1.1 prefix along with the added content-length
payload = payload[payload.index(b'--'):]
else:
part_message.add_header("Content-Type", "application/http")
part_message.add_header("Content-Transfer-Encoding", "binary")
part_message.add_header("Content-ID", str(content_index))
payload = req.serialize()
content_index += 1
part_message.set_payload(payload)
main_message.attach(part_message)
try:
from email.policy import HTTP
full_message = main_message.as_bytes(policy=HTTP)
eol = b"\r\n"
except ImportError: # Python 2.7
# Right now we decide to not support Python 2.7 on serialization, since
# it doesn't serialize a valid HTTP request (and our main scenario Storage refuses it)
raise NotImplementedError(
"Multipart request are not supported on Python 2.7")
# full_message = main_message.as_string()
# eol = b'\n'
_, _, body = full_message.split(eol, 2)
self.set_bytes_body(body)
self.headers["Content-Type"] = ("multipart/mixed; boundary=" +
main_message.get_boundary())
return content_index
def handle_reply(envelope, smtp: SMTP, msg: Message,
rcpt_to: str) -> (bool, str):
"""
return whether an email has been delivered and
the smtp status ("250 Message accepted", "550 Non-existent email address", etc)
"""
reply_email = rcpt_to.lower().strip()
# reply_email must end with EMAIL_DOMAIN
if not reply_email.endswith(EMAIL_DOMAIN):
LOG.warning(f"Reply email {reply_email} has wrong domain")
return False, "550 SL E2"
contact = Contact.get_by(reply_email=reply_email)
if not contact:
LOG.warning(f"No such forward-email with {reply_email} as reply-email")
return False, "550 SL E4 Email not exist"
alias = contact.alias
address: str = contact.alias.email
alias_domain = address[address.find("@") + 1:]
# alias must end with one of the ALIAS_DOMAINS or custom-domain
if not email_belongs_to_alias_domains(alias.email):
if not CustomDomain.get_by(domain=alias_domain):
return False, "550 SL E5"
user = alias.user
mail_from = envelope.mail_from.lower().strip()
# bounce email initiated by Postfix
# can happen in case emails cannot be delivered to user-email
# in this case Postfix will try to send a bounce report to original sender, which is
# the "reply email"
if mail_from == "<>":
LOG.warning(
"Bounce when sending to alias %s from %s, user %s",
alias,
contact,
user,
)
handle_bounce(contact, alias, msg, user)
return False, "550 SL E6"
mailbox = Mailbox.get_by(email=mail_from, user_id=user.id)
if not mailbox or mailbox not in alias.mailboxes:
# only mailbox can send email to the reply-email
handle_unknown_mailbox(envelope, msg, reply_email, user, alias)
return False, "550 SL E7"
if ENFORCE_SPF and mailbox.force_spf:
ip = msg[_IP_HEADER]
if not spf_pass(ip, envelope, mailbox, user, alias,
contact.website_email, msg):
# cannot use 4** here as sender will retry. 5** because that generates bounce report
return True, "250 SL E11"
delete_header(msg, _IP_HEADER)
delete_header(msg, "DKIM-Signature")
delete_header(msg, "Received")
# make the email comes from alias
from_header = alias.email
# add alias name from alias
if alias.name:
LOG.d("Put alias name in from header")
from_header = formataddr((alias.name, alias.email))
elif alias.custom_domain:
LOG.d("Put domain default alias name in from header")
# add alias name from domain
if alias.custom_domain.name:
from_header = formataddr((alias.custom_domain.name, alias.email))
add_or_replace_header(msg, "From", from_header)
# some email providers like ProtonMail adds automatically the Reply-To field
# make sure to delete it
delete_header(msg, "Reply-To")
# remove sender header if present as this could reveal user real email
delete_header(msg, "Sender")
delete_header(msg, "X-Sender")
replace_header_when_reply(msg, alias, "To")
replace_header_when_reply(msg, alias, "Cc")
# Received-SPF is injected by postfix-policyd-spf-python can reveal user original email
delete_header(msg, "Received-SPF")
LOG.d(
"send email from %s to %s, mail_options:%s,rcpt_options:%s",
alias.email,
contact.website_email,
envelope.mail_options,
envelope.rcpt_options,
)
# replace "*****@*****.**" by the contact email in the email body
# as this is usually included when replying
if user.replace_reverse_alias:
if msg.is_multipart():
for part in msg.walk():
if part.get_content_maintype() != "text":
continue
part = replace_str_in_msg(part, reply_email,
contact.website_email)
else:
msg = replace_str_in_msg(msg, reply_email, contact.website_email)
if alias_domain in ALIAS_DOMAINS:
add_dkim_signature(msg, alias_domain)
# add DKIM-Signature for custom-domain alias
else:
custom_domain: CustomDomain = CustomDomain.get_by(domain=alias_domain)
if custom_domain.dkim_verified:
add_dkim_signature(msg, alias_domain)
# create PGP email if needed
if contact.pgp_finger_print and user.is_premium():
LOG.d("Encrypt message for contact %s", contact)
try:
msg = prepare_pgp_message(msg, contact.pgp_finger_print)
except PGPException:
LOG.exception("Cannot encrypt message %s -> %s. %s %s", alias,
contact, mailbox, user)
# so the client can retry later
return False, "421 SL E13 Retry later"
try:
smtp.sendmail(
alias.email,
contact.website_email,
msg.as_bytes(),
envelope.mail_options,
envelope.rcpt_options,
)
except Exception:
LOG.exception("Cannot send email from %s to %s", alias, contact)
send_email(
mailbox.email,
f"Email cannot be sent to {contact.email} from {alias.email}",
render(
"transactional/reply-error.txt",
user=user,
alias=alias,
contact=contact,
contact_domain=get_email_domain_part(contact.email),
),
render(
"transactional/reply-error.html",
user=user,
alias=alias,
contact=contact,
contact_domain=get_email_domain_part(contact.email),
),
)
else:
EmailLog.create(contact_id=contact.id,
is_reply=True,
user_id=contact.user_id)
db.session.commit()
return True, "250 Message accepted for delivery"
# 真正删除邮件
if isinstance(mail_ids, list):
for one in mail_ids:
encode_id = "{}".format(one)
self.add_flags(encode_id.encode(), '\Deleted')
self.imap.expunge()
return True
return False
def logout(self):
# 关闭imap连接
self.imap.close() # 关闭打开的目录，如INBOX
self.imap.logout() # 退出登陆
if __name__ == '__main__':
from email.message import Message
from email.utils import localtime, format_datetime
new_message = Message()
new_message['Subject'] = 'subject2'
new_message['From'] = '*****@*****.**'
new_message['To'] = '*****@*****.**'
new_message['Date'] = format_datetime(localtime())
new_message.set_payload('This is the body of the message.\n')
print(new_message.as_bytes(unixfrom=True))
mail_server = MyMail(store=True)
mail_server.imap.append('Sent', '', imaplib.Time2Internaldate(time.time()),
new_message.as_bytes(unixfrom=True))
