def check_outlook_message_id(self, msg, target=None):
message_id = msg.msg.get("Message-ID")
if not message_id:
return
msg_regex = Regex(r"^<[0-9a-f]{4}([0-9a-f]{8})\$[0-9a-f]{8}\$["
r"0-9a-f]{8}\@")
regex = msg_regex.search(message_id)
if not regex:
return False
timetocken = int(regex.group(1), 16)
date = msg.msg.get("Date")
x = 0.0023283064365387
y = 27111902.8329849
mail_date = time.mktime(email.utils.parsedate(date))
expected = int((mail_date * x) + y)
if abs(timetocken - expected) < 250:
return False
received = msg.msg.get("Received")
received_regex = Regex(r"(\s.?\d+ \S\S\S \d+ \d+:\d+:\d+ \S+).*?$")
regex = received_regex.search(received)
received_date = 0
if regex:
received_date = time.mktime(email.utils.parsedate(regex.group()))
expected = int((received_date * x) + y)
return abs(timetocken - expected) >= 250
def check_for_msn_groups_headers(self, msg, target=None):
"""Check if the email's destination is a msn group"""
to = ''.join(msg.get_decoded_header('To'))
if not Regex(r"<(\S+)\@groups\.msn\.com>").search(to):
return False
listname = Regex(r"<(\S+)\@groups\.msn\.com>").match(to).groups()[0]
server_rgx = Regex(r"from mail pickup service by "
r"((?:p\d\d\.)groups\.msn\.com)\b")
server = ''
for rcvd in msg.get_decoded_header('Received'):
if server_rgx.search(rcvd):
server = server_rgx.search(rcvd).groups()[0]
break
if not server:
return False
message_id = ''.join(msg.get_decoded_header('Message-Id'))
if listname == "notifications":
if not Regex(r"^<\S+\@{0}".format(server)).search(message_id):
return False
else:
msn_addr = Regex(r"^<{0}-\S+\@groups\.msn\.com>".format(listname))
if not msn_addr.search(message_id):
return False
msn_addr = "{0}[email protected]".format(listname)
if msg.sender_address != msn_addr:
return False
return True
def check_for_forged_juno_received_headers(self, msg, target=None):
from_addr = ''.join(msg.get_all_addr_header("From"))
if not from_addr.rsplit("@", 1)[-1].endswith("juno.com"):
return False
if self.gated_through_received_hdr_remover(msg):
return False
xorig = ''.join(msg.get_decoded_header("X-Originating-IP"))
xmailer = ''.join(msg.get_decoded_header("X-Mailer"))
rcvd = ''.join(msg.get_decoded_header("Received"))
if xorig != "":
juno_re = Regex(r"from.*\b(?:juno|untd)\.com.*"
r"[\[\(]{0}[\]\)].*by".format(IP_ADDRESS.pattern), re.X)
cookie_re = Regex(r" cookie\.(?:juno|untd)\.com ")
if not juno_re.search(rcvd) and not cookie_re.search(rcvd):
return True
if "Juno " not in xmailer:
return True
else:
mail_com_re = Regex(r"from.*\bmail\.com.*\[{}\].*by".format(
IP_ADDRESS.pattern), re.X)
untd_com_re = Regex(r"from\s+(webmail\S+\.untd"
r"\.com)\s+\(\1\s+\[{}\]\)\s+by".format(
IP_ADDRESS.pattern), re.X)
if mail_com_re.search(rcvd) and not Regex(r"\bmail\.com").search(
xmailer):
return True
elif untd_com_re.search(rcvd) and not Regex(
r"^Webmail Version \d").search(xmailer):
return True
else:
return True
return False
def check_freemail_header(self, msg, header, regex=None, target=None):
"""Check all possible 'from' headers to see if sender
is freemail. It is possible to provide a regex
rule to match against too.
Returns True if it is or False otherwise
"""
self.ctxt.log.debug("FreeMail::Plugin check_freemail_header"
" %s", 'with regex: ' + regex if regex else '')
if not header:
self.ctxt.log.warn("FreeMail::Plugin check_freemail_header"
" requires an argument")
return False
if regex:
try:
check_re = Regex(regex).compile()
except re.error:
self.ctxt.log.warn("FreeMail::Plugin check_freemail_header"
" regex error")
return False
else:
check_re = None
if not msg.msg.get(header, None):
self.ctxt.log.debug(
"FreeMail::Plugin check_freemail_header"
" header: %s not found", header)
return False
header_emails = self.get_global('email_re').findall(msg.msg[header])
if not header_emails:
self.ctxt.log.debug(
"FreeMail::Plugin check_freemail_header"
" no emails found in header: %s", header)
return False
for email in header_emails:
if self._is_freemail(email):
if check_re and not check_re.search(email):
return False
elif check_re and check_re.search(email):
self.ctxt.log.debug(
"FreeMail::Plugin check_freemail_header"
" HIT! %s is freemail and matches regex", email)
result = ("Header " + header +
" is freemail and matches regex")
if self["freemail_add_describe_email"]:
_email = "(" + email.replace("@", "[at]") + ")"
result = result + "\n\t" + _email
return str(result)
self.ctxt.log.debug(
"FreeMail::Plugin check_freemail_header"
" HIT! %s is freemail", email)
result = "Header " + header + " is freemail"
if self["freemail_add_describe_email"]:
_email = "(" + email.replace("@", "[at]") + ")"
result = result + "\n\t" + _email
return str(result)
return False
def check_freemail_from(self, msg, regex=None, target=None):
"""Check if in specified header gave as parameter
is a freemail or no. It is possible to provide a regex
rule to match against too.
Returns True if it is or False otherwise
"""
self.ctxt.log.debug(
"FreeMail::Plugin Eval rule check_freemail_from"
" %s", 'with regex: ' + regex if regex else '')
all_from_headers = [
'From', 'Envelope-Sender', 'Resent-Sender', 'X-Envelope-From',
'EnvelopeFrom', 'Resent-From'
]
header_emails = []
if regex:
try:
check_re = Regex(regex)
except re.error:
self.ctxt.log.warn("FreeMail::Plugin check_freemail_from"
" regex error")
return False
else:
check_re = None
header_emails = msg.get_all_from_headers_addr()
header_emails = sorted(set(header_emails))
if not header_emails:
self.ctxt.log.debug(
"FreeMail::Plugin check_freemail_from"
" no emails found in from headers: %s", all_from_headers)
return False
for email in header_emails:
if self._is_freemail(email):
if check_re and not check_re.search(email):
return False
elif check_re and check_re.search(email):
self.ctxt.log.debug(
"FreeMail::Plugin check_freemail_from"
" HIT! %s is freemail and matches regex", email)
result = "Sender address is freemail and matches regex"
if self["freemail_add_describe_email"]:
_email = "(" + email.replace("@", "[at]") + ")"
result = result + "\n\t" + _email
return str(result)
self.ctxt.log.debug(
"FreeMail::Plugin check_freemail_from"
" HIT! %s is freemail", email)
result = "Sender address is freemail"
if self["freemail_add_describe_email"]:
_email = "(" + email.replace("@", "[at]") + ")"
result = result + "\n\t" + _email
return str(result)
return False
def gated_through_received_hdr_remover(self, msg, target=None):
"""Check if the email is gated through ezmlm"""
txt = ''.join(msg.get_decoded_header("Mailing-List"))
rcvd = ''.join(msg.get_decoded_header("Received"))
if Regex(r"^contact \S+\@\S+\; run by ezmlm$").search(txt):
dlto = ''.join(msg.get_decoded_header("Delivered-To"))
mailing_list_re = Regex(r"^mailing list \S+\@\S+")
qmail_re = Regex(r"qmail \d+ invoked (?:from "
r"network|by .{3,20})\); \d+ ... \d+")
if mailing_list_re.search(dlto) and qmail_re.search(rcvd):
return True
if not rcvd:
return True
if Regex(r"from groups\.msn\.com \(\S+\.msn\.com ").search(rcvd):
return True
return False
def check_for_to_in_subject(self, msg, test, target=None):
"""
Check if to address is in Subject field.
If it is called with 'address', check if full address is in subject,
else if the parameter is 'user', then check if user name is in subject.
"""
full_to = msg.get_all_addr_header('To')
if not full_to:
return False
subject = msg.msg.get('Subject', "")
for to in full_to:
if test == "address":
subject_regex = Regex(r".*" + re.escape(to) + r".*", re.I)
if subject_regex.search(subject):
return True
elif test == "user":
regex = re.match("(\S+)@.*", to)
if regex:
to = regex.group(1)
if Regex(r"^" + re.escape(to) + "$").search(subject):
return True
if Regex(r"(?:re|fw):\s*(?:\w+\s+)?" + re.escape(to) + "$")\
.search(subject):
return True
if Regex(r"\s*" + re.escape(to) + "[,:;!?-]$")\
.search(subject):
return True
if Regex(r"^" + re.escape(to) + "\s*[,:;!?-](\s).*")\
.search(subject):
return True
return False
def check_for_forged_yahoo_received_headers(self, msg, target=None):
"""Check for forged yahoo received headers"""
from_addr = ''.join(msg.get_all_addr_header("From"))
rcvd = ''.join(msg.get_decoded_header("Received"))
if "yahoo.com" not in from_addr:
return False
if (msg.get_decoded_header("Resent-From") and
msg.get_decoded_header("Resent-To")):
xrcvd = ''.join(msg.get_decoded_header("X-Received"))
rcvd = xrcvd if xrcvd else rcvd
if self.gated_through_received_hdr_remover(msg):
return False
for relay in msg.untrusted_relays + msg.trusted_relays:
rdns = relay.get("rdns")
if rdns and "yahoo.com" in rdns:
return False
if Regex(r"by web\S+\.mail\S*\.yahoo\.com via HTTP").search(rcvd):
return False
if Regex(r"by smtp\S+\.yahoo\.com with SMTP").search(rcvd):
return False
yahoo_ip_re = Regex(
r"from\s+\[{}\]\s+by\s+\S+\."
r"(?:groups|scd|dcn)\.yahoo\.com\s+with\s+NNFMP".format(
IP_ADDRESS.pattern), re.X)
if yahoo_ip_re.search(rcvd):
return False
if (Regex(r"\bmailer\d+\.bulk\.scd\.yahoo\.com\b").search(rcvd) and
from_addr.rsplit("@", 1)[-1] == "reply.yahoo.com"):
return False
if Regex("by \w+\.\w+\.yahoo\.com \(\d+\.\d+\.\d+\/\d+\.\d+\.\d+\)"
"(?: with ESMTP)? id \w+").search(rcvd):
return False
return True
def _check_for_forged_hotmail_received_headers(self, msg):
self.hotmail_addr_but_no_hotmail_received = 0
self.hotmail_addr_with_forged_hotmail_received = 0
rcvd = msg.msg.get("Received")
if not rcvd:
return False
pickup_service_regex = Regex(r"from mail pickup service by hotmail"
r"\.com with Microsoft SMTPSVC;")
if pickup_service_regex.search(rcvd):
return False
if self.check_for_msn_groups_headers(msg):
return False
ip_header = msg.msg.get("X-ORIGINATING-IP")
if ip_header and IP_ADDRESS.search(ip_header):
FORGED_REGEX = Regex(
r"from\s+(?:\S*\.)?hotmail.com\s+\(\S+\.hotmail("
r"?:\.msn)?\.com[\)]|"
r"from\s+\S*\.hotmail\.com\s+\(\[{IP_ADDRESS}\]|"
r"from\s+\S+\s+by\s+\S+\.hotmail(?:\.msn)?\.com\s+with\s+ "
r"HTTP\;|"
r"from\s+\[66\.218.\S+\]\s+by\s+\S+\.yahoo\.com"
r"".format(IP_ADDRESS=IP_ADDRESS.pattern), re.I | re.X)
if FORGED_REGEX.search(rcvd):
return False
if self.gated_through_received_hdr_remover(msg):
return False
helo_hotmail_regex = Regex(r"(?:from |HELO |helo=)\S*hotmail\.com\b")
if helo_hotmail_regex.search(rcvd):
self.hotmail_addr_with_forged_hotmail_received = 1
else:
from_address = msg.msg.get("From")
if not from_address:
from_address = ""
if "hotmail.com" not in from_address:
return False
self.hotmail_addr_but_no_hotmail_received = 1
def _update_base64_text_stats(self, msg, content_type,
content_transfer_encoding,
content_disposition, charset):
text_charset_re = Regex(r"(us-ascii|ansi_x3\.4-1968|iso-ir-6|"
r"ansi_x3\.4-1986|iso_646\.irv:1991|"
r"ascii|iso646-us|us|ibm367|cp367|"
r"csascii)")
charset_check = not charset or text_charset_re.search(charset)
cdisposition_check = not (content_disposition
and content_disposition.strip()
in ("inline", "attachment"))
if ("base64" in content_transfer_encoding and charset_check
and cdisposition_check):
self.set_local(msg, "mime_base64_encoded_text", True)
def check_freemail_body(self, msg, regex=None, target=None):
"""
Check if there are free emails in body parts
of the message
"""
self.ctxt.log.debug("FreeMail::Plugin check_freemail_body"
" %s", 'with regex: ' + regex if regex else '')
body_emails = self.get_global('body_emails')
if not len(body_emails):
self.ctxt.log.debug("FreeMail::Plugin check_freemail_body "
"No emails found in body of the message")
return False
if regex:
try:
check_re = Regex(regex).compile()
except re.error:
self.ctxt.log.warn("FreeMail::Plugin check_freemail_from"
" regex error")
return False
else:
check_re = None
if not self._parse_body():
return False
if check_re:
for email in self.get_global("freemail_body_emails"):
if check_re.search(email):
self.ctxt.log.debug(
"FreeMail::Plugin check_freemail_body"
" HIT! %s is freemail and matches regex", email)
result = "Address from body is freemail and matches regex"
if self["freemail_add_describe_email"]:
_email = "(" + email.replace("@", "[at]") + ")"
result = result + "\n\t" + _email
return str(result)
else:
if len(self.get_global("freemail_body_emails")):
emails = " ,".join(self.get_global("freemail_body_emails"))
self.ctxt.log.debug(
"FreeMail::Plugin check_freemail_body"
" HIT! body has freemails: %s", emails)
result = "Body has freemails"
if self["freemail_add_describe_email"]:
_emails = "(" + emails.replace("@", "[at]") + ")"
result = result + "\n\t" + _emails
return str(result)
return False
def check_start(self, msg):
"""Verify that the domains are valid and separate wildcard
domains from the rest."""
domain_re = Regex(r'^[a-z0-9.*?-]+$')
freemail_domains = self.get_global('freemail_domains')
freemail_temp_wc = []
for domain in freemail_domains[:]:
if not domain_re.search(domain):
freemail_domains.remove(domain)
self.ctxt.log.warn(
"FreeMail::Plugin Invalid freemail domain: %s", domain)
if '*' in domain:
temp = domain.replace('.', '\.')
temp = temp.replace('?', '.')
temp = temp.replace('*', '[^.]*')
freemail_temp_wc.append(temp)
if freemail_temp_wc:
wild_doms = r'\@(?:{0})$'.format('|'.join(freemail_temp_wc))
self.set_global('freemail_domains_re', Regex(wild_doms))
self.set_global('freemail_domains', freemail_domains)
valid_tlds = (self.get_global('util_rb_tld') +
self.get_global('util_rb_2tld') +
self.get_global('util_rb_3tld'))
tlds_re = r'(?:{0})'.format("|".join(valid_tlds))
email_re = Regex(
r"""
(?=.{{0,64}}\@) # limit userpart to 64 chars (and speed up searching?)
(?<![a-z0-9!#\$%&'*+\/=?^_`{{|}}~-]) # start boundary
( # capture email
[a-z0-9!#\$%&'*+\/=?^_`{{|}}~-]+ # no dot in beginning
(?:\.[a-z0-9!#\$%&'*+\/=?^_`{{|}}~-]+)* # no consecutive dots, no ending dot
\@
(?:[a-z0-9](?:[a-z0-9-]{{0,59}}[a-z0-9])?\.){{1,4}} # max 4x61 char parts (should be enough?)
{tld} # ends with valid tld
)
(?!(?:[a-z0-9-]|\.[a-z0-9])) # make sure domain ends here
""".format(tld=tlds_re), re.X | re.I)
self.set_global('email_re', email_re)
self.set_global('body_emails', set())
self.set_global("check_if_parsed", False)
def check_for_unique_subject_id(self, msg, target=None):
"""Check if in subject appears an unique id"""
subject = "".join(msg.get_decoded_header("Subject"))
id = None
unique_id_re_list = [
r"[-_\.\s]{7,}([-a-z0-9]{4,})$",
r"\s{10,}(?:\S\s)?(\S+)$",
r"\s{3,}[-:\#\(\[]+([-a-z0-9]{4,})[\]\)]+$",
r"\s{3,}[-:\#]([a-z0-9]{5,})$",
r"[\s._]{3,}([^0\s._]\d{3,})$",
r"[\s._]{3,}\[(\S+)\]$",
# (7217vPhZ0-478TLdy5829qicU9-0@26) and similar
r"\(([-\w]{7,}\@\d+)\)$",
r"\b(\d{7,})\s*$",
# stuff at end of line after "!" or "?" is usually an id
r"[!\?]\s*(\d{4,}|\w+(-\w+)+)\s*$",
# 9095IPZK7-095wsvp8715rJgY8-286-28 and similar
# excluding 'Re:', etc and the first word
r"(?:\w{2,3}:\s)?\w+\s+(\w{7,}-\w{7,}(-\w+)*)\s*$",
# #30D7 and similar
r"\s#\s*([a-f0-9]{4,})\s*$"
]
for rgx in unique_id_re_list:
match = Regex(rgx, re.I).search(subject)
if match:
id = match.group()
break
if not id:
return False
comercial_re = Regex(r"(?:item|invoice|order|number|confirmation)"
r".{1,6}%s\s*$" % id, re.X | re.I)
if Regex(r"\d{5,}").search(id) and comercial_re.search(subject):
return False
return True
def _get_received_header_times(self, msg):
try:
return self.get_local(msg, "received_header_times")
except KeyError:
pass
received_times = []
self.set_local(msg, "received_header_times", received_times)
date_re = Regex(r"(\s.?\d+ \S\S\S \d+ \d+:\d+:\d+ \S+)")
for rcvd in msg.get_decoded_header("Received"):
try:
date = date_re.search(rcvd).group()
except (AttributeError, TypeError):
date = None
if not date:
continue
self.ctxt.log.debug("eval: trying Received header date for "
"real time: %s", date)
received_time = self._parse_rfc822_date(date)
if received_time:
received_times.append(received_time)
return received_times
def _update_mime_bad_iso_charset(self, msg, charset):
is_iso_re = Regex(r"iso-.*-.*\b")
good_iso_re = Regex(r"iso-(?:8859-\d{1,2}|2022-(?:jp|kr))\b")
if is_iso_re.search(charset) and not good_iso_re.search(charset):
self.set_local(msg, "mime_bad_iso_charset", True)