def check_for_forged_yahoo_received_headers(self, msg, target=None):
"""Check for forged yahoo received headers"""
from_addr = ''.join(msg.get_all_addr_header("From"))
rcvd = ''.join(msg.get_decoded_header("Received"))
if from_addr.rsplit("@", 1)[-1] != "yahoo.com":
return False
if (msg.get_decoded_header("Resent-From") and
msg.get_decoded_header("Resent-To")):
xrcvd = ''.join(msg.get_decoded_header("X-Received"))
rcvd = xrcvd if xrcvd else rcvd
if self.gated_through_received_hdr_remover(msg):
return False
for relay in msg.untrusted_relays + msg.trusted_relays:
rdns = relay.get("rdns")
if rdns and "yahoo.com" in rdns:
return False
if Regex(r"by web\S+\.mail\S*\.yahoo\.com via HTTP").search(rcvd):
return False
if Regex(r"by smtp\S+\.yahoo\.com with SMTP").search(rcvd):
return False
yahoo_ip_re = Regex(
r"from \[{}\] by \S+\."
r"(?:groups|scd|dcn)\.yahoo\.com with NNFMP".format(
IP_ADDRESS.pattern), re.X)
if yahoo_ip_re.search(rcvd):
return False
if (Regex(r"\bmailer\d+\.bulk\.scd\.yahoo\.com\b").search(rcvd) and
from_addr.rsplit("@", 1)[-1] == "reply.yahoo.com"):
return False
if Regex("by \w+\.\w+\.yahoo\.com \(\d+\.\d+\.\d+\/\d+\.\d+\.\d+\)"
"(?: with ESMTP)? id \w+").search(rcvd):
return False
return True
def check_for_msn_groups_headers(self, msg, target=None):
"""Check if the email's destination is a msn group"""
to = ''.join(msg.get_decoded_header('To'))
if not Regex(r"<(\S+)\@groups\.msn\.com>").search(to):
return False
listname = Regex(r"<(\S+)\@groups\.msn\.com>").match(to).groups()[0]
server_rgx = Regex(r"from mail pickup service by "
r"((?:p\d\d\.)groups\.msn\.com)\b")
server = ''
for rcvd in msg.get_decoded_header('Received'):
if server_rgx.search(rcvd):
server = server_rgx.search(rcvd).groups()[0]
break
if not server:
return False
message_id = ''.join(msg.get_decoded_header('Message-Id'))
if listname == "notifications":
if not Regex(r"^<\S+\@{0}".format(server)).search(message_id):
return False
else:
msn_addr = Regex(r"^<{0}-\S+\@groups\.msn\.com>".format(listname))
if not msn_addr.search(message_id):
return False
msn_addr = "{0}[email protected]".format(listname)
if msg.sender_address != msn_addr:
return False
return True
def _check_for_forged_hotmail_received_headers(self, msg):
self.hotmail_addr_but_no_hotmail_received = 0
self.hotmail_addr_with_forged_hotmail_received = 0
rcvd = msg.msg.get("Received")
if re.search(r"from mail pickup service by hotmail"
r"\.com with Microsoft SMTPSVC;", rcvd):
return False
if self.check_for_msn_groups_headers(msg):
return False
ip_header = msg.msg.get("X-ORIGINATING-IP")
if ip_header and IP_ADDRESS.search(ip_header):
FORGED_REGEX = Regex(
r"from\s+(?:\S*\.)?hotmail.com\s+\(\S+\.hotmail("
r"?:\.msn)?\.com[\)]|"
r"from\s+\S*\.hotmail\.com\s+\(\[{IP_ADDRESS}\]|"
r"from\s+\S+\s+by\s+\S+\.hotmail(?:\.msn)?\.com\s+with\s+ "
r"HTTP\;|"
r"from\s+\[66\.218.\S+\]\s+by\s+\S+\.yahoo\.com"
r"".format(IP_ADDRESS=IP_ADDRESS.pattern), re.I | re.X)
if FORGED_REGEX.search(rcvd):
return False
if self.gated_through_received_hdr_remover(msg):
return False
if re.search(r"(?:from |HELO |helo=)\S*hotmail\.com\b", rcvd):
self.hotmail_addr_with_forged_hotmail_received = 1
else:
from_address = msg.msg.get("From")
if not from_address:
from_address = ""
if not re.search(r"\bhotmail\.com$", from_address):
return False
self.hotmail_addr_but_no_hotmail_received = 1
def _check_rbl(self, msg, rbl_server, qtype="A", subtest=None):
"""Checks all the IPs of this message on the specified
list.
:param msg: The message that we perform the check on.
:param rbl_server: The RBL list to check
:param qtype: The DNS record type to check
:param subtest: If specified then an additional check
is done on the result of the DNS lookup by matching
this regular expression against the result.
:return: True if there is a match and the subtest
passes and False otherwise.
"""
if self.ctxt.skip_rbl_checks:
return False
if subtest is not None:
try:
subtest = Regex(subtest)
except re.error as e:
self.ctxt.err("Invalid regex %s: %s", subtest, e)
return False
for ip in msg.get_untrusted_ips():
rev = self.ctxt.dns.reverse_ip(ip)
results = self.ctxt.dns.query("%s.%s" % (rev, rbl_server), qtype)
if results and not subtest:
return True
for result in results:
if subtest.match(str(result)):
return True
return False
def check_freemail_from(self, msg, regex=None, target=None):
"""Check if in specified header gave as parameter
is a freemail or no. It is possible to provide a regex
rule to match against too.
Returns True if it is or False otherwise
"""
self.ctxt.log.debug("FreeMail::Plugin Eval rule check_freemail_from"
" %s", 'with regex: ' + regex if regex else '')
all_from_headers = ['From', 'Envelope-Sender',
'Resent-Sender', 'X-Envelope-From',
'EnvelopeFrom', 'Resent-From']
header_emails = []
if regex:
try:
check_re = Regex(regex)
except re.error:
self.ctxt.log.warn("FreeMail::Plugin check_freemail_from"
" regex error")
return False
else:
check_re = None
header_emails = msg.get_all_from_headers_addr()
header_emails = sorted(set(header_emails))
if not header_emails:
self.ctxt.log.debug("FreeMail::Plugin check_freemail_from"
" no emails found in from headers: %s",
all_from_headers)
return False
for email in header_emails:
if self._is_freemail(email):
if check_re and not check_re.search(email):
return False
elif check_re and check_re.search(email):
self.ctxt.log.debug(
"FreeMail::Plugin check_freemail_from"
" HIT! %s is freemail and matches regex", email)
result = "Sender address is freemail and matches regex"
if self["freemail_add_describe_email"]:
_email = "(" + email.replace("@", "[at]") + ")"
result = result + "\n\t" + _email
return str(result)
self.ctxt.log.debug("FreeMail::Plugin check_freemail_from"
" HIT! %s is freemail", email)
result = "Sender address is freemail"
if self["freemail_add_describe_email"]:
_email = "(" + email.replace("@", "[at]") + ")"
result = result + "\n\t" + _email
return str(result)
return False
def _get_received_header_times(self, msg):
self.set_local(msg, "received_header_times", list())
received = msg.get_decoded_header("Received")
if not len(received):
return
# handle fetchmail headers
local = []
from_local_re = Regex(r"\bfrom (?:localhost\s|(?:\S+ ){1,2}\S*\b127\.0\.0\.1\b)")
qmail_re = Regex(r"qmail \d+ invoked by uid \d+")
if from_local_re.search(received[0]) or qmail_re.search(received[0]):
local.append(received[0])
del received[0]
local_with_fetch_re = Regex(r"\bby localhost with \w+ \(fetchmail-[\d.]+")
if received and local_with_fetch_re.search(received[0]):
local.append(received[0])
del received[0]
elif local:
received.insert(0, local[0])
del local[0]
fetchmail_times = []
date_re = Regex(r"(\s.?\d+ \S\S\S \d+ \d+:\d+:\d+ \S+)")
for rcvd in local:
try:
date = date_re.search(rcvd).group()
except TypeError:
date = None
if date:
self.ctxt.log.debug(
"eval: trying Received fetchmail "
"header date for real time: %s", date)
received_time = self._parse_rfc822_date(date)
current_time = datetime.datetime.utcnow()
if received_time and current_time >= received_time:
self.ctxt.log.debug("eval: time_t from date=%s, rcvd=%s",
received_time, date)
fetchmail_times.append(received_time)
if len(fetchmail_times) > 1:
self.set_local(msg, "received_fetchmail_time",
sorted(fetchmail_times, reverse=True)[0])
header_times = []
for rcvd in received:
try:
date = date_re.search(rcvd).group()
except (AttributeError, TypeError):
date = None
if not date:
continue
self.ctxt.log.debug("eval: trying Received header date for "
"real time: %s", date)
received_time = self._parse_rfc822_date(date)
if received_time:
header_times.append(received_time)
if header_times:
self.set_local(msg, "received_header_times", header_times)
else:
self.ctxt.log.debug("eval: no dates found in Received headers")
def check_for_forged_juno_received_headers(self, msg, target=None):
from_addr = ''.join(msg.get_all_addr_header("From"))
if from_addr.rsplit("@", 1)[-1] != "juno.com":
return False
if self.gated_through_received_hdr_remover(msg):
return False
xorig = ''.join(msg.get_decoded_header("X-Originating-IP"))
xmailer = ''.join(msg.get_decoded_header(""))
rcvd = ''.join(msg.get_decoded_header("Received"))
if xorig != "":
juno_re = Regex(r"from.*\b(?:juno|untd)\.com.*"
r"[\[\(]{0}[\]\)].*by".format(IP_ADDRESS.pattern), re.X)
cookie_re = Regex(r" cookie\.(?:juno|untd)\.com ")
if not juno_re.search(rcvd) and cookie_re.search(rcvd):
return True
if "Juno " not in xmailer:
return True
else:
mail_com_re = Regex(r"from.*\bmail\.com.*\[{}\].*by".format(
IP_ADDRESS.pattern), re.X)
untd_com_re = Regex(r"from (webmail\S+\.untd"
r"\.com) \(\1 \[\d+.\d+.\d+.\d+\]\) by")
if mail_com_re.search(rcvd) and not Regex(r"\bmail\.com").search(
xmailer):
return True
elif untd_com_re.search(rcvd) and not Regex(
r"^Webmail Version \d").search(xmailer):
return True
return False
def gated_through_received_hdr_remover(self, msg, target=None):
"""Check if the email is gated through ezmlm"""
txt = ''.join(msg.get_decoded_header("Mailing-List"))
rcvd = ''.join(msg.get_decoded_header("Received"))
if Regex(r"^contact \S+\@\S+\; run by ezmlm$").search(txt):
dlto = ''.join(msg.get_decoded_header("Delivered-To"))
mailing_list_re = Regex(r"^mailing list \S+\@\S+")
qmail_re = Regex(r"qmail \d+ invoked (?:from "
r"network|by .{3,20})\); \d+ ... \d+")
if mailing_list_re.search(dlto) and qmail_re.search(rcvd):
return True
if not rcvd:
return True
if Regex(r"from groups\.msn\.com \(\S+\.msn\.com ").search(rcvd):
return True
return False
def check_for_forged_gw05_received_headers(self, msg, target=None):
gw05_re = Regex(r"from\s(\S+)\sby\s(\S+)\swith\sESMTP\;\s+\S\S\S,"
r"\s+\d+\s+\S\S\S\s+\d{4}\s+\d\d:\d\d:\d\d\s+[-+]*"
r"\d{4}", re.X | re.I)
for rcv in msg.get_decoded_header("Received"):
h1 = ""
h2 = ""
try:
match = gw05_re.match(rcv)
if match:
h1, h2 = match.groups()
if h1 and h2 and h2 != ".":
return True
except IndexError:
continue
return False
def _check_for_forged_received(self, msg):
mismatch_from = 0
mismatch_ip_helo = 0
hostname_re = Regex(r"^\w+(?:[\w.-]+\.)+\w+$")
ip_re = Regex(r"^(\d+\.\d+)\.\d+\.\d+")
for index, relay in enumerate(msg.untrusted_relays):
from_ip = relay.get("ip")
from_host = self.hostname_to_domain(relay.get("rdns"))
by_host = self.hostname_to_domain(relay.get("by"))
helo_host = self.hostname_to_domain(relay.get("helo"))
if not by_host or not hostname_re.match(by_host):
continue
if from_host and from_ip == '127.0.0.1':
from_host = "undef"
self.ctxt.log.debug("eval: forged-HELO: from=%s helo=%s by=%s" % (
from_host if from_host else "(undef)",
helo_host if helo_host else "(undef)",
by_host if by_host else "(undef)"
))
try:
ip_netmask_16 = ipaddress.IPv4Network(from_ip).supernet(16)
except ValueError:
ip_netmask_16 = ""
try:
helo_netmask_16 = ipaddress.IPv4Network(helo_host).supernet(16)
except ValueError:
helo_netmask_16 = ""
if ip_netmask_16 and helo_netmask_16 and from_ip != helo_host:
if (ip_netmask_16 != helo_netmask_16 and
not IP_PRIVATE.match(helo_host)):
self.ctxt.log.debug("eval: forged-HELO: massive mismatch "
"on IP-addr HELO: %s != %s" %
(helo_host, from_ip))
mismatch_ip_helo += 1
prev = msg.untrusted_relays[index - 1]
if prev and index > 0:
prev_from_host = prev.get("rdns")
if (hostname_re.match(prev_from_host)
and by_host != prev_from_host
and not self._helo_forgery_whitelisted(by_host,
prev_from_host)):
self.ctxt.log.debug("eval: forged-HELO: mismatch on from: "
"%s != %s" % (prev_from_host, by_host))
mismatch_from += 1
self.set_global("mismatch_from", mismatch_from)
self.set_global("mismatch_ip_helo", mismatch_ip_helo)
def subject_is_all_caps(self, msg, target=None):
"""Checks if the subject is all capital letters.
This eval rule ignore short subjects, one word subject and
the prepended notations. (E.g. ``Re:``)
"""
for subject in msg.get_decoded_header("Subject"):
# Remove the Re/Fwd notations in the subject
subject = Regex(r"^(Re|Fwd|Fw|Aw|Antwort|Sv):", re.I).sub("",
subject)
subject = subject.strip()
if len(subject) < 10:
# Don't match short subjects
continue
if len(subject.split()) == 1:
# Don't match one word subjects
continue
if subject.isupper():
return True
return False
def check_for_no_rdns_dotcom_helo(self, msg, option=None, target=None):
"""Check untrusted relays and verify if latest relay it has helo from
a big email provider like lycos, hotmail, excite, caramail, cs, aol,
msn, yahoo, drizzle"""
no_rdns_dotcom_helo = False
for relay in msg.untrusted_relays:
if IP_PRIVATE.match(relay.get("ip")):
continue
from_host = relay.get("rdns")
helo_host = relay.get("helo")
if not helo_host:
continue
no_rdns_dotcom_helo = False
big_isp_re = Regex(
r".*(?:\.|^)(lycos\.com|lycos\.co\.uk|hotmail\.com"
r"|localhost\.com|excite\.com|caramail\.com|"
r"cs\.com|aol\.com|msn\.com|yahoo\.com|"
r"drizzle\.com)$")
if big_isp_re.match(helo_host):
if not from_host:
no_rdns_dotcom_helo = True
return no_rdns_dotcom_helo
def check_for_unique_subject_id(self, msg, target=None):
"""Check if in subject appears an unique id"""
subject = "".join(msg.get_decoded_header("Subject"))
id = None
unique_id_re_list = [
r"[-_\.\s]{7,}([-a-z0-9]{4,})$",
r"\s{10,}(?:\S\s)?(\S+)$",
r"\s{3,}[-:\#\(\[]+([-a-z0-9]{4,})[\]\)]+$",
r"\s{3,}[-:\#]([a-z0-9]{5,})$",
r"[\s._]{3,}([^0\s._]\d{3,})$",
r"[\s._]{3,}\[(\S+)\]$",
# (7217vPhZ0-478TLdy5829qicU9-0@26) and similar
r"\(([-\w]{7,}\@\d+)\)$",
r"\b(\d{7,})\s*$",
# stuff at end of line after "!" or "?" is usually an id
r"[!\?]\s*(\d{4,}|\w+(-\w+)+)\s*$",
# 9095IPZK7-095wsvp8715rJgY8-286-28 and similar
# excluding 'Re:', etc and the first word
r"(?:\w{2,3}:\s)?\w+\s+(\w{7,}-\w{7,}(-\w+)*)\s*$",
# #30D7 and similar
r"\s#\s*([a-f0-9]{4,})\s*$"
]
for rgx in unique_id_re_list:
match = Regex(rgx, re.I).search(subject)
if match:
id = match.group()
break
if not id:
return False
comercial_re = Regex(r"(?:item|invoice|order|number|confirmation)"
r".{1,6}%s\s*$" % id, re.X | re.I)
if Regex(r"\d{5,}").search(id) and comercial_re.search(subject):
return False
return True
def _check_rbl_addr(self, addresses, rbl_server, subtest=None):
"""Checks the specified addresses on the specified list.
:param addresses: A list of addresses to check
:param rbl_server: The RBL list to check
:param subtest: If specified then an additional check
is done on the result of the DNS lookup by matching
this regular expression against the result.
:return: True if there is a match and the subtest
passes and False otherwise.
"""
if self.ctxt.skip_rbl_checks:
return False
if subtest is not None:
try:
subtest = Regex(subtest)
except re.error as e:
self.ctxt.err("Invalid regex %s: %s", subtest, e)
return False
for addr in addresses:
if "@" in addr:
domain = addr.rsplit("@", 1)[1].strip()
else:
domain = addr.strip()
results = self.ctxt.dns.query("%s.%s" % (domain, rbl_server), "A")
if results and not subtest:
return True
for result in results:
if subtest.match(str(result)):
return True
return False
