def check_for_forged_yahoo_received_headers(self, msg, target=None): """Check for forged yahoo received headers""" from_addr = ''.join(msg.get_all_addr_header("From")) rcvd = ''.join(msg.get_decoded_header("Received")) if from_addr.rsplit("@", 1)[-1] != "yahoo.com": return False if (msg.get_decoded_header("Resent-From") and msg.get_decoded_header("Resent-To")): xrcvd = ''.join(msg.get_decoded_header("X-Received")) rcvd = xrcvd if xrcvd else rcvd if self.gated_through_received_hdr_remover(msg): return False for relay in msg.untrusted_relays + msg.trusted_relays: rdns = relay.get("rdns") if rdns and "yahoo.com" in rdns: return False if Regex(r"by web\S+\.mail\S*\.yahoo\.com via HTTP").search(rcvd): return False if Regex(r"by smtp\S+\.yahoo\.com with SMTP").search(rcvd): return False yahoo_ip_re = Regex( r"from \[{}\] by \S+\." r"(?:groups|scd|dcn)\.yahoo\.com with NNFMP".format( IP_ADDRESS.pattern), re.X) if yahoo_ip_re.search(rcvd): return False if (Regex(r"\bmailer\d+\.bulk\.scd\.yahoo\.com\b").search(rcvd) and from_addr.rsplit("@", 1)[-1] == "reply.yahoo.com"): return False if Regex("by \w+\.\w+\.yahoo\.com \(\d+\.\d+\.\d+\/\d+\.\d+\.\d+\)" "(?: with ESMTP)? id \w+").search(rcvd): return False return True
def check_for_msn_groups_headers(self, msg, target=None): """Check if the email's destination is a msn group""" to = ''.join(msg.get_decoded_header('To')) if not Regex(r"<(\S+)\@groups\.msn\.com>").search(to): return False listname = Regex(r"<(\S+)\@groups\.msn\.com>").match(to).groups()[0] server_rgx = Regex(r"from mail pickup service by " r"((?:p\d\d\.)groups\.msn\.com)\b") server = '' for rcvd in msg.get_decoded_header('Received'): if server_rgx.search(rcvd): server = server_rgx.search(rcvd).groups()[0] break if not server: return False message_id = ''.join(msg.get_decoded_header('Message-Id')) if listname == "notifications": if not Regex(r"^<\S+\@{0}".format(server)).search(message_id): return False else: msn_addr = Regex(r"^<{0}-\S+\@groups\.msn\.com>".format(listname)) if not msn_addr.search(message_id): return False msn_addr = "{0}[email protected]".format(listname) if msg.sender_address != msn_addr: return False return True
def _check_for_forged_hotmail_received_headers(self, msg): self.hotmail_addr_but_no_hotmail_received = 0 self.hotmail_addr_with_forged_hotmail_received = 0 rcvd = msg.msg.get("Received") if re.search(r"from mail pickup service by hotmail" r"\.com with Microsoft SMTPSVC;", rcvd): return False if self.check_for_msn_groups_headers(msg): return False ip_header = msg.msg.get("X-ORIGINATING-IP") if ip_header and IP_ADDRESS.search(ip_header): FORGED_REGEX = Regex( r"from\s+(?:\S*\.)?hotmail.com\s+\(\S+\.hotmail(" r"?:\.msn)?\.com[\)]|" r"from\s+\S*\.hotmail\.com\s+\(\[{IP_ADDRESS}\]|" r"from\s+\S+\s+by\s+\S+\.hotmail(?:\.msn)?\.com\s+with\s+ " r"HTTP\;|" r"from\s+\[66\.218.\S+\]\s+by\s+\S+\.yahoo\.com" r"".format(IP_ADDRESS=IP_ADDRESS.pattern), re.I | re.X) if FORGED_REGEX.search(rcvd): return False if self.gated_through_received_hdr_remover(msg): return False if re.search(r"(?:from |HELO |helo=)\S*hotmail\.com\b", rcvd): self.hotmail_addr_with_forged_hotmail_received = 1 else: from_address = msg.msg.get("From") if not from_address: from_address = "" if not re.search(r"\bhotmail\.com$", from_address): return False self.hotmail_addr_but_no_hotmail_received = 1
def _check_rbl(self, msg, rbl_server, qtype="A", subtest=None): """Checks all the IPs of this message on the specified list. :param msg: The message that we perform the check on. :param rbl_server: The RBL list to check :param qtype: The DNS record type to check :param subtest: If specified then an additional check is done on the result of the DNS lookup by matching this regular expression against the result. :return: True if there is a match and the subtest passes and False otherwise. """ if self.ctxt.skip_rbl_checks: return False if subtest is not None: try: subtest = Regex(subtest) except re.error as e: self.ctxt.err("Invalid regex %s: %s", subtest, e) return False for ip in msg.get_untrusted_ips(): rev = self.ctxt.dns.reverse_ip(ip) results = self.ctxt.dns.query("%s.%s" % (rev, rbl_server), qtype) if results and not subtest: return True for result in results: if subtest.match(str(result)): return True return False
def check_freemail_from(self, msg, regex=None, target=None): """Check if in specified header gave as parameter is a freemail or no. It is possible to provide a regex rule to match against too. Returns True if it is or False otherwise """ self.ctxt.log.debug("FreeMail::Plugin Eval rule check_freemail_from" " %s", 'with regex: ' + regex if regex else '') all_from_headers = ['From', 'Envelope-Sender', 'Resent-Sender', 'X-Envelope-From', 'EnvelopeFrom', 'Resent-From'] header_emails = [] if regex: try: check_re = Regex(regex) except re.error: self.ctxt.log.warn("FreeMail::Plugin check_freemail_from" " regex error") return False else: check_re = None header_emails = msg.get_all_from_headers_addr() header_emails = sorted(set(header_emails)) if not header_emails: self.ctxt.log.debug("FreeMail::Plugin check_freemail_from" " no emails found in from headers: %s", all_from_headers) return False for email in header_emails: if self._is_freemail(email): if check_re and not check_re.search(email): return False elif check_re and check_re.search(email): self.ctxt.log.debug( "FreeMail::Plugin check_freemail_from" " HIT! %s is freemail and matches regex", email) result = "Sender address is freemail and matches regex" if self["freemail_add_describe_email"]: _email = "(" + email.replace("@", "[at]") + ")" result = result + "\n\t" + _email return str(result) self.ctxt.log.debug("FreeMail::Plugin check_freemail_from" " HIT! %s is freemail", email) result = "Sender address is freemail" if self["freemail_add_describe_email"]: _email = "(" + email.replace("@", "[at]") + ")" result = result + "\n\t" + _email return str(result) return False
def _get_received_header_times(self, msg): self.set_local(msg, "received_header_times", list()) received = msg.get_decoded_header("Received") if not len(received): return # handle fetchmail headers local = [] from_local_re = Regex(r"\bfrom (?:localhost\s|(?:\S+ ){1,2}\S*\b127\.0\.0\.1\b)") qmail_re = Regex(r"qmail \d+ invoked by uid \d+") if from_local_re.search(received[0]) or qmail_re.search(received[0]): local.append(received[0]) del received[0] local_with_fetch_re = Regex(r"\bby localhost with \w+ \(fetchmail-[\d.]+") if received and local_with_fetch_re.search(received[0]): local.append(received[0]) del received[0] elif local: received.insert(0, local[0]) del local[0] fetchmail_times = [] date_re = Regex(r"(\s.?\d+ \S\S\S \d+ \d+:\d+:\d+ \S+)") for rcvd in local: try: date = date_re.search(rcvd).group() except TypeError: date = None if date: self.ctxt.log.debug( "eval: trying Received fetchmail " "header date for real time: %s", date) received_time = self._parse_rfc822_date(date) current_time = datetime.datetime.utcnow() if received_time and current_time >= received_time: self.ctxt.log.debug("eval: time_t from date=%s, rcvd=%s", received_time, date) fetchmail_times.append(received_time) if len(fetchmail_times) > 1: self.set_local(msg, "received_fetchmail_time", sorted(fetchmail_times, reverse=True)[0]) header_times = [] for rcvd in received: try: date = date_re.search(rcvd).group() except (AttributeError, TypeError): date = None if not date: continue self.ctxt.log.debug("eval: trying Received header date for " "real time: %s", date) received_time = self._parse_rfc822_date(date) if received_time: header_times.append(received_time) if header_times: self.set_local(msg, "received_header_times", header_times) else: self.ctxt.log.debug("eval: no dates found in Received headers")
def check_for_forged_juno_received_headers(self, msg, target=None): from_addr = ''.join(msg.get_all_addr_header("From")) if from_addr.rsplit("@", 1)[-1] != "juno.com": return False if self.gated_through_received_hdr_remover(msg): return False xorig = ''.join(msg.get_decoded_header("X-Originating-IP")) xmailer = ''.join(msg.get_decoded_header("")) rcvd = ''.join(msg.get_decoded_header("Received")) if xorig != "": juno_re = Regex(r"from.*\b(?:juno|untd)\.com.*" r"[\[\(]{0}[\]\)].*by".format(IP_ADDRESS.pattern), re.X) cookie_re = Regex(r" cookie\.(?:juno|untd)\.com ") if not juno_re.search(rcvd) and cookie_re.search(rcvd): return True if "Juno " not in xmailer: return True else: mail_com_re = Regex(r"from.*\bmail\.com.*\[{}\].*by".format( IP_ADDRESS.pattern), re.X) untd_com_re = Regex(r"from (webmail\S+\.untd" r"\.com) \(\1 \[\d+.\d+.\d+.\d+\]\) by") if mail_com_re.search(rcvd) and not Regex(r"\bmail\.com").search( xmailer): return True elif untd_com_re.search(rcvd) and not Regex( r"^Webmail Version \d").search(xmailer): return True return False
def gated_through_received_hdr_remover(self, msg, target=None): """Check if the email is gated through ezmlm""" txt = ''.join(msg.get_decoded_header("Mailing-List")) rcvd = ''.join(msg.get_decoded_header("Received")) if Regex(r"^contact \S+\@\S+\; run by ezmlm$").search(txt): dlto = ''.join(msg.get_decoded_header("Delivered-To")) mailing_list_re = Regex(r"^mailing list \S+\@\S+") qmail_re = Regex(r"qmail \d+ invoked (?:from " r"network|by .{3,20})\); \d+ ... \d+") if mailing_list_re.search(dlto) and qmail_re.search(rcvd): return True if not rcvd: return True if Regex(r"from groups\.msn\.com \(\S+\.msn\.com ").search(rcvd): return True return False
def check_for_forged_gw05_received_headers(self, msg, target=None): gw05_re = Regex(r"from\s(\S+)\sby\s(\S+)\swith\sESMTP\;\s+\S\S\S," r"\s+\d+\s+\S\S\S\s+\d{4}\s+\d\d:\d\d:\d\d\s+[-+]*" r"\d{4}", re.X | re.I) for rcv in msg.get_decoded_header("Received"): h1 = "" h2 = "" try: match = gw05_re.match(rcv) if match: h1, h2 = match.groups() if h1 and h2 and h2 != ".": return True except IndexError: continue return False
def _check_for_forged_received(self, msg): mismatch_from = 0 mismatch_ip_helo = 0 hostname_re = Regex(r"^\w+(?:[\w.-]+\.)+\w+$") ip_re = Regex(r"^(\d+\.\d+)\.\d+\.\d+") for index, relay in enumerate(msg.untrusted_relays): from_ip = relay.get("ip") from_host = self.hostname_to_domain(relay.get("rdns")) by_host = self.hostname_to_domain(relay.get("by")) helo_host = self.hostname_to_domain(relay.get("helo")) if not by_host or not hostname_re.match(by_host): continue if from_host and from_ip == '127.0.0.1': from_host = "undef" self.ctxt.log.debug("eval: forged-HELO: from=%s helo=%s by=%s" % ( from_host if from_host else "(undef)", helo_host if helo_host else "(undef)", by_host if by_host else "(undef)" )) try: ip_netmask_16 = ipaddress.IPv4Network(from_ip).supernet(16) except ValueError: ip_netmask_16 = "" try: helo_netmask_16 = ipaddress.IPv4Network(helo_host).supernet(16) except ValueError: helo_netmask_16 = "" if ip_netmask_16 and helo_netmask_16 and from_ip != helo_host: if (ip_netmask_16 != helo_netmask_16 and not IP_PRIVATE.match(helo_host)): self.ctxt.log.debug("eval: forged-HELO: massive mismatch " "on IP-addr HELO: %s != %s" % (helo_host, from_ip)) mismatch_ip_helo += 1 prev = msg.untrusted_relays[index - 1] if prev and index > 0: prev_from_host = prev.get("rdns") if (hostname_re.match(prev_from_host) and by_host != prev_from_host and not self._helo_forgery_whitelisted(by_host, prev_from_host)): self.ctxt.log.debug("eval: forged-HELO: mismatch on from: " "%s != %s" % (prev_from_host, by_host)) mismatch_from += 1 self.set_global("mismatch_from", mismatch_from) self.set_global("mismatch_ip_helo", mismatch_ip_helo)
def subject_is_all_caps(self, msg, target=None): """Checks if the subject is all capital letters. This eval rule ignore short subjects, one word subject and the prepended notations. (E.g. ``Re:``) """ for subject in msg.get_decoded_header("Subject"): # Remove the Re/Fwd notations in the subject subject = Regex(r"^(Re|Fwd|Fw|Aw|Antwort|Sv):", re.I).sub("", subject) subject = subject.strip() if len(subject) < 10: # Don't match short subjects continue if len(subject.split()) == 1: # Don't match one word subjects continue if subject.isupper(): return True return False
def check_for_no_rdns_dotcom_helo(self, msg, option=None, target=None): """Check untrusted relays and verify if latest relay it has helo from a big email provider like lycos, hotmail, excite, caramail, cs, aol, msn, yahoo, drizzle""" no_rdns_dotcom_helo = False for relay in msg.untrusted_relays: if IP_PRIVATE.match(relay.get("ip")): continue from_host = relay.get("rdns") helo_host = relay.get("helo") if not helo_host: continue no_rdns_dotcom_helo = False big_isp_re = Regex( r".*(?:\.|^)(lycos\.com|lycos\.co\.uk|hotmail\.com" r"|localhost\.com|excite\.com|caramail\.com|" r"cs\.com|aol\.com|msn\.com|yahoo\.com|" r"drizzle\.com)$") if big_isp_re.match(helo_host): if not from_host: no_rdns_dotcom_helo = True return no_rdns_dotcom_helo
def check_for_unique_subject_id(self, msg, target=None): """Check if in subject appears an unique id""" subject = "".join(msg.get_decoded_header("Subject")) id = None unique_id_re_list = [ r"[-_\.\s]{7,}([-a-z0-9]{4,})$", r"\s{10,}(?:\S\s)?(\S+)$", r"\s{3,}[-:\#\(\[]+([-a-z0-9]{4,})[\]\)]+$", r"\s{3,}[-:\#]([a-z0-9]{5,})$", r"[\s._]{3,}([^0\s._]\d{3,})$", r"[\s._]{3,}\[(\S+)\]$", # (7217vPhZ0-478TLdy5829qicU9-0@26) and similar r"\(([-\w]{7,}\@\d+)\)$", r"\b(\d{7,})\s*$", # stuff at end of line after "!" or "?" is usually an id r"[!\?]\s*(\d{4,}|\w+(-\w+)+)\s*$", # 9095IPZK7-095wsvp8715rJgY8-286-28 and similar # excluding 'Re:', etc and the first word r"(?:\w{2,3}:\s)?\w+\s+(\w{7,}-\w{7,}(-\w+)*)\s*$", # #30D7 and similar r"\s#\s*([a-f0-9]{4,})\s*$" ] for rgx in unique_id_re_list: match = Regex(rgx, re.I).search(subject) if match: id = match.group() break if not id: return False comercial_re = Regex(r"(?:item|invoice|order|number|confirmation)" r".{1,6}%s\s*$" % id, re.X | re.I) if Regex(r"\d{5,}").search(id) and comercial_re.search(subject): return False return True
def _check_rbl_addr(self, addresses, rbl_server, subtest=None): """Checks the specified addresses on the specified list. :param addresses: A list of addresses to check :param rbl_server: The RBL list to check :param subtest: If specified then an additional check is done on the result of the DNS lookup by matching this regular expression against the result. :return: True if there is a match and the subtest passes and False otherwise. """ if self.ctxt.skip_rbl_checks: return False if subtest is not None: try: subtest = Regex(subtest) except re.error as e: self.ctxt.err("Invalid regex %s: %s", subtest, e) return False for addr in addresses: if "@" in addr: domain = addr.rsplit("@", 1)[1].strip() else: domain = addr.strip() results = self.ctxt.dns.query("%s.%s" % (domain, rbl_server), "A") if results and not subtest: return True for result in results: if subtest.match(str(result)): return True return False