def check_outlook_message_id(self, msg, target=None): message_id = msg.msg.get("Message-ID") if not message_id: return msg_regex = Regex(r"^<[0-9a-f]{4}([0-9a-f]{8})\$[0-9a-f]{8}\$[" r"0-9a-f]{8}\@") regex = msg_regex.search(message_id) if not regex: return False timetocken = int(regex.group(1), 16) date = msg.msg.get("Date") x = 0.0023283064365387 y = 27111902.8329849 mail_date = time.mktime(email.utils.parsedate(date)) expected = int((mail_date * x) + y) if abs(timetocken - expected) < 250: return False received = msg.msg.get("Received") received_regex = Regex(r"(\s.?\d+ \S\S\S \d+ \d+:\d+:\d+ \S+).*?$") regex = received_regex.search(received) received_date = 0 if regex: received_date = time.mktime(email.utils.parsedate(regex.group())) expected = int((received_date * x) + y) return abs(timetocken - expected) >= 250
def check_for_msn_groups_headers(self, msg, target=None): """Check if the email's destination is a msn group""" to = ''.join(msg.get_decoded_header('To')) if not Regex(r"<(\S+)\@groups\.msn\.com>").search(to): return False listname = Regex(r"<(\S+)\@groups\.msn\.com>").match(to).groups()[0] server_rgx = Regex(r"from mail pickup service by " r"((?:p\d\d\.)groups\.msn\.com)\b") server = '' for rcvd in msg.get_decoded_header('Received'): if server_rgx.search(rcvd): server = server_rgx.search(rcvd).groups()[0] break if not server: return False message_id = ''.join(msg.get_decoded_header('Message-Id')) if listname == "notifications": if not Regex(r"^<\S+\@{0}".format(server)).search(message_id): return False else: msn_addr = Regex(r"^<{0}-\S+\@groups\.msn\.com>".format(listname)) if not msn_addr.search(message_id): return False msn_addr = "{0}[email protected]".format(listname) if msg.sender_address != msn_addr: return False return True
def check_for_forged_juno_received_headers(self, msg, target=None): from_addr = ''.join(msg.get_all_addr_header("From")) if not from_addr.rsplit("@", 1)[-1].endswith("juno.com"): return False if self.gated_through_received_hdr_remover(msg): return False xorig = ''.join(msg.get_decoded_header("X-Originating-IP")) xmailer = ''.join(msg.get_decoded_header("X-Mailer")) rcvd = ''.join(msg.get_decoded_header("Received")) if xorig != "": juno_re = Regex(r"from.*\b(?:juno|untd)\.com.*" r"[\[\(]{0}[\]\)].*by".format(IP_ADDRESS.pattern), re.X) cookie_re = Regex(r" cookie\.(?:juno|untd)\.com ") if not juno_re.search(rcvd) and not cookie_re.search(rcvd): return True if "Juno " not in xmailer: return True else: mail_com_re = Regex(r"from.*\bmail\.com.*\[{}\].*by".format( IP_ADDRESS.pattern), re.X) untd_com_re = Regex(r"from\s+(webmail\S+\.untd" r"\.com)\s+\(\1\s+\[{}\]\)\s+by".format( IP_ADDRESS.pattern), re.X) if mail_com_re.search(rcvd) and not Regex(r"\bmail\.com").search( xmailer): return True elif untd_com_re.search(rcvd) and not Regex( r"^Webmail Version \d").search(xmailer): return True else: return True return False
def check_freemail_header(self, msg, header, regex=None, target=None): """Check all possible 'from' headers to see if sender is freemail. It is possible to provide a regex rule to match against too. Returns True if it is or False otherwise """ self.ctxt.log.debug("FreeMail::Plugin check_freemail_header" " %s", 'with regex: ' + regex if regex else '') if not header: self.ctxt.log.warn("FreeMail::Plugin check_freemail_header" " requires an argument") return False if regex: try: check_re = Regex(regex).compile() except re.error: self.ctxt.log.warn("FreeMail::Plugin check_freemail_header" " regex error") return False else: check_re = None if not msg.msg.get(header, None): self.ctxt.log.debug( "FreeMail::Plugin check_freemail_header" " header: %s not found", header) return False header_emails = self.get_global('email_re').findall(msg.msg[header]) if not header_emails: self.ctxt.log.debug( "FreeMail::Plugin check_freemail_header" " no emails found in header: %s", header) return False for email in header_emails: if self._is_freemail(email): if check_re and not check_re.search(email): return False elif check_re and check_re.search(email): self.ctxt.log.debug( "FreeMail::Plugin check_freemail_header" " HIT! %s is freemail and matches regex", email) result = ("Header " + header + " is freemail and matches regex") if self["freemail_add_describe_email"]: _email = "(" + email.replace("@", "[at]") + ")" result = result + "\n\t" + _email return str(result) self.ctxt.log.debug( "FreeMail::Plugin check_freemail_header" " HIT! %s is freemail", email) result = "Header " + header + " is freemail" if self["freemail_add_describe_email"]: _email = "(" + email.replace("@", "[at]") + ")" result = result + "\n\t" + _email return str(result) return False
def check_freemail_from(self, msg, regex=None, target=None): """Check if in specified header gave as parameter is a freemail or no. It is possible to provide a regex rule to match against too. Returns True if it is or False otherwise """ self.ctxt.log.debug( "FreeMail::Plugin Eval rule check_freemail_from" " %s", 'with regex: ' + regex if regex else '') all_from_headers = [ 'From', 'Envelope-Sender', 'Resent-Sender', 'X-Envelope-From', 'EnvelopeFrom', 'Resent-From' ] header_emails = [] if regex: try: check_re = Regex(regex) except re.error: self.ctxt.log.warn("FreeMail::Plugin check_freemail_from" " regex error") return False else: check_re = None header_emails = msg.get_all_from_headers_addr() header_emails = sorted(set(header_emails)) if not header_emails: self.ctxt.log.debug( "FreeMail::Plugin check_freemail_from" " no emails found in from headers: %s", all_from_headers) return False for email in header_emails: if self._is_freemail(email): if check_re and not check_re.search(email): return False elif check_re and check_re.search(email): self.ctxt.log.debug( "FreeMail::Plugin check_freemail_from" " HIT! %s is freemail and matches regex", email) result = "Sender address is freemail and matches regex" if self["freemail_add_describe_email"]: _email = "(" + email.replace("@", "[at]") + ")" result = result + "\n\t" + _email return str(result) self.ctxt.log.debug( "FreeMail::Plugin check_freemail_from" " HIT! %s is freemail", email) result = "Sender address is freemail" if self["freemail_add_describe_email"]: _email = "(" + email.replace("@", "[at]") + ")" result = result + "\n\t" + _email return str(result) return False
def gated_through_received_hdr_remover(self, msg, target=None): """Check if the email is gated through ezmlm""" txt = ''.join(msg.get_decoded_header("Mailing-List")) rcvd = ''.join(msg.get_decoded_header("Received")) if Regex(r"^contact \S+\@\S+\; run by ezmlm$").search(txt): dlto = ''.join(msg.get_decoded_header("Delivered-To")) mailing_list_re = Regex(r"^mailing list \S+\@\S+") qmail_re = Regex(r"qmail \d+ invoked (?:from " r"network|by .{3,20})\); \d+ ... \d+") if mailing_list_re.search(dlto) and qmail_re.search(rcvd): return True if not rcvd: return True if Regex(r"from groups\.msn\.com \(\S+\.msn\.com ").search(rcvd): return True return False
def check_for_to_in_subject(self, msg, test, target=None): """ Check if to address is in Subject field. If it is called with 'address', check if full address is in subject, else if the parameter is 'user', then check if user name is in subject. """ full_to = msg.get_all_addr_header('To') if not full_to: return False subject = msg.msg.get('Subject', "") for to in full_to: if test == "address": subject_regex = Regex(r".*" + re.escape(to) + r".*", re.I) if subject_regex.search(subject): return True elif test == "user": regex = re.match("(\S+)@.*", to) if regex: to = regex.group(1) if Regex(r"^" + re.escape(to) + "$").search(subject): return True if Regex(r"(?:re|fw):\s*(?:\w+\s+)?" + re.escape(to) + "$")\ .search(subject): return True if Regex(r"\s*" + re.escape(to) + "[,:;!?-]$")\ .search(subject): return True if Regex(r"^" + re.escape(to) + "\s*[,:;!?-](\s).*")\ .search(subject): return True return False
def check_for_forged_yahoo_received_headers(self, msg, target=None): """Check for forged yahoo received headers""" from_addr = ''.join(msg.get_all_addr_header("From")) rcvd = ''.join(msg.get_decoded_header("Received")) if "yahoo.com" not in from_addr: return False if (msg.get_decoded_header("Resent-From") and msg.get_decoded_header("Resent-To")): xrcvd = ''.join(msg.get_decoded_header("X-Received")) rcvd = xrcvd if xrcvd else rcvd if self.gated_through_received_hdr_remover(msg): return False for relay in msg.untrusted_relays + msg.trusted_relays: rdns = relay.get("rdns") if rdns and "yahoo.com" in rdns: return False if Regex(r"by web\S+\.mail\S*\.yahoo\.com via HTTP").search(rcvd): return False if Regex(r"by smtp\S+\.yahoo\.com with SMTP").search(rcvd): return False yahoo_ip_re = Regex( r"from\s+\[{}\]\s+by\s+\S+\." r"(?:groups|scd|dcn)\.yahoo\.com\s+with\s+NNFMP".format( IP_ADDRESS.pattern), re.X) if yahoo_ip_re.search(rcvd): return False if (Regex(r"\bmailer\d+\.bulk\.scd\.yahoo\.com\b").search(rcvd) and from_addr.rsplit("@", 1)[-1] == "reply.yahoo.com"): return False if Regex("by \w+\.\w+\.yahoo\.com \(\d+\.\d+\.\d+\/\d+\.\d+\.\d+\)" "(?: with ESMTP)? id \w+").search(rcvd): return False return True
def _check_for_forged_hotmail_received_headers(self, msg): self.hotmail_addr_but_no_hotmail_received = 0 self.hotmail_addr_with_forged_hotmail_received = 0 rcvd = msg.msg.get("Received") if not rcvd: return False pickup_service_regex = Regex(r"from mail pickup service by hotmail" r"\.com with Microsoft SMTPSVC;") if pickup_service_regex.search(rcvd): return False if self.check_for_msn_groups_headers(msg): return False ip_header = msg.msg.get("X-ORIGINATING-IP") if ip_header and IP_ADDRESS.search(ip_header): FORGED_REGEX = Regex( r"from\s+(?:\S*\.)?hotmail.com\s+\(\S+\.hotmail(" r"?:\.msn)?\.com[\)]|" r"from\s+\S*\.hotmail\.com\s+\(\[{IP_ADDRESS}\]|" r"from\s+\S+\s+by\s+\S+\.hotmail(?:\.msn)?\.com\s+with\s+ " r"HTTP\;|" r"from\s+\[66\.218.\S+\]\s+by\s+\S+\.yahoo\.com" r"".format(IP_ADDRESS=IP_ADDRESS.pattern), re.I | re.X) if FORGED_REGEX.search(rcvd): return False if self.gated_through_received_hdr_remover(msg): return False helo_hotmail_regex = Regex(r"(?:from |HELO |helo=)\S*hotmail\.com\b") if helo_hotmail_regex.search(rcvd): self.hotmail_addr_with_forged_hotmail_received = 1 else: from_address = msg.msg.get("From") if not from_address: from_address = "" if "hotmail.com" not in from_address: return False self.hotmail_addr_but_no_hotmail_received = 1
def _update_base64_text_stats(self, msg, content_type, content_transfer_encoding, content_disposition, charset): text_charset_re = Regex(r"(us-ascii|ansi_x3\.4-1968|iso-ir-6|" r"ansi_x3\.4-1986|iso_646\.irv:1991|" r"ascii|iso646-us|us|ibm367|cp367|" r"csascii)") charset_check = not charset or text_charset_re.search(charset) cdisposition_check = not (content_disposition and content_disposition.strip() in ("inline", "attachment")) if ("base64" in content_transfer_encoding and charset_check and cdisposition_check): self.set_local(msg, "mime_base64_encoded_text", True)
def check_freemail_body(self, msg, regex=None, target=None): """ Check if there are free emails in body parts of the message """ self.ctxt.log.debug("FreeMail::Plugin check_freemail_body" " %s", 'with regex: ' + regex if regex else '') body_emails = self.get_global('body_emails') if not len(body_emails): self.ctxt.log.debug("FreeMail::Plugin check_freemail_body " "No emails found in body of the message") return False if regex: try: check_re = Regex(regex).compile() except re.error: self.ctxt.log.warn("FreeMail::Plugin check_freemail_from" " regex error") return False else: check_re = None if not self._parse_body(): return False if check_re: for email in self.get_global("freemail_body_emails"): if check_re.search(email): self.ctxt.log.debug( "FreeMail::Plugin check_freemail_body" " HIT! %s is freemail and matches regex", email) result = "Address from body is freemail and matches regex" if self["freemail_add_describe_email"]: _email = "(" + email.replace("@", "[at]") + ")" result = result + "\n\t" + _email return str(result) else: if len(self.get_global("freemail_body_emails")): emails = " ,".join(self.get_global("freemail_body_emails")) self.ctxt.log.debug( "FreeMail::Plugin check_freemail_body" " HIT! body has freemails: %s", emails) result = "Body has freemails" if self["freemail_add_describe_email"]: _emails = "(" + emails.replace("@", "[at]") + ")" result = result + "\n\t" + _emails return str(result) return False
def check_start(self, msg): """Verify that the domains are valid and separate wildcard domains from the rest.""" domain_re = Regex(r'^[a-z0-9.*?-]+$') freemail_domains = self.get_global('freemail_domains') freemail_temp_wc = [] for domain in freemail_domains[:]: if not domain_re.search(domain): freemail_domains.remove(domain) self.ctxt.log.warn( "FreeMail::Plugin Invalid freemail domain: %s", domain) if '*' in domain: temp = domain.replace('.', '\.') temp = temp.replace('?', '.') temp = temp.replace('*', '[^.]*') freemail_temp_wc.append(temp) if freemail_temp_wc: wild_doms = r'\@(?:{0})$'.format('|'.join(freemail_temp_wc)) self.set_global('freemail_domains_re', Regex(wild_doms)) self.set_global('freemail_domains', freemail_domains) valid_tlds = (self.get_global('util_rb_tld') + self.get_global('util_rb_2tld') + self.get_global('util_rb_3tld')) tlds_re = r'(?:{0})'.format("|".join(valid_tlds)) email_re = Regex( r""" (?=.{{0,64}}\@) # limit userpart to 64 chars (and speed up searching?) (?<![a-z0-9!#\$%&'*+\/=?^_`{{|}}~-]) # start boundary ( # capture email [a-z0-9!#\$%&'*+\/=?^_`{{|}}~-]+ # no dot in beginning (?:\.[a-z0-9!#\$%&'*+\/=?^_`{{|}}~-]+)* # no consecutive dots, no ending dot \@ (?:[a-z0-9](?:[a-z0-9-]{{0,59}}[a-z0-9])?\.){{1,4}} # max 4x61 char parts (should be enough?) {tld} # ends with valid tld ) (?!(?:[a-z0-9-]|\.[a-z0-9])) # make sure domain ends here """.format(tld=tlds_re), re.X | re.I) self.set_global('email_re', email_re) self.set_global('body_emails', set()) self.set_global("check_if_parsed", False)
def check_for_unique_subject_id(self, msg, target=None): """Check if in subject appears an unique id""" subject = "".join(msg.get_decoded_header("Subject")) id = None unique_id_re_list = [ r"[-_\.\s]{7,}([-a-z0-9]{4,})$", r"\s{10,}(?:\S\s)?(\S+)$", r"\s{3,}[-:\#\(\[]+([-a-z0-9]{4,})[\]\)]+$", r"\s{3,}[-:\#]([a-z0-9]{5,})$", r"[\s._]{3,}([^0\s._]\d{3,})$", r"[\s._]{3,}\[(\S+)\]$", # (7217vPhZ0-478TLdy5829qicU9-0@26) and similar r"\(([-\w]{7,}\@\d+)\)$", r"\b(\d{7,})\s*$", # stuff at end of line after "!" or "?" is usually an id r"[!\?]\s*(\d{4,}|\w+(-\w+)+)\s*$", # 9095IPZK7-095wsvp8715rJgY8-286-28 and similar # excluding 'Re:', etc and the first word r"(?:\w{2,3}:\s)?\w+\s+(\w{7,}-\w{7,}(-\w+)*)\s*$", # #30D7 and similar r"\s#\s*([a-f0-9]{4,})\s*$" ] for rgx in unique_id_re_list: match = Regex(rgx, re.I).search(subject) if match: id = match.group() break if not id: return False comercial_re = Regex(r"(?:item|invoice|order|number|confirmation)" r".{1,6}%s\s*$" % id, re.X | re.I) if Regex(r"\d{5,}").search(id) and comercial_re.search(subject): return False return True
def _get_received_header_times(self, msg): try: return self.get_local(msg, "received_header_times") except KeyError: pass received_times = [] self.set_local(msg, "received_header_times", received_times) date_re = Regex(r"(\s.?\d+ \S\S\S \d+ \d+:\d+:\d+ \S+)") for rcvd in msg.get_decoded_header("Received"): try: date = date_re.search(rcvd).group() except (AttributeError, TypeError): date = None if not date: continue self.ctxt.log.debug("eval: trying Received header date for " "real time: %s", date) received_time = self._parse_rfc822_date(date) if received_time: received_times.append(received_time) return received_times
def _update_mime_bad_iso_charset(self, msg, charset): is_iso_re = Regex(r"iso-.*-.*\b") good_iso_re = Regex(r"iso-(?:8859-\d{1,2}|2022-(?:jp|kr))\b") if is_iso_re.search(charset) and not good_iso_re.search(charset): self.set_local(msg, "mime_bad_iso_charset", True)