def test_no_policy_match(self):
espf = EnforceSpf()
espf.set_enforcement('fail', match_code='550')
class TestSession(object):
address = ('1.2.3.4', 56789)
envelope = None
ehlo_as = 'testehlo'
class TestValidators(object):
def __init__(self):
self.session = TestSession()
@espf.check
def validate_mail(self, reply, sender):
pass
spf.check2(i='1.2.3.4', s='*****@*****.**',
h='testehlo').AndReturn(('none', 'the reason'))
self.mox.ReplayAll()
validators = TestValidators()
reply = Reply('250', '2.0.0 Ok')
validators.validate_mail(reply, '*****@*****.**')
self.assertEqual('250', reply.code)
self.assertEqual('2.0.0 Ok', reply.message)
def test_on_validate_rcpt(self):
espf = EnforceSpf()
espf.set_enforcement('fail', match_code='550')
class TestSession(object):
address = ('1.2.3.4', 56789)
envelope = Envelope('*****@*****.**')
ehlo_as = 'testehlo'
class TestValidators(object):
def __init__(self):
self.session = TestSession()
@espf.check
def validate_rcpt(self, reply, recipient):
pass
spf.check2(i='1.2.3.4', s='*****@*****.**',
h='testehlo').AndReturn(('fail', 'the reason'))
self.mox.ReplayAll()
validators = TestValidators()
reply = Reply('250', '2.0.0 Ok')
validators.validate_rcpt(reply, 'asdf')
self.assertEqual('550', reply.code)
self.assertEqual('5.7.1 Access denied', reply.message)
def examine(self,suspect):
if not have_spf:
return DUNNO
client_address=suspect.get_value('client_address')
if client_address is None:
self.logger.error('No client address found')
return DUNNO
if have_netaddr:
ip_whitelist=self.config.get('SPFPlugin','ip_whitelist')
ip_whitelist=[IPNetwork(i.strip()) for i in ip_whitelist.split(',')]
for net in ip_whitelist:
if IPAddress(client_address) in net:
return DUNNO
sender=suspect.get_value('sender')
if sender is None:
self.logger.warning('No RCPT address found')
return DEFER_IF_PERMIT,'internal policy error (no from address)'
sender_email = strip_address(sender)
sender_domain = extract_domain(sender_email)
domain_whitelist=self.config.get('SPFPlugin','domain_whitelist')
domain_whitelist=[i.strip() for i in domain_whitelist.split(',')]
if sender_domain in domain_whitelist:
return DUNNO
helo_name=suspect.get_value('helo_name')
if helo_name is None:
self.logger.error('No SMTP HELO name found')
return DUNNO
on_softfail=self.config.get('SPFPlugin','on_softfail')
softfail = DUNNO
if on_softfail == 'DEFER':
softfail = DEFER_IF_PERMIT
on_softerror=self.config.get('SPFPlugin','on_softerror')
softerror = DUNNO
if on_softerror == 'REJECT':
softerror = REJECT
result, explanation = spf.check2(client_address, sender_email, helo_name)
self.logger.debug('Postomaat SPF check: ip: %s, from: %s, helo: %s, result: %s' % (client_address, sender_email, helo_name, result))
action = DUNNO
message = None
if result == 'fail':
action = REJECT
message = explanation
elif result == 'temperror':
action = DEFER_IF_PERMIT
message = explanation
elif result == 'softfail' and softfail != DUNNO:
action = softfail
message = explanation
elif result == 'softerror' and softerror != DUNNO:
action = softerror
message = explanation
return action, message
def check_spf(domain, mx, record):
"""Test the SPF records on a given domain."""
results = {
'test': 'spf',
'passed': None,
'records': [{'domain': domain, 'type': 'TXT', 'value': record}],
'messages': []
}
try:
for ip, helo in mx:
spf_result, message = spf.check2(i=ip, s='admin@%s' % domain, h=helo)
results['succeeded'] = True
if spf_result == "none" and message == "":
message = "No SPF record set! Please set one"
results['messages'].append("(%s/%s) %s: %s" % (ip, helo, spf_result, message))
if spf_result == "pass" and results['passed'] in (None, True):
results['passed'] = True
else:
results['passed'] = False
except spf.AmbiguityWarning as ex:
results['succeeded'] = True
results['passed'] = False
results['message'] = str(ex)
except (spf.TempError, spf.PermError) as ex:
results['succeeded'] = False
results['message'] = str(ex)
return results
def _query_spf(self, timeout, ip, mx, sender_address):
self.ctxt.log.debug("SPF::Plugin %s",
"Querying the dns server(%s, %s, %s)..."
% (ip, mx, sender_address))
result, comment = spf.check2(i=ip, s=sender_address,
h=mx, timeout=timeout)
return result
def examine(self, suspect):
if not PYSPF_AVAILABLE:
suspect.debug("pyspf not available, can not check")
self._logger().warning(
"%s: SPF Check skipped, pyspf unavailable" % (suspect.id))
suspect.set_tag('SPF.status', 'skipped')
suspect.set_tag("SPF.explanation", 'missing dependency')
return DUNNO
clientinfo = suspect.get_client_info(self.config)
if clientinfo is None:
suspect.debug("client info not available for SPF check")
self._logger().warning(
"%s: SPF Check skipped, could not get client info" % (suspect.id))
suspect.set_tag('SPF.status', 'skipped')
suspect.set_tag(
"SPF.explanation", 'could not extract client information')
return DUNNO
helo, ip, revdns = clientinfo
result, explanation = spf.check2(ip, suspect.from_address, helo)
suspect.set_tag("SPF.status", result)
suspect.set_tag("SPF.explanation", explanation)
suspect.debug("SPF status: %s (%s)" % (result, explanation))
return DUNNO
def examine(self, suspect):
if not PYSPF_AVAILABLE:
suspect.debug("pyspf not available, can not check")
self._logger().warning(
"%s: SPF Check skipped, pyspf unavailable" % (suspect.id))
suspect.set_tag('SPF.status', 'skipped')
suspect.set_tag("SPF.explanation", 'missing dependency')
return DUNNO
clientinfo = suspect.get_client_info(self.config)
if clientinfo == None:
suspect.debug("client info not available for SPF check")
self._logger().warning(
"%s: SPF Check skipped, could not get client info" % (suspect.id))
suspect.set_tag('SPF.status', 'skipped')
suspect.set_tag(
"SPF.explanation", 'could not extract client information')
return DUNNO
helo, ip, revdns = clientinfo
result, explanation = spf.check2(ip, suspect.from_address, helo)
suspect.set_tag("SPF.status", result)
suspect.set_tag("SPF.explanation", explanation)
suspect.debug("SPF status: %s (%s)" % (result, explanation))
return DUNNO
def spf_pass(
envelope,
mailbox: Mailbox,
user: User,
alias: Alias,
contact_email: str,
msg: Message,
) -> bool:
ip = msg[_IP_HEADER]
if ip:
LOG.d("Enforce SPF on %s %s", ip, envelope.mail_from)
try:
r = spf.check2(i=ip, s=envelope.mail_from, h=None)
except Exception:
LOG.exception("SPF error, mailbox %s, ip %s", mailbox.email, ip)
else:
# TODO: Handle temperr case (e.g. dns timeout)
# only an absolute pass, or no SPF policy at all is 'valid'
if r[0] not in ["pass", "none"]:
LOG.w(
"SPF fail for mailbox %s, reason %s, failed IP %s",
mailbox.email,
r[0],
ip,
)
subject = get_header_unicode(msg["Subject"])
send_email_with_rate_control(
user,
ALERT_SPF,
mailbox.email,
f"SimpleLogin Alert: attempt to send emails from your alias {alias.email} from unknown IP Address",
render(
"transactional/spf-fail.txt",
alias=alias.email,
ip=ip,
mailbox_url=URL + f"/dashboard/mailbox/{mailbox.id}#spf",
to_email=contact_email,
subject=subject,
time=arrow.now(),
),
render(
"transactional/spf-fail.html",
ip=ip,
mailbox_url=URL + f"/dashboard/mailbox/{mailbox.id}#spf",
to_email=contact_email,
subject=subject,
time=arrow.now(),
),
)
return False
else:
LOG.w(
"Could not find %s header %s -> %s",
_IP_HEADER,
mailbox.email,
contact_email,
)
return True
async def handle_MAIL(self, server, session, envelope, address,
mail_options):
ip = session.peer[0]
result, description = spf.check2(ip, address, session.host_name)
envelope.mail_from = address
envelope.mail_options.extend(mail_options)
return "250 OK"
def _query_spf(self, timeout, ip, mx, sender_address):
self.ctxt.log.debug(
"SPF::Plugin %s", "Querying the dns server(%s, %s, %s)..." %
(ip, mx, sender_address))
result, comment = spf.check2(i=ip,
s=sender_address,
h=mx,
timeout=timeout)
return result
def _spf_lookup(self, ip, from_address, helo, retries=3):
spf.MAX_LOOKUP = self.config.getint(self.section, 'max_lookups')
result, explanation = spf.check2(ip, from_address, helo)
if result == 'temperror' and retries > 0:
time.sleep(self.config.getint(self.section, 'temperror_sleep'))
retries -= 1
result, explanation = self._spf_lookup(ip, from_address, helo,
retries)
return result, explanation
def main():
if len(sys.argv) != 4:
print('[' + os.path.basename(__file__) +
'] invalid number of arguments.')
sys.exit(1)
result, explanation = spf.check2(sys.argv[1], sys.argv[2], sys.argv[3])
print(result + ',' + explanation)
sys.exit(0)
def test_reason_in_message(self):
espf = EnforceSpf()
espf.set_enforcement('pass', match_code='250', match_message='{reason}')
class TestSession(object):
address = ('1.2.3.4', 56789)
envelope = None
ehlo_as = 'testehlo'
class TestValidators(object):
def __init__(self):
self.session = TestSession()
@espf.check
def validate_mail(self, reply, sender):
pass
spf.check2(i='1.2.3.4', s='*****@*****.**', h='testehlo').AndReturn(('pass', 'the reason'))
self.mox.ReplayAll()
validators = TestValidators()
reply = Reply('250', '2.0.0 Ok')
validators.validate_mail(reply, '*****@*****.**')
self.assertEqual('250', reply.code)
self.assertEqual('2.0.0 the reason', reply.message)
def test_on_validate_rcpt(self):
espf = EnforceSpf()
espf.set_enforcement('fail', match_code='550')
class TestSession(object):
address = ('1.2.3.4', 56789)
envelope = Envelope('*****@*****.**')
ehlo_as = 'testehlo'
class TestValidators(object):
def __init__(self):
self.session = TestSession()
@espf.check
def validate_rcpt(self, reply, recipient):
pass
spf.check2(i='1.2.3.4', s='*****@*****.**', h='testehlo').AndReturn(('fail', 'the reason'))
self.mox.ReplayAll()
validators = TestValidators()
reply = Reply('250', '2.0.0 Ok')
validators.validate_rcpt(reply, 'asdf')
self.assertEqual('550', reply.code)
self.assertEqual('5.7.1 Access denied', reply.message)
def main():
if len(sys.argv) != 4:
print('[' + os.path.basename(__file__) + '] invalid number of arguments.')
sys.exit(64)
result, explanation = spf.check2(sys.argv[1], sys.argv[2], sys.argv[3])
print('[' + os.path.basename(__file__) + '] (' + result + ', ' + explanation + ')')
if result == 'pass':
sys.exit(0)
else:
# Invalid spf, exit with code 11.
sys.exit(11)
def query(self, sender, ip, ehlo_as):
"""Performs a direct query to check the sender's domain to see if the
given IP and EHLO string are authorized to send for that domain.
:param sender: The sender address.
:param ip: The IP address string of the sending client.
:param ehlo_as: The EHLO string given by the sending client.
:returns: A tuple of the result and reason strings.
"""
result, reason = 'temperror', 'Timed out'
with gevent.Timeout(self.timeout, False):
result, reason = spf.check2(i=ip, s=sender, h=ehlo_as)
return result, reason
def query(self, sender, ip, ehlo_as):
"""Performs a direct query to check the sender's domain to see if the
given IP and EHLO string are authorized to send for that domain.
:param sender: The sender address.
:param ip: The IP address string of the sending client.
:param ehlo_as: The EHLO string given by the sending client.
:returns: A tuple of the result and reason strings.
"""
result, reason = 'temperror', 'Timed out'
with gevent.Timeout(self.timeout, False):
result, reason = spf.check2(i=ip, s=sender, h=ehlo_as)
return result, reason
def examine(self, suspect):
if not HAVE_SPF:
return DUNNO
client_address = suspect.get_value('client_address')
helo_name = suspect.get_value('helo_name')
sender = suspect.get_value('sender')
if client_address is None or helo_name is None or sender is None:
self.logger.error('missing client_address or helo or sender')
return DUNNO
if self.ip_whitelisted(client_address):
self.logger.info("Client %s is whitelisted - no SPF check" %
client_address)
return DUNNO
sender_email = strip_address(sender)
if sender_email == '' or sender_email is None:
return DUNNO
sender_domain = extract_domain(sender_email)
if sender_domain is None:
self.logger.error('no domain found in sender address %s' %
sender_email)
return DUNNO
if not self.check_this_domain(sender_domain):
self.logger.debug('skipping SPF check for %s' % sender_domain)
return DUNNO
result, explanation = spf.check2(client_address, sender_email,
helo_name)
suspect.tags['spf'] = result
if result != 'none':
self.logger.info(
'SPF client=%s, sender=%s, h=%s result=%s : %s' %
(client_address, sender_email, helo_name, result, explanation))
action = DUNNO
message = apply_template(
self.config.get(self.section, 'messagetemplate'), suspect,
dict(result=result, explanation=explanation))
configopt = 'on_%s' % result
if self.config.has_option(self.section, configopt):
action = string_to_actioncode(
self.config.get(self.section, configopt))
return action, message
def main():
if len(sys.argv) != 4:
print('[' + os.path.basename(__file__) +
'] invalid number of arguments.')
sys.exit(64)
result, explanation = spf.check2(sys.argv[1], sys.argv[2], sys.argv[3])
print('[' + os.path.basename(__file__) + '] (' + result + ', ' +
explanation + ')')
if result == 'pass':
sys.exit(0)
else:
# Invalid spf, exit with code 11.
sys.exit(11)
def examine(self,suspect):
if not have_spf:
return DUNNO
client_address=suspect.get_value('client_address')
helo_name=suspect.get_value('helo_name')
sender=suspect.get_value('sender')
if client_address is None or helo_name is None or sender is None:
self.logger.error('missing client_address or helo or sender')
return DUNNO
if self.ip_whitelisted(client_address):
self.logger.info("Client %s is whitelisted - no SPF check"%client_address)
return DUNNO
sender_email = strip_address(sender)
if sender_email=='' or sender_email is None:
return DUNNO
selective_sender_domain_file=self.config.get(self.section,'domain_selective_spf_file')
if selective_sender_domain_file!='':
if self.selective_domain_loader is None:
self.selective_domain_loader=FileList(selective_sender_domain_file,lowercase=True)
try:
sender_domain = extract_domain(sender_email)
if sender_domain is None:
return DUNNO
except ValueError as e:
self.logger.warning(str(e))
return DUNNO
if not sender_domain.lower() in self.selective_domain_loader.get_list():
return DUNNO
result, explanation = spf.check2(client_address, sender_email, helo_name)
suspect.tags['spf']=result
if result!='none':
self.logger.info('SPF client=%s, sender=%s, h=%s result=%s : %s' % (client_address, sender_email, helo_name, result,explanation))
action = DUNNO
message = apply_template(self.config.get(self.section,'messagetemplate'),suspect,dict(result=result,explanation=explanation))
configopt='on_%s'%result
if self.config.has_option(self.section,configopt):
action=self.config.get(self.section,configopt)
return action, message
def get_info(self, data):
try:
ip = ''
sender = ''
helo = ''
inputs = data.split('|')
ip = inputs[0]
sender = inputs[1]
if (len(inputs) == 3):
helo = inputs[2]
result = spf.check2(i=ip, s=sender, h=helo)
except SyntaxError:
print("Syntax Error")
return {"SpfCheck": result}
def doFilter(bodyFile, controlFileList):
"""Use the SPF mechanism to blacklist email."""
try:
sendersMta = courier.control.getSendersMta(controlFileList)
sendersIp = courier.control.getSendersIP(controlFileList)
sender = courier.control.getSender(controlFileList)
except:
return '451 Internal failure locating control files'
# Don't waste time on DSNs.
if sender == '':
return ''
helo = sendersMta.split(' ')[1]
(spfResult, spfExplanation) = spf.check2(sendersIp, sender, helo)
if spfResult == 'fail':
return '517 SPF returns deny'
return ''
async def handle_MAIL(self, server: SMTP, session: Session,
envelope: Envelope, address: str,
mail_options: list):
ip = session.peer[0]
result, description = spf.check2(ip, address, session.host_name)
valid_spf = result == 'pass'
envelope.spf = valid_spf
log.info("SPF: %s, %s", result, description)
if self.verify_spf and not valid_spf:
return '550 SPF validation failed'
envelope.mail_from = address
envelope.mail_options.extend(mail_options)
return '250 OK'
def joomla_version(target, port, timeout_sec, log_in_file, language,
time_sleep, thread_tmp_filename, socks_proxy, scan_id,
scan_cmd):
try:
s = conn(target, port, timeout_sec, socks_proxy)
if not s:
return False
else:
try:
global result
host_ip = socket.gethostbyname(target)
if version() is 2:
host_ip = host_ip.decode("utf-8")
result = spf.check2(host_ip, "admin@" + target, target)
return True
except Exception:
return False
except Exception as e:
return False
def examine(self,suspect):
if not HAVE_SPF:
return DUNNO
client_address=suspect.get_value('client_address')
helo_name=suspect.get_value('helo_name')
sender=suspect.get_value('sender')
if client_address is None or helo_name is None or sender is None:
self.logger.error('missing client_address or helo or sender')
return DUNNO
if self.ip_whitelisted(client_address):
self.logger.info("Client %s is whitelisted - no SPF check"%client_address)
return DUNNO
sender_email = strip_address(sender)
if sender_email=='' or sender_email is None:
return DUNNO
sender_domain = extract_domain(sender_email)
if sender_domain is None:
self.logger.error('no domain found in sender address %s' % sender_email)
return DUNNO
if not self.check_this_domain(sender_domain):
self.logger.debug('skipping SPF check for %s' % sender_domain)
return DUNNO
result, explanation = spf.check2(client_address, sender_email, helo_name)
suspect.tags['spf'] = result
if result != 'none':
self.logger.info('SPF client=%s, sender=%s, h=%s result=%s : %s' % (client_address, sender_email, helo_name, result,explanation))
action = DUNNO
message = apply_template(self.config.get(self.section, 'messagetemplate'), suspect, dict(result=result, explanation=explanation))
configopt = 'on_%s' % result
if self.config.has_option(self.section, configopt):
action=string_to_actioncode(self.config.get(self.section, configopt))
return action, message
if up_srv_helo == up_srv_hostn[0]:
iprev_res = "pass"
iprev_hn = up_srv_hostn[0]
else:
iprev_res = "fail"
except:
iprev_res = "temperror"
iprev_result = authres.IPRevAuthenticationResult(result=iprev_res,
policy_iprev=up_srv_ip,
policy_iprev_comment=iprev_hn)
results_list += [iprev_result]
### SPF CHECK
spf_result = spf.check2(i=up_srv_ip, s=sender_address, h=up_srv_helo)
spf_res = authres.SPFAuthenticationResult(result=spf_result[0],
smtp_mailfrom=sender_address,
smtp_helo=up_srv_helo,
reason=spf_result[1])
results_list += [spf_res]
# Write Received-SPF header (must be added ABOVE Received: lines for this server)
sys.stdout.write(
'Received-SPF: {0} ({1}) receiver={2}; client-ip={3}; helo={4}; envelope-from={5};'
.format(spf_result[0], spf_result[1], AUTHSERV_HOSTNAME, up_srv_ip,
up_srv_helo, sender_address).encode("utf-8") + linesep)
### ARC SIGNATURE
#import logging
#logging.basicConfig(level=10)
def runCheck(ip, s, h):
result = []
return result.append(spf.check2(ip, s, h))
# Exit if we can not get client IP
if len(remoteip) == 0:
print
exit()
# Exit if the host did not say hel(l)o
if not os.environ.get('SMTPHELOHOST'):
print
exit()
# Exit if there is no sender
if not os.environ.get('SMTPMAILFROM'):
print
exit()
# Run the SPF check
result, explanation = spf.check2(i=remoteip,
s=os.environ['SMTPMAILFROM'],
h=os.environ['SMTPHELOHOST'])
# Only fails will be rejected
if result == "fail":
print >> sys.stderr, "%d SPF check failed for %s" % (
os.getppid(), os.environ['SMTPMAILFROM'])
print "E451 SPF check failed [%s]" % explanation
else:
print
exit()
return(( 'DUNNO', '' ))
try:
import spf
except Exception, e:
syslog.syslog('ERROR: Unable to import SPF module.')
return(( 'DUNNO', '' ))
if self.debug >= 3:
syslog.syslog(syslog.LOG_DEBUG, 'SPF info: %s,%s,%s'
% ( self.data['client_address'], self.data['sender_domain'],
'unknown' ))
# SPF version 2 check
if hasattr(spf, 'check2'):
try:
ret = spf.check2(i = self.data['client_address'],
s = self.data['sender_domain'], h = 'unknown')
except Exception, e:
syslog.syslog('ERROR: SPF check failed: %s' % str(e))
return(( 'DUNNO', '' ))
spfResult = string.strip(ret[0])
spfReason = string.strip(ret[1])
spfResult = spfResult.lower()
spfResult = spfResult.capitalize()
if self.data.get('sender_domain'): identity = 'identity=mailfrom; '
else: identity = 'identity=helo; '
spfDetail = (identity + 'client-ip=%s; helo=%s; envelope-from=%s; '
'receiver=%s; '
% ( self.data.get('client_address', '<UNKNOWN>'),
self.data.get('helo_name', '<UNKNOWN>'),
self.data.get('sender', '<UNKNOWN>'),
self.data.get('recipient', '<UNKNOWN>'),
def email_analysis(filename, exclude_private_ip, check_spf):
urlList = []
hopList = []
hopListIP = []
domainList = []
attachmentsList = []
hopListIPnoPrivate = []
resultmeioc = {
"filename": os.path.basename(filename),
"from": None,
"sender": None,
"x-sender": None,
"to": None,
"cc": None,
"bcc": None,
"envelope-to": None,
"delivered-to": None,
"subject": None,
"x-originating-ip": None,
"relay_full": None,
"relay_ip": None,
"spf": None,
"urls": None,
"domains": None,
"attachments": None
}
with open(filename, "rb") as fp:
msg = BytesParser(policy=policy.default).parse(fp)
if msg:
#
# Header analysis
#
if msg["From"]:
# A sender obfuscation technique involves entering two e-mails. Only the last one is the real one. Example:
#
# Sender Name: Mario Rossi <*****@*****.**>
# Sender Mail: [email protected]
mail_from = re.findall(
"[A-Za-z0-9.!#$%&'*+\/=?^_`{|}~\-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,6}",
msg["From"], re.IGNORECASE)
if mail_from:
resultmeioc["from"] = mail_from[-1]
if msg["Sender"]:
mail_sender = re.findall(
"[A-Za-z0-9.!#$%&'*+\/=?^_`{|}~\-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,6}",
msg["Sender"], re.IGNORECASE)
if mail_sender:
resultmeioc["sender"] = mail_sender[-1]
if msg["X-Sender"]:
mail_xsender = re.findall(
"[A-Za-z0-9.!#$%&'*+\/=?^_`{|}~\-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,6}",
msg["X-Sender"], re.IGNORECASE)
if mail_xsender:
resultmeioc["x-sender"] = mail_xsender[-1]
if msg["To"]:
mail_to = re.findall(
"[A-Za-z0-9.!#$%&'*+\/=?^_`{|}~\-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,6}",
msg["To"], re.IGNORECASE)
if mail_to:
# Remove possible duplicates and create a numbered dictionary
mail_to = dict(
zip(range(len(list(set(mail_to)))), list(set(mail_to))))
resultmeioc["to"] = mail_to
if msg["Bcc"]:
resultmeioc["bcc"] = msg["Bcc"]
if msg["Cc"]:
mail_cc = re.findall(
"[A-Za-z0-9.!#$%&'*+\/=?^_`{|}~\-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,6}",
msg["Cc"], re.IGNORECASE)
if mail_cc:
# Remove possible duplicates and create a numbered dictionary
mail_cc = dict(
zip(range(len(list(set(mail_cc)))), list(set(mail_cc))))
resultmeioc["cc"] = mail_cc
if msg["Envelope-to"]:
mail_envelopeto = re.findall(
"[A-Za-z0-9.!#$%&'*+\/=?^_`{|}~\-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,6}",
msg["Envelope-to"], re.IGNORECASE)
if mail_envelopeto:
# Remove possible duplicates and create a numbered dictionary
mail_envelopeto = dict(
zip(range(len(list(set(mail_envelopeto)))),
list(set(mail_envelopeto))))
resultmeioc["envelope-to"] = mail_envelopeto
if msg["Delivered-To"]:
resultmeioc["delivered-to"] = msg["Delivered-To"]
if msg["X-Originating-IP"]:
# Usually the IP is in square brackets, I remove them if present.
mail_xorigip = msg["X-Originating-IP"].replace("[", "").replace(
"]", "")
resultmeioc["x-originating-ip"] = mail_xorigip
if msg["Subject"]:
resultmeioc["subject"] = msg["Subject"]
# Identify each relay
received = msg.get_all("Received")
if received:
received.reverse()
for line in received:
hops = re.findall(
"from\s+(.*?)\s+by(.*?)(?:(?:with|via)(.*?)(?:id|$)|id|$)",
line, re.DOTALL | re.X)
for hop in hops:
ipv4_address = re.findall(r"[0-9]+(?:\.[0-9]+){3}", hop[0],
re.DOTALL | re.X)
# https://gist.github.com/dfee/6ed3a4b05cfe7a6faf40a2102408d5d8
ipv6_address = re.findall(
r"(?:(?:(?:(?:[0-9a-fA-F]){1,4}):){1,4}:[^\s:](?:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])))|(?:::(?:ffff(?::0{1,4}){0,1}:){0,1}[^\s:](?:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])))|(?:fe80:(?::(?:(?:[0-9a-fA-F]){1,4})){0,4}%[0-9a-zA-Z]{1,})|(?::(?:(?::(?:(?:[0-9a-fA-F]){1,4})){1,7}|:))|(?:(?:(?:[0-9a-fA-F]){1,4}):(?:(?::(?:(?:[0-9a-fA-F]){1,4})){1,6}))|(?:(?:(?:(?:[0-9a-fA-F]){1,4}):){1,2}(?::(?:(?:[0-9a-fA-F]){1,4})){1,5})|(?:(?:(?:(?:[0-9a-fA-F]){1,4}):){1,3}(?::(?:(?:[0-9a-fA-F]){1,4})){1,4})|(?:(?:(?:(?:[0-9a-fA-F]){1,4}):){1,4}(?::(?:(?:[0-9a-fA-F]){1,4})){1,3})|(?:(?:(?:(?:[0-9a-fA-F]){1,4}):){1,5}(?::(?:(?:[0-9a-fA-F]){1,4})){1,2})|(?:(?:(?:(?:[0-9a-fA-F]){1,4}):){1,6}:(?:(?:[0-9a-fA-F]){1,4}))|(?:(?:(?:(?:[0-9a-fA-F]){1,4}):){1,7}:)|(?:(?:(?:(?:[0-9a-fA-F]){1,4}):){7,7}(?:(?:[0-9a-fA-F]){1,4}))",
hop[0], re.DOTALL | re.X)
if ipv4_address:
for ipv4 in ipv4_address:
if ipaddress.ip_address(ipv4):
hopListIP.append(ipv4)
if not ipaddress.ip_address(ipv4).is_private:
hopListIPnoPrivate.append(ipv4)
if ipv6_address:
for ipv6 in ipv6_address:
if ipaddress.ip_address(ipv6) and not "6::":
hopListIP.append(ipv6)
if not ipaddress.ip_address(ipv6).is_private:
hopListIPnoPrivate.append(ipv6)
if hop[0]:
hopList.append(hop[0])
if hopList:
resultmeioc["relay_full"] = dict(zip(range(len(hopList)), hopList))
if hopListIP:
if exclude_private_ip:
resultmeioc["relay_ip"] = dict(
zip(range(len(hopListIPnoPrivate)), hopListIPnoPrivate))
else:
resultmeioc["relay_ip"] = dict(
zip(range(len(hopListIP)), hopListIP))
#
# Body analysis
#
for part in msg.walk():
if part.get_content_type() == "text/plain":
# https://gist.github.com/dperini/729294
urlList.extend(
re.findall(
"(?:(?:(?:https?|ftp):)?\/\/)(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z0-9\u00a1-\uffff][a-z0-9\u00a1-\uffff_-]{0,62})?[a-z0-9\u00a1-\uffff]\.)+(?:[a-z\u00a1-\uffff]{2,}\.?))(?::\d{2,5})?(?:[/?#]\S*)?",
part.get_content(),
re.UNICODE | re.IGNORECASE | re.MULTILINE))
if part.get_content_type() == "text/html":
# The try/except is necessary, if the body of the eMail contains an incorrect or unencoded HTML code the script freeezes.
try:
soup = BeautifulSoup(part.get_content(), "html.parser")
tags = soup.find_all("a", href=True)
for url in tags:
urlList.append(url.get("href"))
except:
pass
if part.get_filename():
if part.get_payload(decode=True):
filename = part.get_filename()
filemd5 = hashlib.md5(
part.get_payload(decode=True)).hexdigest()
filesha1 = hashlib.sha1(
part.get_payload(decode=True)).hexdigest()
filesha256 = hashlib.sha256(
part.get_payload(decode=True)).hexdigest()
attachmentsList.append({
"filename": filename,
"MD5": filemd5,
"SHA1": filesha1,
"SHA256": filesha256
})
# Identify each domain reported in the eMail body
for url in urlList:
analyzeddomain = tldcache(url).registered_domain
if analyzeddomain:
domainList.append(analyzeddomain)
# Remove Duplicate
urlList = list(set(urlList))
domainList = list(set(domainList))
if urlList:
resultmeioc["urls"] = dict(zip(range(len(urlList)), urlList))
resultmeioc["domains"] = dict(
zip(range(len(domainList)), domainList))
if attachmentsList:
resultmeioc["attachments"] = attachmentsList
#
# Verify the SPF record if requested
#
if check_spf:
testspf = False
resultspf = ""
for ip in hopListIPnoPrivate:
if not testspf and "mail_from" in locals():
resultspf = spf.check2(ip, mail_from[-1],
mail_from[-1].split("@")[1])[0]
try:
resultspf = spf.check2(ip, mail_from[-1],
mail_from[-1].split("@")[1])[0]
except:
pass
if resultspf == "pass":
testspf = True
else:
testspf = False
resultmeioc["spf"] = testspf
print(json.dumps(resultmeioc, indent=4))
def check_spf(ip, mail_from, helo):
res, reason = spf.check2(ip, mail_from, helo)
return SPFAuthenticationResult(result=res,
reason=reason,
smtp_mailfrom=mail_from,
smtp_helo=helo)
logger_1.debug("The CustomerNumber:" + k + " has no domains associated with it")
TeamDictonary[CustomerNumber] = ParsingList
x += 1
for k, v in TeamDictonary.items():
z = 0
eachcounter = 0
ValuesLength = len(v)
if ValuesLength > 0:
while z < ValuesLength:
for each in IPList:
if v[z] != "" and v[z] != None:
result = spf.check2(
each, "test@" + v[z], v[z]
) ##This is where we will call to check this specific domain result should return true/false
logger_1.debug("trying " + each)
if result[0] != "pass":
newresult = spf.check2(each, "test@" + v[z], v[z]) ##double check
if newresult[0] != "pass":
logger_1.debug(
"The domain "
+ v[z]
+ " has failed validation on "
+ each
+ ". Located in CustomerNumber:"
+ k
)
else:
logger_1.debug(
error = True
if sender is None:
print("No sender (-s) passed!")
error = True
if code is None:
print("No RFC4408-compliant validation response (-r) passed!")
error = True
if error:
print("Exiting because of errors...\n")
exit(EXIT_UNKNOWN)
try:
results = spf.check2(i=ip,s=sender,h=domain,timeout=float(timeout))
except:
print "Exception ocurred when trying to check SPF record: ", sys.exc_info()
exit(EXIT_UNKNOWN)
real_code = results[0]
msg = results[1]
detail = Template(detail_template).substitute(ip=ip,sender=sender,domain=domain,code=real_code)
if code == real_code:
print "CHECK_SPF OK:",detail
exit(EXIT_OK)
elif warning_mode:
print "CHECK_SPF WARNING:",detail
exit(EXIT_WARNING)
else:
