def process_message(msgMailRequest):
"""
Shiva - This function gets called only when a spam has to be relayed
"""
# Bound to send a parameter to QueueReceiver as its __init__ needs at least one argument
# Passing it variable queuePath as, anyway, variable queue_dir in server.py gets the same string from settings.py module
processMessage = server.QueueReceiver(queuePath) # Instantiating an object
processMessage.process_message(
msgMailRequest
) # process_message sends msgMailRequest to router for delivery
def main(mailFields, matchedHash, key, msgMailRequest):
logging.info("[+]Inside shivaprocessold Module.")
relay_enabled = server.shivaconf.getboolean('analyzer', 'relay')
records = server.QueueReceiver.records
for record in records:
if record['s_id'] == matchedHash:
if mailFields['attachmentFileMd5']:
i = 0
while i < len(mailFields['attachmentFileMd5']):
if mailFields['attachmentFileMd5'][i] not in record['attachmentFileMd5']:
record['attachmentFile'].append(mailFields['attachmentFile'][i])
record['attachmentFileMd5'].append(mailFields['attachmentFileMd5'][i])
record['attachmentFileName'].append(mailFields['attachmentFileName'][i])
i += 1
if mailFields['links']:
for newLink in mailFields['links']:
if newLink not in record['links']:
record['links'].append(newLink)
if record['inlineFileMd5'] != mailFields['inlineFileMd5']:
i = 0
while i < len(mailFields['inlineFileMd5']):
if mailFields['inlineFileMd5'][i] not in record['inlineFileMd5']:
record['inlineFile'].append(mailFields['inlineFile'][i])
record['inlineFileMd5'].append(mailFields['inlineFileMd5'][i])
record['inlineFileName'].append(mailFields['inlineFileName'][i])
i += 1
ipList = record['sourceIP'].split(", ")
if mailFields['sourceIP'] not in ipList:
record['sourceIP'] = record['sourceIP'] + ", " + mailFields['sourceIP']
sensorIDs = record['sensorID'].split(", ")
if mailFields['sensorID'] not in sensorIDs:
record['sensorID'] = mailFields['sensorID'] + ", " + record['sensorID']
recipients = record['to'].split(",")
if mailFields['to'] not in recipients:
record['to'] = record['to'] + "," + mailFields['to']
record['counter'] += 1
logging.info("value of record counter has reached: %s" % record['counter'])
if relay_enabled is True:
relaycounter = server.shivaconf.getint('analyzer', 'globalcounter')
if (int(server.QueueReceiver.totalRelay) > relaycounter):
logging.info("[+]shivaprocessold Module: Limit reached. No relay.")
#individualcounter = server.shivaconf.getint('analyzer', 'individualcounter')
elif next((i for i, sublist in enumerate([myval for myval in server.whitelist_ids.values()]) if mailFields['to'] in sublist), -1) > -1:
logging.info("[+]shivaprocessold Module: Recipient found in white list - relaying")
# Following 3 lines does the relaying
queuePath = server.shivaconf.get('global', 'queuepath')
processMessage = server.QueueReceiver(queuePath)
processMessage.process_message(msgMailRequest)
record['relayed'] += 1
server.QueueReceiver.totalRelay += 1
else:
if record['counter'] <= 11:
if record['counter'] == 11:
logging.info("counter is = 11")
logging.info("automated scanning has started - Not relaying anymore")
server.whitelist_ids.pop(mailFields['s_id'], None)
logging.info("poping automated key")
for key, value in server.whitelist_ids.items():
logging.info("key: %s, value: %s" % (key, value))
else:
logging.info("[+]shivaprocessold Module: Adding recipient to whitelist and relaying")
if mailFields['s_id'] in server.whitelist_ids:
logging.info("spam-id in whitlist - extending")
server.whitelist_ids[mailFields['s_id']].append(mailFields['to'])
#mailFields['attachmentFileName'].append(fileName)
else:
logging.info("spam-id not in whitelist - adding")
server.whitelist_ids[mailFields['s_id']] = mailFields['to'].split()
logging.info("\n\nprocessold after adding new recipient\n\n")
for key, value in server.whitelist_ids.items():
logging.info("key: %s, value: %s" % (key, value))
# Following 3 lines does the relaying
queuePath = server.shivaconf.get('global', 'queuepath')
processMessage = server.QueueReceiver(queuePath)
processMessage.process_message(msgMailRequest)
record['relayed'] += 1
server.QueueReceiver.totalRelay += 1
def main(mailFields, key, msgMailRequest):
"""Main function.
Stores the parsed fields as dictionary and then appends it to our
temporary list.
"""
logging.info("Inside shivaaddnewrecord Module.")
rawspampath = server.shivaconf.get('analyzer', 'rawspampath')
queuepath = server.shivaconf.get('global', 'queuepath')
relay_enabled = server.shivaconf.getboolean('analyzer', 'relay')
records = server.QueueReceiver.records
source = queuepath + "/new/" + key
filename = mailFields['s_id'] + "-" + key
destination = rawspampath + filename
shutil.copy2(source, destination) # shutil.copy2() copies the meta-data too
newRecord = { 'headers':mailFields['headers'],
'to':mailFields['to'],
'from':mailFields['from'],
'subject':mailFields['subject'],
'date':mailFields['date'],
'firstSeen':mailFields['firstSeen'],
'lastSeen':mailFields['lastSeen'],
'firstRelayed':mailFields['firstRelayed'],
'lastRelayed':mailFields['lastRelayed'],
'sourceIP':mailFields['sourceIP'],
'sensorID':mailFields['sensorID'],
'text':mailFields['text'],
'html':mailFields['html'],
'inlineFileName':mailFields['inlineFileName'],
'inlineFile':mailFields['inlineFile'],
'inlineFileMd5':mailFields['inlineFileMd5'],
'attachmentFileName': mailFields['attachmentFileName'],
'attachmentFile':mailFields['attachmentFile'],
'attachmentFileMd5':mailFields['attachmentFileMd5'],
'links':mailFields['links'],
'ssdeep':mailFields['ssdeep'],
's_id':mailFields['s_id'],
'len':mailFields['len'],
'counter':1,
'relayed':0 }
if relay_enabled is True:
relaycounter = server.shivaconf.getint('analyzer', 'globalcounter')
if (int(server.QueueReceiver.totalRelay) > relaycounter):
logging.info("[+]shivaaddnewrecord Module: Limit reached. No relay.")
elif next((i for i, sublist in enumerate([myval for myval in server.whitelist_ids.values()]) if mailFields['to'] in sublist), -1) > -1:
logging.info("[+]shivaaddnewrecord Module: Recipient found in white list - relaying")
# Following 3 lines does the relaying
queuePath = server.shivaconf.get('global', 'queuepath')
processMessage = server.QueueReceiver(queuePath)
processMessage.process_message(msgMailRequest)
newRecord['relayed'] += 1
server.QueueReceiver.totalRelay += 1
else:
logging.info("[+]shivaaddnewrecord Module: Adding recipient to whitelist and relaying")
server.whitelist_ids[mailFields['s_id']] = mailFields['to'].split()
for key, value in server.whitelist_ids.items():
logging.info("key: %s, value: %s" % (key, value))
# Following 3 lines does the relaying
queuePath = server.shivaconf.get('global', 'queuepath')
processMessage = server.QueueReceiver(queuePath)
processMessage.process_message(msgMailRequest)
newRecord['relayed'] += 1
server.QueueReceiver.totalRelay += 1
records.insert(0, newRecord) #Inserting new record at the first position.
del newRecord
def main(self, mailFields, matchedHash, key, msgMailRequest):
logging.info("[+]Inside artemisprocessold Module.")
records = server.QueueReceiver.records
for record in records:
if record['s_id'] == matchedHash:
if mailFields['attachmentFileMd5']:
i = 0
while i < len(mailFields['attachmentFileMd5']):
if mailFields['attachmentFileMd5'][i] not in record[
'attachmentFileMd5']:
record['attachmentFile'].append(
mailFields['attachmentFile'][i])
record['attachmentFileMd5'].append(
mailFields['attachmentFileMd5'][i])
record['attachmentFileName'].append(
mailFields['attachmentFileName'][i])
i += 1
if mailFields['links']:
for newLink in mailFields['links']:
if newLink not in record['links']:
record['links'].append(newLink)
if record['inlineFileMd5'] != mailFields['inlineFileMd5']:
i = 0
while i < len(mailFields['inlineFileMd5']):
if mailFields['inlineFileMd5'][i] not in record[
'inlineFileMd5']:
record['inlineFile'].append(
mailFields['inlineFile'][i])
record['inlineFileMd5'].append(
mailFields['inlineFileMd5'][i])
record['inlineFileName'].append(
mailFields['inlineFileName'][i])
i += 1
ipList = record['sourceIP'].split(", ")
if mailFields['sourceIP'] not in ipList:
record['sourceIP'] = record[
'sourceIP'] + ", " + mailFields['sourceIP']
sensorIDs = record['sensorID'].split(", ")
if mailFields['sensorID'] not in sensorIDs:
record['sensorID'] = mailFields[
'sensorID'] + ", " + record['sensorID']
recipients = record['to'].split(",")
if mailFields['to'] not in recipients:
record['to'] = record['to'] + "," + mailFields['to']
user_list = record['user'].split(", ")
if mailFields['user'] not in user_list:
record['user'] = record['user'] + ", " + mailFields['user']
record['counter'] += 1
logging.info("value of record counter has reached: %s" %
record['counter'])
if self.relay is True:
if mailFields['to'].split(
"@")[1] in self.blackhole_domains:
logging.info(
"Email in blackhole_domains, skipping relay")
else:
if (int(server.QueueReceiver.totalRelay) >
self.globalcounter):
logging.info(
"[+]artemisprocessold Module: Limit reached. No relay."
)
elif next((i for i, sublist in enumerate(
[myval for myval in server.whitelist_ids.values()])
if mailFields['to'] in sublist), -1) > -1:
logging.info(
"[+]artemisprocessold Module: Recipient found in white list - relaying"
)
# Following 2 lines do the relaying
processMessage = server.QueueReceiver(
self.queuepath)
processMessage.process_message(msgMailRequest)
record['relayed'] += 1
server.QueueReceiver.totalRelay += 1
else:
if record['counter'] <= 11:
if record['counter'] == 11:
logging.info("counter is = 11")
logging.info(
"automated scanning has started - Not relaying anymore"
)
server.whitelist_ids.pop(
mailFields['s_id'], None)
logging.info("poping automated key")
for key, value in server.whitelist_ids.items(
):
logging.info("key: %s, value: %s" %
(key, value))
else:
logging.info(
"[+]artemisprocessold Module: Adding recipient to whitelist and relaying"
)
if mailFields[
's_id'] in server.whitelist_ids:
logging.info(
"spam-id in whitlist - extending")
server.whitelist_ids[
mailFields['s_id']].append(
mailFields['to'])
else:
logging.info(
"spam-id not in whitelist - adding"
)
server.whitelist_ids[
mailFields['s_id']] = mailFields[
'to'].split()
logging.info(
"\n\nprocessold after adding new recipient\n\n"
)
for key, value in server.whitelist_ids.items(
):
logging.info("key: %s, value: %s" %
(key, value))
# Following 2 lines do the relaying
processMessage = server.QueueReceiver(
self.queuepath)
processMessage.process_message(
msgMailRequest)
record['relayed'] += 1
server.QueueReceiver.totalRelay += 1
def main(mailFields, key, msgMailRequest):
"""Main function.
Stores the parsed fields as dictionary and then appends it to our
temporary list.
"""
logging.info("Inside shivaaddnewrecord Module.")
rawspampath = server.shivaconf.get('analyzer', 'rawspampath')
queuepath = server.shivaconf.get('global', 'queuepath')
relay_enabled = server.shivaconf.getboolean('analyzer', 'relay')
records = server.QueueReceiver.records
source = queuepath + "/new/" + key
filename = mailFields['s_id'] + "-" + key
probability_tuple = (0, 0)
url_phishing = False
phish_flag = None
phishing_human_check = None
# check whether email is imported manually
sensor = mailFields['sensorID']
if not sensor:
sensor = 'default'
if re.match('.*phishingImport.*', sensor):
probability_tuple = (-1, -1)
phish_flag = True
phishing_human_check = True
elif re.match('.*spamImport.*', sensor):
probability_tuple = (-1, -1)
phish_flag = False
phishing_human_check = False
else:
# email is not manually imported, compute score
email_verdict = check_mail(mailFields)
probability_tuple = (email_verdict['shiva_prob'],
email_verdict['sa_prob'])
url_phishing = email_verdict['urlPhishing']
phish_flag = email_verdict['verdict']
if phish_flag:
destination = rawspampath + "phishing/" + filename
else:
destination = rawspampath + "spam/" + filename
shutil.copy2(source,
destination) # shutil.copy2() copies the meta-data too
newRecord = {
'headers': mailFields['headers'],
'to': mailFields['to'],
'from': mailFields['from'],
'subject': mailFields['subject'],
'date': mailFields['date'],
'firstSeen': mailFields['firstSeen'],
'lastSeen': mailFields['lastSeen'],
'firstRelayed': mailFields['firstRelayed'],
'lastRelayed': mailFields['lastRelayed'],
'sourceIP': mailFields['sourceIP'],
'sensorID': mailFields['sensorID'],
'text': mailFields['text'],
'html': mailFields['html'],
'inlineFileName': mailFields['inlineFileName'],
'inlineFile': mailFields['inlineFile'],
'inlineFileMd5': mailFields['inlineFileMd5'],
'attachmentFileName': mailFields['attachmentFileName'],
'attachmentFile': mailFields['attachmentFile'],
'attachmentFileMd5': mailFields['attachmentFileMd5'],
'links': mailFields['links'],
'ssdeep': mailFields['ssdeep'],
's_id': mailFields['s_id'],
'len': mailFields['len'],
'phishingHumanCheck': phishing_human_check,
'derivedPhishingStatus': phish_flag,
'shivaScore': probability_tuple[0],
'spamassassinScore': probability_tuple[1],
'urlPhishing': url_phishing,
'counter': 1,
'relayed': 0
}
if relay_enabled is True:
relaycounter = server.shivaconf.getint('analyzer', 'globalcounter')
if (int(server.QueueReceiver.totalRelay) > relaycounter):
logging.info(
"[+]shivaaddnewrecord Module: Limit reached. No relay.")
elif next((i for i, sublist in enumerate(
[myval for myval in server.whitelist_ids.values()])
if mailFields['to'] in sublist), -1) > -1:
logging.info(
"[+]shivaaddnewrecord Module: Recipient found in white list - relaying"
)
# Following 3 lines does the relaying
queuePath = server.shivaconf.get('global', 'queuepath')
processMessage = server.QueueReceiver(queuePath)
processMessage.process_message(msgMailRequest)
newRecord['relayed'] += 1
server.QueueReceiver.totalRelay += 1
else:
logging.info(
"[+]shivaaddnewrecord Module: Adding recipient to whitelist and relaying"
)
server.whitelist_ids[mailFields['s_id']] = mailFields['to'].split()
for key, value in server.whitelist_ids.items():
logging.info("key: %s, value: %s" % (key, value))
# Following 3 lines does the relaying
queuePath = server.shivaconf.get('global', 'queuepath')
processMessage = server.QueueReceiver(queuePath)
processMessage.process_message(msgMailRequest)
newRecord['relayed'] += 1
server.QueueReceiver.totalRelay += 1
records.insert(0, newRecord) #Inserting new record at the first position.
del newRecord
def main(self, mailFields, key, msgMailRequest):
"""Main function.
Stores the parsed fields as dictionary and then appends it to our
temporary list.
"""
logging.info("Inside artemisaddnewrecord Module.")
records = server.QueueReceiver.records
source = self.queuepath + "/new/" + key
filename = mailFields['s_id'] + "-" + key
destination = self.rawspampath + filename
shutil.copy2(source,
destination) # shutil.copy2() copies the meta-data too
newRecord = {
'headers': mailFields['headers'],
'to': mailFields['to'],
'from': mailFields['from'],
'subject': mailFields['subject'],
'date': mailFields['date'],
'firstSeen': mailFields['firstSeen'],
'lastSeen': mailFields['lastSeen'],
'firstRelayed': mailFields['firstRelayed'],
'lastRelayed': mailFields['lastRelayed'],
'sourceIP': mailFields['sourceIP'],
'sensorID': mailFields['sensorID'],
'text': mailFields['text'],
'html': mailFields['html'],
'inlineFileName': mailFields['inlineFileName'],
'inlineFile': mailFields['inlineFile'],
'inlineFileMd5': mailFields['inlineFileMd5'],
'attachmentFileName': mailFields['attachmentFileName'],
'attachmentFile': mailFields['attachmentFile'],
'attachmentFileMd5': mailFields['attachmentFileMd5'],
'links': mailFields['links'],
'ssdeep': mailFields['ssdeep'],
's_id': mailFields['s_id'],
'len': mailFields['len'],
'user': mailFields['user'],
'counter': 1,
'relayed': 0
}
if self.relay is True:
if mailFields['to'].split("@")[1] in self.blackhole_domains:
logging.info("Email in blackhole_domains, skipping relay")
else:
if (int(server.QueueReceiver.totalRelay) > self.globalcounter):
logging.info(
"[+]artemisaddnewrecord Module: Limit reached. No relay."
)
elif next((i for i, sublist in enumerate(
[myval for myval in server.whitelist_ids.values()])
if mailFields['to'] in sublist), -1) > -1:
logging.info(
"[+]artemisaddnewrecord Module: Recipient found in white list - relaying"
)
# Following 2 lines do the relaying
processMessage = server.QueueReceiver(self.queuepath)
processMessage.process_message(msgMailRequest)
newRecord['relayed'] += 1
server.QueueReceiver.totalRelay += 1
else:
logging.info(
"[+]artemisaddnewrecord Module: Adding recipient to whitelist and relaying"
)
server.whitelist_ids[
mailFields['s_id']] = mailFields['to'].split()
for key, value in server.whitelist_ids.items():
logging.info("key: %s, value: %s" % (key, value))
# Following 2 lines do the relaying
processMessage = server.QueueReceiver(self.queuepath)
processMessage.process_message(msgMailRequest)
newRecord['relayed'] += 1
server.QueueReceiver.totalRelay += 1
#Inserting new record at the first position
records.insert(0, newRecord)
del newRecord