def run(self):
try:
while True:
# Wait for a message from the channel that a TW was modified
message = self.c1.get_message(timeout=self.timeout)
#print('Message received from channel {} with data {}'.format(message['channel'], message['data']))
if message['data'] == 'stop_process':
return True
elif message['channel'] == 'tw_modified':
# Get the profileid and twid
try:
profileid = message['data'].split(':')[0]
twid = message['data'].split(':')[1]
# Start of the port scan detection
self.print(
'Running the detection of portscans in profile {} TW {}'
.format(profileid, twid), 6, 0)
# For port scan detection, we will measure different things:
# 1. Vertical port scan:
# - 1 srcip sends not established flows to > 3 dst ports in the same dst ip. Any number of packets
# 2. Horizontal port scan:
# - 1 srcip sends not established flows to the same dst ports in > 3 dst ip.
# 3. Too many connections???:
# - 1 srcip sends not established flows to the same dst ports, > 3 pkts, to the same dst ip
# 4. Slow port scan. Same as the others but distributed in multiple time windows
# Remember that in slips all these port scans can happen for traffic going IN to an IP or going OUT from the IP.
# Get the list of dports that we connected as client using TCP not established
direction = 'Dst'
state = 'NotEstablished'
protocol = 'TCP'
role = 'Client'
type_data = 'Ports'
data = __database__.getDataFromProfileTW(
profileid, twid, direction, state, protocol, role,
type_data)
# For each port, see if the amount is over the threshold
for dport in data.keys():
"""
###
# PortScan Type 3. Direction OUT
# Considering all the flows in this TW, for all the Dst IP, get the sum of all the pkts send to each dst port TCP No tEstablished
totalpkts = int(data[dport]['totalpkt'])
# If for each port, more than X amount of packets were sent, report an evidence
if totalpkts > 3:
# Type of evidence
type_evidence = 'PortScanType3'
# Key
key = 'dport' + ':' + dport + ':' + type_evidence
# Description
description = 'Too Many Not Estab TCP to same port {} from IP: {}. Amount: {}'.format(dport, profileid.split('_')[1], totalpkts)
# Threat level
threat_level = 50
# Confidence. By counting how much we are over the threshold.
if totalpkts >= 10:
# 10 pkts or more, receive the max confidence
confidence = 1
else:
# Between 3 and 10 pkts compute a kind of linear grow
confidence = totalpkts / 10.0
__database__.setEvidence(profileid, twid, type_evidence, threat_level, confidence)
self.print('Too Many Not Estab TCP to same port {} from IP: {}. Amount: {}'.format(dport, profileid.split('_')[1], totalpkts),6,0)
"""
### PortScan Type 2. Direction OUT
dstips = data[dport]['dstips']
amount_of_dips = len(dstips)
# If we contacted more than 3 dst IPs on this port with not established connections.. we have evidence
#self.print('Horizontal Portscan check. Amount of dips: {}. Threshold=3'.format(amount_of_dips), 3, 0)
# Type of evidence
type_evidence = 'PortScanType2'
# Key
key = 'dport' + ':' + dport + ':' + type_evidence
# Threat level
threat_level = 50
# Compute the confidence
pkts_sent = 0
# We detect a scan every Threshold. So we detect when there is 3, 6, 9, 12, etc. dips per port.
# The idea is that after X dips we detect a connection. And then we 'reset' the counter until we see again X more.
cache_key = profileid + ':' + twid + ':' + key
try:
prev_amount_dips = self.cache_det_thresholds[
cache_key]
except KeyError:
prev_amount_dips = 0
#self.print('Key: {}. Prev dips: {}, Current: {}'.format(cache_key, prev_amount_dips, amount_of_dips))
if amount_of_dips % 3 == 0 and prev_amount_dips < amount_of_dips:
for dip in dstips:
# Get the total amount of pkts sent to the same port to all IPs
pkts_sent += dstips[dip]
if pkts_sent > 10:
confidence = 1
else:
# Between 3 and 10 pkts compute a kind of linear grow
confidence = pkts_sent / 10.0
# Description
description = 'New horizontal port scan detected to port {}. Not Estab TCP from IP: {}. Tot pkts sent all IPs: {}'.format(
dport,
profileid.split(self.fieldseparator)[1],
pkts_sent, confidence)
__database__.setEvidence(key,
threat_level,
confidence,
description,
profileid=profileid,
twid=twid)
self.print(description, 3, 0)
# Store in our local cache how many dips were there:
self.cache_det_thresholds[
cache_key] = amount_of_dips
# Get the list of dstips that we connected as client using TCP not established, and their ports
direction = 'Dst'
state = 'NotEstablished'
protocol = 'TCP'
role = 'Client'
type_data = 'IPs'
data = __database__.getDataFromProfileTW(
profileid, twid, direction, state, protocol, role,
type_data)
# For each dstip, see if the amount of ports connections is over the threshold
for dstip in data.keys():
### PortScan Type 1. Direction OUT
# dstports is a dict
dstports = data[dstip]['dstports']
amount_of_dports = len(dstports)
#self.print('Vertical Portscan check. Amount of dports: {}. Threshold=3'.format(amount_of_dports), 3, 0)
# Type of evidence
type_evidence = 'PortScanType1'
# Key
key = 'dstip' + ':' + dstip + ':' + type_evidence
# Threat level
threat_level = 50
# We detect a scan every Threshold. So we detect when there is 3, 6, 9, 12, etc. dports per dip.
# The idea is that after X dips we detect a connection. And then we 'reset' the counter until we see again X more.
cache_key = profileid + ':' + twid + ':' + key
try:
prev_amount_dports = self.cache_det_thresholds[
cache_key]
except KeyError:
prev_amount_dports = 0
#self.print('Key: {}, Prev dports: {}, Current: {}'.format(cache_key, prev_amount_dports, amount_of_dports))
if amount_of_dports % 3 == 0 and prev_amount_dports < amount_of_dports:
# Compute the confidence
pkts_sent = 0
for dport in dstports:
# Get the total amount of pkts sent to the same port to all IPs
pkts_sent += dstports[dport]
if pkts_sent > 10:
confidence = 1
else:
# Between 3 and 10 pkts compute a kind of linear grow
confidence = pkts_sent / 10.0
# Description
description = 'New vertical port scan detected to IP {} from {}. Total {} dst ports. Not Estab TCP. Tot pkts sent all ports: {}'.format(
dstip,
profileid.split(self.fieldseparator)[1],
amount_of_dports, pkts_sent, confidence)
__database__.setEvidence(key,
threat_level,
confidence,
description,
profileid=profileid,
twid=twid)
self.print(description, 3, 0)
# Store in our local cache how many dips were there:
self.cache_det_thresholds[
cache_key] = amount_of_dports
except AttributeError:
# When the channel is created the data '1' is sent
continue
except KeyboardInterrupt:
self.print('Stopping the process', 0, 1)
return True
except Exception as inst:
self.print('Error in run() of {}'.format(inst), 0, 1)
self.print(type(inst), 0, 1)
self.print(inst, 0, 1)
def process_global_data(self):
"""
This is the main function called by the timer process
Read the global data and output it on logs
"""
try:
#1. Get the list of profiles modified
# How many profiles we have?
profilesLen = str(__database__.getProfilesLen())
# Get the list of all the modifed TW for all the profiles
TWModifiedforProfile = __database__.getModifiedTWLogs()
amount_of_modified = len(TWModifiedforProfile)
self.outputqueue.put(
'20|logs|[Logs] Number of Profiles in DB: {}. Modified TWs: {}. ({})'
.format(profilesLen, amount_of_modified,
datetime.now().strftime('%Y-%m-%d--%H:%M:%S')))
last_profile_id = None
description_of_malicious_ip_profile = None
for profileTW in TWModifiedforProfile:
# Get the profileid and twid
profileid = profileTW.split(
self.fieldseparator
)[0] + self.fieldseparator + profileTW.split(
self.fieldseparator)[1]
twid = profileTW.split(self.fieldseparator)[2]
# Get the time of this TW. For the file name
twtime = __database__.getTimeTW(profileid, twid)
twtime = time.strftime('%Y-%m-%dT%H:%M:%S',
time.localtime(twtime))
self.print(
'\tStoring Profile: {}. TW {}. Time: {}'.format(
profileid, twid, twtime), 0, 2)
#self.print('\tProfile: {} has {} timewindows'.format(profileid, twLen), 0, 3)
# Create the folder for this profile if it doesn't exist
profilefolder = self.createProfileFolder(profileid)
# Create the TW log file
twlog = twtime + '.' + twid
# First Erase its file and save the data again
self.addDataToFile(profilefolder + '/' + twlog,
'',
file_mode='w+',
data_mode='raw')
# Save in the log file all parts of the profile
# 0. Write the profileID for people getting know what they see in the file.
self.addDataToFile(profilefolder + '/' + twlog,
'ProfileID: {}\n'.format(profileid),
file_mode='a+',
data_type='text')
# 0. Is a ip of this profile stored as malicious?
# If it still one profile do not ask again the database for each new time_window.
if last_profile_id != profileid:
description_of_malicious_ip_profile = __database__.is_profile_malicious(
profileid)
if description_of_malicious_ip_profile:
ip_of_profile = profileid.split(self.separator)[1]
text_data = '[THREAT INTELIGENCE] IP of this profile: {} was detected as malicious. Description: "{}"\n'.format(
ip_of_profile, description_of_malicious_ip_profile)
self.addDataToFile(profilefolder + '/' + twlog,
text_data,
file_mode='a+',
data_type='text')
# 1. Detections to block. The getBlockingRequest function return {True, False}
blocking = __database__.getBlockingRequest(profileid, twid)
if blocking:
text_data = 'Was requested to block in this time window: ' + str(
blocking)
self.addDataToFile(profilefolder + '/' + twlog,
text_data,
file_mode='a+',
data_type='json')
self.outputqueue.put(
'03|logs|\t\t[Logs] Blocking Request: ' +
str(blocking))
# 2. Info about the evidence so far for this TW.
evidence = __database__.getEvidenceForTW(profileid, twid)
if evidence:
evidence = json.loads(evidence)
self.addDataToFile(profilefolder + '/' + twlog,
'Evidence of detections in this TW:',
file_mode='a+',
data_type='text')
self.outputqueue.put(
'03|logs|\t\t[Logs] Evidence of detections in this TW:'
)
for data in evidence:
self.addDataToFile(profilefolder + '/' + twlog,
'\tEvidence: {}'.format(data[0]),
file_mode='a+',
data_type='text')
self.outputqueue.put(
'03|logs|\t\t\t Evidence: {}'.format(data[0]))
# 3. DstIPs
dstips = __database__.getDstIPsfromProfileTW(profileid, twid)
if dstips:
# Add dstips to log file
self.addDataToFile(profilefolder + '/' + twlog,
'DstIP:',
file_mode='a+',
data_type='text')
self.outputqueue.put('03|logs|\t\t[Logs] DstIP:')
data = json.loads(dstips)
# Better printing of data
for key in data:
ip_info = __database__.getIPData(key)
if ip_info:
printable_ip_info = ', '.join(
'{} {}'.format(k, v)
for k, v in ip_info.items())
else:
printable_ip_info = '-'
self.addDataToFile(profilefolder + '/' + twlog,
'\t{} ({} times). Info: {}'.format(
key, data[key],
printable_ip_info),
file_mode='a+',
data_type='text')
self.outputqueue.put('03|logs|\t\t[Logs] DstIP: ' + dstips)
# 4. SrcIPs
srcips = __database__.getSrcIPsfromProfileTW(profileid, twid)
if srcips:
# Add srcips
self.addDataToFile(profilefolder + '/' + twlog,
'SrcIP:',
file_mode='a+',
data_type='text')
self.outputqueue.put('03|logs|\t\t[Logs] SrcIP:')
data = json.loads(srcips)
for key in data:
ip_info = __database__.getIPData(key)
if ip_info:
printable_ip_info = ', '.join(
'{} {}'.format(k, v)
for k, v in ip_info.items())
else:
printable_ip_info = '-'
self.addDataToFile(profilefolder + '/' + twlog,
'\t{} ({} times). Info: {}'.format(
key, data[key],
printable_ip_info),
file_mode='a+',
data_type='text')
self.outputqueue.put(
'03|logs|\t\t\t[Logs] {} ({} times)'.format(
key, data[key]))
# 5. OutTuples
out_tuples = __database__.getOutTuplesfromProfileTW(
profileid, twid)
if out_tuples:
# Add tuples
self.addDataToFile(profilefolder + '/' + twlog,
'OutTuples:',
file_mode='a+',
data_type='text')
self.outputqueue.put('03|logs|\t\t[Logs] OutTuples:')
data = json.loads(out_tuples)
for key in data:
self.addDataToFile(profilefolder + '/' + twlog,
'\t{} ({})'.format(key, data[key]),
file_mode='a+',
data_type='text')
self.outputqueue.put(
'03|logs|\t\t\t[Logs] {} ({})'.format(
key, data[key]))
self.outputqueue.put('03|logs|\t\t[Logs] Tuples: ' +
out_tuples)
# 6. InTuples
in_tuples = __database__.getInTuplesfromProfileTW(
profileid, twid)
if in_tuples:
# Add in tuples
self.addDataToFile(profilefolder + '/' + twlog,
'InTuples:',
file_mode='a+',
data_type='text')
self.outputqueue.put('03|logs|\t\t[Logs] InTuples:')
data = json.loads(in_tuples)
for key in data:
self.addDataToFile(profilefolder + '/' + twlog,
'\t{} ({})'.format(key, data[key]),
file_mode='a+',
data_type='text')
self.outputqueue.put(
'03|logs|\t\t\t[Logs] {} ({})'.format(
key, data[key]))
# 7. Print the port data
all_roles = ['Client', 'Server']
all_protocols = ['TCP', 'UDP', 'ICMP', 'IPV6ICMP']
all_states = ['Established', 'NotEstablished']
all_directions = ['Dst', 'Src']
type_data = 'Ports'
for role in all_roles:
for protocol in all_protocols:
for state in all_states:
for direction in all_directions:
text_data = 'As {}, {} {} {} ports:'.format(
role, protocol, state, direction)
self.outputqueue.put('03|logs|\t\t\t[Logs]: ' +
text_data)
data = __database__.getDataFromProfileTW(
profileid, twid, direction, state,
protocol, role, type_data)
#self.print('LOGING data: {}'.format(data))
if data:
self.addDataToFile(profilefolder + '/' +
twlog,
text_data,
file_mode='a+',
data_type='text')
for port in data:
text_data = '\tPort {}. Total Flows: {}. Total Pkts: {}. TotalBytes: {}.'.format(
port, data[port]['totalflows'],
data[port]['totalpkt'],
data[port]['totalbytes'])
self.addDataToFile(profilefolder +
'/' + twlog,
text_data,
file_mode='a+',
data_type='text')
self.outputqueue.put(
'03|logs|\t\t\t[Logs]: ' +
text_data)
# 8. Info about the evidence so far for this TW.
evidence = __database__.getEvidenceForTW(profileid, twid)
if evidence:
evidence = json.loads(evidence)
self.addDataToFile(profilefolder + '/' + twlog,
'Evidence of detections in this TW:',
file_mode='a+',
data_type='text')
for key in evidence:
self.addDataToFile(
profilefolder + '/' + twlog,
'\tEvidence Description: {}. Confidence: {}. Threat Level: {} (key:{})'
.format(evidence[key][2], evidence[key][0],
evidence[key][1], key),
file_mode='a+',
data_type='text')
# Add free line between tuple info and information about ports and IP.
self.addDataToFile(profilefolder + '/' + twlog,
'',
file_mode='a+',
data_type='text')
"""
Dst ports and Src ports
"""
"""
flow_type_key = [Src,Dst] + [Port,IP] + [Client,Server] + [TCP,UDP, ICMP, ICMP6] + [Established, NotEstablished]
Example: flow_type_key = 'SrcPortClientTCPEstablished'
"""
flow_types_dict = self.create_all_flow_possibilities()
hash_key = profileid + self.separator + twid
for flow_type_key, sentence in flow_types_dict.items():
data = __database__.get_data_from_profile_tw(
hash_key, flow_type_key)
if data:
self.addDataToFile(profilefolder + '/' + twlog,
sentence,
file_mode='a+',
data_type='text')
for port, sample in data.items():
type = 'IP' if 'ip' in flow_type_key.lower(
) else 'Port'
text_data = '\t{} {}. Total Flows: {}. Total Pkts: {}. TotalBytes: {}.'.format(
type, port, sample['totalflows'],
sample['totalpkt'], sample['totalbytes'])
self.addDataToFile(profilefolder + '/' + twlog,
text_data,
file_mode='a+',
data_type='text')
self.outputqueue.put('03|logs|\t\t\t[Logs]: ' +
text_data)
# 9. This should be last. Detections to block
blocking = __database__.getBlockingRequest(profileid, twid)
if blocking:
self.addDataToFile(
profilefolder + '/' + twlog,
'Was requested to block in this time window: ' +
str(blocking),
file_mode='a+',
data_type='json')
self.outputqueue.put(
'03|logs|\t\t[Logs] Blocking Request: ' +
str(blocking))
# Mark it as not modified anymore
__database__.markProfileTWAsNotModifiedLogs(profileid, twid)
###########
# Create Timeline for each profile
# Store the timeline from the DB in a file
# The timeline file is unique for all timewindows. Much easier to read this way.
# Get all the TW for this profile
tws = __database__.getTWsfromProfile(profileid)
ip = profileid.split('_')[1]
timeline_path = profilefolder + '/' + 'Complete-timeline-outgoing-actions.txt'
# If the file does not exists yet, create it
if not os.path.isfile(timeline_path):
self.addDataToFile(
timeline_path,
'Complete TimeLine of IP {}\n'.format(ip),
file_mode='w+')
for twid_tuple in tws:
(twid, starttime) = twid_tuple
hash_key = profileid + self.separator + twid
first_index = self.timeline_first_index.get(hash_key, 0)
data, first_index = __database__.get_timeline_last_lines(
profileid, twid, first_index)
self.timeline_first_index[hash_key] = first_index
if data:
self.print(
'Adding to the profile line {} {}, data {}'.format(
profileid, twid, data), 6, 0)
self.addDataToFile(
profilefolder + '/' +
'Complete-timeline-outgoing-actions.txt',
data,
file_mode='a+',
data_type='lines',
data_mode='raw')
last_profile_id = profileid
# Create the file of the blocked profiles and TW
TWforProfileBlocked = __database__.getBlockedTW()
# Create the file of blocked data
if TWforProfileBlocked:
self.addDataToFile('Blocked.txt',
'Detections:\n',
file_mode='w+',
data_type='text')
for blocked in TWforProfileBlocked:
self.addDataToFile('Blocked.txt',
'\t' + str(blocked).split('_')[1] +
': ' + str(blocked).split('_')[2],
file_mode='a+',
data_type='json')
#self.outputqueue.put('03|logs|\t\t[Logs]: Blocked file updated: {}'.format(TWforProfileBlocked))
# Create a file with information about the capture in general
#self.addDataToFile('Information.txt', 'Information about this slips run', file_mode='w+', data_type='text')
#self.addDataToFile('Information.txt', '================================\n', file_mode='a+', data_type='text')
#self.addDataToFile('Information.txt', 'Type of input: ' + , file_mode='a+', data_type='text')
except KeyboardInterrupt:
return True
except Exception as inst:
self.outputqueue.put(
'01|logs|\t[Logs] Error in process_global_data in LogsProcess')
self.outputqueue.put('01|logs|\t[Logs] {}'.format(type(inst)))
self.outputqueue.put('01|logs|\t[Logs] {}'.format(inst))
sys.exit(1)