def __init__(self, outputqueue, config):
multiprocessing.Process.__init__(self)
# All the printing output should be sent to the outputqueue. The outputqueue is connected to another process called OutputProcess
self.outputqueue = outputqueue
# In case you need to read the slips.conf configuration file for your own configurations
self.config = config
# Start the DB
__database__.start(self.config)
self.separator = __database__.getFieldSeparator()
# Subscribe to 'new_flow' channel
self.c1 = __database__.subscribe('new_flow')
# To store the timelines of each profileid_twid
self.profiles_tw = {}
# Load the list of common known ports
self.load_ports()
# Store malicious IPs. We do not make alert everytime we receive flow with thi IP but only once.
self.alerted_malicous_ips_dict = {}
# Read information how we should print timestamp.
self.is_human_timestamp = bool(self.read_configuration('modules', 'timeline_human_timestamp'))
self.analysis_direction = self.config.get('parameters', 'analysis_direction')
# Wait a little so we give time to have something to print
# Set the timeout based on the platform. This is because the pyredis lib does not have officially recognized the timeout=None as it works in only macos and timeout=-1 as it only works in linux
if platform.system() == 'Darwin':
# macos
self.timeout = None
elif platform.system() == 'Linux':
# linux
self.timeout = None
else:
#??
self.timeout = None
def __init__(self, outputqueue, config):
multiprocessing.Process.__init__(self)
# All the printing output should be sent to the outputqueue.
# The outputqueue is connected to another process called OutputProcess
self.outputqueue = outputqueue
# In case you need to read the slips.conf configuration file for
# your own configurations
self.config = config
# Start the DB
__database__.start(self.config)
# Retrieve the labels
self.normal_label = __database__.normal_label
self.malicious_label = __database__.malicious_label
# To which channels do you wnat to subscribe? When a message
# arrives on the channel the module will wakeup
# The options change, so the last list is on the
# slips/core/database.py file. However common options are:
# - new_ip
# - tw_modified
# - evidence_added
self.c1 = __database__.subscribe('tw_closed')
# get separator
self.separator = __database__.separator
# Set the timeout based on the platform. This is because the
# pyredis lib does not have officially recognized the
# timeout=None as it works in only macos and timeout=-1 as it only works in linux
if platform.system() == 'Darwin':
# macos
self.timeout = None
elif platform.system() == 'Linux':
# linux
self.timeout = None
else:
# Other systems
self.timeout = None
def __init__(self, outputqueue, config):
multiprocessing.Process.__init__(self)
# All the printing output should be sent to the outputqueue. The outputqueue is connected to another process called OutputProcess
self.outputqueue = outputqueue
# In case you need to read the slips.conf configuration file for your own configurations
self.config = config
# Start the DB
__database__.start(self.config)
# Open the maminddb offline db
try:
self.reader = maxminddb.open_database(
'modules/geoip/GeoLite2-Country.mmdb')
except:
self.print(
'Error opening the geolite2 db in ./GeoLite2-Country_20190402/GeoLite2-Country.mmdb. Please download it from https://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz. Please note it must be the MaxMind DB version.'
)
# To which channels do you wnat to subscribe? When a message arrives on the channel the module will wakeup
self.c1 = __database__.subscribe('new_ip')
# Set the timeout based on the platform. This is because the pyredis lib does not have officially recognized the timeout=None as it works in only macos and timeout=-1 as it only works in linux
if platform.system() == 'Darwin':
# macos
self.timeout = None
elif platform.system() == 'Linux':
self.timeout = None
else:
#??
self.timeout = None
def __init__(self, outputqueue, config):
multiprocessing.Process.__init__(self)
self.outputqueue = outputqueue
# This dictionary will hold each malicious ip to store in the db
self.malicious_ips_dict = {}
# In case you need to read the slips.conf configuration file for your own configurations
self.config = config
self.separator = __database__.getFieldSeparator()
# This default path is only used in case there is no path in the configuration file
self.path_to_malicious_ip_folder = 'modules/ThreatIntelligence1/malicious_ips_files/'
# Subscribe to the new_ip channel
__database__.start(self.config)
self.c1 = __database__.subscribe('new_ip')
# Create the update manager. This manager takes care of the re-downloading of the list of IoC when needed.
self.update_manager = UpdateIPManager(self.outputqueue)
# Update the remote file containing malicious IPs.
self.__update_remote_malicious_file()
# Set the timeout based on the platform. This is because the pyredis lib does not have officially recognized the timeout=None as it works in only macos and timeout=-1 as it only works in linux
if platform.system() == 'Darwin':
# macos
self.timeout = None
elif platform.system() == 'Linux':
self.timeout = -1
else:
#??
self.timeout = None
def __init__(self, outputqueue, config):
multiprocessing.Process.__init__(self)
# All the printing output should be sent to the outputqueue. The outputqueue is connected to another process called OutputProcess
self.outputqueue = outputqueue
# In case you need to read the slips.conf configuration file for your own configurations
self.config = config
# Start the DB
__database__.start(self.config)
# Subscribe to the channel
self.c1 = __database__.subscribe('new_flow')
self.fieldseparator = __database__.getFieldSeparator()
# Set the output queue of our database instance
__database__.setOutputQueue(self.outputqueue)
# Read the configuration
self.read_configuration()
# To know when to retrain. We store the number of labels when we last retrain
self.retrain = 0
if platform.system() == 'Darwin':
# macos
self.timeout = None
elif platform.system() == 'Linux':
# linux
self.timeout = None
else:
# ??
self.timeout = None
def __init__(self, outputqueue, config):
multiprocessing.Process.__init__(self)
self.outputqueue = outputqueue
self.config = config
# Start the DB
__database__.start(self.config)
# Set the output queue of our database instance
__database__.setOutputQueue(self.outputqueue)
# Get from the database the separator used to separate the IP and the word profile
self.fieldseparator = __database__.getFieldSeparator()
# To which channels do you wnat to subscribe? When a message arrives on the channel the module will wakeup
self.c1 = __database__.subscribe('tw_modified')
# We need to know that after a detection, if we receive another flow that does not modify the count for the detection, we are not
# re-detecting again only becase the threshold was overcomed last time.
self.cache_det_thresholds = {}
# Set the timeout based on the platform. This is because the pyredis lib does not have officially recognized the timeout=None as it works in only macos and timeout=-1 as it only works in linux
if platform.system() == 'Darwin':
# macos
self.timeout = None
elif platform.system() == 'Linux':
self.timeout = None
else:
#??
self.timeout = None
self.separator = '_'
def __init__(self, outputqueue, config):
multiprocessing.Process.__init__(self)
# All the printing output should be sent to the outputqueue.
# The outputqueue is connected to another process called OutputProcess
self.outputqueue = outputqueue
# In case you need to read the slips.conf configuration file for
# your own configurations
self.config = config
# Read the conf
self.read_configuration()
# Start the DB
__database__.start(self.config)
self.c1 = __database__.subscribe('core_messages')
# Update file manager
self.update_manager = UpdateFileManager(self.outputqueue, config)
# Timer to update the ThreatIntelligence files
self.timer_manager = InfiniteTimer(self.update_period,
self.update_malicious_files)
# Set the timeout based on the platform. This is because the
# pyredis lib does not have officially recognized the
# timeout=None as it works in only macos and timeout=-1 as it only works in linux
if platform.system() == 'Darwin':
# macos
self.timeout = None
elif platform.system() == 'Linux':
# linux
self.timeout = None
else:
# Other systems
self.timeout = None
def __init__(self, outputqueue, config):
self.outputqueue = outputqueue
self.config = config
# For now, read the malicious IPs from here
self.name = 'UpdateManager'
self.new_update_time = float('-inf')
# Start the database
__database__.start(self.config)
# Get a separator from the database
self.separator = __database__.getFieldSeparator()
# Read the conf
self.read_configuration()
def __init__(self, outputqueue, config, testing=False):
multiprocessing.Process.__init__(self)
# All the printing output should be sent to the outputqueue, which is connected to OutputProcess
self.outputqueue = outputqueue
# In case you need to read the slips.conf configuration file for your own configurations
self.config = config
# Start the DB
# This line might not be needed when running SLIPS, but when VT module is run standalone, it still uses the
# database and this line is necessary. Do not delete it, instead move it to line 21.
__database__.start(
self.config) # TODO: What does this line do? It changes nothing.
# To which channels do you want to subscribe? When a message arrives on the channel the module will wake up
# The options change, so the last list is on the slips/core/database.py file. However common options are:
# - new_ip
# - tw_modified
# - evidence_added
self.c1 = __database__.subscribe('new_ip')
# VT api URL for querying IPs
self.url = 'https://www.virustotal.com/vtapi/v2/ip-address/report'
# Read the conf file
key_file = self.__read_configuration("virustotal", "api_key_file")
self.key = None
try:
with open(key_file, "r") as f:
self.key = f.read(64)
except FileNotFoundError:
self.print("The file with API key (" + key_file +
") could not be loaded. VT module is stopping.")
# query counter for debugging purposes
self.counter = 0
# Pool manager to make HTTP requests with urllib3
# The certificate provides a bundle of trusted CAs, the certificates are located in certifi.where()
self.http = urllib3.PoolManager(cert_reqs="CERT_REQUIRED",
ca_certs=certifi.where())
# Set the timeout based on the platform. This is because the pyredis lib does not have officially recognized the timeout=None as it works in only macos and timeout=-1 as it only works in linux
if platform.system() == 'Darwin':
# macos
self.timeout = None
elif platform.system() == 'Linux':
# linux
self.timeout = None
else:
#??
self.timeout = None
def init_experiment(base_dir, exp_id, exp_suffix=""):
config = get_default_config()
output_process_queue = Queue()
output_process_thread = OutputProcess(output_process_queue, 1, 1, config)
output_process_thread.start()
# Start the DB
__database__.start(config)
__database__.setOutputQueue(output_process_queue)
exp_dir = base_dir + str(exp_id) + exp_suffix + "/"
if not os.path.exists(exp_dir):
os.mkdir(exp_dir)
return config, output_process_queue, output_process_thread, exp_dir
def __init__(self, outputqueue, config):
multiprocessing.Process.__init__(self)
self.outputqueue = outputqueue
# In case you need to read the slips.conf configuration file for your own configurations
self.config = config
# Subscribe to the channel
__database__.start(self.config)
# Get a separator from the database
self.separator = __database__.getFieldSeparator()
self.c1 = __database__.subscribe('give_threat_intelligence')
# Set the timeout based on the platform. This is because the pyredis lib does not have officially recognized the timeout=None as it works in only macos and timeout=-1 as it only works in linux
if platform.system() == 'Darwin':
# macos
self.timeout = None
elif platform.system() == 'Linux':
self.timeout = None
else:
self.timeout = None
self.__read_configuration()
def __init__(self, inputqueue, outputqueue, verbose, debug, config):
self.name = 'Logs'
multiprocessing.Process.__init__(self)
self.verbose = verbose
self.debug = debug
self.config = config
# Start the DB
__database__.start(self.config)
self.separator = '_'
# From the config, read the timeout to read logs. Now defaults to 5 seconds
self.inputqueue = inputqueue
self.outputqueue = outputqueue
# Read the configuration
self.read_configuration()
self.fieldseparator = __database__.getFieldSeparator()
# For some weird reason the database loses its outputqueue and we have to re set it here.......
__database__.setOutputQueue(self.outputqueue)
self.timeline_first_index = {}
self.is_timline_file = False
def __init__(self, outputqueue, profilerqueue, input_type,
input_information, config, packet_filter):
multiprocessing.Process.__init__(self)
self.outputqueue = outputqueue
self.profilerqueue = profilerqueue
self.config = config
# Start the DB
__database__.start(self.config)
self.input_type = input_type
self.input_information = input_information
self.zeek_folder = './zeek_files'
self.nfdump_output_file = 'nfdump_output.txt'
self.nfdump_timeout = None
self.name = 'input'
# Read the configuration
self.read_configuration()
# If we were given something from command line, has preference over the configuration file
if packet_filter:
self.packet_filter = "'" + packet_filter + "'"
self.event_handler = None
self.event_observer = None
def __init__(self, inputqueue, outputqueue, config):
self.name = 'Evidence'
multiprocessing.Process.__init__(self)
self.inputqueue = inputqueue
self.outputqueue = outputqueue
self.config = config
# Start the DB
__database__.start(self.config)
self.separator = __database__.separator
# Read the configuration
self.read_configuration()
# Subscribe to channel 'tw_modified'
self.c1 = __database__.subscribe('evidence_added')
# Set the timeout based on the platform. This is because the pyredis lib does not have officially recognized the timeout=None as it works in only macos and timeout=-1 as it only works in linux
if platform.system() == 'Darwin':
# macos
self.timeout = None
elif platform.system() == 'Linux':
# now linux also needs to be non-negative
self.timeout = None
else:
self.timeout = None
def __init__(self, config):
super().__init__(self.REGEX)
self.config = config
# Start the DB
__database__.start(self.config)