def __init__(self, outputqueue, config, testing=False): multiprocessing.Process.__init__(self) # All the printing output should be sent to the outputqueue, which is connected to OutputProcess self.outputqueue = outputqueue # In case you need to read the slips.conf configuration file for your own configurations self.config = config # Start the DB # This line might not be needed when running SLIPS, but when VT module is run standalone, it still uses the # database and this line is necessary. Do not delete it, instead move it to line 21. __database__.start( self.config) # TODO: What does this line do? It changes nothing. # To which channels do you want to subscribe? When a message arrives on the channel the module will wake up # The options change, so the last list is on the slips/core/database.py file. However common options are: # - new_ip # - tw_modified # - evidence_added self.c1 = __database__.subscribe('new_flow') self.c2 = __database__.subscribe('new_dns_flow') # VT api URL for querying IPs self.url = 'https://www.virustotal.com/vtapi/v2/ip-address/report' # Read the conf file self.__read_configuration() self.key = None try: with open(self.key_file, "r") as f: self.key = f.read(64) except FileNotFoundError: self.print("The file with API key (" + self.key_file + ") could not be loaded. VT module is stopping.") # query counter for debugging purposes self.counter = 0 # Pool manager to make HTTP requests with urllib3 # The certificate provides a bundle of trusted CAs, the certificates are located in certifi.where() self.http = urllib3.PoolManager(cert_reqs="CERT_REQUIRED", ca_certs=certifi.where()) # Set the timeout based on the platform. This is because the pyredis lib does not have officially recognized the timeout=None as it works in only macos and timeout=-1 as it only works in linux if platform.system() == 'Darwin': # macos self.timeout = None elif platform.system() == 'Linux': # linux self.timeout = None else: #?? self.timeout = None
def __init__(self, outputqueue, config): multiprocessing.Process.__init__(self) # All the printing output should be sent to the outputqueue. The outputqueue is connected to another process called OutputProcess self.outputqueue = outputqueue # In case you need to read the slips.conf configuration file for your own configurations self.config = config # Start the DB __database__.start(self.config) self.separator = __database__.getFieldSeparator() # Subscribe to 'new_flow' channel self.c1 = __database__.subscribe('new_flow') # To store the timelines of each profileid_twid self.profiles_tw = {} # Load the list of common known ports self.load_ports() # Store malicious IPs. We do not make alert everytime we receive flow with thi IP but only once. self.alerted_malicous_ips_dict = {} # Read information how we should print timestamp. self.is_human_timestamp = bool(self.read_configuration('modules', 'timeline_human_timestamp')) self.analysis_direction = self.config.get('parameters', 'analysis_direction') # Wait a little so we give time to have something to print # Set the timeout based on the platform. This is because the pyredis lib does not have officially recognized the timeout=None as it works in only macos and timeout=-1 as it only works in linux if platform.system() == 'Darwin': # macos self.timeout = None elif platform.system() == 'Linux': # linux self.timeout = None else: #?? self.timeout = None
def __init__(self, outputqueue, config): multiprocessing.Process.__init__(self) # All the printing output should be sent to the outputqueue. The outputqueue is connected to another process called OutputProcess self.outputqueue = outputqueue # In case you need to read the slips.conf configuration file for your own configurations self.config = config # Start the DB __database__.start(self.config) # Open the maminddb offline db try: self.reader = maxminddb.open_database( 'modules/geoip/GeoLite2-Country.mmdb') except: self.print( 'Error opening the geolite2 db in ./GeoLite2-Country_20190402/GeoLite2-Country.mmdb. Please download it from https://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz. Please note it must be the MaxMind DB version.' ) # To which channels do you wnat to subscribe? When a message arrives on the channel the module will wakeup self.c1 = __database__.subscribe('new_ip') # Set the timeout based on the platform. This is because the pyredis lib does not have officially recognized the timeout=None as it works in only macos and timeout=-1 as it only works in linux if platform.system() == 'Darwin': # macos self.timeout = None elif platform.system() == 'Linux': self.timeout = None else: #?? self.timeout = None
def __init__(self, outputqueue, config): multiprocessing.Process.__init__(self) # All the printing output should be sent to the outputqueue. # The outputqueue is connected to another process called OutputProcess self.outputqueue = outputqueue # In case you need to read the slips.conf configuration file for # your own configurations self.config = config # Start the DB __database__.start(self.config) # Retrieve the labels self.normal_label = __database__.normal_label self.malicious_label = __database__.malicious_label # To which channels do you wnat to subscribe? When a message # arrives on the channel the module will wakeup # The options change, so the last list is on the # slips/core/database.py file. However common options are: # - new_ip # - tw_modified # - evidence_added self.c1 = __database__.subscribe('tw_closed') # get separator self.separator = __database__.separator # Set the timeout based on the platform. This is because the # pyredis lib does not have officially recognized the # timeout=None as it works in only macos and timeout=-1 as it only works in linux if platform.system() == 'Darwin': # macos self.timeout = None elif platform.system() == 'Linux': # linux self.timeout = None else: # Other systems self.timeout = None
def __init__(self, outputqueue, config): multiprocessing.Process.__init__(self) self.outputqueue = outputqueue # This dictionary will hold each malicious ip to store in the db self.malicious_ips_dict = {} # In case you need to read the slips.conf configuration file for your own configurations self.config = config self.separator = __database__.getFieldSeparator() # This default path is only used in case there is no path in the configuration file self.path_to_malicious_ip_folder = 'modules/ThreatIntelligence1/malicious_ips_files/' # Subscribe to the new_ip channel __database__.start(self.config) self.c1 = __database__.subscribe('new_ip') # Create the update manager. This manager takes care of the re-downloading of the list of IoC when needed. self.update_manager = UpdateIPManager(self.outputqueue) # Update the remote file containing malicious IPs. self.__update_remote_malicious_file() # Set the timeout based on the platform. This is because the pyredis lib does not have officially recognized the timeout=None as it works in only macos and timeout=-1 as it only works in linux if platform.system() == 'Darwin': # macos self.timeout = None elif platform.system() == 'Linux': self.timeout = -1 else: #?? self.timeout = None
def __init__(self, outputqueue, config): multiprocessing.Process.__init__(self) # All the printing output should be sent to the outputqueue. The outputqueue is connected to another process called OutputProcess self.outputqueue = outputqueue # In case you need to read the slips.conf configuration file for your own configurations self.config = config # Start the DB __database__.start(self.config) # Subscribe to the channel self.c1 = __database__.subscribe('new_flow') self.fieldseparator = __database__.getFieldSeparator() # Set the output queue of our database instance __database__.setOutputQueue(self.outputqueue) # Read the configuration self.read_configuration() # To know when to retrain. We store the number of labels when we last retrain self.retrain = 0 if platform.system() == 'Darwin': # macos self.timeout = None elif platform.system() == 'Linux': # linux self.timeout = None else: # ?? self.timeout = None
def __init__(self, outputqueue, config): multiprocessing.Process.__init__(self) self.outputqueue = outputqueue self.config = config # Start the DB __database__.start(self.config) # Set the output queue of our database instance __database__.setOutputQueue(self.outputqueue) # Get from the database the separator used to separate the IP and the word profile self.fieldseparator = __database__.getFieldSeparator() # To which channels do you wnat to subscribe? When a message arrives on the channel the module will wakeup self.c1 = __database__.subscribe('tw_modified') # We need to know that after a detection, if we receive another flow that does not modify the count for the detection, we are not # re-detecting again only becase the threshold was overcomed last time. self.cache_det_thresholds = {} # Set the timeout based on the platform. This is because the pyredis lib does not have officially recognized the timeout=None as it works in only macos and timeout=-1 as it only works in linux if platform.system() == 'Darwin': # macos self.timeout = None elif platform.system() == 'Linux': self.timeout = None else: #?? self.timeout = None self.separator = '_'
def __init__(self, outputqueue, config): multiprocessing.Process.__init__(self) # All the printing output should be sent to the outputqueue. # The outputqueue is connected to another process called OutputProcess self.outputqueue = outputqueue # In case you need to read the slips.conf configuration file for # your own configurations self.config = config # Read the conf self.read_configuration() # Start the DB __database__.start(self.config) self.c1 = __database__.subscribe('core_messages') # Update file manager self.update_manager = UpdateFileManager(self.outputqueue, config) # Timer to update the ThreatIntelligence files self.timer_manager = InfiniteTimer(self.update_period, self.update_malicious_files) # Set the timeout based on the platform. This is because the # pyredis lib does not have officially recognized the # timeout=None as it works in only macos and timeout=-1 as it only works in linux if platform.system() == 'Darwin': # macos self.timeout = None elif platform.system() == 'Linux': # linux self.timeout = None else: # Other systems self.timeout = None
def __init__(self, outputqueue, config): multiprocessing.Process.__init__(self) self.outputqueue = outputqueue # In case you need to read the slips.conf configuration file for your own configurations self.config = config # Subscribe to the channel __database__.start(self.config) # Get a separator from the database self.separator = __database__.getFieldSeparator() self.c1 = __database__.subscribe('give_threat_intelligence') # Set the timeout based on the platform. This is because the pyredis lib does not have officially recognized the timeout=None as it works in only macos and timeout=-1 as it only works in linux if platform.system() == 'Darwin': # macos self.timeout = None elif platform.system() == 'Linux': self.timeout = None else: self.timeout = None self.__read_configuration()
def __init__(self, inputqueue, outputqueue, config): self.name = 'Evidence' multiprocessing.Process.__init__(self) self.inputqueue = inputqueue self.outputqueue = outputqueue self.config = config # Start the DB __database__.start(self.config) self.separator = __database__.separator # Read the configuration self.read_configuration() # Subscribe to channel 'tw_modified' self.c1 = __database__.subscribe('evidence_added') # Set the timeout based on the platform. This is because the pyredis lib does not have officially recognized the timeout=None as it works in only macos and timeout=-1 as it only works in linux if platform.system() == 'Darwin': # macos self.timeout = None elif platform.system() == 'Linux': # now linux also needs to be non-negative self.timeout = None else: self.timeout = None