def status_agent(): """ Retrieve the status of the agent processes :return: A tuple, where the first element is the zeek process status (string), and second element are the FileBeats and PF_RING status """ # Load service processes zeek_p = zeek.ZeekProcess() filebeat_p = filebeat.FileBeatProcess() # Load service profilers pf_ring_profiler = pf_ring.PFRingProfiler(stderr=False) filebeat_profiler = filebeat.FileBeatProfiler(stderr=False) zeek_profiler = zeek.ZeekProfiler(stderr=False) suricata_profiler = suricata.SuricataProfiler(stderr=False) if not (filebeat_profiler.is_installed or zeek_profiler.is_installed): sys.stderr.write('[-] Could not start agent. Is it installed?\n') sys.stderr.write('[-] dynamite install agent\n') return False agent_status = dict( agent_processes={ 'zeek': zeek_p.status(), 'pf_ring': pf_ring_profiler.get_profile(), 'filebeat': filebeat_p.status() }) if suricata_profiler.is_installed: # Load Suricata process suricata_p = suricata.SuricataProcess() agent_status['agent_processes']['suricata'] = suricata_p.status() return agent_status
def profile_agent(): """ Get information about installation/running processes within the agent stack :return: A dictionary containing the status of each component """ pf_ring_profiler = pf_ring.PFRingProfiler() filebeat_profiler = filebeat.FileBeatProfiler() suricata_profiler = suricata.SuricataProfiler() zeek_profiler = zeek.ZeekProfiler() return dict(PF_RING=pf_ring_profiler.get_profile(), FILEBEAT=filebeat_profiler.get_profile(), SURICATA=suricata_profiler.get_profile(), ZEEK=zeek_profiler.get_profile())
def start_agent(): """ Start the Zeek (BroCtl) and FileBeats processes :return: True, if started successfully """ # Load service profilers pf_ring_profiler = pf_ring.PFRingProfiler(stderr=False) filebeat_profiler = filebeat.FileBeatProfiler(stderr=False) zeek_profiler = zeek.ZeekProfiler(stderr=False) suricata_profiler = suricata.SuricataProfiler(stderr=False) # Load service processes filebeat_p = filebeat.FileBeatProcess( install_directory=environment_variables.get('FILEBEAT_HOME')) zeek_p = zeek.ZeekProcess( install_directory=environment_variables.get('ZEEK_HOME')) if not (filebeat_profiler.is_installed or zeek_profiler.is_installed): sys.stderr.write('[-] Could not start agent. Is it installed?\n') sys.stderr.write('[-] dynamite install agent\n') return False if not pf_ring_profiler.is_running: sys.stderr.write( '[-] PF_RING kernel modules were not loaded. Try running ' '\'modprobe pf_ring min_num_slots=32768\' as root.\n') return False sys.stdout.write('[+] Starting agent processes.\n') if suricata_profiler.is_installed: # Load Suricata process suricata_p = suricata.SuricataProcess( install_directory=environment_variables.get('SURICATA_HOME'), configuration_directory=environment_variables.get( 'SURICATA_CONFIG')) if not suricata_p.start(stdout=True): sys.stderr.write('[-] Could not start agent.suricata_process.\n') return False if not zeek_p.start(stdout=True): sys.stderr.write('[-] Could not start agent.zeek_process.\n') return False if not filebeat_p.start(stdout=True): sys.stderr.write('[-] Could not start agent.filebeat.\n') return False return True
def status_agent(): """ Retrieve the status of the agent processes :return: A tuple, where the first element is the zeek process status (string), and second element are the FileBeats and PF_RING status """ # Load service processes zeek_p = zeek.ZeekProcess( install_directory=environment_variables.get('ZEEK_HOME')) filebeat_p = filebeat.FileBeatProcess( environment_variables.get('FILEBEAT_HOME')) # Load service profilers pf_ring_profiler = pf_ring.PFRingProfiler(stderr=False) filebeat_profiler = filebeat.FileBeatProfiler(stderr=False) zeek_profiler = zeek.ZeekProfiler(stderr=False) suricata_profiler = suricata.SuricataProfiler(stderr=False) if not (filebeat_profiler.is_installed or zeek_profiler.is_installed): sys.stderr.write('[-] Could not start agent. Is it installed?\n') sys.stderr.write('[-] dynamite install agent\n') return False agent_status = dict(agent_processes={ 'pf_ring': pf_ring_profiler.get_profile(), 'filebeat': filebeat_p.status() }) if suricata_profiler.is_installed: # Load Suricata process suricata_p = suricata.SuricataProcess( install_directory=environment_variables.get('SURICATA_HOME'), configuration_directory=environment_variables.get( 'SURICATA_CONFIG')) agent_status['agent_processes']['suricata'] = suricata_p.status() return zeek_p.status(), agent_status
def setup_suricata(self, network_interface=None, stdout=False): """ Setup Suricata IDS with PF_RING support :param stdout: Print output to console :param network_interface: The interface to listen on :return: True, if setup successful """ if not network_interface: network_interface = utilities.get_network_interface_names()[0] if network_interface not in utilities.get_network_interface_names(): sys.stderr.write( '[-] The network interface that your defined: \'{}\' is invalid. Valid network interfaces: {}\n'.format( network_interface, utilities.get_network_interface_names())) return False self._copy_suricata_files_and_directories(stdout=stdout) pf_ring_install = pf_ring.PFRingInstaller() if not pf_ring.PFRingProfiler().is_installed: if stdout: sys.stdout.write('[+] Installing PF_RING kernel modules and dependencies.\n') sys.stdout.flush() time.sleep(1) pf_ring_install.download_pf_ring(stdout=True) pf_ring_install.extract_pf_ring(stdout=True) pf_ring_install.setup_pf_ring(stdout=True) try: os.symlink(os.path.join(pf_ring_install.install_directory, 'lib', 'libpcap.so.1'), '/lib/libpcap.so.1') except Exception as e: sys.stderr.write('[-] Failed to re-link libpcap.so.1 -> /lib/libpcap.so.1: {}\n'.format(e)) if 'exists' not in str(e).lower(): return False try: os.symlink(os.path.join(pf_ring_install.install_directory, 'lib', 'libpfring.so'), '/lib/libpfring.so.1') except Exception as e: sys.stderr.write('[-] Failed to re-link libpfring.so -> /lib/libpfring.so.1: {}\n'.format(e)) if 'exists' not in str(e).lower(): return False # CentOS linker libraries try: os.symlink(os.path.join(pf_ring_install.install_directory, 'lib', 'libpcap.so.1'), '/usr/local/lib/libpcap.so.1') except Exception as e: sys.stderr.write('[-] Failed to re-link libpcap.so.1 -> /usr/local/lib/libpcap.so.1: {}\n'.format(e)) if 'exists' not in str(e).lower(): return False try: os.symlink(os.path.join(pf_ring_install.install_directory, 'lib', 'libpfring.so'), '/usr/local/lib/libpfring.so.1') except Exception as e: sys.stderr.write('[-] Failed to re-link libpfring.so -> /usr/local/lib/libpfring.so.1: {}\n'.format(e)) if 'exists' not in str(e).lower(): return False time.sleep(5) suricata_compiled = self._configure_and_compile_suricata(pf_ring_installer=pf_ring_install, stdout=stdout) if not suricata_compiled: return False if 'SURICATA_HOME' not in open('/etc/dynamite/environment').read(): if stdout: sys.stdout.write('[+] Updating Suricata default home path [{}]\n'.format( self.install_directory)) subprocess.call('echo SURICATA_HOME="{}" >> /etc/dynamite/environment'.format(self.install_directory), shell=True) if 'SURICATA_CONFIG' not in open('/etc/dynamite/environment').read(): if stdout: sys.stdout.write('[+] Updating Suricata default config path [{}]\n'.format( self.configuration_directory)) subprocess.call('echo SURICATA_CONFIG="{}" >> /etc/dynamite/environment'.format(self.configuration_directory), shell=True) suricata_rules_installed = self._setup_suricata_rules(stdout=stdout) if not suricata_rules_installed: return False config = SuricataConfigurator(self.configuration_directory) config.pfring_interfaces = [] config.add_pfring_interface(network_interface, threads='auto', cluster_id=99) config.default_log_directory = self.log_directory config.default_rules_directory = os.path.join(self.configuration_directory, 'rules') config.reference_config_file = os.path.join(self.configuration_directory, 'reference.config') config.classification_file = os.path.join(self.configuration_directory, 'rules', 'classification.config') config.write_config() return True
def uninstall_agent(prompt_user=True): """ Uninstall the agent :param prompt_user: Print a warning before continuing :return: True, if uninstall succeeded """ environment_variables = utilities.get_environment_file_dict() filebeat_profiler = filebeat.FileBeatProfiler() pf_profiler = pf_ring.PFRingProfiler() zeek_profiler = zeek.ZeekProfiler() suricata_profiler = suricata.SuricataProfiler() if not (filebeat_profiler.is_installed or zeek_profiler.is_installed or suricata_profiler.is_installed): sys.stderr.write('[-] No agent installation detected.\n') return False if filebeat_profiler.is_installed: filebeat_config = filebeat.FileBeatConfigurator( install_directory=environment_variables.get('FILEBEAT_HOME')) if prompt_user: sys.stderr.write( '[-] WARNING! REMOVING THE AGENT WILL RESULT IN EVENTS NO LONGER BEING SENT TO {}.\n' .format(filebeat_config.get_logstash_targets())) resp = utilities.prompt_input( 'Are you sure you wish to continue? ([no]|yes): ') while resp not in ['', 'no', 'yes']: resp = utilities.prompt_input( 'Are you sure you wish to continue? ([no]|yes): ') if resp != 'yes': sys.stdout.write('[+] Exiting\n') return False if filebeat_profiler.is_running: filebeat.FileBeatProcess().stop(stdout=True) if zeek_profiler.is_running: zeek.ZeekProcess().stop() if pf_profiler.is_installed: shutil.rmtree(environment_variables.get('PF_RING_HOME')) os.remove('/opt/dynamite/.agent_environment_prepared') if filebeat_profiler.is_installed: shutil.rmtree(environment_variables.get('FILEBEAT_HOME'), ignore_errors=True) if zeek_profiler.is_installed: shutil.rmtree(environment_variables.get('ZEEK_HOME'), ignore_errors=True) shutil.rmtree(environment_variables.get('ZEEK_SCRIPTS'), ignore_errors=True) if suricata_profiler.is_installed: shutil.rmtree(environment_variables.get('SURICATA_HOME'), ignore_errors=True) shutil.rmtree(environment_variables.get('SURICATA_CONFIG'), ignore_errors=True) shutil.rmtree(environment_variables.get('OINKMASTER_HOME'), ignore_errors=True) shutil.rmtree(const.INSTALL_CACHE, ignore_errors=True) env_lines = '' for line in open('/etc/dynamite/environment').readlines(): if 'FILEBEAT_HOME' in line: continue elif 'ZEEK_HOME' in line: continue elif 'ZEEK_SCRIPTS' in line: continue elif 'SURICATA_HOME' in line: continue elif 'SURICATA_CONFIG' in line: continue elif 'PF_RING_HOME' in line: continue elif 'OINKMASTER_HOME' in line: continue elif line.strip() == '': continue env_lines += line.strip() + '\n' with open('/etc/dynamite/environment', 'w') as f: f.write(env_lines) return True
def install_agent(network_interface, agent_label, logstash_target): """ :param network_interface: The network interface that the agent should analyze traffic on :param agent_label: A descriptive label representing the segment/location on your network that your agent is monitoring :param logstash_target: The host port combination for the target Logstash server (E.G "localhost:5044") :return: True, if install succeeded """ zeek_installer = zeek.ZeekInstaller() zeek_profiler = zeek.ZeekProfiler(stderr=True) suricata_profiler = suricata.SuricataProfiler() filebeat_installer = filebeat.FileBeatInstaller() filebeat_profiler = filebeat.FileBeatProfiler() # === Check running processes/prerequisites if not is_agent_environment_prepared(): sys.stderr.write( '[-] The environment must first be prepared prior to agent installation. \n' ) sys.stderr.write( '[-] This includes the installation of kernel development headers, ' 'required for PF_RING kernel modules to be loaded. \n') sys.stderr.write( '[-] To prepare the agent environment run \'dynamite prepare agent\'.\n' ) sys.stderr.flush() return False if zeek_profiler.is_running or filebeat_profiler.is_running: sys.stderr.write( '[-] Please stop the agent before attempting re-installation.\n') return False elif suricata_profiler.is_running: sys.stderr.write( '[-] Please stop the agent before attempting re-installation.\n') return False # === Install Suricata === suricata_installer = suricata.SuricataInstaller() if not suricata_profiler.is_downloaded: suricata_installer.download_suricata(stdout=True) suricata_installer.extract_suricata(stdout=True) else: sys.stdout.write( '[+] Suricata has already been downloaded to local cache. Skipping Suricata Download.\n' ) if not suricata_profiler.is_installed: suricata_installer.install_dependencies() suricata_installer.setup_suricata(network_interface=network_interface, stdout=True) else: sys.stdout.write( '[+] Suricata has already been installed on this system. ' 'Skipping Suricata Installation.\n') # === Install Zeek === if not zeek_profiler.is_downloaded: zeek_installer.download_zeek(stdout=True) zeek_installer.extract_zeek(stdout=True) else: sys.stdout.write( '[+] Zeek has already been downloaded to local cache. Skipping Zeek Download.\n' ) if not zeek_profiler.is_installed: if not zeek_installer.install_dependencies(): sys.stderr.write( '[-] Could not find a native package manager. Currently [APT-GET/YUM are supported]\n' ) return False zeek_installer.setup_zeek(network_interface=network_interface, stdout=True) else: sys.stdout.write( '[+] Zeek has already been installed on this system. Skipping Zeek Installation.\n' ) # === Install Filebeat === if not filebeat_profiler.is_downloaded: filebeat_installer.download_filebeat(stdout=True) filebeat_installer.extract_filebeat(stdout=True) else: sys.stdout.write( '[+] FileBeat has already been downloaded to local cache. Skipping FileBeat Download.\n' ) if not filebeat_profiler.is_installed: environment_variables = utilities.get_environment_file_dict() monitored_paths = [ os.path.join(environment_variables.get('ZEEK_HOME'), 'logs/current/*.log') ] suricata_config = suricata.SuricataConfigurator( configuration_directory=environment_variables.get( 'SURICATA_CONFIG')) monitored_paths.append( os.path.join(suricata_config.default_log_directory, 'eve.json')) filebeat_installer.setup_filebeat(stdout=True) filebeat_config = filebeat.FileBeatConfigurator() filebeat_config.set_logstash_targets([logstash_target]) filebeat_config.set_monitor_target_paths(monitored_paths) filebeat_config.set_agent_tag(agent_label) filebeat_config.write_config() else: sys.stdout.write( '[+] FileBeat has already been installed on this system. Skipping FileBeat Installation.\n' ) # === Post installation checks === pf_ring_post_install_profiler = pf_ring.PFRingProfiler() zeek_post_install_profiler = zeek.ZeekProfiler() suricata_post_profiler = suricata.SuricataProfiler() filebeat_post_install_profiler = filebeat.FileBeatProfiler() if not pf_ring_post_install_profiler.is_running: sys.stderr.write( '[-] PF_RING kernel module was not loaded properly.\n') return False if zeek_post_install_profiler.is_installed and filebeat_post_install_profiler.is_installed: if suricata_post_profiler.is_installed: sys.stdout.write( '[+] Agent installation complete. Start the agent: \'dynamite start agent\'.\n' ) sys.stdout.flush() return True else: sys.stderr.write( '[-] Agent installation failed. Suricata did not install properly.\n' ) sys.stderr.flush() return False return False
def setup_zeek(self, network_interface=None, stdout=False): """ Setup Zeek NSM with PF_RING support :param stdout: Print output to console :param network_interface: The interface to listen on :return: True, if setup successful """ if not network_interface: network_interface = utilities.get_network_interface_names()[0] if network_interface not in utilities.get_network_interface_names(): sys.stderr.write( '[-] The network interface that your defined: \'{}\' is invalid. Valid network interfaces: {}\n' .format(network_interface, utilities.get_network_interface_names())) return False if stdout: sys.stdout.write( '[+] Creating zeek install|configuration|logging directories.\n' ) subprocess.call('mkdir -p {}'.format(self.install_directory), shell=True) subprocess.call('mkdir -p {}'.format(self.configuration_directory), shell=True) pf_ring_install = pf_ring.PFRingInstaller() if not pf_ring.PFRingProfiler().is_installed: if stdout: sys.stdout.write( '[+] Installing PF_RING kernel modules and dependencies.\n' ) sys.stdout.flush() time.sleep(1) pf_ring_install.download_pf_ring(stdout=True) pf_ring_install.extract_pf_ring(stdout=True) pf_ring_install.setup_pf_ring(stdout=True) if stdout: sys.stdout.write( '\n\n[+] Compiling Zeek from source. This can take up to 30 minutes. Have a cup of coffee.' '\n\n') sys.stdout.flush() time.sleep(5) subprocess.call( './configure --prefix={} --scriptdir={} --with-pcap={}'.format( self.install_directory, self.configuration_directory, pf_ring_install.install_directory), shell=True, cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME)) subprocess.call('make; make install', shell=True, cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME)) if 'ZEEK_HOME' not in open('/etc/environment').read(): if stdout: sys.stdout.write( '[+] Updating Zeek default home path [{}]\n'.format( self.install_directory)) subprocess.call('echo ZEEK_HOME="{}" >> /etc/environment'.format( self.install_directory), shell=True) if 'ZEEK_SCRIPTS' not in open('/etc/environment').read(): if stdout: sys.stdout.write( '[+] Updating Zeek default script path [{}]\n'.format( self.configuration_directory)) subprocess.call( 'echo ZEEK_SCRIPTS="{}" >> /etc/environment'.format( self.configuration_directory), shell=True) if stdout: sys.stdout.write( '[+] Overwriting default Script | Node configurations.\n') shutil.copy( os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'broctl-nodes.cfg'), os.path.join(self.install_directory, 'etc', 'node.cfg')) shutil.copy( os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'local.bro'), os.path.join(self.configuration_directory, 'site', 'local.bro')) ZeekScriptConfigurator().write_config() node_config = ZeekNodeConfigurator(self.install_directory) available_cpus = utilities.get_cpu_core_count() workers_cpu_grps = [ range(0, available_cpus)[n:n + 2] for n in range(0, len(range(0, available_cpus)), 2) ] for i, cpu_group in enumerate(workers_cpu_grps): node_config.add_worker(name='dynamite-worker-{}'.format(i + 1), host='localhost', interface=network_interface, lb_procs=10, pin_cpus=cpu_group) node_config.write_config()