def get_optimal_zeek_worker_config( interface_names: List[str], available_cpus: Optional[Tuple] = None) -> node.Workers: """Algorithm for determining the assignment of CPUs for Zeek workers Args: interface_names: A list of network interface names available_cpus: If None, we'll derive this by looking at the cpu core count, otherwise a list of cpu cores Returns: A node.Workers object """ zeek_worker_configs = node.Workers() if not available_cpus: # Reserve CPU 0 for KERNEL operations available_cpus = [ c for c in range(1, utilities.get_cpu_core_count()) ] for cpu_affinity_group in utilities.get_optimal_cpu_interface_config( interface_names=interface_names, available_cpus=available_cpus): net_interface = cpu_affinity_group['interface_name'] pinned_cpus = cpu_affinity_group['pin_cpus'] lb_processes = cpu_affinity_group['thread_count'] zeek_worker_configs.add_worker( node.Worker(worker_name='dynamite-worker-' + net_interface, host='localhost', interface_name=net_interface, load_balance_processes=lb_processes, pinned_cpus=pinned_cpus, cluster_id=randint(1, 32768), cluster_type='AF_Packet::FANOUT_QM')) return zeek_worker_configs
def add_worker(self, name, interface, host, lb_procs=10, pin_cpus=(0, 1), af_packet_fanout_id=None, af_packet_fanout_mode=None): """ :param name: The name of the worker :param interface: The interface that the worker should be monitoring :param host: The host on which the worker is running :param lb_procs: The number of Zeek processes associated with a given worker :param pin_cpus: Core affinity for the processes (iterable), :param af_packet_fanout_id: To scale processing across threads, packet sockets can form a fanout group. In this mode, each matching packet is enqueued onto only one socket in the group. A socket joins a fanout group by calling setsockopt(2) with level SOL_PACKET and option PACKET_FANOUT. Each network namespace can have up to 65536 independent groups. :param af_packet_fanout_mode: The algorithm used to spread traffic between sockets. """ valid_fanout_modes = [ 'FANOUT_HASH', # The default mode, PACKET_FANOUT_HASH, sends packets from # the same flow to the same socket to maintain per-flow # ordering. For each packet, it chooses a socket by taking # the packet flow hash modulo the number of sockets in the # group, where a flow hash is a hash over network-layer # address and optional transport-layer port fields. 'FANOUT_CPU', # selects the socket based on the CPU that the packet arrived on 'FANOUT_QM' # (available since Linux 3.14) selects the socket using the recorded # queue_mapping of the received skb. ] if not str(interface).startswith('af_packet::'): interface = 'af_packet::' + interface if not af_packet_fanout_id: af_packet_fanout_id = random.randint(1, 32768) if not af_packet_fanout_mode: af_packet_fanout_mode = 'AF_Packet::FANOUT_HASH' else: if str(af_packet_fanout_mode).upper() in valid_fanout_modes: af_packet_fanout_mode = 'AF_Packet::' + str( af_packet_fanout_mode).upper() else: af_packet_fanout_mode = 'AF_Packet::FANOUT_HASH' if max(pin_cpus) < utilities.get_cpu_core_count() and min( pin_cpus) >= 0: pin_cpus = [str(cpu_n) for cpu_n in pin_cpus] self.node_config[name] = { 'type': 'worker', 'interface': interface, 'lb_method': 'custom', 'lb_procs': lb_procs, 'pin_cpus': ','.join(pin_cpus), 'host': host, 'af_packet_fanout_id': af_packet_fanout_id, 'af_packet_fanout_mode': af_packet_fanout_mode }
def add_worker(self, name, interface, host, lb_procs=10, pin_cpus=(0, 1)): """ :param name: The name of the worker :param interface: The interface that the worker should be monitoring :param host: The host on which the worker is running :param lb_procs: The number of Zeek processes associated with a given worker :param pin_cpus: Core affinity for the processes (iterable) """ if not str(interface).startswith('af_packet::'): interface = 'af_packet::' + interface if max(pin_cpus) < utilities.get_cpu_core_count() and min( pin_cpus) >= 0: pin_cpus = [str(cpu_n) for cpu_n in pin_cpus] self.node_config[name] = { 'type': 'worker', 'interface': interface, 'lb_method': 'custom', 'lb_procs': lb_procs, 'pin_cpus': ','.join(pin_cpus), 'host': host }
def add_worker(self, name, interface, host, lb_procs=10, pin_cpus=(0, 1)): """ :param name: The name of the worker :param interface: The interface that the worker should be monitoring :param host: The host on which the worker is running :param lb_procs: The number of Zeek processes associated with a given worker :param pin_cpus: Core affinity for the processes (iterable) :return: True, if added successfully """ if max(pin_cpus) < utilities.get_cpu_core_count() and min( pin_cpus) >= 0: pin_cpus = [str(cpu_n) for cpu_n in pin_cpus] self.node_config[name] = { 'type': 'worker', 'interface': interface, 'lb_method': 'pf_ring', 'lb_procs': lb_procs, 'pin_cpus': ','.join(pin_cpus), 'host': host } return True return False
def get_available_cpus(self) -> list: """Get the CPU core numbers that are not currently being utilized Returns: A list of available CPUs """ zeek_profiler = zeek_profile.ProcessProfiler() suricata_profiler = suricata_profile.ProcessProfiler() available_cpus = [c for c in range(0, utilities.get_cpu_core_count())] reserved_cpus = [] zeek_cpus = [] suricata_cpus = [] if zeek_profiler.is_installed(): zeek_node_config_mng = zeek_config.NodeConfigManager( install_directory=self.zeek_install_directory, stdout=self.stdout, verbose=self.verbose) for worker in zeek_node_config_mng.workers: zeek_cpus.extend(worker.pinned_cpus) if suricata_profiler.is_installed(): suricata_config_mng = suricata_config.ConfigManager( configuration_directory=self.suricata_configuration_directory, stdout=self.stdout, verbose=self.verbose) if suricata_config_mng.threading.worker_cpu_set: suricata_cpus.extend( suricata_config_mng.threading.worker_cpu_set) if suricata_config_mng.threading.receive_cpu_set: suricata_cpus.extend( suricata_config_mng.threading.receive_cpu_set) if suricata_config_mng.threading.management_cpu_set: suricata_cpus.extend( suricata_config_mng.threading.management_cpu_set) reserved_cpus.extend(suricata_cpus) reserved_cpus.extend(zeek_cpus) return list(set([c for c in available_cpus if c not in reserved_cpus]))
def setup_zeek(self): """ Setup Zeek NSM with PF_RING support """ env_file = os.path.join(const.CONFIG_PATH, 'environment') self.logger.info('Creating Zeek installation, configuration and logging directories.') try: utilities.makedirs(self.install_directory, exist_ok=True) utilities.makedirs(self.configuration_directory, exist_ok=True) except Exception as e: self.logger.error('General error occurred while attempting to create root directories.') self.logger.debug("General error occurred while attempting to create root directories; {}".format(e)) raise zeek_exceptions.InstallZeekError( "General error occurred while attempting to create root directories; {}".format(e)) self.logger.info('Compiling Zeek from source. This can take up to 30 minutes.') if self.stdout: utilities.print_coffee_art() time.sleep(1) self.logger.info('Configuring Zeek.') if self.verbose: zeek_config_p = subprocess.Popen('./configure --prefix={} --scriptdir={} --enable-jemalloc'.format( self.install_directory, self.configuration_directory), shell=True, cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME)) else: zeek_config_p = subprocess.Popen('./configure --prefix={} --scriptdir={} --enable-jemalloc'.format( self.install_directory, self.configuration_directory), shell=True, cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME), stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: zeek_config_p.communicate() except Exception as e: self.logger.error("General error occurred while configuring Zeek.") self.logger.debug("General error occurred while configuring Zeek; {}".format(e)) raise zeek_exceptions.InstallZeekError( "General error occurred while configuring Zeek; {}".format(e)) if zeek_config_p.returncode != 0: self.logger.error( "Zeek configuration process returned non-zero; exit-code: {}".format(zeek_config_p.returncode)) raise zeek_exceptions.InstallZeekError( "Zeek configuration process returned non-zero; exit-code: {}".format(zeek_config_p.returncode)) time.sleep(1) self.logger.info("Compiling Zeek.") if utilities.get_cpu_core_count() > 1: parallel_threads = utilities.get_cpu_core_count() - 1 else: parallel_threads = 1 if self.verbose: compile_zeek_process = subprocess.Popen('make -j {}; make install'.format(parallel_threads), shell=True, cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME)) try: compile_zeek_process.communicate() except Exception as e: self.logger.error("General error occurred while compiling Zeek.") self.logger.debug("General error occurred while compiling Zeek; {}".format(e)) raise zeek_exceptions.InstallZeekError( "General error occurred while compiling Zeek; {}".format(e)) compile_zeek_return_code = compile_zeek_process.returncode else: compile_zeek_process = subprocess.Popen('make -j {}; make install'.format(parallel_threads), shell=True, cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME), stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: compile_zeek_return_code = utilities.run_subprocess_with_status(compile_zeek_process, expected_lines=6779) except Exception as e: self.logger.error("General error occurred while compiling Zeek.") self.logger.debug("General error occurred while compiling Zeek; {}".format(e)) raise zeek_exceptions.InstallZeekError( "General error occurred while compiling Zeek; {}".format(e)) if compile_zeek_return_code != 0: self.logger.error( "Failed to compile Zeek from source; error code: {}; run with --verbose flag for more info.".format( compile_zeek_return_code)) raise zeek_exceptions.InstallZeekError( "Zeek compilation process returned non-zero; exit-code: {}".format(compile_zeek_return_code)) try: with open(env_file) as env_f: if 'ZEEK_HOME' not in env_f.read(): self.logger.info('Updating Zeek default home path [{}]'.format(self.install_directory)) subprocess.call('echo ZEEK_HOME="{}" >> {}'.format(self.install_directory, env_file), shell=True) if 'ZEEK_SCRIPTS' not in env_f.read(): self.logger.info('Updating Zeek default script path [{}]'.format(self.configuration_directory)) subprocess.call('echo ZEEK_SCRIPTS="{}" >> {}'.format(self.configuration_directory, env_file), shell=True) except IOError as e: self.logger.error("Failed to open {} for reading.".format(env_file)) self.logger.debug("Failed to open {} for reading; {}".format(env_file, e)) raise zeek_exceptions.InstallZeekError( "Failed to open {} for reading; {}".format(env_file, e)) except Exception as e: self.logger.error("General error while creating environment variables in {}.".format(env_file)) self.logger.debug("General error while creating environment variables in {}; {}".format(env_file, e)) raise zeek_exceptions.InstallZeekError( "General error while creating environment variables in {}; {}".format(env_file, e)) self.logger.info("Overwriting Zeek node.cfg file with our changes.") try: shutil.copy(os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'broctl-nodes.cfg'), os.path.join(self.install_directory, 'etc', 'node.cfg')) shutil.copy(os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'local.zeek'), os.path.join(self.configuration_directory, 'site', 'local.zeek')) except Exception as e: self.logger.error("General error occurred while copying default Zeek configurations.") self.logger.debug("General error occurred while copying default Zeek configurations; {}".format(e)) raise zeek_exceptions.InstallZeekError( "General error occurred while copying default Zeek configurations; {}".format(e)) try: node_config = zeek_configs.NodeConfigManager(self.install_directory) except zeek_exceptions.ReadsZeekConfigError: self.logger.error("An error occurred while reading Zeek configurations.") raise zeek_exceptions.InstallZeekError("An error occurred while reading Zeek configurations.") # Clear out pre-set workers. for key in list(node_config.node_config): if node_config.node_config[key]['type'] == 'worker': del node_config.node_config[key] # Calculate new workers. for worker in node_config.get_optimal_zeek_worker_config(self.capture_network_interfaces, stdout=self.stdout, verbose=self.verbose): node_config.add_worker(name=worker['name'], host=worker['host'], interface=worker['interface'], lb_procs=worker['lb_procs'], pin_cpus=worker['pinned_cpus'] ) try: node_config.write_config() except zeek_exceptions.WriteZeekConfigError: self.logger.error("An error occurred while writing Zeek configurations.") raise zeek_exceptions.InstallZeekError("An error occurred while writing Zeek configurations.") try: sysctl = systemctl.SystemCtl() except general_exceptions.CallProcessError: raise zeek_exceptions.InstallZeekError("Could not find systemctl.") self.logger.info("Installing Zeek systemd service.") if not sysctl.install_and_enable(os.path.join(const.DEFAULT_CONFIGS, 'systemd', 'zeek.service')): raise zeek_exceptions.InstallZeekError("Failed to install Zeek systemd service.")
def setup_zeek_community_id_plugin(self): bro_commmunity_id_plugin_path = \ os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'uncompiled_scripts', 'zeek-community-id') self.logger.info('Configuring Zeek Corelight_CommunityID plugin.') if self.verbose: config_zeek_community_id_script_process = subprocess.Popen( './configure --zeek-dist={} --install-root={}'.format( os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME), self.configuration_directory), shell=True, cwd=bro_commmunity_id_plugin_path ) else: config_zeek_community_id_script_process = subprocess.Popen( './configure --zeek-dist={} --install-root={}'.format( os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME), self.configuration_directory), shell=True, cwd=bro_commmunity_id_plugin_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: config_zeek_community_id_script_process.communicate() except Exception as e: self.logger.error('General error occurred while starting Corelight_CommunityID configuration.') self.logger.debug('General error occurred while starting Corelight_CommunityID configuration; {}'.format(e)) raise zeek_exceptions.InstallZeekError( "General error occurred while starting Corelight_CommunityID configuration; {}".format(e)) if config_zeek_community_id_script_process.returncode != 0: self.logger.debug("Corelight_CommunityID configuration returned non-zero; exit-code: {}".format( config_zeek_community_id_script_process.returncode)) raise zeek_exceptions.InstallZeekError( "Corelight_CommunityID configuration returned non-zero; exit-code: {}".format( config_zeek_community_id_script_process.returncode)) self.logger.info('Compiling Zeek Corelight_CommunityID [PATCHED] plugin.') if utilities.get_cpu_core_count() > 1: parallel_threads = utilities.get_cpu_core_count() - 1 else: parallel_threads = 1 if self.verbose: compile_zeek_community_id_script_process = subprocess.Popen( 'make -j {}; make install'.format(parallel_threads), shell=True, cwd=bro_commmunity_id_plugin_path) else: compile_zeek_community_id_script_process = subprocess.Popen( 'make -j {}; make install'.format(parallel_threads), shell=True, cwd=bro_commmunity_id_plugin_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: compile_zeek_community_id_script_process.communicate() except Exception as e: self.logger.error('General error occurred while compiling Corelight_CommunityID [PATCHED].') self.logger.debug("General error occurred while compiling Corelight_CommunityID [PATCHED]; {}".format(e)) raise zeek_exceptions.InstallZeekError( "General error occurred while compiling Corelight_CommunityID [PATCHED]; {}".format(e)) if compile_zeek_community_id_script_process.returncode != 0: self.logger.error("General error occurred while compiling Corelight_CommunityID [PATCHED]; {}".format( compile_zeek_community_id_script_process.returncode)) raise zeek_exceptions.InstallZeekError( "Corelight_CommunityID [PATCHED] compilation process returned non-zero; exit-code: {}".format( compile_zeek_community_id_script_process.returncode)) try: shutil.copytree(os.path.join(self.configuration_directory, 'Corelight_CommunityID'), os.path.join(self.install_directory, 'lib', 'zeek', 'plugins', 'Corelight_CommunityID')) except Exception as e: if 'FileExist' not in str(e): self.logger.error("General error occurred while installing Corelight_CommunityID [PATCHED] plugin.") self.logger.debug("General error occurred while installing Corelight_CommunityID [PATCHED] plugin; " "{}".format(e)) raise zeek_exceptions.InstallZeekError( "General error occurred while installing Corelight_CommunityID [PATCHED] plugin; {}".format( e))
def setup_zeek(self, network_interface=None, stdout=False): """ Setup Zeek NSM with PF_RING support :param stdout: Print output to console :param network_interface: The interface to listen on :return: True, if setup successful """ if not network_interface: network_interface = utilities.get_network_interface_names()[0] if network_interface not in utilities.get_network_interface_names(): sys.stderr.write( '[-] The network interface that your defined: \'{}\' is invalid. Valid network interfaces: {}\n' .format(network_interface, utilities.get_network_interface_names())) return False if stdout: sys.stdout.write( '[+] Creating zeek install|configuration|logging directories.\n' ) subprocess.call('mkdir -p {}'.format(self.install_directory), shell=True) subprocess.call('mkdir -p {}'.format(self.configuration_directory), shell=True) pf_ring_install = pf_ring.PFRingInstaller() if not pf_ring.PFRingProfiler().is_installed: if stdout: sys.stdout.write( '[+] Installing PF_RING kernel modules and dependencies.\n' ) sys.stdout.flush() time.sleep(1) pf_ring_install.download_pf_ring(stdout=True) pf_ring_install.extract_pf_ring(stdout=True) pf_ring_install.setup_pf_ring(stdout=True) if stdout: sys.stdout.write( '\n\n[+] Compiling Zeek from source. This can take up to 30 minutes. Have a cup of coffee.' '\n\n') sys.stdout.flush() time.sleep(5) subprocess.call( './configure --prefix={} --scriptdir={} --with-pcap={}'.format( self.install_directory, self.configuration_directory, pf_ring_install.install_directory), shell=True, cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME)) subprocess.call('make; make install', shell=True, cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME)) if 'ZEEK_HOME' not in open('/etc/environment').read(): if stdout: sys.stdout.write( '[+] Updating Zeek default home path [{}]\n'.format( self.install_directory)) subprocess.call('echo ZEEK_HOME="{}" >> /etc/environment'.format( self.install_directory), shell=True) if 'ZEEK_SCRIPTS' not in open('/etc/environment').read(): if stdout: sys.stdout.write( '[+] Updating Zeek default script path [{}]\n'.format( self.configuration_directory)) subprocess.call( 'echo ZEEK_SCRIPTS="{}" >> /etc/environment'.format( self.configuration_directory), shell=True) if stdout: sys.stdout.write( '[+] Overwriting default Script | Node configurations.\n') shutil.copy( os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'broctl-nodes.cfg'), os.path.join(self.install_directory, 'etc', 'node.cfg')) shutil.copy( os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'local.bro'), os.path.join(self.configuration_directory, 'site', 'local.bro')) ZeekScriptConfigurator().write_config() node_config = ZeekNodeConfigurator(self.install_directory) available_cpus = utilities.get_cpu_core_count() workers_cpu_grps = [ range(0, available_cpus)[n:n + 2] for n in range(0, len(range(0, available_cpus)), 2) ] for i, cpu_group in enumerate(workers_cpu_grps): node_config.add_worker(name='dynamite-worker-{}'.format(i + 1), host='localhost', interface=network_interface, lb_procs=10, pin_cpus=cpu_group) node_config.write_config()
def optimize(self, available_cpus: Optional[List[int]] = None) -> None: """Apply the best CPU-affinity related configurations to Zeek and Suricata Returns: None """ zeek_profiler = zeek_profile.ProcessProfiler() suricata_profiler = suricata_profile.ProcessProfiler() if not available_cpus: available_cpus = [ c for c in range(0, utilities.get_cpu_core_count()) ] self.logger.info(f'{len(available_cpus)} CPU cores detected.') if zeek_profiler.is_attached_to_network( ) and suricata_profiler.is_attached_to_network(): self.logger.info( 'Both Zeek and Suricata are installed. Allocating 60% of resources to Zeek, ' '30% to Suricata, and 10% to Kernel.') kern_alloc, zeek_alloc, suricata_alloc = .1, .6, .3 elif zeek_profiler.is_attached_to_network(): self.logger.info( 'Only Zeek is installed. Allocating 90% of resources to it and 10% to Kernel.' ) kern_alloc, zeek_alloc, suricata_alloc = .1, .9, 0 elif suricata_profiler.is_attached_to_network(): self.logger.info( 'Only Suricata is installed. Allocating 90% of resources to it and 10% to Kernel.' ) kern_alloc, zeek_alloc, suricata_alloc = .1, 0, .9 else: self.logger.error( 'Neither Zeek nor Suricata is installed. You must install at least one of these in order ' 'to run this command.') return None if len(available_cpus) > 4: round_func = math.ceil else: round_func = math.floor kern_cpu_count = math.ceil(kern_alloc * len(available_cpus)) zeek_cpu_count = round_func(zeek_alloc * len(available_cpus)) suricata_cpu_count = round_func(suricata_alloc * len(available_cpus)) zeek_cpus = [ c for c in available_cpus[kern_cpu_count:kern_cpu_count + zeek_cpu_count] ] suricata_cpus = [ c for c in available_cpus[kern_cpu_count + zeek_cpu_count:kern_cpu_count + zeek_cpu_count + suricata_cpu_count] ] if zeek_profiler.is_attached_to_network(): zeek_node_config_mng = zeek_config.NodeConfigManager( install_directory=self.zeek_install_directory, stdout=self.stdout, verbose=self.verbose) zeek_node_config_mng.workers = zeek_node_config_mng.get_optimal_zeek_worker_config( zeek_profiler.get_attached_interfaces(), available_cpus=tuple(zeek_cpus)) zeek_node_config_mng.commit() if suricata_profiler.is_attached_to_network(): suricata_config_mng = suricata_config.ConfigManager( configuration_directory=self.suricata_configuration_directory, stdout=self.stdout, verbose=self.verbose) suricata_config_mng.threading = suricata_config_mng.get_optimal_suricata_threading_config( available_cpus=tuple(suricata_cpus)) suricata_config_mng.runmode = 'workers' for suricata_iface in suricata_config_mng.af_packet_interfaces: suricata_iface.threads = round_func( len(suricata_config_mng.threading.worker_cpu_set) / len(suricata_profiler.get_attached_interfaces())) suricata_iface.cluster_type = 'cluster_qm' suricata_config_mng.commit()
def _configure_and_compile_suricata(self): if self.configuration_directory.endswith('/'): suricata_config_parent = '/'.join( self.configuration_directory.split('/')[:-2]) else: suricata_config_parent = '/'.join( self.configuration_directory.split('/')[:-1]) self.logger.info( 'Compiling Suricata from source. This can take up to 5 to 10 minutes.' ) if self.stdout: utilities.print_coffee_art() time.sleep(1) self.logger.info('Configuring Suricata.') if self.verbose: suricata_config_p = subprocess.Popen( './configure --prefix={} --sysconfdir={} --localstatedir=/var/dynamite/suricata' .format(self.install_directory, suricata_config_parent), shell=True, cwd=os.path.join(const.INSTALL_CACHE, const.SURICATA_DIRECTORY_NAME)) else: suricata_config_p = subprocess.Popen( './configure --prefix={} --sysconfdir={} --localstatedir=/var/dynamite/suricata' .format(self.install_directory, suricata_config_parent), shell=True, cwd=os.path.join(const.INSTALL_CACHE, const.SURICATA_DIRECTORY_NAME), stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: suricata_config_p.communicate() except Exception as e: self.logger.error( "General error occurred while configuring Suricata.") self.logger.debug( "General error occurred while configuring Suricata; {}".format( e)) raise suricata_exceptions.InstallSuricataError( "General error occurred while configuring Suricata; {}".format( e)) if suricata_config_p.returncode != 0: self.logger.error( "Zeek configuration process returned non-zero; exit-code: {}". format(suricata_config_p.returncode)) raise suricata_exceptions.InstallSuricataError( "Suricata configuration process returned non-zero; exit-code: {}" .format(suricata_config_p.returncode)) time.sleep(1) self.logger.info("Compiling Suricata.") if utilities.get_cpu_core_count() > 1: parallel_threads = utilities.get_cpu_core_count() - 1 else: parallel_threads = 1 if self.verbose: compile_suricata_process = subprocess.Popen( 'make -j {}; make install; make install-conf'.format( parallel_threads), shell=True, cwd=os.path.join(const.INSTALL_CACHE, const.SURICATA_DIRECTORY_NAME)) try: compile_suricata_process.communicate() except Exception as e: self.logger.error( "General error occurred while compiling Suricata.") self.logger.debug( "General error occurred while compiling Suricata; {}". format(e)) raise suricata_exceptions.InstallSuricataError( "General error occurred while compiling Suricata; {}". format(e)) compile_suricata_return_code = compile_suricata_process.returncode else: compile_suricata_process = subprocess.Popen( 'make -j {}; make install; make install-conf'.format( parallel_threads), shell=True, cwd=os.path.join(const.INSTALL_CACHE, const.SURICATA_DIRECTORY_NAME), stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: compile_suricata_return_code = utilities.run_subprocess_with_status( compile_suricata_process, expected_lines=935) except Exception as e: self.logger.error( "General error occurred while compiling Suricata.") self.logger.debug( "General error occurred while compiling Suricata; {}". format(e)) raise suricata_exceptions.InstallSuricataError( "General error occurred while compiling Suricata; {}". format(e)) if compile_suricata_return_code != 0: self.logger.error( "Failed to compile Suricata from source; error code: {}; run with --verbose flag for more info." .format(compile_suricata_return_code)) raise suricata_exceptions.InstallSuricataError( "Suricata compilation process returned non-zero; exit-code: {}" .format(compile_suricata_return_code))
def get_zeek_workers(network_capture_interfaces, strategy="aggressive", stdout=True, verbose=False): """ Algorithm for determining the assignment of CPUs for Zeek workers :param network_capture_interfaces: A list of network interface names :param strategy: 'aggressive', results in more CPUs pinned per interface, sometimes overshoots resources 'conservative', results in less CPUs pinned per interface, but never overshoots resources :param stdout: Print the output to console :param verbose: Include detailed debug messages :return: A dictionary containing Zeek worker configuration """ log_level = logging.INFO if verbose: log_level = logging.DEBUG logger = get_logger('ZEEK', level=log_level, stdout=stdout) cpus = [c for c in range(0, utilities.get_cpu_core_count())] logger.info( "Calculating optimal Zeek worker strategy [strategy: {}].".format( strategy)) logger.debug("Detected CPU Cores: {}".format(cpus)) # Reserve 0 for KERNEL/Userland opts available_cpus = cpus[1:] def grouper(n, iterable): args = [iter(iterable)] * n return zip_longest(*args) def create_workers(net_interfaces, avail_cpus): idx = 0 zeek_worker_configs = [] for net_interface in net_interfaces: if idx >= len(avail_cpus): idx = 0 if isinstance(avail_cpus[idx], int): avail_cpus[idx] = [avail_cpus[idx]] zeek_worker_configs.append( dict(name='dynamite-worker-' + net_interface, host='localhost', interface=net_interface, lb_procs=len(avail_cpus[idx]), pinned_cpus=avail_cpus[idx])) idx += 1 return zeek_worker_configs if len(available_cpus) <= len(network_capture_interfaces): # Wrap the number of CPUs around the number of network interfaces; # Since there are more network interfaces than CPUs; CPUs will be assigned more than once # lb_procs will always be 1 zeek_workers = create_workers(network_capture_interfaces, available_cpus) else: # In this scenario we choose from one of two strategies # 1. Aggressive: # - Take the ratio of network_interfaces to available CPUS; ** ROUND UP **. # - Group the available CPUs by this integer # (if the ratio == 2 create as many groupings of 2 CPUs as possible) # - Apply the same wrapping logic used above, but with the CPU groups instead of single CPU instances # 2. Conservative: # - Take the ratio of network_interfaces to available CPUS; ** ROUND DOWN **. # - Group the available CPUs by this integer # (if the ratio == 2 create as many groupings of 2 CPUs as possible) # - Apply the same wrapping logic used above, but with the CPU groups instead of single CPU instances aggressive_ratio = int( math.ceil( len(available_cpus) / float(len(network_capture_interfaces)))) conservative_ratio = int( math.floor( len(available_cpus) / len(network_capture_interfaces))) if strategy == 'aggressive': cpu_groups = grouper(aggressive_ratio, available_cpus) else: cpu_groups = grouper(conservative_ratio, available_cpus) temp_cpu_groups = [] for cpu_group in cpu_groups: cpu_group = [c for c in cpu_group if c] temp_cpu_groups.append(cpu_group) cpu_groups = temp_cpu_groups zeek_workers = create_workers(network_capture_interfaces, cpu_groups) logger.info('Zeek Worker Count: {}'.format(len(zeek_workers))) logger.debug('Zeek Workers: {}'.format(zeek_workers)) return zeek_workers
def setup_zeek(self, network_interface=None): """ Setup Zeek NSM with PF_RING support :param stdout: Print output to console :param network_interface: The interface to listen on :return: True, if setup successful """ if not network_interface: network_interface = utilities.get_network_interface_names()[0] if network_interface not in utilities.get_network_interface_names(): sys.stderr.write( '[-] The network interface that your defined: \'{}\' is invalid. Valid network interfaces: {}\n' .format(network_interface, utilities.get_network_interface_names())) raise Exception( 'Invalid network interface {}'.format(network_interface)) if self.stdout: sys.stdout.write( '[+] Creating zeek install|configuration|logging directories.\n' ) subprocess.call('mkdir -p {}'.format(self.install_directory), shell=True) subprocess.call('mkdir -p {}'.format(self.configuration_directory), shell=True) pf_ring_profiler = pf_ring.PFRingProfiler() pf_ring_install = pf_ring.PFRingInstaller( downlaod_pf_ring_archive=not pf_ring_profiler.is_downloaded, stdout=self.stdout, verbose=self.verbose) if not pf_ring_profiler.is_installed: if self.stdout: sys.stdout.write( '[+] Installing PF_RING kernel modules and dependencies.\n' ) sys.stdout.flush() time.sleep(1) if self.stdout: sys.stdout.write( '[+] Compiling Zeek from source. This can take up to 30 minutes. ' 'Have another cup of coffee.\n') sys.stdout.flush() utilities.print_coffee_art() time.sleep(1) sys.stdout.write('[+] Configuring...\n') sys.stdout.flush() if self.verbose: subprocess.call( './configure --prefix={} --scriptdir={} --with-pcap={}'.format( self.install_directory, self.configuration_directory, pf_ring_install.install_directory), shell=True, cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME)) else: subprocess.call( './configure --prefix={} --scriptdir={} --with-pcap={}'.format( self.install_directory, self.configuration_directory, pf_ring_install.install_directory), shell=True, cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME), stdout=subprocess.PIPE, stderr=subprocess.PIPE) sys.stdout.write('[+] Compiling...\n') sys.stdout.flush() time.sleep(1) if self.verbose: compile_zeek_process = subprocess.Popen( 'make; make install', shell=True, cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME)) compile_zeek_process.communicate() compile_return_code = compile_zeek_process.returncode else: compile_zeek_process = subprocess.Popen( 'make; make install', shell=True, cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME), stdout=subprocess.PIPE, stderr=subprocess.PIPE) compile_return_code = utilities.run_subprocess_with_status( compile_zeek_process, expected_lines=6596) if compile_return_code != 0: sys.stderr.write( '[-] Failed to compile Zeek from source; error code: {}; ; run with ' '--debug flag for more info.\n'.format( compile_zeek_process.returncode)) return False if 'ZEEK_HOME' not in open('/etc/dynamite/environment').read(): if self.stdout: sys.stdout.write( '[+] Updating Zeek default home path [{}]\n'.format( self.install_directory)) subprocess.call( 'echo ZEEK_HOME="{}" >> /etc/dynamite/environment'.format( self.install_directory), shell=True) if 'ZEEK_SCRIPTS' not in open('/etc/dynamite/environment').read(): if self.stdout: sys.stdout.write( '[+] Updating Zeek default script path [{}]\n'.format( self.configuration_directory)) subprocess.call( 'echo ZEEK_SCRIPTS="{}" >> /etc/dynamite/environment'.format( self.configuration_directory), shell=True) if self.stdout: sys.stdout.write( '[+] Overwriting default Script | Node configurations.\n') shutil.copy( os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'broctl-nodes.cfg'), os.path.join(self.install_directory, 'etc', 'node.cfg')) shutil.copy( os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'local.bro'), os.path.join(self.configuration_directory, 'site', 'local.bro')) node_config = ZeekNodeConfigurator(self.install_directory) cpu_count = utilities.get_cpu_core_count() cpus = [c for c in range(0, cpu_count)] if cpu_count > 1: pinned_cpus = cpus[:-1] lb_procs = len(pinned_cpus) else: pinned_cpus = cpus lb_procs = 1 node_config.add_worker(name='dynamite-worker-1', host='localhost', interface=network_interface, lb_procs=lb_procs, pin_cpus=pinned_cpus) node_config.write_config()
def setup(self, inspect_interfaces: List[str]): """Install Suricata Args: inspect_interfaces: A list of network interfaces to capture on (E.G ["mon0", "mon1"]) Returns: None """ if not self.skip_interface_validation: if not self.validate_inspect_interfaces(inspect_interfaces): raise install.NetworkInterfaceNotFound(inspect_interfaces) sysctl = systemctl.SystemCtl() self.install_suricata_dependencies() self.create_update_suricata_environment_variables() self.logger.debug(f'Creating directory: {self.configuration_directory}') utilities.makedirs(self.configuration_directory) self.logger.debug(f'Creating directory: {self.install_directory}') utilities.makedirs(self.install_directory) self.logger.debug(f'Creating directory: {self.log_directory}') utilities.makedirs(self.log_directory) self.copy_suricata_files_and_directories() self.logger.info('Setting up Suricata from source. This can a few minutes.') if self.stdout: utilities.print_coffee_art() self.configure_compile_suricata() self.copy_file_or_directory_to_destination( f'{const.DEFAULT_CONFIGS}/suricata/suricata.yaml', self.configuration_directory ) suricata_config = config.ConfigManager(self.configuration_directory, stdout=self.stdout, verbose=self.verbose) suricata_config.default_log_directory = self.log_directory suricata_config.suricata_log_output_file = os.path.join(self.log_directory, 'suricata.log') suricata_config.default_rules_directory = os.path.join(self.configuration_directory, 'rules') suricata_config.reference_config_file = os.path.join(self.configuration_directory, 'reference.config') suricata_config.classification_file = os.path.join(self.configuration_directory, 'rules', 'classification.config') suricata_config.af_packet_interfaces = misc.AfPacketInterfaces() for interface in inspect_interfaces: suricata_config.af_packet_interfaces.add( misc.AfPacketInterface( interface_name=interface, threads='auto', cluster_id=random.randint(1, 50000), cluster_type='cluster_qm' ) ) suricata_config.threading = suricata_config.get_optimal_suricata_threading_config( tuple([i for i in range(0, utilities.get_cpu_core_count() - 1)])) suricata_config.commit() self.logger.info('Applying Suricata configuration.') self.logger.debug(suricata_config.af_packet_interfaces) suricata_config.commit() # Fix Permissions self.logger.info('Setting up file permissions.') utilities.set_ownership_of_file(self.configuration_directory, user='******', group='dynamite') utilities.set_ownership_of_file(self.install_directory, user='******', group='dynamite') utilities.set_ownership_of_file(self.log_directory, user='******', group='dynamite') post_install_bootstrap_updater(self.install_directory, stdout=self.stdout, verbose=self.verbose) self.logger.info(f'Installing service -> {const.DEFAULT_CONFIGS}/systemd/suricata.service') sysctl.install_and_enable(os.path.join(const.DEFAULT_CONFIGS, 'systemd', 'suricata.service'))
def get_parallel_threads() -> int: parallel_threads = 1 cpu_available_cores = utilities.get_cpu_core_count() if cpu_available_cores > 1: parallel_threads = cpu_available_cores - 1 return parallel_threads