def get_attached_interfaces(self) -> List[str]:
conf_mng = zeek_config.NodeConfigManager(
install_directory=self.install_directory,
stdout=False,
verbose=False)
if not conf_mng.workers:
return []
return [
worker.interface for worker in conf_mng.workers
if worker.interface in utilities.get_network_interface_names()
]
def setup(self, inspect_interfaces: List[str]):
"""Setup Zeek
Args:
inspect_interfaces: A list of network interfaces to capture on (E.G ["mon0", "mon1"])
Returns:
None
"""
if not self.skip_interface_validation:
if not self.validate_inspect_interfaces(inspect_interfaces):
raise install.NetworkInterfaceNotFound(inspect_interfaces)
sysctl = systemctl.SystemCtl()
self.install_zeek_dependencies()
self.create_update_zeek_environment_variables()
self.logger.debug(f'Creating directory: {self.configuration_directory}')
utilities.makedirs(self.configuration_directory)
self.logger.debug(f'Creating directory: {self.install_directory}')
utilities.makedirs(self.install_directory)
self.logger.info('Setting up Zeek from source. This can take up to 15 minutes.')
if self.stdout:
utilities.print_coffee_art()
self.configure_compile_zeek()
self.logger.info('Setting up Zeek package manager.')
zkg_installer = zkg_install.InstallManager()
zkg_installer.setup()
package.InstallPackageManager(const.ZEEK_PACKAGES, stdout=self.stdout, verbose=self.verbose).setup()
self.copy_file_or_directory_to_destination(f'{const.DEFAULT_CONFIGS}/zeek/broctl-nodes.cfg',
f'{self.install_directory}/etc/node.cfg')
self.copy_file_or_directory_to_destination(f'{const.DEFAULT_CONFIGS}/zeek/local.zeek',
f'{self.configuration_directory}/site/local.zeek')
# Optimize Configurations
site_local_config = config.SiteLocalConfigManager(self.configuration_directory, stdout=self.stdout,
verbose=self.verbose)
node_config = config.NodeConfigManager(self.install_directory, stdout=self.stdout, verbose=self.verbose)
node_config.workers = node.Workers()
for worker in node_config.get_optimal_zeek_worker_config(inspect_interfaces):
node_config.workers.add_worker(
worker=worker
)
self.logger.info('Applying node configuration.')
node_config.commit()
# Fix Permissions
self.logger.info('Setting up file permissions.')
utilities.set_ownership_of_file(self.configuration_directory, user='******', group='dynamite')
utilities.set_ownership_of_file(self.install_directory, user='******', group='dynamite')
self.logger.info(f'Installing service -> {const.DEFAULT_CONFIGS}/systemd/zeek.service')
sysctl.install_and_enable(os.path.join(const.DEFAULT_CONFIGS, 'systemd', 'zeek.service'))
def onStart(self):
npyscreen.setTheme(npyscreen.Themes.ColorfulTheme)
env_vars = get_environment_file_dict()
self.zeek_config = config.NodeConfigManager(env_vars['ZEEK_HOME'])
self.addForm('MAIN',
ZeekNodeSettingsForm,
name='Zeek Cluster Configuration')
self.addForm('EDITWORKERFM', EditWorkerForm, name='Edit Zeek Worker')
self.addForm('EDITLOGGERFM',
EditLoggerManagerProxy,
name='Edit Logger',
component_type='logger')
self.addForm('EDITMANAGERFM',
EditLoggerManagerProxy,
name='Edit Manager',
component_type='manager')
self.addForm('EDITPROXYFM',
EditLoggerManagerProxy,
name='Edit Proxy',
component='proxy')
def get_available_cpus(self) -> list:
"""Get the CPU core numbers that are not currently being utilized
Returns:
A list of available CPUs
"""
zeek_profiler = zeek_profile.ProcessProfiler()
suricata_profiler = suricata_profile.ProcessProfiler()
available_cpus = [c for c in range(0, utilities.get_cpu_core_count())]
reserved_cpus = []
zeek_cpus = []
suricata_cpus = []
if zeek_profiler.is_installed():
zeek_node_config_mng = zeek_config.NodeConfigManager(
install_directory=self.zeek_install_directory,
stdout=self.stdout,
verbose=self.verbose)
for worker in zeek_node_config_mng.workers:
zeek_cpus.extend(worker.pinned_cpus)
if suricata_profiler.is_installed():
suricata_config_mng = suricata_config.ConfigManager(
configuration_directory=self.suricata_configuration_directory,
stdout=self.stdout,
verbose=self.verbose)
if suricata_config_mng.threading.worker_cpu_set:
suricata_cpus.extend(
suricata_config_mng.threading.worker_cpu_set)
if suricata_config_mng.threading.receive_cpu_set:
suricata_cpus.extend(
suricata_config_mng.threading.receive_cpu_set)
if suricata_config_mng.threading.management_cpu_set:
suricata_cpus.extend(
suricata_config_mng.threading.management_cpu_set)
reserved_cpus.extend(suricata_cpus)
reserved_cpus.extend(zeek_cpus)
return list(set([c for c in available_cpus if c not in reserved_cpus]))
def setup_zeek(self):
"""
Setup Zeek NSM with PF_RING support
"""
env_file = os.path.join(const.CONFIG_PATH, 'environment')
self.logger.info('Creating Zeek installation, configuration and logging directories.')
try:
utilities.makedirs(self.install_directory, exist_ok=True)
utilities.makedirs(self.configuration_directory, exist_ok=True)
except Exception as e:
self.logger.error('General error occurred while attempting to create root directories.')
self.logger.debug("General error occurred while attempting to create root directories; {}".format(e))
raise zeek_exceptions.InstallZeekError(
"General error occurred while attempting to create root directories; {}".format(e))
self.logger.info('Compiling Zeek from source. This can take up to 30 minutes.')
if self.stdout:
utilities.print_coffee_art()
time.sleep(1)
self.logger.info('Configuring Zeek.')
if self.verbose:
zeek_config_p = subprocess.Popen('./configure --prefix={} --scriptdir={} --enable-jemalloc'.format(
self.install_directory, self.configuration_directory),
shell=True, cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME))
else:
zeek_config_p = subprocess.Popen('./configure --prefix={} --scriptdir={} --enable-jemalloc'.format(
self.install_directory, self.configuration_directory),
shell=True, cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME), stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
try:
zeek_config_p.communicate()
except Exception as e:
self.logger.error("General error occurred while configuring Zeek.")
self.logger.debug("General error occurred while configuring Zeek; {}".format(e))
raise zeek_exceptions.InstallZeekError(
"General error occurred while configuring Zeek; {}".format(e))
if zeek_config_p.returncode != 0:
self.logger.error(
"Zeek configuration process returned non-zero; exit-code: {}".format(zeek_config_p.returncode))
raise zeek_exceptions.InstallZeekError(
"Zeek configuration process returned non-zero; exit-code: {}".format(zeek_config_p.returncode))
time.sleep(1)
self.logger.info("Compiling Zeek.")
if utilities.get_cpu_core_count() > 1:
parallel_threads = utilities.get_cpu_core_count() - 1
else:
parallel_threads = 1
if self.verbose:
compile_zeek_process = subprocess.Popen('make -j {}; make install'.format(parallel_threads), shell=True,
cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME))
try:
compile_zeek_process.communicate()
except Exception as e:
self.logger.error("General error occurred while compiling Zeek.")
self.logger.debug("General error occurred while compiling Zeek; {}".format(e))
raise zeek_exceptions.InstallZeekError(
"General error occurred while compiling Zeek; {}".format(e))
compile_zeek_return_code = compile_zeek_process.returncode
else:
compile_zeek_process = subprocess.Popen('make -j {}; make install'.format(parallel_threads), shell=True,
cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME),
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
try:
compile_zeek_return_code = utilities.run_subprocess_with_status(compile_zeek_process,
expected_lines=6779)
except Exception as e:
self.logger.error("General error occurred while compiling Zeek.")
self.logger.debug("General error occurred while compiling Zeek; {}".format(e))
raise zeek_exceptions.InstallZeekError(
"General error occurred while compiling Zeek; {}".format(e))
if compile_zeek_return_code != 0:
self.logger.error(
"Failed to compile Zeek from source; error code: {}; run with --verbose flag for more info.".format(
compile_zeek_return_code))
raise zeek_exceptions.InstallZeekError(
"Zeek compilation process returned non-zero; exit-code: {}".format(compile_zeek_return_code))
try:
with open(env_file) as env_f:
if 'ZEEK_HOME' not in env_f.read():
self.logger.info('Updating Zeek default home path [{}]'.format(self.install_directory))
subprocess.call('echo ZEEK_HOME="{}" >> {}'.format(self.install_directory, env_file),
shell=True)
if 'ZEEK_SCRIPTS' not in env_f.read():
self.logger.info('Updating Zeek default script path [{}]'.format(self.configuration_directory))
subprocess.call('echo ZEEK_SCRIPTS="{}" >> {}'.format(self.configuration_directory, env_file),
shell=True)
except IOError as e:
self.logger.error("Failed to open {} for reading.".format(env_file))
self.logger.debug("Failed to open {} for reading; {}".format(env_file, e))
raise zeek_exceptions.InstallZeekError(
"Failed to open {} for reading; {}".format(env_file, e))
except Exception as e:
self.logger.error("General error while creating environment variables in {}.".format(env_file))
self.logger.debug("General error while creating environment variables in {}; {}".format(env_file, e))
raise zeek_exceptions.InstallZeekError(
"General error while creating environment variables in {}; {}".format(env_file, e))
self.logger.info("Overwriting Zeek node.cfg file with our changes.")
try:
shutil.copy(os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'broctl-nodes.cfg'),
os.path.join(self.install_directory, 'etc', 'node.cfg'))
shutil.copy(os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'local.zeek'),
os.path.join(self.configuration_directory, 'site', 'local.zeek'))
except Exception as e:
self.logger.error("General error occurred while copying default Zeek configurations.")
self.logger.debug("General error occurred while copying default Zeek configurations; {}".format(e))
raise zeek_exceptions.InstallZeekError(
"General error occurred while copying default Zeek configurations; {}".format(e))
try:
node_config = zeek_configs.NodeConfigManager(self.install_directory)
except zeek_exceptions.ReadsZeekConfigError:
self.logger.error("An error occurred while reading Zeek configurations.")
raise zeek_exceptions.InstallZeekError("An error occurred while reading Zeek configurations.")
# Clear out pre-set workers.
for key in list(node_config.node_config):
if node_config.node_config[key]['type'] == 'worker':
del node_config.node_config[key]
# Calculate new workers.
for worker in node_config.get_optimal_zeek_worker_config(self.capture_network_interfaces, stdout=self.stdout,
verbose=self.verbose):
node_config.add_worker(name=worker['name'],
host=worker['host'],
interface=worker['interface'],
lb_procs=worker['lb_procs'],
pin_cpus=worker['pinned_cpus']
)
try:
node_config.write_config()
except zeek_exceptions.WriteZeekConfigError:
self.logger.error("An error occurred while writing Zeek configurations.")
raise zeek_exceptions.InstallZeekError("An error occurred while writing Zeek configurations.")
try:
sysctl = systemctl.SystemCtl()
except general_exceptions.CallProcessError:
raise zeek_exceptions.InstallZeekError("Could not find systemctl.")
self.logger.info("Installing Zeek systemd service.")
if not sysctl.install_and_enable(os.path.join(const.DEFAULT_CONFIGS, 'systemd', 'zeek.service')):
raise zeek_exceptions.InstallZeekError("Failed to install Zeek systemd service.")
def optimize(self, available_cpus: Optional[List[int]] = None) -> None:
"""Apply the best CPU-affinity related configurations to Zeek and Suricata
Returns:
None
"""
zeek_profiler = zeek_profile.ProcessProfiler()
suricata_profiler = suricata_profile.ProcessProfiler()
if not available_cpus:
available_cpus = [
c for c in range(0, utilities.get_cpu_core_count())
]
self.logger.info(f'{len(available_cpus)} CPU cores detected.')
if zeek_profiler.is_attached_to_network(
) and suricata_profiler.is_attached_to_network():
self.logger.info(
'Both Zeek and Suricata are installed. Allocating 60% of resources to Zeek, '
'30% to Suricata, and 10% to Kernel.')
kern_alloc, zeek_alloc, suricata_alloc = .1, .6, .3
elif zeek_profiler.is_attached_to_network():
self.logger.info(
'Only Zeek is installed. Allocating 90% of resources to it and 10% to Kernel.'
)
kern_alloc, zeek_alloc, suricata_alloc = .1, .9, 0
elif suricata_profiler.is_attached_to_network():
self.logger.info(
'Only Suricata is installed. Allocating 90% of resources to it and 10% to Kernel.'
)
kern_alloc, zeek_alloc, suricata_alloc = .1, 0, .9
else:
self.logger.error(
'Neither Zeek nor Suricata is installed. You must install at least one of these in order '
'to run this command.')
return None
if len(available_cpus) > 4:
round_func = math.ceil
else:
round_func = math.floor
kern_cpu_count = math.ceil(kern_alloc * len(available_cpus))
zeek_cpu_count = round_func(zeek_alloc * len(available_cpus))
suricata_cpu_count = round_func(suricata_alloc * len(available_cpus))
zeek_cpus = [
c for c in available_cpus[kern_cpu_count:kern_cpu_count +
zeek_cpu_count]
]
suricata_cpus = [
c for c in available_cpus[kern_cpu_count +
zeek_cpu_count:kern_cpu_count +
zeek_cpu_count + suricata_cpu_count]
]
if zeek_profiler.is_attached_to_network():
zeek_node_config_mng = zeek_config.NodeConfigManager(
install_directory=self.zeek_install_directory,
stdout=self.stdout,
verbose=self.verbose)
zeek_node_config_mng.workers = zeek_node_config_mng.get_optimal_zeek_worker_config(
zeek_profiler.get_attached_interfaces(),
available_cpus=tuple(zeek_cpus))
zeek_node_config_mng.commit()
if suricata_profiler.is_attached_to_network():
suricata_config_mng = suricata_config.ConfigManager(
configuration_directory=self.suricata_configuration_directory,
stdout=self.stdout,
verbose=self.verbose)
suricata_config_mng.threading = suricata_config_mng.get_optimal_suricata_threading_config(
available_cpus=tuple(suricata_cpus))
suricata_config_mng.runmode = 'workers'
for suricata_iface in suricata_config_mng.af_packet_interfaces:
suricata_iface.threads = round_func(
len(suricata_config_mng.threading.worker_cpu_set) /
len(suricata_profiler.get_attached_interfaces()))
suricata_iface.cluster_type = 'cluster_qm'
suricata_config_mng.commit()