def get_optimal_zeek_worker_config(
interface_names: List[str],
available_cpus: Optional[Tuple] = None) -> node.Workers:
"""Algorithm for determining the assignment of CPUs for Zeek workers
Args:
interface_names: A list of network interface names
available_cpus: If None, we'll derive this by looking at the cpu core count, otherwise a list of cpu cores
Returns:
A node.Workers object
"""
zeek_worker_configs = node.Workers()
if not available_cpus:
# Reserve CPU 0 for KERNEL operations
available_cpus = [
c for c in range(1, utilities.get_cpu_core_count())
]
for cpu_affinity_group in utilities.get_optimal_cpu_interface_config(
interface_names=interface_names,
available_cpus=available_cpus):
net_interface = cpu_affinity_group['interface_name']
pinned_cpus = cpu_affinity_group['pin_cpus']
lb_processes = cpu_affinity_group['thread_count']
zeek_worker_configs.add_worker(
node.Worker(worker_name='dynamite-worker-' + net_interface,
host='localhost',
interface_name=net_interface,
load_balance_processes=lb_processes,
pinned_cpus=pinned_cpus,
cluster_id=randint(1, 32768),
cluster_type='AF_Packet::FANOUT_QM'))
return zeek_worker_configs
def add_worker(self,
name,
interface,
host,
lb_procs=10,
pin_cpus=(0, 1),
af_packet_fanout_id=None,
af_packet_fanout_mode=None):
"""
:param name: The name of the worker
:param interface: The interface that the worker should be monitoring
:param host: The host on which the worker is running
:param lb_procs: The number of Zeek processes associated with a given worker
:param pin_cpus: Core affinity for the processes (iterable),
:param af_packet_fanout_id: To scale processing across threads, packet sockets can form a
fanout group. In this mode, each matching packet is enqueued
onto only one socket in the group. A socket joins a fanout
group by calling setsockopt(2) with level SOL_PACKET and
option PACKET_FANOUT. Each network namespace can have up to
65536 independent groups.
:param af_packet_fanout_mode: The algorithm used to spread traffic between sockets.
"""
valid_fanout_modes = [
'FANOUT_HASH', # The default mode, PACKET_FANOUT_HASH, sends packets from
# the same flow to the same socket to maintain per-flow
# ordering. For each packet, it chooses a socket by taking
# the packet flow hash modulo the number of sockets in the
# group, where a flow hash is a hash over network-layer
# address and optional transport-layer port fields.
'FANOUT_CPU', # selects the socket based on the CPU that the packet arrived on
'FANOUT_QM' # (available since Linux 3.14) selects the socket using the recorded
# queue_mapping of the received skb.
]
if not str(interface).startswith('af_packet::'):
interface = 'af_packet::' + interface
if not af_packet_fanout_id:
af_packet_fanout_id = random.randint(1, 32768)
if not af_packet_fanout_mode:
af_packet_fanout_mode = 'AF_Packet::FANOUT_HASH'
else:
if str(af_packet_fanout_mode).upper() in valid_fanout_modes:
af_packet_fanout_mode = 'AF_Packet::' + str(
af_packet_fanout_mode).upper()
else:
af_packet_fanout_mode = 'AF_Packet::FANOUT_HASH'
if max(pin_cpus) < utilities.get_cpu_core_count() and min(
pin_cpus) >= 0:
pin_cpus = [str(cpu_n) for cpu_n in pin_cpus]
self.node_config[name] = {
'type': 'worker',
'interface': interface,
'lb_method': 'custom',
'lb_procs': lb_procs,
'pin_cpus': ','.join(pin_cpus),
'host': host,
'af_packet_fanout_id': af_packet_fanout_id,
'af_packet_fanout_mode': af_packet_fanout_mode
}
def add_worker(self, name, interface, host, lb_procs=10, pin_cpus=(0, 1)):
"""
:param name: The name of the worker
:param interface: The interface that the worker should be monitoring
:param host: The host on which the worker is running
:param lb_procs: The number of Zeek processes associated with a given worker
:param pin_cpus: Core affinity for the processes (iterable)
"""
if not str(interface).startswith('af_packet::'):
interface = 'af_packet::' + interface
if max(pin_cpus) < utilities.get_cpu_core_count() and min(
pin_cpus) >= 0:
pin_cpus = [str(cpu_n) for cpu_n in pin_cpus]
self.node_config[name] = {
'type': 'worker',
'interface': interface,
'lb_method': 'custom',
'lb_procs': lb_procs,
'pin_cpus': ','.join(pin_cpus),
'host': host
}
def add_worker(self, name, interface, host, lb_procs=10, pin_cpus=(0, 1)):
"""
:param name: The name of the worker
:param interface: The interface that the worker should be monitoring
:param host: The host on which the worker is running
:param lb_procs: The number of Zeek processes associated with a given worker
:param pin_cpus: Core affinity for the processes (iterable)
:return: True, if added successfully
"""
if max(pin_cpus) < utilities.get_cpu_core_count() and min(
pin_cpus) >= 0:
pin_cpus = [str(cpu_n) for cpu_n in pin_cpus]
self.node_config[name] = {
'type': 'worker',
'interface': interface,
'lb_method': 'pf_ring',
'lb_procs': lb_procs,
'pin_cpus': ','.join(pin_cpus),
'host': host
}
return True
return False
def get_available_cpus(self) -> list:
"""Get the CPU core numbers that are not currently being utilized
Returns:
A list of available CPUs
"""
zeek_profiler = zeek_profile.ProcessProfiler()
suricata_profiler = suricata_profile.ProcessProfiler()
available_cpus = [c for c in range(0, utilities.get_cpu_core_count())]
reserved_cpus = []
zeek_cpus = []
suricata_cpus = []
if zeek_profiler.is_installed():
zeek_node_config_mng = zeek_config.NodeConfigManager(
install_directory=self.zeek_install_directory,
stdout=self.stdout,
verbose=self.verbose)
for worker in zeek_node_config_mng.workers:
zeek_cpus.extend(worker.pinned_cpus)
if suricata_profiler.is_installed():
suricata_config_mng = suricata_config.ConfigManager(
configuration_directory=self.suricata_configuration_directory,
stdout=self.stdout,
verbose=self.verbose)
if suricata_config_mng.threading.worker_cpu_set:
suricata_cpus.extend(
suricata_config_mng.threading.worker_cpu_set)
if suricata_config_mng.threading.receive_cpu_set:
suricata_cpus.extend(
suricata_config_mng.threading.receive_cpu_set)
if suricata_config_mng.threading.management_cpu_set:
suricata_cpus.extend(
suricata_config_mng.threading.management_cpu_set)
reserved_cpus.extend(suricata_cpus)
reserved_cpus.extend(zeek_cpus)
return list(set([c for c in available_cpus if c not in reserved_cpus]))
def setup_zeek(self):
"""
Setup Zeek NSM with PF_RING support
"""
env_file = os.path.join(const.CONFIG_PATH, 'environment')
self.logger.info('Creating Zeek installation, configuration and logging directories.')
try:
utilities.makedirs(self.install_directory, exist_ok=True)
utilities.makedirs(self.configuration_directory, exist_ok=True)
except Exception as e:
self.logger.error('General error occurred while attempting to create root directories.')
self.logger.debug("General error occurred while attempting to create root directories; {}".format(e))
raise zeek_exceptions.InstallZeekError(
"General error occurred while attempting to create root directories; {}".format(e))
self.logger.info('Compiling Zeek from source. This can take up to 30 minutes.')
if self.stdout:
utilities.print_coffee_art()
time.sleep(1)
self.logger.info('Configuring Zeek.')
if self.verbose:
zeek_config_p = subprocess.Popen('./configure --prefix={} --scriptdir={} --enable-jemalloc'.format(
self.install_directory, self.configuration_directory),
shell=True, cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME))
else:
zeek_config_p = subprocess.Popen('./configure --prefix={} --scriptdir={} --enable-jemalloc'.format(
self.install_directory, self.configuration_directory),
shell=True, cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME), stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
try:
zeek_config_p.communicate()
except Exception as e:
self.logger.error("General error occurred while configuring Zeek.")
self.logger.debug("General error occurred while configuring Zeek; {}".format(e))
raise zeek_exceptions.InstallZeekError(
"General error occurred while configuring Zeek; {}".format(e))
if zeek_config_p.returncode != 0:
self.logger.error(
"Zeek configuration process returned non-zero; exit-code: {}".format(zeek_config_p.returncode))
raise zeek_exceptions.InstallZeekError(
"Zeek configuration process returned non-zero; exit-code: {}".format(zeek_config_p.returncode))
time.sleep(1)
self.logger.info("Compiling Zeek.")
if utilities.get_cpu_core_count() > 1:
parallel_threads = utilities.get_cpu_core_count() - 1
else:
parallel_threads = 1
if self.verbose:
compile_zeek_process = subprocess.Popen('make -j {}; make install'.format(parallel_threads), shell=True,
cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME))
try:
compile_zeek_process.communicate()
except Exception as e:
self.logger.error("General error occurred while compiling Zeek.")
self.logger.debug("General error occurred while compiling Zeek; {}".format(e))
raise zeek_exceptions.InstallZeekError(
"General error occurred while compiling Zeek; {}".format(e))
compile_zeek_return_code = compile_zeek_process.returncode
else:
compile_zeek_process = subprocess.Popen('make -j {}; make install'.format(parallel_threads), shell=True,
cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME),
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
try:
compile_zeek_return_code = utilities.run_subprocess_with_status(compile_zeek_process,
expected_lines=6779)
except Exception as e:
self.logger.error("General error occurred while compiling Zeek.")
self.logger.debug("General error occurred while compiling Zeek; {}".format(e))
raise zeek_exceptions.InstallZeekError(
"General error occurred while compiling Zeek; {}".format(e))
if compile_zeek_return_code != 0:
self.logger.error(
"Failed to compile Zeek from source; error code: {}; run with --verbose flag for more info.".format(
compile_zeek_return_code))
raise zeek_exceptions.InstallZeekError(
"Zeek compilation process returned non-zero; exit-code: {}".format(compile_zeek_return_code))
try:
with open(env_file) as env_f:
if 'ZEEK_HOME' not in env_f.read():
self.logger.info('Updating Zeek default home path [{}]'.format(self.install_directory))
subprocess.call('echo ZEEK_HOME="{}" >> {}'.format(self.install_directory, env_file),
shell=True)
if 'ZEEK_SCRIPTS' not in env_f.read():
self.logger.info('Updating Zeek default script path [{}]'.format(self.configuration_directory))
subprocess.call('echo ZEEK_SCRIPTS="{}" >> {}'.format(self.configuration_directory, env_file),
shell=True)
except IOError as e:
self.logger.error("Failed to open {} for reading.".format(env_file))
self.logger.debug("Failed to open {} for reading; {}".format(env_file, e))
raise zeek_exceptions.InstallZeekError(
"Failed to open {} for reading; {}".format(env_file, e))
except Exception as e:
self.logger.error("General error while creating environment variables in {}.".format(env_file))
self.logger.debug("General error while creating environment variables in {}; {}".format(env_file, e))
raise zeek_exceptions.InstallZeekError(
"General error while creating environment variables in {}; {}".format(env_file, e))
self.logger.info("Overwriting Zeek node.cfg file with our changes.")
try:
shutil.copy(os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'broctl-nodes.cfg'),
os.path.join(self.install_directory, 'etc', 'node.cfg'))
shutil.copy(os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'local.zeek'),
os.path.join(self.configuration_directory, 'site', 'local.zeek'))
except Exception as e:
self.logger.error("General error occurred while copying default Zeek configurations.")
self.logger.debug("General error occurred while copying default Zeek configurations; {}".format(e))
raise zeek_exceptions.InstallZeekError(
"General error occurred while copying default Zeek configurations; {}".format(e))
try:
node_config = zeek_configs.NodeConfigManager(self.install_directory)
except zeek_exceptions.ReadsZeekConfigError:
self.logger.error("An error occurred while reading Zeek configurations.")
raise zeek_exceptions.InstallZeekError("An error occurred while reading Zeek configurations.")
# Clear out pre-set workers.
for key in list(node_config.node_config):
if node_config.node_config[key]['type'] == 'worker':
del node_config.node_config[key]
# Calculate new workers.
for worker in node_config.get_optimal_zeek_worker_config(self.capture_network_interfaces, stdout=self.stdout,
verbose=self.verbose):
node_config.add_worker(name=worker['name'],
host=worker['host'],
interface=worker['interface'],
lb_procs=worker['lb_procs'],
pin_cpus=worker['pinned_cpus']
)
try:
node_config.write_config()
except zeek_exceptions.WriteZeekConfigError:
self.logger.error("An error occurred while writing Zeek configurations.")
raise zeek_exceptions.InstallZeekError("An error occurred while writing Zeek configurations.")
try:
sysctl = systemctl.SystemCtl()
except general_exceptions.CallProcessError:
raise zeek_exceptions.InstallZeekError("Could not find systemctl.")
self.logger.info("Installing Zeek systemd service.")
if not sysctl.install_and_enable(os.path.join(const.DEFAULT_CONFIGS, 'systemd', 'zeek.service')):
raise zeek_exceptions.InstallZeekError("Failed to install Zeek systemd service.")
def setup_zeek_community_id_plugin(self):
bro_commmunity_id_plugin_path = \
os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'uncompiled_scripts', 'zeek-community-id')
self.logger.info('Configuring Zeek Corelight_CommunityID plugin.')
if self.verbose:
config_zeek_community_id_script_process = subprocess.Popen(
'./configure --zeek-dist={} --install-root={}'.format(
os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME), self.configuration_directory),
shell=True, cwd=bro_commmunity_id_plugin_path
)
else:
config_zeek_community_id_script_process = subprocess.Popen(
'./configure --zeek-dist={} --install-root={}'.format(
os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME), self.configuration_directory),
shell=True, cwd=bro_commmunity_id_plugin_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
try:
config_zeek_community_id_script_process.communicate()
except Exception as e:
self.logger.error('General error occurred while starting Corelight_CommunityID configuration.')
self.logger.debug('General error occurred while starting Corelight_CommunityID configuration; {}'.format(e))
raise zeek_exceptions.InstallZeekError(
"General error occurred while starting Corelight_CommunityID configuration; {}".format(e))
if config_zeek_community_id_script_process.returncode != 0:
self.logger.debug("Corelight_CommunityID configuration returned non-zero; exit-code: {}".format(
config_zeek_community_id_script_process.returncode))
raise zeek_exceptions.InstallZeekError(
"Corelight_CommunityID configuration returned non-zero; exit-code: {}".format(
config_zeek_community_id_script_process.returncode))
self.logger.info('Compiling Zeek Corelight_CommunityID [PATCHED] plugin.')
if utilities.get_cpu_core_count() > 1:
parallel_threads = utilities.get_cpu_core_count() - 1
else:
parallel_threads = 1
if self.verbose:
compile_zeek_community_id_script_process = subprocess.Popen(
'make -j {}; make install'.format(parallel_threads), shell=True,
cwd=bro_commmunity_id_plugin_path)
else:
compile_zeek_community_id_script_process = subprocess.Popen(
'make -j {}; make install'.format(parallel_threads), shell=True,
cwd=bro_commmunity_id_plugin_path,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
try:
compile_zeek_community_id_script_process.communicate()
except Exception as e:
self.logger.error('General error occurred while compiling Corelight_CommunityID [PATCHED].')
self.logger.debug("General error occurred while compiling Corelight_CommunityID [PATCHED]; {}".format(e))
raise zeek_exceptions.InstallZeekError(
"General error occurred while compiling Corelight_CommunityID [PATCHED]; {}".format(e))
if compile_zeek_community_id_script_process.returncode != 0:
self.logger.error("General error occurred while compiling Corelight_CommunityID [PATCHED]; {}".format(
compile_zeek_community_id_script_process.returncode))
raise zeek_exceptions.InstallZeekError(
"Corelight_CommunityID [PATCHED] compilation process returned non-zero; exit-code: {}".format(
compile_zeek_community_id_script_process.returncode))
try:
shutil.copytree(os.path.join(self.configuration_directory, 'Corelight_CommunityID'),
os.path.join(self.install_directory, 'lib', 'zeek', 'plugins', 'Corelight_CommunityID'))
except Exception as e:
if 'FileExist' not in str(e):
self.logger.error("General error occurred while installing Corelight_CommunityID [PATCHED] plugin.")
self.logger.debug("General error occurred while installing Corelight_CommunityID [PATCHED] plugin; "
"{}".format(e))
raise zeek_exceptions.InstallZeekError(
"General error occurred while installing Corelight_CommunityID [PATCHED] plugin; {}".format(
e))
def setup_zeek(self, network_interface=None, stdout=False):
"""
Setup Zeek NSM with PF_RING support
:param stdout: Print output to console
:param network_interface: The interface to listen on
:return: True, if setup successful
"""
if not network_interface:
network_interface = utilities.get_network_interface_names()[0]
if network_interface not in utilities.get_network_interface_names():
sys.stderr.write(
'[-] The network interface that your defined: \'{}\' is invalid. Valid network interfaces: {}\n'
.format(network_interface,
utilities.get_network_interface_names()))
return False
if stdout:
sys.stdout.write(
'[+] Creating zeek install|configuration|logging directories.\n'
)
subprocess.call('mkdir -p {}'.format(self.install_directory),
shell=True)
subprocess.call('mkdir -p {}'.format(self.configuration_directory),
shell=True)
pf_ring_install = pf_ring.PFRingInstaller()
if not pf_ring.PFRingProfiler().is_installed:
if stdout:
sys.stdout.write(
'[+] Installing PF_RING kernel modules and dependencies.\n'
)
sys.stdout.flush()
time.sleep(1)
pf_ring_install.download_pf_ring(stdout=True)
pf_ring_install.extract_pf_ring(stdout=True)
pf_ring_install.setup_pf_ring(stdout=True)
if stdout:
sys.stdout.write(
'\n\n[+] Compiling Zeek from source. This can take up to 30 minutes. Have a cup of coffee.'
'\n\n')
sys.stdout.flush()
time.sleep(5)
subprocess.call(
'./configure --prefix={} --scriptdir={} --with-pcap={}'.format(
self.install_directory, self.configuration_directory,
pf_ring_install.install_directory),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME))
subprocess.call('make; make install',
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.ZEEK_DIRECTORY_NAME))
if 'ZEEK_HOME' not in open('/etc/environment').read():
if stdout:
sys.stdout.write(
'[+] Updating Zeek default home path [{}]\n'.format(
self.install_directory))
subprocess.call('echo ZEEK_HOME="{}" >> /etc/environment'.format(
self.install_directory),
shell=True)
if 'ZEEK_SCRIPTS' not in open('/etc/environment').read():
if stdout:
sys.stdout.write(
'[+] Updating Zeek default script path [{}]\n'.format(
self.configuration_directory))
subprocess.call(
'echo ZEEK_SCRIPTS="{}" >> /etc/environment'.format(
self.configuration_directory),
shell=True)
if stdout:
sys.stdout.write(
'[+] Overwriting default Script | Node configurations.\n')
shutil.copy(
os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'broctl-nodes.cfg'),
os.path.join(self.install_directory, 'etc', 'node.cfg'))
shutil.copy(
os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'local.bro'),
os.path.join(self.configuration_directory, 'site', 'local.bro'))
ZeekScriptConfigurator().write_config()
node_config = ZeekNodeConfigurator(self.install_directory)
available_cpus = utilities.get_cpu_core_count()
workers_cpu_grps = [
range(0, available_cpus)[n:n + 2]
for n in range(0, len(range(0, available_cpus)), 2)
]
for i, cpu_group in enumerate(workers_cpu_grps):
node_config.add_worker(name='dynamite-worker-{}'.format(i + 1),
host='localhost',
interface=network_interface,
lb_procs=10,
pin_cpus=cpu_group)
node_config.write_config()
def optimize(self, available_cpus: Optional[List[int]] = None) -> None:
"""Apply the best CPU-affinity related configurations to Zeek and Suricata
Returns:
None
"""
zeek_profiler = zeek_profile.ProcessProfiler()
suricata_profiler = suricata_profile.ProcessProfiler()
if not available_cpus:
available_cpus = [
c for c in range(0, utilities.get_cpu_core_count())
]
self.logger.info(f'{len(available_cpus)} CPU cores detected.')
if zeek_profiler.is_attached_to_network(
) and suricata_profiler.is_attached_to_network():
self.logger.info(
'Both Zeek and Suricata are installed. Allocating 60% of resources to Zeek, '
'30% to Suricata, and 10% to Kernel.')
kern_alloc, zeek_alloc, suricata_alloc = .1, .6, .3
elif zeek_profiler.is_attached_to_network():
self.logger.info(
'Only Zeek is installed. Allocating 90% of resources to it and 10% to Kernel.'
)
kern_alloc, zeek_alloc, suricata_alloc = .1, .9, 0
elif suricata_profiler.is_attached_to_network():
self.logger.info(
'Only Suricata is installed. Allocating 90% of resources to it and 10% to Kernel.'
)
kern_alloc, zeek_alloc, suricata_alloc = .1, 0, .9
else:
self.logger.error(
'Neither Zeek nor Suricata is installed. You must install at least one of these in order '
'to run this command.')
return None
if len(available_cpus) > 4:
round_func = math.ceil
else:
round_func = math.floor
kern_cpu_count = math.ceil(kern_alloc * len(available_cpus))
zeek_cpu_count = round_func(zeek_alloc * len(available_cpus))
suricata_cpu_count = round_func(suricata_alloc * len(available_cpus))
zeek_cpus = [
c for c in available_cpus[kern_cpu_count:kern_cpu_count +
zeek_cpu_count]
]
suricata_cpus = [
c for c in available_cpus[kern_cpu_count +
zeek_cpu_count:kern_cpu_count +
zeek_cpu_count + suricata_cpu_count]
]
if zeek_profiler.is_attached_to_network():
zeek_node_config_mng = zeek_config.NodeConfigManager(
install_directory=self.zeek_install_directory,
stdout=self.stdout,
verbose=self.verbose)
zeek_node_config_mng.workers = zeek_node_config_mng.get_optimal_zeek_worker_config(
zeek_profiler.get_attached_interfaces(),
available_cpus=tuple(zeek_cpus))
zeek_node_config_mng.commit()
if suricata_profiler.is_attached_to_network():
suricata_config_mng = suricata_config.ConfigManager(
configuration_directory=self.suricata_configuration_directory,
stdout=self.stdout,
verbose=self.verbose)
suricata_config_mng.threading = suricata_config_mng.get_optimal_suricata_threading_config(
available_cpus=tuple(suricata_cpus))
suricata_config_mng.runmode = 'workers'
for suricata_iface in suricata_config_mng.af_packet_interfaces:
suricata_iface.threads = round_func(
len(suricata_config_mng.threading.worker_cpu_set) /
len(suricata_profiler.get_attached_interfaces()))
suricata_iface.cluster_type = 'cluster_qm'
suricata_config_mng.commit()
def _configure_and_compile_suricata(self):
if self.configuration_directory.endswith('/'):
suricata_config_parent = '/'.join(
self.configuration_directory.split('/')[:-2])
else:
suricata_config_parent = '/'.join(
self.configuration_directory.split('/')[:-1])
self.logger.info(
'Compiling Suricata from source. This can take up to 5 to 10 minutes.'
)
if self.stdout:
utilities.print_coffee_art()
time.sleep(1)
self.logger.info('Configuring Suricata.')
if self.verbose:
suricata_config_p = subprocess.Popen(
'./configure --prefix={} --sysconfdir={} --localstatedir=/var/dynamite/suricata'
.format(self.install_directory, suricata_config_parent),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.SURICATA_DIRECTORY_NAME))
else:
suricata_config_p = subprocess.Popen(
'./configure --prefix={} --sysconfdir={} --localstatedir=/var/dynamite/suricata'
.format(self.install_directory, suricata_config_parent),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.SURICATA_DIRECTORY_NAME),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
try:
suricata_config_p.communicate()
except Exception as e:
self.logger.error(
"General error occurred while configuring Suricata.")
self.logger.debug(
"General error occurred while configuring Suricata; {}".format(
e))
raise suricata_exceptions.InstallSuricataError(
"General error occurred while configuring Suricata; {}".format(
e))
if suricata_config_p.returncode != 0:
self.logger.error(
"Zeek configuration process returned non-zero; exit-code: {}".
format(suricata_config_p.returncode))
raise suricata_exceptions.InstallSuricataError(
"Suricata configuration process returned non-zero; exit-code: {}"
.format(suricata_config_p.returncode))
time.sleep(1)
self.logger.info("Compiling Suricata.")
if utilities.get_cpu_core_count() > 1:
parallel_threads = utilities.get_cpu_core_count() - 1
else:
parallel_threads = 1
if self.verbose:
compile_suricata_process = subprocess.Popen(
'make -j {}; make install; make install-conf'.format(
parallel_threads),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.SURICATA_DIRECTORY_NAME))
try:
compile_suricata_process.communicate()
except Exception as e:
self.logger.error(
"General error occurred while compiling Suricata.")
self.logger.debug(
"General error occurred while compiling Suricata; {}".
format(e))
raise suricata_exceptions.InstallSuricataError(
"General error occurred while compiling Suricata; {}".
format(e))
compile_suricata_return_code = compile_suricata_process.returncode
else:
compile_suricata_process = subprocess.Popen(
'make -j {}; make install; make install-conf'.format(
parallel_threads),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.SURICATA_DIRECTORY_NAME),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
try:
compile_suricata_return_code = utilities.run_subprocess_with_status(
compile_suricata_process, expected_lines=935)
except Exception as e:
self.logger.error(
"General error occurred while compiling Suricata.")
self.logger.debug(
"General error occurred while compiling Suricata; {}".
format(e))
raise suricata_exceptions.InstallSuricataError(
"General error occurred while compiling Suricata; {}".
format(e))
if compile_suricata_return_code != 0:
self.logger.error(
"Failed to compile Suricata from source; error code: {}; run with --verbose flag for more info."
.format(compile_suricata_return_code))
raise suricata_exceptions.InstallSuricataError(
"Suricata compilation process returned non-zero; exit-code: {}"
.format(compile_suricata_return_code))
def get_zeek_workers(network_capture_interfaces,
strategy="aggressive",
stdout=True,
verbose=False):
"""
Algorithm for determining the assignment of CPUs for Zeek workers
:param network_capture_interfaces: A list of network interface names
:param strategy: 'aggressive', results in more CPUs pinned per interface, sometimes overshoots resources
'conservative', results in less CPUs pinned per interface, but never overshoots resources
:param stdout: Print the output to console
:param verbose: Include detailed debug messages
:return: A dictionary containing Zeek worker configuration
"""
log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
logger = get_logger('ZEEK', level=log_level, stdout=stdout)
cpus = [c for c in range(0, utilities.get_cpu_core_count())]
logger.info(
"Calculating optimal Zeek worker strategy [strategy: {}].".format(
strategy))
logger.debug("Detected CPU Cores: {}".format(cpus))
# Reserve 0 for KERNEL/Userland opts
available_cpus = cpus[1:]
def grouper(n, iterable):
args = [iter(iterable)] * n
return zip_longest(*args)
def create_workers(net_interfaces, avail_cpus):
idx = 0
zeek_worker_configs = []
for net_interface in net_interfaces:
if idx >= len(avail_cpus):
idx = 0
if isinstance(avail_cpus[idx], int):
avail_cpus[idx] = [avail_cpus[idx]]
zeek_worker_configs.append(
dict(name='dynamite-worker-' + net_interface,
host='localhost',
interface=net_interface,
lb_procs=len(avail_cpus[idx]),
pinned_cpus=avail_cpus[idx]))
idx += 1
return zeek_worker_configs
if len(available_cpus) <= len(network_capture_interfaces):
# Wrap the number of CPUs around the number of network interfaces;
# Since there are more network interfaces than CPUs; CPUs will be assigned more than once
# lb_procs will always be 1
zeek_workers = create_workers(network_capture_interfaces,
available_cpus)
else:
# In this scenario we choose from one of two strategies
# 1. Aggressive:
# - Take the ratio of network_interfaces to available CPUS; ** ROUND UP **.
# - Group the available CPUs by this integer
# (if the ratio == 2 create as many groupings of 2 CPUs as possible)
# - Apply the same wrapping logic used above, but with the CPU groups instead of single CPU instances
# 2. Conservative:
# - Take the ratio of network_interfaces to available CPUS; ** ROUND DOWN **.
# - Group the available CPUs by this integer
# (if the ratio == 2 create as many groupings of 2 CPUs as possible)
# - Apply the same wrapping logic used above, but with the CPU groups instead of single CPU instances
aggressive_ratio = int(
math.ceil(
len(available_cpus) /
float(len(network_capture_interfaces))))
conservative_ratio = int(
math.floor(
len(available_cpus) / len(network_capture_interfaces)))
if strategy == 'aggressive':
cpu_groups = grouper(aggressive_ratio, available_cpus)
else:
cpu_groups = grouper(conservative_ratio, available_cpus)
temp_cpu_groups = []
for cpu_group in cpu_groups:
cpu_group = [c for c in cpu_group if c]
temp_cpu_groups.append(cpu_group)
cpu_groups = temp_cpu_groups
zeek_workers = create_workers(network_capture_interfaces,
cpu_groups)
logger.info('Zeek Worker Count: {}'.format(len(zeek_workers)))
logger.debug('Zeek Workers: {}'.format(zeek_workers))
return zeek_workers
def setup_zeek(self, network_interface=None):
"""
Setup Zeek NSM with PF_RING support
:param stdout: Print output to console
:param network_interface: The interface to listen on
:return: True, if setup successful
"""
if not network_interface:
network_interface = utilities.get_network_interface_names()[0]
if network_interface not in utilities.get_network_interface_names():
sys.stderr.write(
'[-] The network interface that your defined: \'{}\' is invalid. Valid network interfaces: {}\n'
.format(network_interface,
utilities.get_network_interface_names()))
raise Exception(
'Invalid network interface {}'.format(network_interface))
if self.stdout:
sys.stdout.write(
'[+] Creating zeek install|configuration|logging directories.\n'
)
subprocess.call('mkdir -p {}'.format(self.install_directory),
shell=True)
subprocess.call('mkdir -p {}'.format(self.configuration_directory),
shell=True)
pf_ring_profiler = pf_ring.PFRingProfiler()
pf_ring_install = pf_ring.PFRingInstaller(
downlaod_pf_ring_archive=not pf_ring_profiler.is_downloaded,
stdout=self.stdout,
verbose=self.verbose)
if not pf_ring_profiler.is_installed:
if self.stdout:
sys.stdout.write(
'[+] Installing PF_RING kernel modules and dependencies.\n'
)
sys.stdout.flush()
time.sleep(1)
if self.stdout:
sys.stdout.write(
'[+] Compiling Zeek from source. This can take up to 30 minutes. '
'Have another cup of coffee.\n')
sys.stdout.flush()
utilities.print_coffee_art()
time.sleep(1)
sys.stdout.write('[+] Configuring...\n')
sys.stdout.flush()
if self.verbose:
subprocess.call(
'./configure --prefix={} --scriptdir={} --with-pcap={}'.format(
self.install_directory, self.configuration_directory,
pf_ring_install.install_directory),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.ZEEK_DIRECTORY_NAME))
else:
subprocess.call(
'./configure --prefix={} --scriptdir={} --with-pcap={}'.format(
self.install_directory, self.configuration_directory,
pf_ring_install.install_directory),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.ZEEK_DIRECTORY_NAME),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
sys.stdout.write('[+] Compiling...\n')
sys.stdout.flush()
time.sleep(1)
if self.verbose:
compile_zeek_process = subprocess.Popen(
'make; make install',
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.ZEEK_DIRECTORY_NAME))
compile_zeek_process.communicate()
compile_return_code = compile_zeek_process.returncode
else:
compile_zeek_process = subprocess.Popen(
'make; make install',
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.ZEEK_DIRECTORY_NAME),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
compile_return_code = utilities.run_subprocess_with_status(
compile_zeek_process, expected_lines=6596)
if compile_return_code != 0:
sys.stderr.write(
'[-] Failed to compile Zeek from source; error code: {}; ; run with '
'--debug flag for more info.\n'.format(
compile_zeek_process.returncode))
return False
if 'ZEEK_HOME' not in open('/etc/dynamite/environment').read():
if self.stdout:
sys.stdout.write(
'[+] Updating Zeek default home path [{}]\n'.format(
self.install_directory))
subprocess.call(
'echo ZEEK_HOME="{}" >> /etc/dynamite/environment'.format(
self.install_directory),
shell=True)
if 'ZEEK_SCRIPTS' not in open('/etc/dynamite/environment').read():
if self.stdout:
sys.stdout.write(
'[+] Updating Zeek default script path [{}]\n'.format(
self.configuration_directory))
subprocess.call(
'echo ZEEK_SCRIPTS="{}" >> /etc/dynamite/environment'.format(
self.configuration_directory),
shell=True)
if self.stdout:
sys.stdout.write(
'[+] Overwriting default Script | Node configurations.\n')
shutil.copy(
os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'broctl-nodes.cfg'),
os.path.join(self.install_directory, 'etc', 'node.cfg'))
shutil.copy(
os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'local.bro'),
os.path.join(self.configuration_directory, 'site', 'local.bro'))
node_config = ZeekNodeConfigurator(self.install_directory)
cpu_count = utilities.get_cpu_core_count()
cpus = [c for c in range(0, cpu_count)]
if cpu_count > 1:
pinned_cpus = cpus[:-1]
lb_procs = len(pinned_cpus)
else:
pinned_cpus = cpus
lb_procs = 1
node_config.add_worker(name='dynamite-worker-1',
host='localhost',
interface=network_interface,
lb_procs=lb_procs,
pin_cpus=pinned_cpus)
node_config.write_config()
def setup(self, inspect_interfaces: List[str]):
"""Install Suricata
Args:
inspect_interfaces: A list of network interfaces to capture on (E.G ["mon0", "mon1"])
Returns:
None
"""
if not self.skip_interface_validation:
if not self.validate_inspect_interfaces(inspect_interfaces):
raise install.NetworkInterfaceNotFound(inspect_interfaces)
sysctl = systemctl.SystemCtl()
self.install_suricata_dependencies()
self.create_update_suricata_environment_variables()
self.logger.debug(f'Creating directory: {self.configuration_directory}')
utilities.makedirs(self.configuration_directory)
self.logger.debug(f'Creating directory: {self.install_directory}')
utilities.makedirs(self.install_directory)
self.logger.debug(f'Creating directory: {self.log_directory}')
utilities.makedirs(self.log_directory)
self.copy_suricata_files_and_directories()
self.logger.info('Setting up Suricata from source. This can a few minutes.')
if self.stdout:
utilities.print_coffee_art()
self.configure_compile_suricata()
self.copy_file_or_directory_to_destination(
f'{const.DEFAULT_CONFIGS}/suricata/suricata.yaml',
self.configuration_directory
)
suricata_config = config.ConfigManager(self.configuration_directory, stdout=self.stdout, verbose=self.verbose)
suricata_config.default_log_directory = self.log_directory
suricata_config.suricata_log_output_file = os.path.join(self.log_directory, 'suricata.log')
suricata_config.default_rules_directory = os.path.join(self.configuration_directory, 'rules')
suricata_config.reference_config_file = os.path.join(self.configuration_directory, 'reference.config')
suricata_config.classification_file = os.path.join(self.configuration_directory, 'rules',
'classification.config')
suricata_config.af_packet_interfaces = misc.AfPacketInterfaces()
for interface in inspect_interfaces:
suricata_config.af_packet_interfaces.add(
misc.AfPacketInterface(
interface_name=interface, threads='auto', cluster_id=random.randint(1, 50000),
cluster_type='cluster_qm'
)
)
suricata_config.threading = suricata_config.get_optimal_suricata_threading_config(
tuple([i for i in range(0, utilities.get_cpu_core_count() - 1)]))
suricata_config.commit()
self.logger.info('Applying Suricata configuration.')
self.logger.debug(suricata_config.af_packet_interfaces)
suricata_config.commit()
# Fix Permissions
self.logger.info('Setting up file permissions.')
utilities.set_ownership_of_file(self.configuration_directory, user='******', group='dynamite')
utilities.set_ownership_of_file(self.install_directory, user='******', group='dynamite')
utilities.set_ownership_of_file(self.log_directory, user='******', group='dynamite')
post_install_bootstrap_updater(self.install_directory, stdout=self.stdout, verbose=self.verbose)
self.logger.info(f'Installing service -> {const.DEFAULT_CONFIGS}/systemd/suricata.service')
sysctl.install_and_enable(os.path.join(const.DEFAULT_CONFIGS, 'systemd', 'suricata.service'))
def get_parallel_threads() -> int:
parallel_threads = 1
cpu_available_cores = utilities.get_cpu_core_count()
if cpu_available_cores > 1:
parallel_threads = cpu_available_cores - 1
return parallel_threads
