def setup(self, inspect_interfaces: List[str]):
"""Setup Zeek
Args:
inspect_interfaces: A list of network interfaces to capture on (E.G ["mon0", "mon1"])
Returns:
None
"""
if not self.skip_interface_validation:
if not self.validate_inspect_interfaces(inspect_interfaces):
raise install.NetworkInterfaceNotFound(inspect_interfaces)
sysctl = systemctl.SystemCtl()
self.install_zeek_dependencies()
self.create_update_zeek_environment_variables()
self.logger.debug(f'Creating directory: {self.configuration_directory}')
utilities.makedirs(self.configuration_directory)
self.logger.debug(f'Creating directory: {self.install_directory}')
utilities.makedirs(self.install_directory)
self.logger.info('Setting up Zeek from source. This can take up to 15 minutes.')
if self.stdout:
utilities.print_coffee_art()
self.configure_compile_zeek()
self.logger.info('Setting up Zeek package manager.')
zkg_installer = zkg_install.InstallManager()
zkg_installer.setup()
package.InstallPackageManager(const.ZEEK_PACKAGES, stdout=self.stdout, verbose=self.verbose).setup()
self.copy_file_or_directory_to_destination(f'{const.DEFAULT_CONFIGS}/zeek/broctl-nodes.cfg',
f'{self.install_directory}/etc/node.cfg')
self.copy_file_or_directory_to_destination(f'{const.DEFAULT_CONFIGS}/zeek/local.zeek',
f'{self.configuration_directory}/site/local.zeek')
# Optimize Configurations
site_local_config = config.SiteLocalConfigManager(self.configuration_directory, stdout=self.stdout,
verbose=self.verbose)
node_config = config.NodeConfigManager(self.install_directory, stdout=self.stdout, verbose=self.verbose)
node_config.workers = node.Workers()
for worker in node_config.get_optimal_zeek_worker_config(inspect_interfaces):
node_config.workers.add_worker(
worker=worker
)
self.logger.info('Applying node configuration.')
node_config.commit()
# Fix Permissions
self.logger.info('Setting up file permissions.')
utilities.set_ownership_of_file(self.configuration_directory, user='******', group='dynamite')
utilities.set_ownership_of_file(self.install_directory, user='******', group='dynamite')
self.logger.info(f'Installing service -> {const.DEFAULT_CONFIGS}/systemd/zeek.service')
sysctl.install_and_enable(os.path.join(const.DEFAULT_CONFIGS, 'systemd', 'zeek.service'))
def setup_zeek(self):
"""
Setup Zeek NSM with PF_RING support
"""
env_file = os.path.join(const.CONFIG_PATH, 'environment')
self.logger.info('Creating Zeek installation, configuration and logging directories.')
try:
utilities.makedirs(self.install_directory, exist_ok=True)
utilities.makedirs(self.configuration_directory, exist_ok=True)
except Exception as e:
self.logger.error('General error occurred while attempting to create root directories.')
self.logger.debug("General error occurred while attempting to create root directories; {}".format(e))
raise zeek_exceptions.InstallZeekError(
"General error occurred while attempting to create root directories; {}".format(e))
self.logger.info('Compiling Zeek from source. This can take up to 30 minutes.')
if self.stdout:
utilities.print_coffee_art()
time.sleep(1)
self.logger.info('Configuring Zeek.')
if self.verbose:
zeek_config_p = subprocess.Popen('./configure --prefix={} --scriptdir={} --enable-jemalloc'.format(
self.install_directory, self.configuration_directory),
shell=True, cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME))
else:
zeek_config_p = subprocess.Popen('./configure --prefix={} --scriptdir={} --enable-jemalloc'.format(
self.install_directory, self.configuration_directory),
shell=True, cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME), stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
try:
zeek_config_p.communicate()
except Exception as e:
self.logger.error("General error occurred while configuring Zeek.")
self.logger.debug("General error occurred while configuring Zeek; {}".format(e))
raise zeek_exceptions.InstallZeekError(
"General error occurred while configuring Zeek; {}".format(e))
if zeek_config_p.returncode != 0:
self.logger.error(
"Zeek configuration process returned non-zero; exit-code: {}".format(zeek_config_p.returncode))
raise zeek_exceptions.InstallZeekError(
"Zeek configuration process returned non-zero; exit-code: {}".format(zeek_config_p.returncode))
time.sleep(1)
self.logger.info("Compiling Zeek.")
if utilities.get_cpu_core_count() > 1:
parallel_threads = utilities.get_cpu_core_count() - 1
else:
parallel_threads = 1
if self.verbose:
compile_zeek_process = subprocess.Popen('make -j {}; make install'.format(parallel_threads), shell=True,
cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME))
try:
compile_zeek_process.communicate()
except Exception as e:
self.logger.error("General error occurred while compiling Zeek.")
self.logger.debug("General error occurred while compiling Zeek; {}".format(e))
raise zeek_exceptions.InstallZeekError(
"General error occurred while compiling Zeek; {}".format(e))
compile_zeek_return_code = compile_zeek_process.returncode
else:
compile_zeek_process = subprocess.Popen('make -j {}; make install'.format(parallel_threads), shell=True,
cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME),
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
try:
compile_zeek_return_code = utilities.run_subprocess_with_status(compile_zeek_process,
expected_lines=6779)
except Exception as e:
self.logger.error("General error occurred while compiling Zeek.")
self.logger.debug("General error occurred while compiling Zeek; {}".format(e))
raise zeek_exceptions.InstallZeekError(
"General error occurred while compiling Zeek; {}".format(e))
if compile_zeek_return_code != 0:
self.logger.error(
"Failed to compile Zeek from source; error code: {}; run with --verbose flag for more info.".format(
compile_zeek_return_code))
raise zeek_exceptions.InstallZeekError(
"Zeek compilation process returned non-zero; exit-code: {}".format(compile_zeek_return_code))
try:
with open(env_file) as env_f:
if 'ZEEK_HOME' not in env_f.read():
self.logger.info('Updating Zeek default home path [{}]'.format(self.install_directory))
subprocess.call('echo ZEEK_HOME="{}" >> {}'.format(self.install_directory, env_file),
shell=True)
if 'ZEEK_SCRIPTS' not in env_f.read():
self.logger.info('Updating Zeek default script path [{}]'.format(self.configuration_directory))
subprocess.call('echo ZEEK_SCRIPTS="{}" >> {}'.format(self.configuration_directory, env_file),
shell=True)
except IOError as e:
self.logger.error("Failed to open {} for reading.".format(env_file))
self.logger.debug("Failed to open {} for reading; {}".format(env_file, e))
raise zeek_exceptions.InstallZeekError(
"Failed to open {} for reading; {}".format(env_file, e))
except Exception as e:
self.logger.error("General error while creating environment variables in {}.".format(env_file))
self.logger.debug("General error while creating environment variables in {}; {}".format(env_file, e))
raise zeek_exceptions.InstallZeekError(
"General error while creating environment variables in {}; {}".format(env_file, e))
self.logger.info("Overwriting Zeek node.cfg file with our changes.")
try:
shutil.copy(os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'broctl-nodes.cfg'),
os.path.join(self.install_directory, 'etc', 'node.cfg'))
shutil.copy(os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'local.zeek'),
os.path.join(self.configuration_directory, 'site', 'local.zeek'))
except Exception as e:
self.logger.error("General error occurred while copying default Zeek configurations.")
self.logger.debug("General error occurred while copying default Zeek configurations; {}".format(e))
raise zeek_exceptions.InstallZeekError(
"General error occurred while copying default Zeek configurations; {}".format(e))
try:
node_config = zeek_configs.NodeConfigManager(self.install_directory)
except zeek_exceptions.ReadsZeekConfigError:
self.logger.error("An error occurred while reading Zeek configurations.")
raise zeek_exceptions.InstallZeekError("An error occurred while reading Zeek configurations.")
# Clear out pre-set workers.
for key in list(node_config.node_config):
if node_config.node_config[key]['type'] == 'worker':
del node_config.node_config[key]
# Calculate new workers.
for worker in node_config.get_optimal_zeek_worker_config(self.capture_network_interfaces, stdout=self.stdout,
verbose=self.verbose):
node_config.add_worker(name=worker['name'],
host=worker['host'],
interface=worker['interface'],
lb_procs=worker['lb_procs'],
pin_cpus=worker['pinned_cpus']
)
try:
node_config.write_config()
except zeek_exceptions.WriteZeekConfigError:
self.logger.error("An error occurred while writing Zeek configurations.")
raise zeek_exceptions.InstallZeekError("An error occurred while writing Zeek configurations.")
try:
sysctl = systemctl.SystemCtl()
except general_exceptions.CallProcessError:
raise zeek_exceptions.InstallZeekError("Could not find systemctl.")
self.logger.info("Installing Zeek systemd service.")
if not sysctl.install_and_enable(os.path.join(const.DEFAULT_CONFIGS, 'systemd', 'zeek.service')):
raise zeek_exceptions.InstallZeekError("Failed to install Zeek systemd service.")
def _configure_and_compile_suricata(self, pf_ring_installer):
if self.configuration_directory.endswith('/'):
suricata_config_parent = '/'.join(
self.configuration_directory.split('/')[:-2])
else:
suricata_config_parent = '/'.join(
self.configuration_directory.split('/')[:-1])
if self.stdout:
sys.stdout.write(
'[+] Compiling Suricata from source. This can take up to 5 to 10 minutes. '
'Have a cup of coffee.\n')
sys.stdout.flush()
utilities.print_coffee_art()
time.sleep(1)
sys.stdout.write('[+] Configuring...\n')
sys.stdout.flush()
if self.verbose:
configure_result = subprocess.call(
'./configure --prefix={} --sysconfdir={} '
'--localstatedir=/var/dynamite/suricata --enable-pfring '
'--with-libpfring-includes={} -with-libpfring-libraries={}'.
format(
self.install_directory, suricata_config_parent,
os.path.join(pf_ring_installer.install_directory,
'include'),
os.path.join(pf_ring_installer.install_directory, 'lib')),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.SURICATA_DIRECTORY_NAME))
else:
configure_result = subprocess.call(
'./configure --prefix={} --sysconfdir={} '
'--localstatedir=/var/dynamite/suricata --enable-pfring '
'--with-libpfring-includes={} -with-libpfring-libraries={}'.
format(
self.install_directory, suricata_config_parent,
os.path.join(pf_ring_installer.install_directory,
'include'),
os.path.join(pf_ring_installer.install_directory, 'lib')),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.SURICATA_DIRECTORY_NAME),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if configure_result != 0:
sys.stderr.write(
'[-] Unable to configure Suricata installation files: {}\n')
return False
sys.stdout.write('[+] Compiling...\n')
sys.stdout.flush()
if self.verbose:
compile_suricata_process = subprocess.Popen(
'make; make install; make install-conf',
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.SURICATA_DIRECTORY_NAME))
compile_suricata_process.communicate()
compile_suricata_return_code = compile_suricata_process.returncode
else:
compile_suricata_process = subprocess.Popen(
'make; make install; make install-conf',
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.SURICATA_DIRECTORY_NAME),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
compile_suricata_return_code = utilities.run_subprocess_with_status(
compile_suricata_process, expected_lines=935)
if compile_suricata_return_code != 0:
sys.stderr.write(
'[-] Unable to compile Suricata installation package; error code {}; run with '
'--debug flag for more info.\n'.format(
compile_suricata_process.returncode))
return False
return True
def _configure_and_compile_suricata(self):
if self.configuration_directory.endswith('/'):
suricata_config_parent = '/'.join(
self.configuration_directory.split('/')[:-2])
else:
suricata_config_parent = '/'.join(
self.configuration_directory.split('/')[:-1])
self.logger.info(
'Compiling Suricata from source. This can take up to 5 to 10 minutes.'
)
if self.stdout:
utilities.print_coffee_art()
time.sleep(1)
self.logger.info('Configuring Suricata.')
if self.verbose:
suricata_config_p = subprocess.Popen(
'./configure --prefix={} --sysconfdir={} --localstatedir=/var/dynamite/suricata'
.format(self.install_directory, suricata_config_parent),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.SURICATA_DIRECTORY_NAME))
else:
suricata_config_p = subprocess.Popen(
'./configure --prefix={} --sysconfdir={} --localstatedir=/var/dynamite/suricata'
.format(self.install_directory, suricata_config_parent),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.SURICATA_DIRECTORY_NAME),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
try:
suricata_config_p.communicate()
except Exception as e:
self.logger.error(
"General error occurred while configuring Suricata.")
self.logger.debug(
"General error occurred while configuring Suricata; {}".format(
e))
raise suricata_exceptions.InstallSuricataError(
"General error occurred while configuring Suricata; {}".format(
e))
if suricata_config_p.returncode != 0:
self.logger.error(
"Zeek configuration process returned non-zero; exit-code: {}".
format(suricata_config_p.returncode))
raise suricata_exceptions.InstallSuricataError(
"Suricata configuration process returned non-zero; exit-code: {}"
.format(suricata_config_p.returncode))
time.sleep(1)
self.logger.info("Compiling Suricata.")
if utilities.get_cpu_core_count() > 1:
parallel_threads = utilities.get_cpu_core_count() - 1
else:
parallel_threads = 1
if self.verbose:
compile_suricata_process = subprocess.Popen(
'make -j {}; make install; make install-conf'.format(
parallel_threads),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.SURICATA_DIRECTORY_NAME))
try:
compile_suricata_process.communicate()
except Exception as e:
self.logger.error(
"General error occurred while compiling Suricata.")
self.logger.debug(
"General error occurred while compiling Suricata; {}".
format(e))
raise suricata_exceptions.InstallSuricataError(
"General error occurred while compiling Suricata; {}".
format(e))
compile_suricata_return_code = compile_suricata_process.returncode
else:
compile_suricata_process = subprocess.Popen(
'make -j {}; make install; make install-conf'.format(
parallel_threads),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.SURICATA_DIRECTORY_NAME),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
try:
compile_suricata_return_code = utilities.run_subprocess_with_status(
compile_suricata_process, expected_lines=935)
except Exception as e:
self.logger.error(
"General error occurred while compiling Suricata.")
self.logger.debug(
"General error occurred while compiling Suricata; {}".
format(e))
raise suricata_exceptions.InstallSuricataError(
"General error occurred while compiling Suricata; {}".
format(e))
if compile_suricata_return_code != 0:
self.logger.error(
"Failed to compile Suricata from source; error code: {}; run with --verbose flag for more info."
.format(compile_suricata_return_code))
raise suricata_exceptions.InstallSuricataError(
"Suricata compilation process returned non-zero; exit-code: {}"
.format(compile_suricata_return_code))
def setup_zeek(self, network_interface=None):
"""
Setup Zeek NSM with PF_RING support
:param stdout: Print output to console
:param network_interface: The interface to listen on
:return: True, if setup successful
"""
if not network_interface:
network_interface = utilities.get_network_interface_names()[0]
if network_interface not in utilities.get_network_interface_names():
sys.stderr.write(
'[-] The network interface that your defined: \'{}\' is invalid. Valid network interfaces: {}\n'
.format(network_interface,
utilities.get_network_interface_names()))
raise Exception(
'Invalid network interface {}'.format(network_interface))
if self.stdout:
sys.stdout.write(
'[+] Creating zeek install|configuration|logging directories.\n'
)
subprocess.call('mkdir -p {}'.format(self.install_directory),
shell=True)
subprocess.call('mkdir -p {}'.format(self.configuration_directory),
shell=True)
pf_ring_profiler = pf_ring.PFRingProfiler()
pf_ring_install = pf_ring.PFRingInstaller(
downlaod_pf_ring_archive=not pf_ring_profiler.is_downloaded,
stdout=self.stdout,
verbose=self.verbose)
if not pf_ring_profiler.is_installed:
if self.stdout:
sys.stdout.write(
'[+] Installing PF_RING kernel modules and dependencies.\n'
)
sys.stdout.flush()
time.sleep(1)
if self.stdout:
sys.stdout.write(
'[+] Compiling Zeek from source. This can take up to 30 minutes. '
'Have another cup of coffee.\n')
sys.stdout.flush()
utilities.print_coffee_art()
time.sleep(1)
sys.stdout.write('[+] Configuring...\n')
sys.stdout.flush()
if self.verbose:
subprocess.call(
'./configure --prefix={} --scriptdir={} --with-pcap={}'.format(
self.install_directory, self.configuration_directory,
pf_ring_install.install_directory),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.ZEEK_DIRECTORY_NAME))
else:
subprocess.call(
'./configure --prefix={} --scriptdir={} --with-pcap={}'.format(
self.install_directory, self.configuration_directory,
pf_ring_install.install_directory),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.ZEEK_DIRECTORY_NAME),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
sys.stdout.write('[+] Compiling...\n')
sys.stdout.flush()
time.sleep(1)
if self.verbose:
compile_zeek_process = subprocess.Popen(
'make; make install',
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.ZEEK_DIRECTORY_NAME))
compile_zeek_process.communicate()
compile_return_code = compile_zeek_process.returncode
else:
compile_zeek_process = subprocess.Popen(
'make; make install',
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.ZEEK_DIRECTORY_NAME),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
compile_return_code = utilities.run_subprocess_with_status(
compile_zeek_process, expected_lines=6596)
if compile_return_code != 0:
sys.stderr.write(
'[-] Failed to compile Zeek from source; error code: {}; ; run with '
'--debug flag for more info.\n'.format(
compile_zeek_process.returncode))
return False
if 'ZEEK_HOME' not in open('/etc/dynamite/environment').read():
if self.stdout:
sys.stdout.write(
'[+] Updating Zeek default home path [{}]\n'.format(
self.install_directory))
subprocess.call(
'echo ZEEK_HOME="{}" >> /etc/dynamite/environment'.format(
self.install_directory),
shell=True)
if 'ZEEK_SCRIPTS' not in open('/etc/dynamite/environment').read():
if self.stdout:
sys.stdout.write(
'[+] Updating Zeek default script path [{}]\n'.format(
self.configuration_directory))
subprocess.call(
'echo ZEEK_SCRIPTS="{}" >> /etc/dynamite/environment'.format(
self.configuration_directory),
shell=True)
if self.stdout:
sys.stdout.write(
'[+] Overwriting default Script | Node configurations.\n')
shutil.copy(
os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'broctl-nodes.cfg'),
os.path.join(self.install_directory, 'etc', 'node.cfg'))
shutil.copy(
os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'local.bro'),
os.path.join(self.configuration_directory, 'site', 'local.bro'))
node_config = ZeekNodeConfigurator(self.install_directory)
cpu_count = utilities.get_cpu_core_count()
cpus = [c for c in range(0, cpu_count)]
if cpu_count > 1:
pinned_cpus = cpus[:-1]
lb_procs = len(pinned_cpus)
else:
pinned_cpus = cpus
lb_procs = 1
node_config.add_worker(name='dynamite-worker-1',
host='localhost',
interface=network_interface,
lb_procs=lb_procs,
pin_cpus=pinned_cpus)
node_config.write_config()
def setup(self, inspect_interfaces: List[str]):
"""Install Suricata
Args:
inspect_interfaces: A list of network interfaces to capture on (E.G ["mon0", "mon1"])
Returns:
None
"""
if not self.skip_interface_validation:
if not self.validate_inspect_interfaces(inspect_interfaces):
raise install.NetworkInterfaceNotFound(inspect_interfaces)
sysctl = systemctl.SystemCtl()
self.install_suricata_dependencies()
self.create_update_suricata_environment_variables()
self.logger.debug(f'Creating directory: {self.configuration_directory}')
utilities.makedirs(self.configuration_directory)
self.logger.debug(f'Creating directory: {self.install_directory}')
utilities.makedirs(self.install_directory)
self.logger.debug(f'Creating directory: {self.log_directory}')
utilities.makedirs(self.log_directory)
self.copy_suricata_files_and_directories()
self.logger.info('Setting up Suricata from source. This can a few minutes.')
if self.stdout:
utilities.print_coffee_art()
self.configure_compile_suricata()
self.copy_file_or_directory_to_destination(
f'{const.DEFAULT_CONFIGS}/suricata/suricata.yaml',
self.configuration_directory
)
suricata_config = config.ConfigManager(self.configuration_directory, stdout=self.stdout, verbose=self.verbose)
suricata_config.default_log_directory = self.log_directory
suricata_config.suricata_log_output_file = os.path.join(self.log_directory, 'suricata.log')
suricata_config.default_rules_directory = os.path.join(self.configuration_directory, 'rules')
suricata_config.reference_config_file = os.path.join(self.configuration_directory, 'reference.config')
suricata_config.classification_file = os.path.join(self.configuration_directory, 'rules',
'classification.config')
suricata_config.af_packet_interfaces = misc.AfPacketInterfaces()
for interface in inspect_interfaces:
suricata_config.af_packet_interfaces.add(
misc.AfPacketInterface(
interface_name=interface, threads='auto', cluster_id=random.randint(1, 50000),
cluster_type='cluster_qm'
)
)
suricata_config.threading = suricata_config.get_optimal_suricata_threading_config(
tuple([i for i in range(0, utilities.get_cpu_core_count() - 1)]))
suricata_config.commit()
self.logger.info('Applying Suricata configuration.')
self.logger.debug(suricata_config.af_packet_interfaces)
suricata_config.commit()
# Fix Permissions
self.logger.info('Setting up file permissions.')
utilities.set_ownership_of_file(self.configuration_directory, user='******', group='dynamite')
utilities.set_ownership_of_file(self.install_directory, user='******', group='dynamite')
utilities.set_ownership_of_file(self.log_directory, user='******', group='dynamite')
post_install_bootstrap_updater(self.install_directory, stdout=self.stdout, verbose=self.verbose)
self.logger.info(f'Installing service -> {const.DEFAULT_CONFIGS}/systemd/suricata.service')
sysctl.install_and_enable(os.path.join(const.DEFAULT_CONFIGS, 'systemd', 'suricata.service'))