def configure_source_package(self, source_root_directory: str, configure_args: Optional[List[str]] = None) -> None:
"""A configure wrapper for the build/make process
Args:
source_root_directory: A directory containing the configuration.in file
configure_args: configure arguments
Returns:
None
"""
temp_config_args = ['./configure']
temp_config_args.extend(configure_args)
temp_config_args = [str(a) for a in temp_config_args]
configure_args = temp_config_args
self.logger.info(f'Configuring build: {source_root_directory}.')
self.logger.debug(" ".join(configure_args))
popen_args = dict(
args=' '.join(configure_args),
shell=True,
cwd=source_root_directory,
)
if not self.verbose:
popen_args['stdout'] = subprocess.PIPE
popen_args['stderr'] = subprocess.PIPE
ret = utilities.run_subprocess_with_status(subprocess.Popen(**popen_args),
expected_lines=None)
else:
p = subprocess.Popen(**popen_args)
p.communicate()
ret = p.returncode
if ret != 0:
self.logger.error(f'Exited: {ret}; Process Info: {configure_args}')
raise exceptions.CallProcessError(f'Exited with {ret}')
def compile_source_package(self, source_root_directory: str, compile_args: Optional[List[str]] = None,
parallel_threads: Optional[int] = None,
expected_lines_printed: Optional[int] = None) -> None:
"""A simple make; make install wrapper
Args:
source_root_directory: The directory containing the MAKEFILE
compile_args: make arguments
parallel_threads: The number of parallel threads to use during compilation (jemalloc)
expected_lines_printed: The number of lines produced by this process (used to generate a progressbar)
Returns: None
"""
if not parallel_threads:
parallel_threads = get_parallel_threads()
if compile_args:
compile_args.extend(['-j', parallel_threads])
else:
compile_args = ['-j', parallel_threads]
temp_compile_args = [f'{const.SYS_BIN}/make']
temp_compile_args.extend(compile_args)
temp_compile_args = [str(a) for a in temp_compile_args]
compile_args = temp_compile_args
compile_args.extend([';', f'{const.SYS_BIN}/make', 'install'])
self.logger.info(f'Compiling: {source_root_directory}.')
self.logger.debug(" ".join(compile_args))
popen_make_args = dict(
args=' '.join(compile_args),
shell=True,
cwd=source_root_directory,
)
if not self.verbose:
popen_make_args['stdout'] = subprocess.PIPE
popen_make_args['stderr'] = subprocess.PIPE
ret = utilities.run_subprocess_with_status(subprocess.Popen(**popen_make_args),
expected_lines=expected_lines_printed)
else:
p = subprocess.Popen(**popen_make_args)
p.communicate()
ret = p.returncode
if ret != 0:
self.logger.error(f'Exited: {ret}; Process Info: {compile_args}')
raise exceptions.CallProcessError(f'Exited with {ret}')
def setup_zeek(self):
"""
Setup Zeek NSM with PF_RING support
"""
env_file = os.path.join(const.CONFIG_PATH, 'environment')
self.logger.info('Creating Zeek installation, configuration and logging directories.')
try:
utilities.makedirs(self.install_directory, exist_ok=True)
utilities.makedirs(self.configuration_directory, exist_ok=True)
except Exception as e:
self.logger.error('General error occurred while attempting to create root directories.')
self.logger.debug("General error occurred while attempting to create root directories; {}".format(e))
raise zeek_exceptions.InstallZeekError(
"General error occurred while attempting to create root directories; {}".format(e))
self.logger.info('Compiling Zeek from source. This can take up to 30 minutes.')
if self.stdout:
utilities.print_coffee_art()
time.sleep(1)
self.logger.info('Configuring Zeek.')
if self.verbose:
zeek_config_p = subprocess.Popen('./configure --prefix={} --scriptdir={} --enable-jemalloc'.format(
self.install_directory, self.configuration_directory),
shell=True, cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME))
else:
zeek_config_p = subprocess.Popen('./configure --prefix={} --scriptdir={} --enable-jemalloc'.format(
self.install_directory, self.configuration_directory),
shell=True, cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME), stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
try:
zeek_config_p.communicate()
except Exception as e:
self.logger.error("General error occurred while configuring Zeek.")
self.logger.debug("General error occurred while configuring Zeek; {}".format(e))
raise zeek_exceptions.InstallZeekError(
"General error occurred while configuring Zeek; {}".format(e))
if zeek_config_p.returncode != 0:
self.logger.error(
"Zeek configuration process returned non-zero; exit-code: {}".format(zeek_config_p.returncode))
raise zeek_exceptions.InstallZeekError(
"Zeek configuration process returned non-zero; exit-code: {}".format(zeek_config_p.returncode))
time.sleep(1)
self.logger.info("Compiling Zeek.")
if utilities.get_cpu_core_count() > 1:
parallel_threads = utilities.get_cpu_core_count() - 1
else:
parallel_threads = 1
if self.verbose:
compile_zeek_process = subprocess.Popen('make -j {}; make install'.format(parallel_threads), shell=True,
cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME))
try:
compile_zeek_process.communicate()
except Exception as e:
self.logger.error("General error occurred while compiling Zeek.")
self.logger.debug("General error occurred while compiling Zeek; {}".format(e))
raise zeek_exceptions.InstallZeekError(
"General error occurred while compiling Zeek; {}".format(e))
compile_zeek_return_code = compile_zeek_process.returncode
else:
compile_zeek_process = subprocess.Popen('make -j {}; make install'.format(parallel_threads), shell=True,
cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME),
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
try:
compile_zeek_return_code = utilities.run_subprocess_with_status(compile_zeek_process,
expected_lines=6779)
except Exception as e:
self.logger.error("General error occurred while compiling Zeek.")
self.logger.debug("General error occurred while compiling Zeek; {}".format(e))
raise zeek_exceptions.InstallZeekError(
"General error occurred while compiling Zeek; {}".format(e))
if compile_zeek_return_code != 0:
self.logger.error(
"Failed to compile Zeek from source; error code: {}; run with --verbose flag for more info.".format(
compile_zeek_return_code))
raise zeek_exceptions.InstallZeekError(
"Zeek compilation process returned non-zero; exit-code: {}".format(compile_zeek_return_code))
try:
with open(env_file) as env_f:
if 'ZEEK_HOME' not in env_f.read():
self.logger.info('Updating Zeek default home path [{}]'.format(self.install_directory))
subprocess.call('echo ZEEK_HOME="{}" >> {}'.format(self.install_directory, env_file),
shell=True)
if 'ZEEK_SCRIPTS' not in env_f.read():
self.logger.info('Updating Zeek default script path [{}]'.format(self.configuration_directory))
subprocess.call('echo ZEEK_SCRIPTS="{}" >> {}'.format(self.configuration_directory, env_file),
shell=True)
except IOError as e:
self.logger.error("Failed to open {} for reading.".format(env_file))
self.logger.debug("Failed to open {} for reading; {}".format(env_file, e))
raise zeek_exceptions.InstallZeekError(
"Failed to open {} for reading; {}".format(env_file, e))
except Exception as e:
self.logger.error("General error while creating environment variables in {}.".format(env_file))
self.logger.debug("General error while creating environment variables in {}; {}".format(env_file, e))
raise zeek_exceptions.InstallZeekError(
"General error while creating environment variables in {}; {}".format(env_file, e))
self.logger.info("Overwriting Zeek node.cfg file with our changes.")
try:
shutil.copy(os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'broctl-nodes.cfg'),
os.path.join(self.install_directory, 'etc', 'node.cfg'))
shutil.copy(os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'local.zeek'),
os.path.join(self.configuration_directory, 'site', 'local.zeek'))
except Exception as e:
self.logger.error("General error occurred while copying default Zeek configurations.")
self.logger.debug("General error occurred while copying default Zeek configurations; {}".format(e))
raise zeek_exceptions.InstallZeekError(
"General error occurred while copying default Zeek configurations; {}".format(e))
try:
node_config = zeek_configs.NodeConfigManager(self.install_directory)
except zeek_exceptions.ReadsZeekConfigError:
self.logger.error("An error occurred while reading Zeek configurations.")
raise zeek_exceptions.InstallZeekError("An error occurred while reading Zeek configurations.")
# Clear out pre-set workers.
for key in list(node_config.node_config):
if node_config.node_config[key]['type'] == 'worker':
del node_config.node_config[key]
# Calculate new workers.
for worker in node_config.get_optimal_zeek_worker_config(self.capture_network_interfaces, stdout=self.stdout,
verbose=self.verbose):
node_config.add_worker(name=worker['name'],
host=worker['host'],
interface=worker['interface'],
lb_procs=worker['lb_procs'],
pin_cpus=worker['pinned_cpus']
)
try:
node_config.write_config()
except zeek_exceptions.WriteZeekConfigError:
self.logger.error("An error occurred while writing Zeek configurations.")
raise zeek_exceptions.InstallZeekError("An error occurred while writing Zeek configurations.")
try:
sysctl = systemctl.SystemCtl()
except general_exceptions.CallProcessError:
raise zeek_exceptions.InstallZeekError("Could not find systemctl.")
self.logger.info("Installing Zeek systemd service.")
if not sysctl.install_and_enable(os.path.join(const.DEFAULT_CONFIGS, 'systemd', 'zeek.service')):
raise zeek_exceptions.InstallZeekError("Failed to install Zeek systemd service.")
def _configure_and_compile_suricata(self, pf_ring_installer):
if self.configuration_directory.endswith('/'):
suricata_config_parent = '/'.join(
self.configuration_directory.split('/')[:-2])
else:
suricata_config_parent = '/'.join(
self.configuration_directory.split('/')[:-1])
if self.stdout:
sys.stdout.write(
'[+] Compiling Suricata from source. This can take up to 5 to 10 minutes. '
'Have a cup of coffee.\n')
sys.stdout.flush()
utilities.print_coffee_art()
time.sleep(1)
sys.stdout.write('[+] Configuring...\n')
sys.stdout.flush()
if self.verbose:
configure_result = subprocess.call(
'./configure --prefix={} --sysconfdir={} '
'--localstatedir=/var/dynamite/suricata --enable-pfring '
'--with-libpfring-includes={} -with-libpfring-libraries={}'.
format(
self.install_directory, suricata_config_parent,
os.path.join(pf_ring_installer.install_directory,
'include'),
os.path.join(pf_ring_installer.install_directory, 'lib')),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.SURICATA_DIRECTORY_NAME))
else:
configure_result = subprocess.call(
'./configure --prefix={} --sysconfdir={} '
'--localstatedir=/var/dynamite/suricata --enable-pfring '
'--with-libpfring-includes={} -with-libpfring-libraries={}'.
format(
self.install_directory, suricata_config_parent,
os.path.join(pf_ring_installer.install_directory,
'include'),
os.path.join(pf_ring_installer.install_directory, 'lib')),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.SURICATA_DIRECTORY_NAME),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if configure_result != 0:
sys.stderr.write(
'[-] Unable to configure Suricata installation files: {}\n')
return False
sys.stdout.write('[+] Compiling...\n')
sys.stdout.flush()
if self.verbose:
compile_suricata_process = subprocess.Popen(
'make; make install; make install-conf',
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.SURICATA_DIRECTORY_NAME))
compile_suricata_process.communicate()
compile_suricata_return_code = compile_suricata_process.returncode
else:
compile_suricata_process = subprocess.Popen(
'make; make install; make install-conf',
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.SURICATA_DIRECTORY_NAME),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
compile_suricata_return_code = utilities.run_subprocess_with_status(
compile_suricata_process, expected_lines=935)
if compile_suricata_return_code != 0:
sys.stderr.write(
'[-] Unable to compile Suricata installation package; error code {}; run with '
'--debug flag for more info.\n'.format(
compile_suricata_process.returncode))
return False
return True
def _configure_and_compile_suricata(self):
if self.configuration_directory.endswith('/'):
suricata_config_parent = '/'.join(
self.configuration_directory.split('/')[:-2])
else:
suricata_config_parent = '/'.join(
self.configuration_directory.split('/')[:-1])
self.logger.info(
'Compiling Suricata from source. This can take up to 5 to 10 minutes.'
)
if self.stdout:
utilities.print_coffee_art()
time.sleep(1)
self.logger.info('Configuring Suricata.')
if self.verbose:
suricata_config_p = subprocess.Popen(
'./configure --prefix={} --sysconfdir={} --localstatedir=/var/dynamite/suricata'
.format(self.install_directory, suricata_config_parent),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.SURICATA_DIRECTORY_NAME))
else:
suricata_config_p = subprocess.Popen(
'./configure --prefix={} --sysconfdir={} --localstatedir=/var/dynamite/suricata'
.format(self.install_directory, suricata_config_parent),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.SURICATA_DIRECTORY_NAME),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
try:
suricata_config_p.communicate()
except Exception as e:
self.logger.error(
"General error occurred while configuring Suricata.")
self.logger.debug(
"General error occurred while configuring Suricata; {}".format(
e))
raise suricata_exceptions.InstallSuricataError(
"General error occurred while configuring Suricata; {}".format(
e))
if suricata_config_p.returncode != 0:
self.logger.error(
"Zeek configuration process returned non-zero; exit-code: {}".
format(suricata_config_p.returncode))
raise suricata_exceptions.InstallSuricataError(
"Suricata configuration process returned non-zero; exit-code: {}"
.format(suricata_config_p.returncode))
time.sleep(1)
self.logger.info("Compiling Suricata.")
if utilities.get_cpu_core_count() > 1:
parallel_threads = utilities.get_cpu_core_count() - 1
else:
parallel_threads = 1
if self.verbose:
compile_suricata_process = subprocess.Popen(
'make -j {}; make install; make install-conf'.format(
parallel_threads),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.SURICATA_DIRECTORY_NAME))
try:
compile_suricata_process.communicate()
except Exception as e:
self.logger.error(
"General error occurred while compiling Suricata.")
self.logger.debug(
"General error occurred while compiling Suricata; {}".
format(e))
raise suricata_exceptions.InstallSuricataError(
"General error occurred while compiling Suricata; {}".
format(e))
compile_suricata_return_code = compile_suricata_process.returncode
else:
compile_suricata_process = subprocess.Popen(
'make -j {}; make install; make install-conf'.format(
parallel_threads),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.SURICATA_DIRECTORY_NAME),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
try:
compile_suricata_return_code = utilities.run_subprocess_with_status(
compile_suricata_process, expected_lines=935)
except Exception as e:
self.logger.error(
"General error occurred while compiling Suricata.")
self.logger.debug(
"General error occurred while compiling Suricata; {}".
format(e))
raise suricata_exceptions.InstallSuricataError(
"General error occurred while compiling Suricata; {}".
format(e))
if compile_suricata_return_code != 0:
self.logger.error(
"Failed to compile Suricata from source; error code: {}; run with --verbose flag for more info."
.format(compile_suricata_return_code))
raise suricata_exceptions.InstallSuricataError(
"Suricata compilation process returned non-zero; exit-code: {}"
.format(compile_suricata_return_code))
def setup_zeek(self, network_interface=None):
"""
Setup Zeek NSM with PF_RING support
:param stdout: Print output to console
:param network_interface: The interface to listen on
:return: True, if setup successful
"""
if not network_interface:
network_interface = utilities.get_network_interface_names()[0]
if network_interface not in utilities.get_network_interface_names():
sys.stderr.write(
'[-] The network interface that your defined: \'{}\' is invalid. Valid network interfaces: {}\n'
.format(network_interface,
utilities.get_network_interface_names()))
raise Exception(
'Invalid network interface {}'.format(network_interface))
if self.stdout:
sys.stdout.write(
'[+] Creating zeek install|configuration|logging directories.\n'
)
subprocess.call('mkdir -p {}'.format(self.install_directory),
shell=True)
subprocess.call('mkdir -p {}'.format(self.configuration_directory),
shell=True)
pf_ring_profiler = pf_ring.PFRingProfiler()
pf_ring_install = pf_ring.PFRingInstaller(
downlaod_pf_ring_archive=not pf_ring_profiler.is_downloaded,
stdout=self.stdout,
verbose=self.verbose)
if not pf_ring_profiler.is_installed:
if self.stdout:
sys.stdout.write(
'[+] Installing PF_RING kernel modules and dependencies.\n'
)
sys.stdout.flush()
time.sleep(1)
if self.stdout:
sys.stdout.write(
'[+] Compiling Zeek from source. This can take up to 30 minutes. '
'Have another cup of coffee.\n')
sys.stdout.flush()
utilities.print_coffee_art()
time.sleep(1)
sys.stdout.write('[+] Configuring...\n')
sys.stdout.flush()
if self.verbose:
subprocess.call(
'./configure --prefix={} --scriptdir={} --with-pcap={}'.format(
self.install_directory, self.configuration_directory,
pf_ring_install.install_directory),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.ZEEK_DIRECTORY_NAME))
else:
subprocess.call(
'./configure --prefix={} --scriptdir={} --with-pcap={}'.format(
self.install_directory, self.configuration_directory,
pf_ring_install.install_directory),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.ZEEK_DIRECTORY_NAME),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
sys.stdout.write('[+] Compiling...\n')
sys.stdout.flush()
time.sleep(1)
if self.verbose:
compile_zeek_process = subprocess.Popen(
'make; make install',
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.ZEEK_DIRECTORY_NAME))
compile_zeek_process.communicate()
compile_return_code = compile_zeek_process.returncode
else:
compile_zeek_process = subprocess.Popen(
'make; make install',
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.ZEEK_DIRECTORY_NAME),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
compile_return_code = utilities.run_subprocess_with_status(
compile_zeek_process, expected_lines=6596)
if compile_return_code != 0:
sys.stderr.write(
'[-] Failed to compile Zeek from source; error code: {}; ; run with '
'--debug flag for more info.\n'.format(
compile_zeek_process.returncode))
return False
if 'ZEEK_HOME' not in open('/etc/dynamite/environment').read():
if self.stdout:
sys.stdout.write(
'[+] Updating Zeek default home path [{}]\n'.format(
self.install_directory))
subprocess.call(
'echo ZEEK_HOME="{}" >> /etc/dynamite/environment'.format(
self.install_directory),
shell=True)
if 'ZEEK_SCRIPTS' not in open('/etc/dynamite/environment').read():
if self.stdout:
sys.stdout.write(
'[+] Updating Zeek default script path [{}]\n'.format(
self.configuration_directory))
subprocess.call(
'echo ZEEK_SCRIPTS="{}" >> /etc/dynamite/environment'.format(
self.configuration_directory),
shell=True)
if self.stdout:
sys.stdout.write(
'[+] Overwriting default Script | Node configurations.\n')
shutil.copy(
os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'broctl-nodes.cfg'),
os.path.join(self.install_directory, 'etc', 'node.cfg'))
shutil.copy(
os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'local.bro'),
os.path.join(self.configuration_directory, 'site', 'local.bro'))
node_config = ZeekNodeConfigurator(self.install_directory)
cpu_count = utilities.get_cpu_core_count()
cpus = [c for c in range(0, cpu_count)]
if cpu_count > 1:
pinned_cpus = cpus[:-1]
lb_procs = len(pinned_cpus)
else:
pinned_cpus = cpus
lb_procs = 1
node_config.add_worker(name='dynamite-worker-1',
host='localhost',
interface=network_interface,
lb_procs=lb_procs,
pin_cpus=pinned_cpus)
node_config.write_config()