def add_bpf_pattern(self, interface_name, bpf_pattern):
"""
Associate a BPF pattern with a valid network interface.
:param interface_name: The name of the network interface to apply the filter to
:param bpf_pattern: A valid BPF filter
"""
if interface_name in utilities.get_network_interface_names():
self.interface_pattern_map[interface_name] = bpf_pattern
def get_attached_interfaces(self) -> List[str]:
conf_mng = suricata_config.ConfigManager(
configuration_directory=self.config_directory,
stdout=False,
verbose=False)
if not conf_mng.af_packet_interfaces:
return []
return [
iface.interface for iface in conf_mng.af_packet_interfaces
if iface.interface in utilities.get_network_interface_names()
]
def get_attached_interfaces(self) -> List[str]:
conf_mng = zeek_config.NodeConfigManager(
install_directory=self.install_directory,
stdout=False,
verbose=False)
if not conf_mng.workers:
return []
return [
worker.interface for worker in conf_mng.workers
if worker.interface in utilities.get_network_interface_names()
]
def validate_inspect_interfaces(inspect_interfaces: List[str]) -> bool:
"""
Determine if one or more capture interface actually exists
Args:
inspect_interfaces: A list of network interface names to evaluate.
Returns:
True, if all interfaces are valid
"""
for interface in inspect_interfaces:
if interface not in utilities.get_network_interface_names():
return False
return True
def _parse_bpf_map_file(self):
"""
Parse bpf_map_file.input configuration file, and determine what bpf filters to apply to each interface
"""
bpf_interface_map_path = os.path.join(self.configuration_directory,
'bpf_map_file.input')
try:
with open(bpf_interface_map_path) as conf_f:
for line in conf_f.readlines():
if line.startswith('#'):
continue
tokenized_line = line.split('\t')
net_interface, bpf_pattern = tokenized_line[0], ''.join(
tokenized_line[1:]).strip()
if net_interface in utilities.get_network_interface_names(
):
self.interface_pattern_map[net_interface] = bpf_pattern
except IOError:
with open(bpf_interface_map_path, 'w') as bpf_input_f:
bpf_input_f.write('')
def setup_suricata(self, network_interface=None, stdout=False):
"""
Setup Suricata IDS with PF_RING support
:param stdout: Print output to console
:param network_interface: The interface to listen on
:return: True, if setup successful
"""
if not network_interface:
network_interface = utilities.get_network_interface_names()[0]
if network_interface not in utilities.get_network_interface_names():
sys.stderr.write(
'[-] The network interface that your defined: \'{}\' is invalid. Valid network interfaces: {}\n'.format(
network_interface, utilities.get_network_interface_names()))
return False
self._copy_suricata_files_and_directories(stdout=stdout)
pf_ring_install = pf_ring.PFRingInstaller()
if not pf_ring.PFRingProfiler().is_installed:
if stdout:
sys.stdout.write('[+] Installing PF_RING kernel modules and dependencies.\n')
sys.stdout.flush()
time.sleep(1)
pf_ring_install.download_pf_ring(stdout=True)
pf_ring_install.extract_pf_ring(stdout=True)
pf_ring_install.setup_pf_ring(stdout=True)
try:
os.symlink(os.path.join(pf_ring_install.install_directory, 'lib', 'libpcap.so.1'), '/lib/libpcap.so.1')
except Exception as e:
sys.stderr.write('[-] Failed to re-link libpcap.so.1 -> /lib/libpcap.so.1: {}\n'.format(e))
if 'exists' not in str(e).lower():
return False
try:
os.symlink(os.path.join(pf_ring_install.install_directory, 'lib', 'libpfring.so'), '/lib/libpfring.so.1')
except Exception as e:
sys.stderr.write('[-] Failed to re-link libpfring.so -> /lib/libpfring.so.1: {}\n'.format(e))
if 'exists' not in str(e).lower():
return False
# CentOS linker libraries
try:
os.symlink(os.path.join(pf_ring_install.install_directory, 'lib', 'libpcap.so.1'),
'/usr/local/lib/libpcap.so.1')
except Exception as e:
sys.stderr.write('[-] Failed to re-link libpcap.so.1 -> /usr/local/lib/libpcap.so.1: {}\n'.format(e))
if 'exists' not in str(e).lower():
return False
try:
os.symlink(os.path.join(pf_ring_install.install_directory, 'lib', 'libpfring.so'),
'/usr/local/lib/libpfring.so.1')
except Exception as e:
sys.stderr.write('[-] Failed to re-link libpfring.so -> /usr/local/lib/libpfring.so.1: {}\n'.format(e))
if 'exists' not in str(e).lower():
return False
time.sleep(5)
suricata_compiled = self._configure_and_compile_suricata(pf_ring_installer=pf_ring_install, stdout=stdout)
if not suricata_compiled:
return False
if 'SURICATA_HOME' not in open('/etc/dynamite/environment').read():
if stdout:
sys.stdout.write('[+] Updating Suricata default home path [{}]\n'.format(
self.install_directory))
subprocess.call('echo SURICATA_HOME="{}" >> /etc/dynamite/environment'.format(self.install_directory),
shell=True)
if 'SURICATA_CONFIG' not in open('/etc/dynamite/environment').read():
if stdout:
sys.stdout.write('[+] Updating Suricata default config path [{}]\n'.format(
self.configuration_directory))
subprocess.call('echo SURICATA_CONFIG="{}" >> /etc/dynamite/environment'.format(self.configuration_directory),
shell=True)
suricata_rules_installed = self._setup_suricata_rules(stdout=stdout)
if not suricata_rules_installed:
return False
config = SuricataConfigurator(self.configuration_directory)
config.pfring_interfaces = []
config.add_pfring_interface(network_interface, threads='auto', cluster_id=99)
config.default_log_directory = self.log_directory
config.default_rules_directory = os.path.join(self.configuration_directory, 'rules')
config.reference_config_file = os.path.join(self.configuration_directory, 'reference.config')
config.classification_file = os.path.join(self.configuration_directory, 'rules', 'classification.config')
config.write_config()
return True
def validate_capture_network_interfaces(network_interfaces):
for interface in network_interfaces:
if interface not in utilities.get_network_interface_names():
return False
return True
def setup_suricata(self, network_interface=None):
"""
Setup Suricata IDS with PF_RING support
:param network_interface: The interface to listen on
"""
if not network_interface:
network_interface = utilities.get_network_interface_names()[0]
if network_interface not in utilities.get_network_interface_names():
sys.stderr.write(
'[-] The network interface that your defined: \'{}\' is invalid. Valid network interfaces: {}\n'
.format(network_interface,
utilities.get_network_interface_names()))
raise Exception(
'Invalid network interface {}'.format(network_interface))
self._copy_suricata_files_and_directories()
pf_ring_profiler = pf_ring.PFRingProfiler()
pf_ring_install = pf_ring.PFRingInstaller(
downlaod_pf_ring_archive=not pf_ring_profiler.is_downloaded,
stdout=self.stdout,
verbose=self.verbose)
if not pf_ring_profiler.is_installed:
if self.stdout:
sys.stdout.write(
'[+] Installing PF_RING kernel modules and dependencies.\n'
)
sys.stdout.flush()
time.sleep(1)
pf_ring_install.setup_pf_ring()
try:
os.symlink(
os.path.join(pf_ring_install.install_directory, 'lib',
'libpcap.so.1'), '/lib/libpcap.so.1')
os.symlink(
os.path.join(pf_ring_install.install_directory, 'lib',
'libpcap.so.1'), '/lib64/libpcap.so.1')
except Exception as e:
sys.stderr.write(
'[-] Failed to re-link libpcap.so.1 -> /lib/libpcap.so.1: {}\n'
.format(e))
if 'exists' not in str(e).lower():
raise Exception(
'An error occurred while linking libpcap.so.1 - {}'.format(
e))
try:
os.symlink(
os.path.join(pf_ring_install.install_directory, 'lib',
'libpfring.so'), '/lib/libpfring.so.1')
os.symlink(
os.path.join(pf_ring_install.install_directory, 'lib',
'libpfring.so'), '/lib64/libpfring.so.1')
except Exception as e:
sys.stderr.write(
'[-] Failed to re-link libpfring.so -> /lib/libpfring.so.1: {}\n'
.format(e))
if 'exists' not in str(e).lower():
raise Exception(
'An error occurred while linking libpfring.so - {}'.format(
e))
# CentOS linker libraries
try:
os.symlink(
os.path.join(pf_ring_install.install_directory, 'lib',
'libpcap.so.1'), '/usr/local/lib/libpcap.so.1')
except Exception as e:
sys.stderr.write(
'[-] Failed to re-link libpcap.so.1 -> /usr/local/lib/libpcap.so.1: {}\n'
.format(e))
if 'exists' not in str(e).lower():
raise Exception(
'An error occurred while linking libpcap.so.1 - {}'.format(
e))
try:
os.symlink(
os.path.join(pf_ring_install.install_directory, 'lib',
'libpfring.so'), '/usr/local/lib/libpfring.so.1')
except Exception as e:
sys.stderr.write(
'[-] Failed to re-link libpfring.so -> /usr/local/lib/libpfring.so.1: {}\n'
.format(e))
if 'exists' not in str(e).lower():
raise Exception(
'An error occurred while linking libpfring.so - {}'.format(
e))
time.sleep(2)
suricata_compiled = self._configure_and_compile_suricata(
pf_ring_installer=pf_ring_install)
if not suricata_compiled:
return False
if 'SURICATA_HOME' not in open('/etc/dynamite/environment').read():
if self.stdout:
sys.stdout.write(
'[+] Updating Suricata default home path [{}]\n'.format(
self.install_directory))
subprocess.call(
'echo SURICATA_HOME="{}" >> /etc/dynamite/environment'.format(
self.install_directory),
shell=True)
if 'SURICATA_CONFIG' not in open('/etc/dynamite/environment').read():
if self.stdout:
sys.stdout.write(
'[+] Updating Suricata default config path [{}]\n'.format(
self.configuration_directory))
subprocess.call(
'echo SURICATA_CONFIG="{}" >> /etc/dynamite/environment'.
format(self.configuration_directory),
shell=True)
self._setup_suricata_rules()
config = SuricataConfigurator(self.configuration_directory)
config.af_packet_interfaces = []
config.add_afpacket_interface(network_interface,
threads='auto',
cluster_id=99)
config.default_log_directory = self.log_directory
config.default_rules_directory = os.path.join(
self.configuration_directory, 'rules')
config.reference_config_file = os.path.join(
self.configuration_directory, 'reference.config')
config.classification_file = os.path.join(self.configuration_directory,
'rules',
'classification.config')
# Disable Unneeded Suricata rules
config.disable_rule('http-events.rules')
config.disable_rule('smtp-events.rules')
config.disable_rule('dns-events.rules')
config.disable_rule('tls-events.rules')
config.disable_rule('drop.rules')
config.disable_rule('emerging-p2p.rules')
config.disable_rule('emerging-pop3.rules')
config.disable_rule('emerging-telnet.rules')
config.disable_rule('emerging-tftp.rules')
config.disable_rule('emerging-voip.rules')
config.write_config()
def setup_zeek(self, network_interface=None, stdout=False):
"""
Setup Zeek NSM with PF_RING support
:param stdout: Print output to console
:param network_interface: The interface to listen on
:return: True, if setup successful
"""
if not network_interface:
network_interface = utilities.get_network_interface_names()[0]
if network_interface not in utilities.get_network_interface_names():
sys.stderr.write(
'[-] The network interface that your defined: \'{}\' is invalid. Valid network interfaces: {}\n'
.format(network_interface,
utilities.get_network_interface_names()))
return False
if stdout:
sys.stdout.write(
'[+] Creating zeek install|configuration|logging directories.\n'
)
subprocess.call('mkdir -p {}'.format(self.install_directory),
shell=True)
subprocess.call('mkdir -p {}'.format(self.configuration_directory),
shell=True)
pf_ring_install = pf_ring.PFRingInstaller()
if not pf_ring.PFRingProfiler().is_installed:
if stdout:
sys.stdout.write(
'[+] Installing PF_RING kernel modules and dependencies.\n'
)
sys.stdout.flush()
time.sleep(1)
pf_ring_install.download_pf_ring(stdout=True)
pf_ring_install.extract_pf_ring(stdout=True)
pf_ring_install.setup_pf_ring(stdout=True)
if stdout:
sys.stdout.write(
'\n\n[+] Compiling Zeek from source. This can take up to 30 minutes. Have a cup of coffee.'
'\n\n')
sys.stdout.flush()
time.sleep(5)
subprocess.call(
'./configure --prefix={} --scriptdir={} --with-pcap={}'.format(
self.install_directory, self.configuration_directory,
pf_ring_install.install_directory),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE, const.ZEEK_DIRECTORY_NAME))
subprocess.call('make; make install',
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.ZEEK_DIRECTORY_NAME))
if 'ZEEK_HOME' not in open('/etc/environment').read():
if stdout:
sys.stdout.write(
'[+] Updating Zeek default home path [{}]\n'.format(
self.install_directory))
subprocess.call('echo ZEEK_HOME="{}" >> /etc/environment'.format(
self.install_directory),
shell=True)
if 'ZEEK_SCRIPTS' not in open('/etc/environment').read():
if stdout:
sys.stdout.write(
'[+] Updating Zeek default script path [{}]\n'.format(
self.configuration_directory))
subprocess.call(
'echo ZEEK_SCRIPTS="{}" >> /etc/environment'.format(
self.configuration_directory),
shell=True)
if stdout:
sys.stdout.write(
'[+] Overwriting default Script | Node configurations.\n')
shutil.copy(
os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'broctl-nodes.cfg'),
os.path.join(self.install_directory, 'etc', 'node.cfg'))
shutil.copy(
os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'local.bro'),
os.path.join(self.configuration_directory, 'site', 'local.bro'))
ZeekScriptConfigurator().write_config()
node_config = ZeekNodeConfigurator(self.install_directory)
available_cpus = utilities.get_cpu_core_count()
workers_cpu_grps = [
range(0, available_cpus)[n:n + 2]
for n in range(0, len(range(0, available_cpus)), 2)
]
for i, cpu_group in enumerate(workers_cpu_grps):
node_config.add_worker(name='dynamite-worker-{}'.format(i + 1),
host='localhost',
interface=network_interface,
lb_procs=10,
pin_cpus=cpu_group)
node_config.write_config()
def __init__(self, purge_config: Optional[bool] = True, stdout: Optional[bool] = False,
verbose: Optional[bool] = False):
"""Uninstall Zeek
Args:
purge_config: If enabled, remove all the configuration files associated with this installation
stdout: Print output to console
verbose: Include detailed debug messages
"""
from dynamite_nsm.services.zeek.process import ProcessManager
env_vars = utilities.get_environment_file_dict()
zeek_directories = [env_vars.get('ZEEK_HOME')]
if purge_config:
zeek_directories.append(env_vars.get('ZEEK_SCRIPTS'))
super().__init__('zeek.uninstall', directories=zeek_directories, sysctl_service_name='zeek.service',
environ_vars=['ZEEK_HOME', 'ZEEK_SCRIPTS'],
process=ProcessManager(stdout=stdout, verbose=verbose), stdout=stdout, verbose=verbose)
if __name__ == '__main__':
install_mngr = InstallManager(
install_directory=f'{const.INSTALL_PATH}/zeek',
configuration_directory=f'{const.CONFIG_PATH}/zeek',
download_zeek_archive=False,
stdout=True,
verbose=True
)
install_mngr.setup(utilities.get_network_interface_names())
def setup_zeek(self, network_interface=None):
"""
Setup Zeek NSM with PF_RING support
:param stdout: Print output to console
:param network_interface: The interface to listen on
:return: True, if setup successful
"""
if not network_interface:
network_interface = utilities.get_network_interface_names()[0]
if network_interface not in utilities.get_network_interface_names():
sys.stderr.write(
'[-] The network interface that your defined: \'{}\' is invalid. Valid network interfaces: {}\n'
.format(network_interface,
utilities.get_network_interface_names()))
raise Exception(
'Invalid network interface {}'.format(network_interface))
if self.stdout:
sys.stdout.write(
'[+] Creating zeek install|configuration|logging directories.\n'
)
subprocess.call('mkdir -p {}'.format(self.install_directory),
shell=True)
subprocess.call('mkdir -p {}'.format(self.configuration_directory),
shell=True)
pf_ring_profiler = pf_ring.PFRingProfiler()
pf_ring_install = pf_ring.PFRingInstaller(
downlaod_pf_ring_archive=not pf_ring_profiler.is_downloaded,
stdout=self.stdout,
verbose=self.verbose)
if not pf_ring_profiler.is_installed:
if self.stdout:
sys.stdout.write(
'[+] Installing PF_RING kernel modules and dependencies.\n'
)
sys.stdout.flush()
time.sleep(1)
if self.stdout:
sys.stdout.write(
'[+] Compiling Zeek from source. This can take up to 30 minutes. '
'Have another cup of coffee.\n')
sys.stdout.flush()
utilities.print_coffee_art()
time.sleep(1)
sys.stdout.write('[+] Configuring...\n')
sys.stdout.flush()
if self.verbose:
subprocess.call(
'./configure --prefix={} --scriptdir={} --with-pcap={}'.format(
self.install_directory, self.configuration_directory,
pf_ring_install.install_directory),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.ZEEK_DIRECTORY_NAME))
else:
subprocess.call(
'./configure --prefix={} --scriptdir={} --with-pcap={}'.format(
self.install_directory, self.configuration_directory,
pf_ring_install.install_directory),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.ZEEK_DIRECTORY_NAME),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
sys.stdout.write('[+] Compiling...\n')
sys.stdout.flush()
time.sleep(1)
if self.verbose:
compile_zeek_process = subprocess.Popen(
'make; make install',
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.ZEEK_DIRECTORY_NAME))
compile_zeek_process.communicate()
compile_return_code = compile_zeek_process.returncode
else:
compile_zeek_process = subprocess.Popen(
'make; make install',
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.ZEEK_DIRECTORY_NAME),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
compile_return_code = utilities.run_subprocess_with_status(
compile_zeek_process, expected_lines=6596)
if compile_return_code != 0:
sys.stderr.write(
'[-] Failed to compile Zeek from source; error code: {}; ; run with '
'--debug flag for more info.\n'.format(
compile_zeek_process.returncode))
return False
if 'ZEEK_HOME' not in open('/etc/dynamite/environment').read():
if self.stdout:
sys.stdout.write(
'[+] Updating Zeek default home path [{}]\n'.format(
self.install_directory))
subprocess.call(
'echo ZEEK_HOME="{}" >> /etc/dynamite/environment'.format(
self.install_directory),
shell=True)
if 'ZEEK_SCRIPTS' not in open('/etc/dynamite/environment').read():
if self.stdout:
sys.stdout.write(
'[+] Updating Zeek default script path [{}]\n'.format(
self.configuration_directory))
subprocess.call(
'echo ZEEK_SCRIPTS="{}" >> /etc/dynamite/environment'.format(
self.configuration_directory),
shell=True)
if self.stdout:
sys.stdout.write(
'[+] Overwriting default Script | Node configurations.\n')
shutil.copy(
os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'broctl-nodes.cfg'),
os.path.join(self.install_directory, 'etc', 'node.cfg'))
shutil.copy(
os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'local.bro'),
os.path.join(self.configuration_directory, 'site', 'local.bro'))
node_config = ZeekNodeConfigurator(self.install_directory)
cpu_count = utilities.get_cpu_core_count()
cpus = [c for c in range(0, cpu_count)]
if cpu_count > 1:
pinned_cpus = cpus[:-1]
lb_procs = len(pinned_cpus)
else:
pinned_cpus = cpus
lb_procs = 1
node_config.add_worker(name='dynamite-worker-1',
host='localhost',
interface=network_interface,
lb_procs=lb_procs,
pin_cpus=pinned_cpus)
node_config.write_config()