def status_agent():
"""
Retrieve the status of the agent processes
:return: A tuple, where the first element is the zeek process status (string), and second element are
the FileBeats and PF_RING status
"""
# Load service processes
zeek_p = zeek.ZeekProcess()
filebeat_p = filebeat.FileBeatProcess()
# Load service profilers
pf_ring_profiler = pf_ring.PFRingProfiler(stderr=False)
filebeat_profiler = filebeat.FileBeatProfiler(stderr=False)
zeek_profiler = zeek.ZeekProfiler(stderr=False)
suricata_profiler = suricata.SuricataProfiler(stderr=False)
if not (filebeat_profiler.is_installed or zeek_profiler.is_installed):
sys.stderr.write('[-] Could not start agent. Is it installed?\n')
sys.stderr.write('[-] dynamite install agent\n')
return False
agent_status = dict(
agent_processes={
'zeek': zeek_p.status(),
'pf_ring': pf_ring_profiler.get_profile(),
'filebeat': filebeat_p.status()
})
if suricata_profiler.is_installed:
# Load Suricata process
suricata_p = suricata.SuricataProcess()
agent_status['agent_processes']['suricata'] = suricata_p.status()
return agent_status
def profile_agent():
"""
Get information about installation/running processes within the agent stack
:return: A dictionary containing the status of each component
"""
pf_ring_profiler = pf_ring.PFRingProfiler()
filebeat_profiler = filebeat.FileBeatProfiler()
suricata_profiler = suricata.SuricataProfiler()
zeek_profiler = zeek.ZeekProfiler()
return dict(PF_RING=pf_ring_profiler.get_profile(),
FILEBEAT=filebeat_profiler.get_profile(),
SURICATA=suricata_profiler.get_profile(),
ZEEK=zeek_profiler.get_profile())
def start_agent():
"""
Start the Zeek (BroCtl) and FileBeats processes
:return: True, if started successfully
"""
# Load service profilers
pf_ring_profiler = pf_ring.PFRingProfiler(stderr=False)
filebeat_profiler = filebeat.FileBeatProfiler(stderr=False)
zeek_profiler = zeek.ZeekProfiler(stderr=False)
suricata_profiler = suricata.SuricataProfiler(stderr=False)
# Load service processes
filebeat_p = filebeat.FileBeatProcess()
zeek_p = zeek.ZeekProcess()
if not (filebeat_profiler.is_installed or zeek_profiler.is_installed):
sys.stderr.write('[-] Could not start agent. Is it installed?\n')
sys.stderr.write('[-] dynamite install agent\n')
return False
if not pf_ring_profiler.is_running:
sys.stderr.write(
'[-] PF_RING kernel modules were not loaded. Try running '
'\'modprobe pf_ring min_num_slots=32768\' as root.\n')
return False
sys.stdout.write('[+] Starting agent processes.\n')
if suricata_profiler.is_installed:
# Load Suricata process
suricata_p = suricata.SuricataProcess()
if not suricata_p.start(stdout=True):
sys.stderr.write('[-] Could not start agent.suricata_process.\n')
return False
if not zeek_p.start(stdout=True):
sys.stderr.write('[-] Could not start agent.zeek_process.\n')
return False
if not filebeat_p.start(stdout=True):
sys.stderr.write('[-] Could not start agent.filebeat.\n')
return False
return True
def setup_suricata(self, network_interface=None):
"""
Setup Suricata IDS with PF_RING support
:param network_interface: The interface to listen on
"""
if not network_interface:
network_interface = utilities.get_network_interface_names()[0]
if network_interface not in utilities.get_network_interface_names():
sys.stderr.write(
'[-] The network interface that your defined: \'{}\' is invalid. Valid network interfaces: {}\n'
.format(network_interface,
utilities.get_network_interface_names()))
raise Exception(
'Invalid network interface {}'.format(network_interface))
self._copy_suricata_files_and_directories()
pf_ring_profiler = pf_ring.PFRingProfiler()
pf_ring_install = pf_ring.PFRingInstaller(
downlaod_pf_ring_archive=not pf_ring_profiler.is_downloaded,
stdout=self.stdout,
verbose=self.verbose)
if not pf_ring_profiler.is_installed:
if self.stdout:
sys.stdout.write(
'[+] Installing PF_RING kernel modules and dependencies.\n'
)
sys.stdout.flush()
time.sleep(1)
pf_ring_install.setup_pf_ring()
try:
os.symlink(
os.path.join(pf_ring_install.install_directory, 'lib',
'libpcap.so.1'), '/lib/libpcap.so.1')
os.symlink(
os.path.join(pf_ring_install.install_directory, 'lib',
'libpcap.so.1'), '/lib64/libpcap.so.1')
except Exception as e:
sys.stderr.write(
'[-] Failed to re-link libpcap.so.1 -> /lib/libpcap.so.1: {}\n'
.format(e))
if 'exists' not in str(e).lower():
raise Exception(
'An error occurred while linking libpcap.so.1 - {}'.format(
e))
try:
os.symlink(
os.path.join(pf_ring_install.install_directory, 'lib',
'libpfring.so'), '/lib/libpfring.so.1')
os.symlink(
os.path.join(pf_ring_install.install_directory, 'lib',
'libpfring.so'), '/lib64/libpfring.so.1')
except Exception as e:
sys.stderr.write(
'[-] Failed to re-link libpfring.so -> /lib/libpfring.so.1: {}\n'
.format(e))
if 'exists' not in str(e).lower():
raise Exception(
'An error occurred while linking libpfring.so - {}'.format(
e))
# CentOS linker libraries
try:
os.symlink(
os.path.join(pf_ring_install.install_directory, 'lib',
'libpcap.so.1'), '/usr/local/lib/libpcap.so.1')
except Exception as e:
sys.stderr.write(
'[-] Failed to re-link libpcap.so.1 -> /usr/local/lib/libpcap.so.1: {}\n'
.format(e))
if 'exists' not in str(e).lower():
raise Exception(
'An error occurred while linking libpcap.so.1 - {}'.format(
e))
try:
os.symlink(
os.path.join(pf_ring_install.install_directory, 'lib',
'libpfring.so'), '/usr/local/lib/libpfring.so.1')
except Exception as e:
sys.stderr.write(
'[-] Failed to re-link libpfring.so -> /usr/local/lib/libpfring.so.1: {}\n'
.format(e))
if 'exists' not in str(e).lower():
raise Exception(
'An error occurred while linking libpfring.so - {}'.format(
e))
time.sleep(2)
suricata_compiled = self._configure_and_compile_suricata(
pf_ring_installer=pf_ring_install)
if not suricata_compiled:
return False
if 'SURICATA_HOME' not in open('/etc/dynamite/environment').read():
if self.stdout:
sys.stdout.write(
'[+] Updating Suricata default home path [{}]\n'.format(
self.install_directory))
subprocess.call(
'echo SURICATA_HOME="{}" >> /etc/dynamite/environment'.format(
self.install_directory),
shell=True)
if 'SURICATA_CONFIG' not in open('/etc/dynamite/environment').read():
if self.stdout:
sys.stdout.write(
'[+] Updating Suricata default config path [{}]\n'.format(
self.configuration_directory))
subprocess.call(
'echo SURICATA_CONFIG="{}" >> /etc/dynamite/environment'.
format(self.configuration_directory),
shell=True)
self._setup_suricata_rules()
config = SuricataConfigurator(self.configuration_directory)
config.af_packet_interfaces = []
config.add_afpacket_interface(network_interface,
threads='auto',
cluster_id=99)
config.default_log_directory = self.log_directory
config.default_rules_directory = os.path.join(
self.configuration_directory, 'rules')
config.reference_config_file = os.path.join(
self.configuration_directory, 'reference.config')
config.classification_file = os.path.join(self.configuration_directory,
'rules',
'classification.config')
# Disable Unneeded Suricata rules
config.disable_rule('http-events.rules')
config.disable_rule('smtp-events.rules')
config.disable_rule('dns-events.rules')
config.disable_rule('tls-events.rules')
config.disable_rule('drop.rules')
config.disable_rule('emerging-p2p.rules')
config.disable_rule('emerging-pop3.rules')
config.disable_rule('emerging-telnet.rules')
config.disable_rule('emerging-tftp.rules')
config.disable_rule('emerging-voip.rules')
config.write_config()
def uninstall_agent(prompt_user=True):
"""
Uninstall the agent
:param prompt_user: Print a warning before continuing
:return: True, if uninstall succeeded
"""
environment_variables = utilities.get_environment_file_dict()
filebeat_profiler = filebeat.FileBeatProfiler()
pf_profiler = pf_ring.PFRingProfiler()
zeek_profiler = zeek.ZeekProfiler()
suricata_profiler = suricata.SuricataProfiler()
if not (filebeat_profiler.is_installed or zeek_profiler.is_installed
or suricata_profiler.is_installed):
sys.stderr.write('[-] No agent installation detected.\n')
return False
if filebeat_profiler.is_installed:
filebeat_config = filebeat.FileBeatConfigurator(
install_directory=environment_variables.get('FILEBEAT_HOME'))
if prompt_user:
sys.stderr.write(
'[-] WARNING! REMOVING THE AGENT WILL RESULT IN EVENTS NO LONGER BEING SENT TO {}.\n'
.format(filebeat_config.get_logstash_targets()))
resp = utilities.prompt_input(
'Are you sure you wish to continue? ([no]|yes): ')
while resp not in ['', 'no', 'yes']:
resp = utilities.prompt_input(
'Are you sure you wish to continue? ([no]|yes): ')
if resp != 'yes':
sys.stdout.write('[+] Exiting\n')
return False
if filebeat_profiler.is_running:
filebeat.FileBeatProcess().stop(stdout=True)
if zeek_profiler.is_running:
zeek.ZeekProcess().stop()
if pf_profiler.is_installed:
shutil.rmtree(environment_variables.get('PF_RING_HOME'))
os.remove('/opt/dynamite/.agent_environment_prepared')
if filebeat_profiler.is_installed:
shutil.rmtree(environment_variables.get('FILEBEAT_HOME'),
ignore_errors=True)
if zeek_profiler.is_installed:
shutil.rmtree(environment_variables.get('ZEEK_HOME'),
ignore_errors=True)
shutil.rmtree(environment_variables.get('ZEEK_SCRIPTS'),
ignore_errors=True)
if suricata_profiler.is_installed:
shutil.rmtree(environment_variables.get('SURICATA_HOME'),
ignore_errors=True)
shutil.rmtree(environment_variables.get('SURICATA_CONFIG'),
ignore_errors=True)
shutil.rmtree(environment_variables.get('OINKMASTER_HOME'),
ignore_errors=True)
shutil.rmtree(const.INSTALL_CACHE, ignore_errors=True)
env_lines = ''
for line in open('/etc/dynamite/environment').readlines():
if 'FILEBEAT_HOME' in line:
continue
elif 'ZEEK_HOME' in line:
continue
elif 'ZEEK_SCRIPTS' in line:
continue
elif 'SURICATA_HOME' in line:
continue
elif 'SURICATA_CONFIG' in line:
continue
elif 'PF_RING_HOME' in line:
continue
elif 'OINKMASTER_HOME' in line:
continue
elif line.strip() == '':
continue
env_lines += line.strip() + '\n'
with open('/etc/dynamite/environment', 'w') as f:
f.write(env_lines)
return True
def install_agent(network_interface,
agent_label,
logstash_target,
verbose=False):
"""
:param network_interface: The network interface that the agent should analyze traffic on
:param agent_label: A descriptive label representing the
segment/location on your network that your agent is monitoring
:param logstash_target: The host port combination for the target Logstash server (E.G "localhost:5044")
:param verbose: Include output from system utilities
:return: True, if install succeeded
"""
zeek_profiler = zeek.ZeekProfiler(stderr=True)
zeek_installer = zeek.ZeekInstaller(
download_zeek_archive=not zeek_profiler.is_downloaded,
stdout=True,
verbose=verbose)
suricata_profiler = suricata.SuricataProfiler()
filebeat_profiler = filebeat.FileBeatProfiler()
filebeat_installer = filebeat.FileBeatInstaller(
download_filebeat_archive=not filebeat_profiler.is_downloaded,
stdout=True)
if zeek_profiler.is_installed and suricata_profiler.is_installed and filebeat_profiler.is_installed:
sys.stderr.write(
'[-] Agent is already installed. If you wish to re-install, first uninstall.\n'
)
return False
# === Check running processes/prerequisites
if not is_agent_environment_prepared():
sys.stderr.write(
'[-] The environment must first be prepared prior to agent installation. \n'
)
sys.stderr.write(
'[-] This includes the installation of kernel development headers, '
'required for PF_RING kernel modules to be loaded. \n')
sys.stderr.write(
'[-] To prepare the agent environment run \'dynamite prepare agent\'.\n'
)
sys.stderr.flush()
return False
if zeek_profiler.is_running or filebeat_profiler.is_running:
sys.stderr.write(
'[-] Please stop the agent before attempting re-installation.\n')
return False
elif suricata_profiler.is_running:
sys.stderr.write(
'[-] Please stop the agent before attempting re-installation.\n')
return False
# === Install Suricata ===
suricata_installer = suricata.SuricataInstaller(
stdout=True,
verbose=verbose,
download_suricata_archive=not suricata_profiler.is_downloaded)
if not suricata_profiler.is_installed:
suricata_installer.setup_suricata(network_interface=network_interface)
else:
sys.stdout.write(
'[+] Suricata has already been installed on this system. '
'Skipping Suricata Installation.\n')
# === Install Zeek ===
if not zeek_profiler.is_installed:
try:
zeek_installer.setup_zeek(network_interface=network_interface)
zeek_installer.setup_dynamite_zeek_scripts()
except Exception as e:
sys.stderr.write(
'[-] An error occurred while trying to install Zeek - {}\n'.
format(e))
return False
else:
sys.stdout.write(
'[+] Zeek has already been installed on this system. Skipping Zeek Installation.\n'
)
if not filebeat_profiler.is_installed:
environment_variables = utilities.get_environment_file_dict()
monitored_paths = [
os.path.join(environment_variables.get('ZEEK_HOME'),
'logs/current/*.log')
]
suricata_config = suricata.SuricataConfigurator(
configuration_directory=environment_variables.get(
'SURICATA_CONFIG'))
monitored_paths.append(
os.path.join(suricata_config.default_log_directory, 'eve.json'))
filebeat_installer.setup_filebeat()
filebeat_config = filebeat.FileBeatConfigurator()
filebeat_config.set_logstash_targets([logstash_target])
filebeat_config.set_monitor_target_paths(monitored_paths)
filebeat_config.set_agent_tag(agent_label)
filebeat_config.write_config()
else:
sys.stdout.write(
'[+] FileBeat has already been installed on this system. Skipping FileBeat Installation.\n'
)
# === Post installation checks ===
pf_ring_post_install_profiler = pf_ring.PFRingProfiler()
zeek_post_install_profiler = zeek.ZeekProfiler()
suricata_post_profiler = suricata.SuricataProfiler()
filebeat_post_install_profiler = filebeat.FileBeatProfiler()
if not pf_ring_post_install_profiler.is_running:
sys.stderr.write(
'[-] PF_RING kernel module was not loaded properly.\n')
return False
if zeek_post_install_profiler.is_installed and filebeat_post_install_profiler.is_installed:
if suricata_post_profiler.is_installed:
sys.stdout.write(
'[+] Agent installation complete. Start the agent: \'dynamite start agent\'.\n'
)
sys.stdout.flush()
return True
else:
sys.stderr.write(
'[-] Agent installation failed. Suricata did not install properly.\n'
)
sys.stderr.flush()
return False
return False
def setup_zeek(self, network_interface=None):
"""
Setup Zeek NSM with PF_RING support
:param stdout: Print output to console
:param network_interface: The interface to listen on
:return: True, if setup successful
"""
if not network_interface:
network_interface = utilities.get_network_interface_names()[0]
if network_interface not in utilities.get_network_interface_names():
sys.stderr.write(
'[-] The network interface that your defined: \'{}\' is invalid. Valid network interfaces: {}\n'
.format(network_interface,
utilities.get_network_interface_names()))
raise Exception(
'Invalid network interface {}'.format(network_interface))
if self.stdout:
sys.stdout.write(
'[+] Creating zeek install|configuration|logging directories.\n'
)
subprocess.call('mkdir -p {}'.format(self.install_directory),
shell=True)
subprocess.call('mkdir -p {}'.format(self.configuration_directory),
shell=True)
pf_ring_profiler = pf_ring.PFRingProfiler()
pf_ring_install = pf_ring.PFRingInstaller(
downlaod_pf_ring_archive=not pf_ring_profiler.is_downloaded,
stdout=self.stdout,
verbose=self.verbose)
if not pf_ring_profiler.is_installed:
if self.stdout:
sys.stdout.write(
'[+] Installing PF_RING kernel modules and dependencies.\n'
)
sys.stdout.flush()
time.sleep(1)
if self.stdout:
sys.stdout.write(
'[+] Compiling Zeek from source. This can take up to 30 minutes. '
'Have another cup of coffee.\n')
sys.stdout.flush()
utilities.print_coffee_art()
time.sleep(1)
sys.stdout.write('[+] Configuring...\n')
sys.stdout.flush()
if self.verbose:
subprocess.call(
'./configure --prefix={} --scriptdir={} --with-pcap={}'.format(
self.install_directory, self.configuration_directory,
pf_ring_install.install_directory),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.ZEEK_DIRECTORY_NAME))
else:
subprocess.call(
'./configure --prefix={} --scriptdir={} --with-pcap={}'.format(
self.install_directory, self.configuration_directory,
pf_ring_install.install_directory),
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.ZEEK_DIRECTORY_NAME),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
sys.stdout.write('[+] Compiling...\n')
sys.stdout.flush()
time.sleep(1)
if self.verbose:
compile_zeek_process = subprocess.Popen(
'make; make install',
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.ZEEK_DIRECTORY_NAME))
compile_zeek_process.communicate()
compile_return_code = compile_zeek_process.returncode
else:
compile_zeek_process = subprocess.Popen(
'make; make install',
shell=True,
cwd=os.path.join(const.INSTALL_CACHE,
const.ZEEK_DIRECTORY_NAME),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
compile_return_code = utilities.run_subprocess_with_status(
compile_zeek_process, expected_lines=6596)
if compile_return_code != 0:
sys.stderr.write(
'[-] Failed to compile Zeek from source; error code: {}; ; run with '
'--debug flag for more info.\n'.format(
compile_zeek_process.returncode))
return False
if 'ZEEK_HOME' not in open('/etc/dynamite/environment').read():
if self.stdout:
sys.stdout.write(
'[+] Updating Zeek default home path [{}]\n'.format(
self.install_directory))
subprocess.call(
'echo ZEEK_HOME="{}" >> /etc/dynamite/environment'.format(
self.install_directory),
shell=True)
if 'ZEEK_SCRIPTS' not in open('/etc/dynamite/environment').read():
if self.stdout:
sys.stdout.write(
'[+] Updating Zeek default script path [{}]\n'.format(
self.configuration_directory))
subprocess.call(
'echo ZEEK_SCRIPTS="{}" >> /etc/dynamite/environment'.format(
self.configuration_directory),
shell=True)
if self.stdout:
sys.stdout.write(
'[+] Overwriting default Script | Node configurations.\n')
shutil.copy(
os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'broctl-nodes.cfg'),
os.path.join(self.install_directory, 'etc', 'node.cfg'))
shutil.copy(
os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'local.bro'),
os.path.join(self.configuration_directory, 'site', 'local.bro'))
node_config = ZeekNodeConfigurator(self.install_directory)
cpu_count = utilities.get_cpu_core_count()
cpus = [c for c in range(0, cpu_count)]
if cpu_count > 1:
pinned_cpus = cpus[:-1]
lb_procs = len(pinned_cpus)
else:
pinned_cpus = cpus
lb_procs = 1
node_config.add_worker(name='dynamite-worker-1',
host='localhost',
interface=network_interface,
lb_procs=lb_procs,
pin_cpus=pinned_cpus)
node_config.write_config()