from logging import DEBUG, INFO
from dynamite_nsm.logger import get_logger
from dynamite_nsm.services.zeek import process
from dynamite_nsm.cmd.zeek.process import interface as zeek_process_interface
if __name__ == '__main__':
parser = zeek_process_interface.get_parser()
args = parser.parse_args()
logger = get_logger('ZEEK',
level=DEBUG if args.verbose else INFO,
stdout=args.stdout)
logger.info(f'Calling zeek.process.{args.entry_method_name}.')
logger.debug(args.__dict__)
result = zeek_process_interface.execute(args)
print(result)
from logging import DEBUG, INFO
from dynamite_nsm.cmd.elasticsearch.process import interface as elasticsearch_process_interface
from dynamite_nsm.logger import get_logger
if __name__ == '__main__':
parser = elasticsearch_process_interface.get_parser()
args = parser.parse_args()
logger = get_logger('elasticsearch.process',
level=DEBUG if args.verbose else INFO,
stdout=args.stdout)
logger.info(f'Calling elasticsearch.process.{args.entry_method_name}.')
logger.debug(args.__dict__)
result = elasticsearch_process_interface.execute(args)
print(result)
from logging import DEBUG, INFO
from dynamite_nsm.logger import get_logger
from dynamite_nsm.cmd.suricata.process import interface as suricata_process_interface
if __name__ == '__main__':
parser = suricata_process_interface.get_parser()
args = parser.parse_args()
logger = get_logger('suricata.process',
level=DEBUG if args.verbose else INFO,
stdout=args.stdout)
logger.info(f'Calling suricata.process.{args.entry_method_name}.')
logger.debug(args.__dict__)
result = suricata_process_interface.execute(args)
print(result)
def uninstall_dynamited(prompt_user=True, stdout=True, verbose=False):
"""
Uninstall dynamited
:param prompt_user: Print a warning before continuing
:param stdout: Print the output to console
:param verbose: Include detailed debug messages
"""
log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
logger = get_logger('DYNAMITED', level=log_level, stdout=stdout)
env_file = os.path.join(const.CONFIG_PATH, 'environment')
environment_variables = utilities.get_environment_file_dict()
dynamited_profiler = dynamited_profile.ProcessProfiler()
if not dynamited_profiler.is_installed():
raise dynamited_exceptions.UninstallDynamiteDaemonError("dynamited is not installed.")
if prompt_user:
sys.stderr.write(
'\n\033[93m[-] WARNING! Removing dynamited will disable various performance metric gathering from '
'occurring.\033[0m\n')
resp = utilities.prompt_input('\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m ')
while resp not in ['', 'no', 'yes']:
resp = utilities.prompt_input('\n\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m ')
if resp != 'yes':
if stdout:
sys.stdout.write('\n[+] Exiting\n')
exit(0)
if dynamited_profiler.is_running():
dynamited_process.ProcessManager().stop()
try:
shutil.rmtree(environment_variables['DYNAMITED_LOGS'])
shutil.rmtree(environment_variables['DYNAMITED_INSTALL'])
shutil.rmtree(environment_variables['DYNAMITED_CONFIG'])
shutil.rmtree(const.INSTALL_CACHE, ignore_errors=True)
env_lines = ''
with open(env_file) as env_fr:
for line in env_fr.readlines():
if 'DYNAMITED_LOGS' in line:
continue
elif 'DYNAMITED_INSTALL' in line:
continue
elif 'DYNAMITED_CONFIG' in line:
continue
elif line.strip() == '':
continue
env_lines += line.strip() + '\n'
with open(env_file, 'w') as env_fw:
env_fw.write(env_lines)
except Exception as e:
logger.error("General error occurred while attempting to uninstall dynamited.".format(e))
logger.debug("General error occurred while attempting to uninstall dynamited; {}".format(e))
raise dynamited_exceptions.UninstallDynamiteDaemonError(
"General error occurred while attempting to uninstall dynamited; {}".format(e))
try:
sysctl = systemctl.SystemCtl()
except general_exceptions.CallProcessError:
raise dynamited_exceptions.UninstallDynamiteDaemonError("Could not find systemctl.")
sysctl.uninstall_and_disable('dynamited')
def install_logstash(configuration_directory,
install_directory,
log_directory,
host='0.0.0.0',
elasticsearch_host='localhost',
elasticsearch_port=9200,
elasticsearch_password='******',
heap_size_gigs=4,
install_jdk=True,
create_dynamite_user=True,
stdout=False,
verbose=False):
"""
Install Logstash with ElastiFlow & Synesis
:param configuration_directory: Path to the configuration directory (E.G /etc/dynamite/logstash/)
:param install_directory: Path to the install directory (E.G /opt/dynamite/logstash/)
:param log_directory: Path to the log directory (E.G /var/log/dynamite/logstash/)
:param host: The IP address to bind LogStash listeners too
:param elasticsearch_password: The password used for authentication across all builtin ES users
:param elasticsearch_host: A hostname/IP of the target elasticsearch instance
:param elasticsearch_port: A port number for the target elasticsearch instance
:param elasticsearch_password: The password used for authentication across all builtin ES users
:param heap_size_gigs: The initial/max java heap space to allocate
:param install_jdk: Install the latest OpenJDK that will be used by Logstash/ElasticSearch
:param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run Logstash/ElasticSearch
:param stdout: Print the output to console
:param verbose: Include output from system utilities
:return: True, if installation succeeded
"""
log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
logger = get_logger('LOGSTASH', level=log_level, stdout=stdout)
ls_profiler = logstash_profile.ProcessProfiler()
if ls_profiler.is_installed():
logger.error('LogStash is already installed.')
raise logstash_exceptions.AlreadyInstalledLogstashError()
if utilities.get_memory_available_bytes() < 6 * (1000**3):
sys.stderr.write(
'\n\033[93m[-] WARNING! LogStash should have at-least 6GB to run '
'currently available [{} GB]\033[0m\n'.format(
utilities.get_memory_available_bytes() / (1000**3)))
if str(utilities.prompt_input(
'\033[93m[?] Continue? [y|N]:\033[0m ')).lower() != 'y':
sys.stdout.write('\n[+] Exiting\n')
exit(0)
ls_installer = InstallManager(
configuration_directory,
install_directory,
log_directory,
host=host,
elasticsearch_host=elasticsearch_host,
elasticsearch_port=elasticsearch_port,
elasticsearch_password=elasticsearch_password,
heap_size_gigs=heap_size_gigs,
download_logstash_archive=not ls_profiler.is_downloaded(),
stdout=stdout,
verbose=verbose)
if install_jdk:
try:
utilities.download_java(stdout=stdout)
utilities.extract_java()
utilities.setup_java()
except Exception as e:
logger.error(
'General error occurred while attempting to setup Java.')
logger.debug(
"General error occurred while attempting to setup Java; {}".
format(e))
raise logstash_exceptions.InstallLogstashError(
"General error occurred while attempting to setup Java; {}".
format(e))
if create_dynamite_user:
utilities.create_dynamite_user(utilities.generate_random_password(50))
ls_installer.setup_logstash()
def uninstall_logstash(prompt_user=True, stdout=True, verbose=False):
"""
Install Logstash with ElastiFlow & Synesis
:param prompt_user: Print a warning before continuing
:param stdout: Print the output to console
:param verbose: Include detailed debug messages
"""
log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
logger = get_logger('LOGSTASH', level=log_level, stdout=stdout)
env_file = os.path.join(const.CONFIG_PATH, 'environment')
environment_variables = utilities.get_environment_file_dict()
configuration_directory = environment_variables.get('LS_PATH_CONF')
ls_profiler = logstash_profile.ProcessProfiler()
ls_config = logstash_config.ConfigManager(
configuration_directory=configuration_directory)
if not ls_profiler.is_installed():
logger.error('LogStash is not installed.')
raise logstash_exceptions.UninstallLogstashError(
"LogStash is not installed.")
if prompt_user:
sys.stderr.write(
'\n\033[93m[-] WARNING! Removing Logstash Will Prevent ElasticSearch From Receiving Events.\033[0m\n'
)
resp = utilities.prompt_input(
'\n\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m '
)
while resp not in ['', 'no', 'yes']:
resp = utilities.prompt_input(
'\033[93m[?] Are you sure you wish to continue? ([no]|yes): \033[0m'
)
if resp != 'yes':
if stdout:
sys.stdout.write('\n[+] Exiting\n')
exit(0)
if ls_profiler.is_running():
logstash_process.ProcessManager().stop()
try:
shutil.rmtree(ls_config.ls_path_conf)
shutil.rmtree(ls_config.ls_home)
shutil.rmtree(ls_config.path_logs)
shutil.rmtree(const.INSTALL_CACHE, ignore_errors=True)
env_lines = ''
with open(env_file) as env_fr:
for line in env_fr.readlines():
if 'LS_PATH_CONF' in line:
continue
elif 'LS_HOME' in line:
continue
elif 'LS_LOGS' in line:
continue
elif 'ELASTIFLOW_' in line:
continue
elif 'SYNLITE_' in line:
continue
elif 'ES_PASSWD' in line:
continue
elif line.strip() == '':
continue
env_lines += line.strip() + '\n'
with open(env_file, 'w') as env_fw:
env_fw.write(env_lines)
except Exception as e:
logger.error(
"General error occurred while attempting to uninstall LogStash.".
format(e))
logger.debug(
"General error occurred while attempting to uninstall LogStash; {}"
.format(e))
raise logstash_exceptions.UninstallLogstashError(
"General error occurred while attempting to uninstall LogStash; {}"
.format(e))
try:
sysctl = systemctl.SystemCtl()
except general_exceptions.CallProcessError:
raise logstash_exceptions.UninstallLogstashError(
"Could not find systemctl.")
sysctl.uninstall_and_disable('logstash')
def install_dependencies(stdout=False, verbose=False):
"""
Install the required dependencies required by Zeek
:param stdout: Print the output to console
:param verbose: Include detailed debug messages
"""
log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
logger = get_logger('ZEEK', level=log_level, stdout=stdout)
logger.info('Installing Dependencies.')
pkt_mng = package_manager.OSPackageManager(stdout=stdout,
verbose=verbose)
packages = None
if pkt_mng.package_manager == 'apt-get':
packages = [
'cmake', 'cmake3', 'make', 'gcc', 'g++', 'flex', 'bison',
'libpcap-dev', 'libssl-dev', 'python-dev', 'swig',
'zlib1g-dev', 'linux-headers-$(uname -r)',
'linux-headers-generic', 'tar'
]
elif pkt_mng.package_manager == 'yum':
packages = [
'cmake', 'cmake3', 'make', 'gcc', 'gcc-c++', 'flex', 'bison',
'libpcap-devel', 'openssl-devel', 'python-devel',
'python2-devel', 'swig', 'zlib-devel',
'kernel-devel-$(uname -r)', 'kernel-devel', 'tar'
]
# Work around for missing dependencies in RHEL/Centos8
try:
pkt_mng.install_packages(['dnf-plugins-core'])
except general_exceptions.OsPackageManagerInstallError as e:
logger.warning(
"Failed to install one or more packages: {}".format(e))
enable_powertools_p = subprocess.Popen(
['yum', 'config-manager', '--set-enabled', 'PowerTools'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
enable_powertools_p.communicate()
if enable_powertools_p.returncode == 0:
logger.info("Installed PowerTools.")
logger.info('Refreshing Package Index.')
try:
pkt_mng.refresh_package_indexes()
except general_exceptions.OsPackageManagerRefreshError as e:
logger.warning("Failed to refresh packages.")
logger.debug("Failed to refresh packages threw: {}".format(e))
raise general_exceptions.OsPackageManagerRefreshError(
'Failed to refresh packages.')
logger.info('Installing the following packages: {}.'.format(packages))
try:
pkt_mng.install_packages(packages)
except general_exceptions.OsPackageManagerInstallError as e:
logger.warning(
"Failed to install packages one or more packages: {}".format(
e))
def install_dependencies(stdout=True, verbose=False):
"""
Install the required dependencies required by Jupyterhub
:param stdout: Print output to console
:param verbose: Include detailed debug messages
"""
log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
logger = get_logger('LAB', level=log_level, stdout=stdout)
pkt_mng = package_manager.OSPackageManager(stdout=stdout,
verbose=verbose)
packages = None
logger.info('Updating Package Indexes.')
pkt_mng.refresh_package_indexes()
logger.info('Installing dependencies.')
if pkt_mng.package_manager == 'apt-get':
packages = [
'build-essential', 'python3', 'python3-pip', 'python3-dev',
'nodejs', 'npm'
]
elif pkt_mng.package_manager == 'yum':
p = subprocess.Popen(
'curl --silent --location https://rpm.nodesource.com/setup_10.x | sudo bash -',
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
close_fds=True)
p.communicate()
if p.returncode != 0:
logger.error('Could not install nodejs source rpm.')
raise general_exceptions.OsPackageManagerInstallError(
"Could not install nodejs from third-party RPM; https://rpm.nodesource.com/setup_10.x"
)
packages = [
'gcc72-c++', 'gcc', 'gcc-c++', 'nodejs', 'python36',
'python36-devel'
]
logger.info('Installing the following packages: {}.'.format(packages))
try:
pkt_mng.install_packages(packages)
except general_exceptions.OsPackageManagerInstallError as e:
logger.warning(
"Failed to install one or more packages: {}".format(e))
logger.info(
'Installing configurable-http-proxy. This may take some time.')
p = subprocess.Popen('npm install -g configurable-http-proxy',
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
p.communicate()
if p.returncode != 0:
try:
err_msg = p.stderr.read()
except ValueError:
err_msg = p.returncode
logger.error(
'Failed to install configurable-http-proxy, ensure npm is installed and in $PATH: {}'
''.format(err_msg))
raise general_exceptions.OsPackageManagerInstallError(
"Could not install configurable-http-proxy via npm; {}".format(
err_msg))
def __init__(self, name, verbose=False, stdout=True):
log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
self.logger = get_logger(str(name).upper(), level=log_level, stdout=stdout)
def get_zeek_workers(network_capture_interfaces,
strategy="aggressive",
stdout=True,
verbose=False):
"""
Algorithm for determining the assignment of CPUs for Zeek workers
:param network_capture_interfaces: A list of network interface names
:param strategy: 'aggressive', results in more CPUs pinned per interface, sometimes overshoots resources
'conservative', results in less CPUs pinned per interface, but never overshoots resources
:param stdout: Print the output to console
:param verbose: Include detailed debug messages
:return: A dictionary containing Zeek worker configuration
"""
log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
logger = get_logger('ZEEK', level=log_level, stdout=stdout)
cpus = [c for c in range(0, utilities.get_cpu_core_count())]
logger.info(
"Calculating optimal Zeek worker strategy [strategy: {}].".format(
strategy))
logger.debug("Detected CPU Cores: {}".format(cpus))
# Reserve 0 for KERNEL/Userland opts
available_cpus = cpus[1:]
def grouper(n, iterable):
args = [iter(iterable)] * n
return zip_longest(*args)
def create_workers(net_interfaces, avail_cpus):
idx = 0
zeek_worker_configs = []
for net_interface in net_interfaces:
if idx >= len(avail_cpus):
idx = 0
if isinstance(avail_cpus[idx], int):
avail_cpus[idx] = [avail_cpus[idx]]
zeek_worker_configs.append(
dict(name='dynamite-worker-' + net_interface,
host='localhost',
interface=net_interface,
lb_procs=len(avail_cpus[idx]),
pinned_cpus=avail_cpus[idx]))
idx += 1
return zeek_worker_configs
if len(available_cpus) <= len(network_capture_interfaces):
# Wrap the number of CPUs around the number of network interfaces;
# Since there are more network interfaces than CPUs; CPUs will be assigned more than once
# lb_procs will always be 1
zeek_workers = create_workers(network_capture_interfaces,
available_cpus)
else:
# In this scenario we choose from one of two strategies
# 1. Aggressive:
# - Take the ratio of network_interfaces to available CPUS; ** ROUND UP **.
# - Group the available CPUs by this integer
# (if the ratio == 2 create as many groupings of 2 CPUs as possible)
# - Apply the same wrapping logic used above, but with the CPU groups instead of single CPU instances
# 2. Conservative:
# - Take the ratio of network_interfaces to available CPUS; ** ROUND DOWN **.
# - Group the available CPUs by this integer
# (if the ratio == 2 create as many groupings of 2 CPUs as possible)
# - Apply the same wrapping logic used above, but with the CPU groups instead of single CPU instances
aggressive_ratio = int(
math.ceil(
len(available_cpus) /
float(len(network_capture_interfaces))))
conservative_ratio = int(
math.floor(
len(available_cpus) / len(network_capture_interfaces)))
if strategy == 'aggressive':
cpu_groups = grouper(aggressive_ratio, available_cpus)
else:
cpu_groups = grouper(conservative_ratio, available_cpus)
temp_cpu_groups = []
for cpu_group in cpu_groups:
cpu_group = [c for c in cpu_group if c]
temp_cpu_groups.append(cpu_group)
cpu_groups = temp_cpu_groups
zeek_workers = create_workers(network_capture_interfaces,
cpu_groups)
logger.info('Zeek Worker Count: {}'.format(len(zeek_workers)))
logger.debug('Zeek Workers: {}'.format(zeek_workers))
return zeek_workers
def install_elasticsearch(configuration_directory,
install_directory,
log_directory,
password='******',
heap_size_gigs=4,
install_jdk=True,
create_dynamite_user=True,
stdout=True,
verbose=False):
"""
Install ElasticSearch
:param configuration_directory: Path to the configuration directory (E.G /etc/dynamite/elasticsearch/)
:param install_directory: Path to the install directory (E.G /opt/dynamite/elasticsearch/)
:param log_directory: Path to the log directory (E.G /var/log/dynamite/elasticsearch/)
:param password: The password used for authentication across all builtin users
:param heap_size_gigs: The initial/max java heap space to allocate
:param install_jdk: Install the latest OpenJDK that will be used by Logstash/ElasticSearch
:param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run Logstash/ElasticSearch
:param stdout: Print the output to console
:param verbose: Include detailed debug messages
"""
log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
logger = get_logger('ELASTICSEARCH', level=log_level, stdout=stdout)
es_profiler = elastic_profile.ProcessProfiler()
if es_profiler.is_installed():
logger.error('ElasticSearch is already installed.')
raise elastic_exceptions.AlreadyInstalledElasticsearchError()
if utilities.get_memory_available_bytes() < 6 * (1000**3):
sys.stderr.write(
'\n\033[93m[-] WARNING! ElasticSearch should have at-least 6GB to run '
'currently available [{} GB]\033[0m\n'.format(
utilities.get_memory_available_bytes() / (1000**3)))
if str(utilities.prompt_input(
'\033[93m[?] Continue? [y|N]:\033[0m ')).lower() != 'y':
sys.stdout.write('\n[+] Exiting\n')
exit(0)
es_installer = InstallManager(
configuration_directory=configuration_directory,
install_directory=install_directory,
log_directory=log_directory,
password=password,
heap_size_gigs=heap_size_gigs,
download_elasticsearch_archive=not es_profiler.is_downloaded(),
stdout=stdout,
verbose=verbose)
if install_jdk:
try:
utilities.download_java(stdout=stdout)
utilities.extract_java()
utilities.setup_java()
except Exception as e:
logger.error(
'General error occurred while attempting to setup Java.')
logger.debug(
"General error occurred while attempting to setup Java; {}".
format(e))
raise elastic_exceptions.InstallElasticsearchError(
"General error occurred while attempting to setup Java; {}".
format(e))
if create_dynamite_user:
utilities.create_dynamite_user(utilities.generate_random_password(50))
es_installer.setup_elasticsearch()
def uninstall_dynamite_lab(prompt_user=True, stdout=True, verbose=False):
"""
Uninstall DynamiteLab
:param prompt_user: Print a warning before continuing
:param stdout: Print the output to console
:param verbose: Include detailed debug messages
"""
log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
logger = get_logger('LAB', level=log_level, stdout=stdout)
env_file = os.path.join(const.CONFIG_PATH, 'environment')
environment_variables = utilities.get_environment_file_dict()
configuration_directory = environment_variables.get('DYNAMITE_LAB_CONFIG')
notebook_home = environment_variables.get('NOTEBOOK_HOME')
dynamite_lab_profiler = lab_profile.ProcessProfiler()
if not (dynamite_lab_profiler.is_installed
and dynamite_lab_profiler.is_configured):
raise lab_exceptions.UninstallLabError("DynamiteLab is not installed.")
dynamite_lab_config = lab_configs.ConfigManager(configuration_directory)
if prompt_user:
sys.stderr.write(
'\n\033[93m[-] WARNING! Removing Dynamite Lab Will Remove All Jupyter Notebooks\033[0m\n'
)
resp = utilities.prompt_input(
'\033[93m[?] Are you sure you wish to continue? ([no]|yes): \033[0m'
)
while resp not in ['', 'no', 'yes']:
resp = utilities.prompt_input(
'\033[93m[?] Are you sure you wish to continue? ([no]|yes): \033[0m'
)
if resp != 'yes':
if stdout:
sys.stdout.write('\n[+] Exiting\n')
return
if dynamite_lab_profiler.is_running:
lab_process.ProcessManager().stop()
shutil.rmtree(configuration_directory)
shutil.rmtree(notebook_home)
shutil.rmtree(const.INSTALL_CACHE, ignore_errors=True)
env_lines = ''
try:
with open(env_file) as env_fr:
for line in env_fr.readlines():
if 'DYNAMITE_LAB_CONFIG' in line:
continue
elif 'NOTEBOOK_HOME' in line:
continue
elif line.strip() == '':
continue
env_lines += line.strip() + '\n'
with open(env_file, 'w') as env_fw:
env_fw.write(env_lines)
except Exception as e:
logger.error(
'General error occurred while attempting to uninstall lab.')
logger.debug(
"General error occurred while attempting to uninstall lab; {}".
format(e))
raise lab_exceptions.UninstallLabError(
"General error occurred while attempting to uninstall lab; {}".
format(e))
InstallManager(
configuration_directory,
notebook_home,
elasticsearch_host=dynamite_lab_config.elasticsearch_url.split(
'//')[1].split(':')[0],
elasticsearch_password=dynamite_lab_config.elasticsearch_password,
elasticsearch_port=dynamite_lab_config.elasticsearch_url.split(
'//')[1].split(':')[1].replace('/', ''),
download_dynamite_sdk_archive=False,
extract_dynamite_sdk_archive=False).uninstall_kibana_lab_icon()
def __init__(self,
configuration_directory,
notebook_home,
elasticsearch_host=None,
elasticsearch_port=None,
elasticsearch_password='******',
jupyterhub_host=None,
jupyterhub_password='******',
download_dynamite_sdk_archive=True,
extract_dynamite_sdk_archive=True,
stdout=True,
verbose=False):
"""
:param configuration_directory: Path to the configuration directory (E.G /etc/dynamite/dynamite_sdk/)
:param notebook_home: The path where Jupyter notebooks are stored
:param elasticsearch_host: A hostname/IP of the target elasticsearch instance
:param elasticsearch_port: A port number for the target elasticsearch instance
:param elasticsearch_password: The password used for authentication across all builtin ES users
:param jupyterhub_host: The host by which users can access this instance;
(Used for creating kibana -> Jupyter hyperlinks)
:param jupyterhub_password: The password used for authenticating to jupyterhub (via jupyter user)
:param download_dynamite_sdk_archive: If True, download the DynamiteSDK archive from a mirror
:param extract_dynamite_sdk_archive: If True, extracts the DynamiteSDK archive
:param stdout: Print output to console
:param verbose: Include detailed debug messages
"""
log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
self.logger = get_logger('LAB', level=log_level, stdout=stdout)
self.elasticsearch_host = elasticsearch_host
self.elasticsearch_port = elasticsearch_port
self.elasticsearch_password = elasticsearch_password
self.jupyterhub_host = jupyterhub_host
self.jupyterhub_password = jupyterhub_password
self.configuration_directory = configuration_directory
self.notebook_home = notebook_home
if download_dynamite_sdk_archive:
try:
self.download_dynamite_sdk(stdout=stdout)
except general_exceptions.DownloadError:
self.logger.error('Failed to download DynamiteSDK archive.')
raise lab_exceptions.InstallLabError(
"Failed to download DynamiteSDK archive.")
if extract_dynamite_sdk_archive:
try:
self.extract_dynamite_sdk()
except general_exceptions.ArchiveExtractionError:
self.logger.error('Failed to extract DynamiteSDK archive.')
raise lab_exceptions.InstallLabError(
"Failed to extract DynamiteSDK archive.")
try:
self.install_dependencies(stdout=stdout, verbose=verbose)
self.install_jupyterhub(stdout=stdout)
except (general_exceptions.InvalidOsPackageManagerDetectedError,
general_exceptions.OsPackageManagerInstallError,
general_exceptions.OsPackageManagerRefreshError):
self.logger.error('One or more OS dependencies failed to install.')
raise lab_exceptions.InstallLabError(
"One or more OS dependencies failed to install.")
self.logger.info('Creating jupyter user in dynamite group.')
utilities.create_jupyter_user(password=self.jupyterhub_password)
self.stdout = stdout
self.verbose = verbose
if not elasticsearch_host:
if elastic_profile.ProcessProfiler().is_installed():
self.elasticsearch_host = 'localhost'
else:
raise lab_exceptions.InstallLabError(
"Elasticsearch must either be installed locally, or a remote host must be specified."
)
from logging import DEBUG, INFO
from dynamite_nsm.logger import get_logger
from dynamite_nsm.cmd.agent.process import interface as agent_process_interface
if __name__ == '__main__':
parser = agent_process_interface.get_parser()
args = parser.parse_args()
logger = get_logger('agent.process', level=DEBUG if args.verbose else INFO, stdout=args.stdout)
logger.info(f'Calling agent.process.{args.entry_method_name}.')
logger.debug(args.__dict__)
result = agent_process_interface.execute(args)
print(result)
def __init__(self,
configuration_directory,
install_directory,
capture_network_interfaces,
download_zeek_archive=True,
stdout=True,
verbose=False):
"""
Install Zeek
:param configuration_directory: Path to the configuration directory (E.G /etc/dynamite/zeek)
:param install_directory: Path to the install directory (E.G /opt/dynamite/zeek/)
:param capture_network_interfaces: A list of network interfaces to capture on (E.G ["mon0", "mon1"])
:param download_zeek_archive: If True, download the Zeek archive from a mirror
:param stdout: Print the output to console
:param verbose: Include detailed debug messages
"""
log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
self.logger = get_logger('ZEEK', level=log_level, stdout=stdout)
self.configuration_directory = configuration_directory
self.install_directory = install_directory
self.capture_network_interfaces = capture_network_interfaces
self.stdout = stdout
self.verbose = verbose
install.BaseInstallManager.__init__(self,
'zeek',
verbose=self.verbose,
stdout=stdout)
utilities.create_dynamite_environment_file()
if download_zeek_archive:
try:
self.logger.info("Attempting to download Zeek archive.")
self.download_from_mirror(const.ZEEK_MIRRORS,
const.ZEEK_ARCHIVE_NAME,
stdout=stdout,
verbose=verbose)
except general_exceptions.DownloadError as e:
self.logger.error("Failed to download Zeek archive.")
self.logger.debug(
"Failed to download Zeek archive, threw: {}.".format(e))
raise zeek_exceptions.InstallZeekError(
"Failed to download Zeek archive.")
try:
self.logger.info("Attempting to extract Zeek archive ({}).".format(
const.ZEEK_ARCHIVE_NAME))
self.extract_archive(
os.path.join(const.INSTALL_CACHE, const.ZEEK_ARCHIVE_NAME))
self.logger.info("Extraction completed.")
except general_exceptions.ArchiveExtractionError as e:
self.logger.error("Failed to extract Zeek archive.")
self.logger.debug(
"Failed to extract Zeek archive, threw: {}.".format(e))
raise zeek_exceptions.InstallZeekError(
"Failed to extract Zeek archive.")
try:
self.install_dependencies(stdout=stdout, verbose=verbose)
except (general_exceptions.InvalidOsPackageManagerDetectedError,
general_exceptions.OsPackageManagerRefreshError):
raise zeek_exceptions.InstallZeekError(
"One or more OS dependencies failed to install.")
if not self.validate_capture_network_interfaces(
self.capture_network_interfaces):
self.logger.error(
"One or more defined network interfaces is invalid: {}".format(
capture_network_interfaces))
raise zeek_exceptions.InstallZeekError(
"One or more defined network interfaces is invalid: {}".format(
capture_network_interfaces))
def uninstall_zeek(prompt_user=True, stdout=True, verbose=False):
"""
Uninstall Zeek
:param prompt_user: Print a warning before continuing
:param stdout: Print the output to console
:param verbose: Include detailed debug messages
"""
log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
logger = get_logger('ZEEK', level=log_level, stdout=stdout)
logger.info("Uninstalling Zeek.")
env_file = os.path.join(const.CONFIG_PATH, 'environment')
environment_variables = utilities.get_environment_file_dict()
zeek_profiler = zeek_profile.ProcessProfiler()
if not zeek_profiler.is_installed():
logger.error("Zeek is not installed. Cannot uninstall.")
raise zeek_exceptions.UninstallZeekError("Zeek is not installed.")
if prompt_user:
sys.stderr.write('\n\033[93m[-] WARNING! Removing Zeek Will Remove Critical Agent Functionality.\033[0m\n')
resp = utilities.prompt_input('\033[93m[?] Are you sure you wish to continue? ([no]|yes): \033[0m')
while resp not in ['', 'no', 'yes']:
resp = utilities.prompt_input('\033[93m[?] Are you sure you wish to continue? ([no]|yes): \033[0m')
if resp != 'yes':
if stdout:
sys.stdout.write('\n[+] Exiting\n')
exit(0)
if zeek_profiler.is_running():
try:
zeek_process.ProcessManager().stop()
except zeek_exceptions.CallZeekProcessError as e:
logger.error("Could not kill Zeek process. Cannot uninstall.")
logger.debug("Could not kill Zeek process. Cannot uninstall; {}".format(e))
raise zeek_exceptions.UninstallZeekError("Could not kill Zeek process; {}".format(e))
install_directory = environment_variables.get('ZEEK_HOME')
config_directory = environment_variables.get('ZEEK_SCRIPTS')
try:
with open(env_file) as env_fr:
env_lines = ''
for line in env_fr.readlines():
if 'ZEEK_HOME' in line:
continue
elif 'ZEEK_SCRIPTS' in line:
continue
elif 'PF_RING_HOME' in line:
continue
elif line.strip() == '':
continue
env_lines += line.strip() + '\n'
with open(env_file, 'w') as env_fw:
env_fw.write(env_lines)
if zeek_profiler.is_installed():
shutil.rmtree(install_directory, ignore_errors=True)
shutil.rmtree(config_directory, ignore_errors=True)
except Exception as e:
logger.error("General error occurred while attempting to uninstall Zeek.")
logger.debug("General error occurred while attempting to uninstall Zeek; {}".format(e))
raise zeek_exceptions.UninstallZeekError(
"General error occurred while attempting to uninstall Zeek; {}".format(e))
try:
sysctl = systemctl.SystemCtl()
except general_exceptions.CallProcessError:
raise zeek_exceptions.UninstallZeekError("Could not find systemctl.")
sysctl.uninstall_and_disable('zeek')
def uninstall_kibana(prompt_user=True, stdout=True, verbose=False):
"""
Uninstall Kibana/ElastiFlow/Synesis Dashboards
:param prompt_user: Print a warning before continuing
:param stdout: Print the output to console
:param verbose: Include detailed debug messages
"""
log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
logger = get_logger('KIBANA', level=log_level, stdout=stdout)
env_file = os.path.join(const.CONFIG_PATH, 'environment')
environment_variables = utilities.get_environment_file_dict()
kb_profiler = kibana_profile.ProcessProfiler()
if not kb_profiler.is_installed():
raise kibana_exceptions.UninstallKibanaError("Kibana is not installed.")
configuration_directory = environment_variables.get('KIBANA_PATH_CONF')
kb_config = kibana_configs.ConfigManager(configuration_directory)
if prompt_user:
sys.stderr.write(
'\n\033[93m[-] WARNING! Removing Kibana will uninstall all visualizations and saved searches previously '
'created.\033[0m\n')
resp = utilities.prompt_input('\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m ')
while resp not in ['', 'no', 'yes']:
resp = utilities.prompt_input('\n\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m ')
if resp != 'yes':
if stdout:
sys.stdout.write('\n[+] Exiting\n')
exit(0)
if kb_profiler.is_running():
kibana_process.ProcessManager().stop()
try:
shutil.rmtree(kb_config.kibana_path_conf)
shutil.rmtree(kb_config.kibana_home)
shutil.rmtree(kb_config.kibana_logs)
shutil.rmtree(const.INSTALL_CACHE, ignore_errors=True)
env_lines = ''
with open(env_file) as env_fr:
for line in env_fr.readlines():
if 'KIBANA_PATH_CONF' in line:
continue
elif 'KIBANA_HOME' in line:
continue
elif 'KIBANA_LOGS' in line:
continue
elif line.strip() == '':
continue
env_lines += line.strip() + '\n'
with open(env_file, 'w') as env_fw:
env_fw.write(env_lines)
except Exception as e:
logger.error("General error occurred while attempting to uninstall Kibana.".format(e))
logger.debug("General error occurred while attempting to uninstall Kibana; {}".format(e))
raise kibana_exceptions.UninstallKibanaError(
"General error occurred while attempting to uninstall kibana; {}".format(e))
try:
sysctl = systemctl.SystemCtl()
except general_exceptions.CallProcessError:
raise kibana_exceptions.UninstallKibanaError("Could not find systemctl.")
sysctl.uninstall_and_disable('kibana')
