def __init__(self, systemd_service: str, name: str, log_path: Optional[str] = None,
create_pid_file: Optional[bool] = False,
stdout: Optional[bool] = True, verbose: Optional[bool] = False,
pretty_print_status: Optional[bool] = False):
"""Manage a service process
Args:
systemd_service: The name of the systemd.service file
name: The name of the process manager
log_path: The path to where the process logs
create_pid_file: If true will attempt to create a PID file
stdout: Print output to console
verbose: Include detailed debug messages
pretty_print_status: If enabled, status will be printed in a tabulated style
"""
log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
self.logger = get_logger(name, level=log_level, stdout=stdout)
self.pid_file = None
self.pid = None
self.systemd_service = systemd_service
self.name = name
self.log_path = log_path
if create_pid_file:
self.pid_file = f'{const.PID_PATH}/{name}.pid'
self.stdout = stdout
self.verbose = verbose
self.pretty_print_status = pretty_print_status
self.sysctl = systemctl.SystemCtl()
if create_pid_file:
self.pid = self._get_pid(self.pid_file)
def setup(self, inspect_interfaces: List[str]):
"""Setup Zeek
Args:
inspect_interfaces: A list of network interfaces to capture on (E.G ["mon0", "mon1"])
Returns:
None
"""
if not self.skip_interface_validation:
if not self.validate_inspect_interfaces(inspect_interfaces):
raise install.NetworkInterfaceNotFound(inspect_interfaces)
sysctl = systemctl.SystemCtl()
self.install_zeek_dependencies()
self.create_update_zeek_environment_variables()
self.logger.debug(f'Creating directory: {self.configuration_directory}')
utilities.makedirs(self.configuration_directory)
self.logger.debug(f'Creating directory: {self.install_directory}')
utilities.makedirs(self.install_directory)
self.logger.info('Setting up Zeek from source. This can take up to 15 minutes.')
if self.stdout:
utilities.print_coffee_art()
self.configure_compile_zeek()
self.logger.info('Setting up Zeek package manager.')
zkg_installer = zkg_install.InstallManager()
zkg_installer.setup()
package.InstallPackageManager(const.ZEEK_PACKAGES, stdout=self.stdout, verbose=self.verbose).setup()
self.copy_file_or_directory_to_destination(f'{const.DEFAULT_CONFIGS}/zeek/broctl-nodes.cfg',
f'{self.install_directory}/etc/node.cfg')
self.copy_file_or_directory_to_destination(f'{const.DEFAULT_CONFIGS}/zeek/local.zeek',
f'{self.configuration_directory}/site/local.zeek')
# Optimize Configurations
site_local_config = config.SiteLocalConfigManager(self.configuration_directory, stdout=self.stdout,
verbose=self.verbose)
node_config = config.NodeConfigManager(self.install_directory, stdout=self.stdout, verbose=self.verbose)
node_config.workers = node.Workers()
for worker in node_config.get_optimal_zeek_worker_config(inspect_interfaces):
node_config.workers.add_worker(
worker=worker
)
self.logger.info('Applying node configuration.')
node_config.commit()
# Fix Permissions
self.logger.info('Setting up file permissions.')
utilities.set_ownership_of_file(self.configuration_directory, user='******', group='dynamite')
utilities.set_ownership_of_file(self.install_directory, user='******', group='dynamite')
self.logger.info(f'Installing service -> {const.DEFAULT_CONFIGS}/systemd/zeek.service')
sysctl.install_and_enable(os.path.join(const.DEFAULT_CONFIGS, 'systemd', 'zeek.service'))
def uninstall(self):
"""Stop and uninstall the service
Returns:
None
"""
sysctl = systemctl.SystemCtl()
if self.process:
self.process.stop()
for dir in self.directories:
self.logger.debug(f'Removing {dir}')
shutil.rmtree(dir, ignore_errors=True)
if self.environ_vars:
for var in self.environ_vars:
self.delete_env_variable(var)
if self.sysctl_service_name:
try:
self.logger.debug(f'Uninstalling {self.sysctl_service_name}')
sysctl.uninstall_and_disable(self.sysctl_service_name)
except FileNotFoundError:
self.logger.debug('Skipping service uninstallation as systemd was not implemented in this setup.')
self.logger.info(f'Successfully uninstalled {self.name}')
def setup(self, node_name: Optional[str] = None, host: Optional[str] = None,
elasticsearch_host: Optional[str] = None, elasticsearch_port: Optional[int] = None,
pipeline_batch_size: Optional[int] = None, pipeline_batch_delay: Optional[int] = None,
heap_size_gigs: Optional[int] = None):
sysctl = systemctl.SystemCtl()
# System patching and directory setup
self.logger.debug('Patching sysctl.')
utilities.update_sysctl()
self.logger.debug('Patching file-handle limits.')
utilities.update_user_file_handle_limits()
utilities.makedirs(self.configuration_directory)
utilities.makedirs(self.install_directory)
utilities.makedirs(self.log_directory)
self.copy_logstash_fills_and_directories()
self.create_update_logstash_environment_variables()
# Overwrite with dynamite default configurations
self.copy_file_or_directory_to_destination(f'{const.DEFAULT_CONFIGS}/logstash/logstash.yml',
self.configuration_directory)
self.copy_file_or_directory_to_destination(f'{const.DEFAULT_CONFIGS}/logstash/jvm.options',
self.configuration_directory)
# Optimize Configurations
ls_main_config = config.ConfigManager(self.configuration_directory)
ls_java_config = config.JavaHeapOptionsConfigManager(self.configuration_directory)
ls_main_config.path_logs = self.log_directory
if not node_name:
node_name = utilities.get_default_es_node_name().replace('es_node', 'ls_node')
if not host:
host = utilities.get_primary_ip_address()
if not elasticsearch_host:
elasticsearch_host = utilities.get_primary_ip_address()
if not elasticsearch_port:
elasticsearch_port = 9200
if not pipeline_batch_size:
pipeline_batch_size = 125
if not pipeline_batch_delay:
pipeline_batch_delay = 50
if not heap_size_gigs:
heap_size_gigs = int((utilities.get_memory_available_bytes() / 10 ** 9) / 2)
self.logger.debug(f'Logstash will connect to Elasticsearch on {elasticsearch_host}:{elasticsearch_port}')
ls_main_config.node_name = node_name
ls_main_config.host = host
ls_main_config.pipeline_batch_size = pipeline_batch_size
ls_main_config.pipeline_batch_delay = pipeline_batch_delay
self.create_update_env_variable('LS_ES_HOST', elasticsearch_host)
self.create_update_env_variable('LS_ES_PORT', elasticsearch_port)
ls_java_config.initial_memory = f'{heap_size_gigs}g'
ls_java_config.maximum_memory = f'{heap_size_gigs}g'
self.logger.debug(f'Java Heap Initial & Max Memory = {heap_size_gigs} GB')
ls_main_config.commit()
ls_java_config.commit()
self.logger.info('Applying configuration.')
# Fix Permissions
self.logger.info('Setting up file permissions.')
utilities.set_ownership_of_file(self.configuration_directory, user='******', group='dynamite')
utilities.set_ownership_of_file(self.install_directory, user='******', group='dynamite')
utilities.set_ownership_of_file(self.log_directory, user='******', group='dynamite')
# Install and enable service
self.logger.info(f'Installing service -> {const.DEFAULT_CONFIGS}/systemd/logstash.service')
sysctl.install_and_enable(f'{const.DEFAULT_CONFIGS}/systemd/logstash.service')
def setup(self, targets: List[str], target_type: Optional[str] = 'elasticsearch',
monitor_log_paths: Optional[List[str]] = None, agent_tag: Optional[str] = None) -> None:
"""Setup Filebeat
Args:
targets: A list of Elasticsearch/Kafka/Logstash targets to forward events to (E.G ["192.168.0.9 5044", ...])
target_type: The target type; current supported: elasticsearch (default), logstash, kafka, redis
monitor_log_paths: A tuple of log paths to monitor
agent_tag: A friendly name for the agent (defaults to the hostname with no spaces and _agt suffix)
Returns:
None
"""
from dynamite_nsm.services.zeek import profile as zeek_profile
from dynamite_nsm.services.suricata import profile as suricata_profile
sysctl = systemctl.SystemCtl()
zeek_log_root, suricata_log_root = None, None
# Directory setup
self.logger.debug(f'Creating directory: {self.install_directory}')
utilities.makedirs(self.install_directory)
utilities.makedirs(f'{self.install_directory}/logs')
self.logger.info('Installing files and directories.')
self.copy_filebeat_files_and_directories()
self.copy_file_or_directory_to_destination(f'{const.DEFAULT_CONFIGS}/filebeat/filebeat.yml',
self.install_directory)
# Overwrite with dynamite default configurations
self.copy_file_or_directory_to_destination(f'{const.DEFAULT_CONFIGS}/filebeat/module/', self.install_directory)
self.copy_file_or_directory_to_destination(f'{const.DEFAULT_CONFIGS}/filebeat/modules.d/',
self.install_directory)
filebeat_config = config.ConfigManager(self.install_directory, verbose=self.verbose, stdout=self.stdout)
if target_type == 'elasticsearch':
filebeat_config.switch_to_elasticsearch_target()
filebeat_config.elasticsearch_targets.target_strings = targets
self.logger.info(f'Enabling Elasticsearch connector: '
f'{filebeat_config.elasticsearch_targets.target_strings}')
elif target_type == 'logstash':
filebeat_config.switch_to_logstash_target()
filebeat_config.logstash_targets.target_strings = targets
elif target_type == 'kafka':
filebeat_config.switch_to_kafka_target()
filebeat_config.kafka_targets.target_strings = targets
elif target_type == 'redis':
filebeat_config.switch_to_redis_target()
filebeat_config.redis_targets.target_strings = targets
filebeat_config.input_logs = misc_filebeat_objs.InputLogs(
monitor_log_paths=[]
)
filebeat_config.field_processors.originating_agent_tag = agent_tag
if not monitor_log_paths:
environ = utilities.get_environment_file_dict()
zeek_log_root = f'{environ.get("ZEEK_HOME", "")}/logs/current/'
suricata_log_root = environ.get('SURICATA_LOGS', '')
zeek_profiler = zeek_profile.ProcessProfiler()
suricata_profiler = suricata_profile.ProcessProfiler()
if zeek_profiler.is_installed():
self.logger.info(f'Zeek installation found; monitoring: {zeek_log_root}*.log')
filebeat_config.input_logs.monitor_log_paths.append(f'{zeek_log_root}*.log')
if suricata_profiler.is_installed():
self.logger.info(f'Suricata installation found; monitoring: {suricata_log_root}/eve.json')
filebeat_config.input_logs.monitor_log_paths.append(f'{suricata_log_root}/eve.json')
else:
filebeat_config.input_logs = misc_filebeat_objs.InputLogs(
monitor_log_paths=monitor_log_paths
)
self.logger.info(f'Monitoring Paths = {filebeat_config.input_logs.monitor_log_paths}')
if not agent_tag:
filebeat_config.field_processors.originating_agent_tag = utilities.get_default_agent_tag()
self.logger.info(f'Agent Tag = {filebeat_config.field_processors.originating_agent_tag}')
self.logger.debug(filebeat_config.elasticsearch_targets.get_raw())
filebeat_config.commit()
self.logger.info('Applying configuration.')
# Fix Permissions
self.logger.info('Installing modules.')
filebeat_config.patch_modules(zeek_log_directory=zeek_log_root, suricata_log_directory=suricata_log_root)
# Setting up permissions
self.logger.info('Setting up file permissions.')
config_file = f'{self.install_directory}/filebeat.yml'
utilities.set_ownership_of_file(self.install_directory, user='******', group='dynamite')
utilities.set_permissions_of_file(f'{self.install_directory}/modules.d/',
unix_permissions_integer='go-w')
utilities.set_permissions_of_file(f'{self.install_directory}/module/', unix_permissions_integer='go-w')
utilities.set_ownership_of_file(config_file, user='******', group='dynamite')
utilities.set_permissions_of_file(config_file, unix_permissions_integer=644)
filebeat_config.enable_ecs_normalization()
# Install and enable service
self.logger.info(f'Installing service -> {const.DEFAULT_CONFIGS}/systemd/filebeat.service')
sysctl.install_and_enable(f'{const.DEFAULT_CONFIGS}/systemd/filebeat.service')
# Update environment file
self.create_update_filebeat_environment_variables()
def setup(self,
host: Optional[str] = None,
port: Optional[int] = None,
elasticsearch_targets: Optional[List[str]] = None) -> None:
"""Setup Kibana
Args:
host: The IP or hostname to listen on
port: The port to listen on
elasticsearch_targets: A list of Elasticsearch urls
Returns:
None
"""
sysctl = systemctl.SystemCtl()
# Directory setup
self.logger.debug(
f'Creating directory: {self.configuration_directory}')
utilities.makedirs(self.configuration_directory)
self.logger.debug(f'Creating directory: {self.install_directory}')
utilities.makedirs(self.install_directory)
self.logger.debug(f'Creating directory: {self.log_directory}')
utilities.makedirs(self.log_directory)
self.copy_kibana_files_and_directories()
self.create_update_kibana_environment_variables()
self.copy_file_or_directory_to_destination(
f'{const.DEFAULT_CONFIGS}/kibana/kibana.yml',
self.configuration_directory)
# Optimize Configurations
kb_main_config = config.ConfigManager(self.configuration_directory)
if not host:
host = utilities.get_primary_ip_address()
if not port:
port = 5601
if not elasticsearch_targets:
elasticsearch_targets = [
f'https://{utilities.get_primary_ip_address()}:9200'
]
self.logger.debug(f'Elasticsearch Targets = {elasticsearch_targets}')
kb_main_config.host = host
kb_main_config.port = port
self.logger.debug(
f'Kibana will listen on {kb_main_config.host}:{kb_main_config.port}'
)
kb_main_config.elasticsearch_targets = elasticsearch_targets
self.logger.info('Applying configuration.')
kb_main_config.commit()
# Fix Permissions
utilities.set_ownership_of_file(self.configuration_directory,
user='******',
group='dynamite')
utilities.set_ownership_of_file(self.install_directory,
user='******',
group='dynamite')
utilities.set_ownership_of_file(self.log_directory,
user='******',
group='dynamite')
# Install and enable service
self.logger.info(
f'Installing service -> {const.DEFAULT_CONFIGS}/systemd/kibana.service'
)
sysctl.install_and_enable(
f'{const.DEFAULT_CONFIGS}/systemd/kibana.service')
self.logger.info('Installing "BaseViews" Kibana package')
task = install_dynamite_base_views.InstallKibanaDynamiteBaseViewsPackage(
username='******', password='******', target=f"http://{host}:{port}")
task.download_and_install()
def setup(self,
node_name: Optional[str] = None,
network_host: Optional[str] = None,
port: Optional[int] = None,
initial_master_nodes: Optional[List[str]] = None,
discover_seed_hosts: Optional[List[str]] = None,
tls_cert_subject: Optional[str] = None,
heap_size_gigs: Optional[int] = None) -> None:
"""Setup Elasticsearch
Args:
node_name: The name of this elasticsearch node
network_host: The IP address to listen on (E.G "0.0.0.0")
port: The port that the ES API is bound to (E.G 9200)
initial_master_nodes: A list of nodes representing master (and master-eligible) nodes in this cluster
discover_seed_hosts: A list of IPs on other hosts you wish to form a cluster with
tls_cert_subject: Denotes the thing being secured; E.G (/C=US/ST=GA/L=Atlanta/O=Dynamite Analytics/OU=R&D/CN=dynamite.ai)
heap_size_gigs: The initial/max java heap space to allocate
Returns:
None
"""
sysctl = systemctl.SystemCtl()
# System patching and directory setup
self.logger.debug('Patching sysctl.')
utilities.update_sysctl()
self.logger.debug('Patching file-handle limits.')
utilities.update_user_file_handle_limits()
self.logger.debug(
f'Creating directory: {self.configuration_directory}')
utilities.makedirs(self.configuration_directory)
self.logger.debug(f'Creating directory: {self.install_directory}')
utilities.makedirs(self.install_directory)
self.logger.debug(f'Creating directory: {self.log_directory}')
utilities.makedirs(self.log_directory)
self.copy_elasticsearch_files_and_directories()
self.create_update_elasticsearch_environment_variables()
# Overwrite with dynamite default configurations
self.copy_file_or_directory_to_destination(
f'{const.DEFAULT_CONFIGS}/elasticsearch/elasticsearch.yml',
self.configuration_directory)
self.copy_file_or_directory_to_destination(
f'{const.DEFAULT_CONFIGS}/elasticsearch/jvm.options',
self.configuration_directory)
self.copy_file_or_directory_to_destination(
f'{const.DEFAULT_CONFIGS}/elasticsearch/security',
self.configuration_directory)
# Optimize Configurations
es_main_config = config.ConfigManager(self.configuration_directory,
verbose=self.verbose,
stdout=self.stdout)
es_java_config = config.JavaHeapOptionsConfigManager(
self.configuration_directory,
verbose=self.verbose,
stdout=self.stdout)
es_main_config.path_logs = self.log_directory
if not node_name:
node_name = utilities.get_default_es_node_name()
if not network_host:
network_host = utilities.get_primary_ip_address()
if not port:
port = 9200
if not initial_master_nodes:
initial_master_nodes = [node_name]
if not discover_seed_hosts:
discover_seed_hosts = [network_host]
if not tls_cert_subject:
tls_cert_subject = '/C=US/ST=GA/L=Atlanta/O=Dynamite/OU=R&D/CN=dynamite.ai'
else:
tls_cert_subject = tls_cert_subject
if not heap_size_gigs:
heap_size_gigs = int(
(utilities.get_memory_available_bytes() / 10**9) / 2)
formatted_subj = tls_cert_subject.lstrip("/").replace("/", ",")
formatted_subj_2 = ','.join(reversed(formatted_subj.split(',')))
es_main_config.node_name = node_name
es_main_config.network_host = network_host
es_main_config.http_port = port
es_main_config.initial_master_nodes = initial_master_nodes
es_main_config.seed_hosts = discover_seed_hosts
es_main_config.authcz_admin_distinguished_names = [
formatted_subj, formatted_subj_2
]
es_java_config.initial_memory = f'{heap_size_gigs}g'
es_java_config.maximum_memory = f'{heap_size_gigs}g'
self.logger.debug(
f'Java Heap Initial & Max Memory = {heap_size_gigs} GB')
es_main_config.commit()
es_java_config.commit()
self.logger.info('Applying configuration.')
# Fix Permissions
self.logger.info('Setting up file permissions.')
utilities.set_ownership_of_file(self.configuration_directory,
user='******',
group='dynamite')
utilities.set_ownership_of_file(self.install_directory,
user='******',
group='dynamite')
utilities.set_ownership_of_file(self.log_directory,
user='******',
group='dynamite')
# Install and enable service
self.logger.info(
f'Installing service -> {const.DEFAULT_CONFIGS}/systemd/elasticsearch.service'
)
sysctl.install_and_enable(
f'{const.DEFAULT_CONFIGS}/systemd/elasticsearch.service')
self.logger.info('Generating SSL Certificates and Keys.')
ssl_gen_task_results = setup_security.GenerateElasticsearchSSLCertificates(
subj=tls_cert_subject).invoke()
for res in ssl_gen_task_results:
cmd, out, err = res
self.logger.debug(f'{" ".join(cmd)} -> out: {out} err: {err}')
self.logger.info('Installing SSL Certificates and Keys')
install_ssl_task_results = setup_security.InstallElasticsearchCertificates(
network_host=network_host).invoke()
for res in install_ssl_task_results:
cmd, out, err = res
self.logger.debug(f'{" ".join(cmd)} -> out: {out} err: {err}')
self.logger.info('Setting up persistent cluster settings.')
configure_cluster_task_results = configure_cluster.UpdateClusterSettings(
network_host=network_host, http_port=port).invoke()
rcode, msg = configure_cluster_task_results
self.logger.debug(f'rcode: {rcode}-> msg: {msg}')
"""
def setup(self, inspect_interfaces: List[str]):
"""Install Suricata
Args:
inspect_interfaces: A list of network interfaces to capture on (E.G ["mon0", "mon1"])
Returns:
None
"""
if not self.skip_interface_validation:
if not self.validate_inspect_interfaces(inspect_interfaces):
raise install.NetworkInterfaceNotFound(inspect_interfaces)
sysctl = systemctl.SystemCtl()
self.install_suricata_dependencies()
self.create_update_suricata_environment_variables()
self.logger.debug(f'Creating directory: {self.configuration_directory}')
utilities.makedirs(self.configuration_directory)
self.logger.debug(f'Creating directory: {self.install_directory}')
utilities.makedirs(self.install_directory)
self.logger.debug(f'Creating directory: {self.log_directory}')
utilities.makedirs(self.log_directory)
self.copy_suricata_files_and_directories()
self.logger.info('Setting up Suricata from source. This can a few minutes.')
if self.stdout:
utilities.print_coffee_art()
self.configure_compile_suricata()
self.copy_file_or_directory_to_destination(
f'{const.DEFAULT_CONFIGS}/suricata/suricata.yaml',
self.configuration_directory
)
suricata_config = config.ConfigManager(self.configuration_directory, stdout=self.stdout, verbose=self.verbose)
suricata_config.default_log_directory = self.log_directory
suricata_config.suricata_log_output_file = os.path.join(self.log_directory, 'suricata.log')
suricata_config.default_rules_directory = os.path.join(self.configuration_directory, 'rules')
suricata_config.reference_config_file = os.path.join(self.configuration_directory, 'reference.config')
suricata_config.classification_file = os.path.join(self.configuration_directory, 'rules',
'classification.config')
suricata_config.af_packet_interfaces = misc.AfPacketInterfaces()
for interface in inspect_interfaces:
suricata_config.af_packet_interfaces.add(
misc.AfPacketInterface(
interface_name=interface, threads='auto', cluster_id=random.randint(1, 50000),
cluster_type='cluster_qm'
)
)
suricata_config.threading = suricata_config.get_optimal_suricata_threading_config(
tuple([i for i in range(0, utilities.get_cpu_core_count() - 1)]))
suricata_config.commit()
self.logger.info('Applying Suricata configuration.')
self.logger.debug(suricata_config.af_packet_interfaces)
suricata_config.commit()
# Fix Permissions
self.logger.info('Setting up file permissions.')
utilities.set_ownership_of_file(self.configuration_directory, user='******', group='dynamite')
utilities.set_ownership_of_file(self.install_directory, user='******', group='dynamite')
utilities.set_ownership_of_file(self.log_directory, user='******', group='dynamite')
post_install_bootstrap_updater(self.install_directory, stdout=self.stdout, verbose=self.verbose)
self.logger.info(f'Installing service -> {const.DEFAULT_CONFIGS}/systemd/suricata.service')
sysctl.install_and_enable(os.path.join(const.DEFAULT_CONFIGS, 'systemd', 'suricata.service'))