def invoke(self, shell: Optional[bool] = False, cwd: Optional[str] = None) -> \
List[Tuple[List, bytes, bytes]]:
utilities.makedirs(f'{self.cert_directory}')
if not cwd:
cwd = self.cert_directory
results = super().invoke(shell=shell, cwd=cwd)
utilities.safely_remove_file(f'{self.cert_directory}/admin-key-temp.pem')
utilities.safely_remove_file(f'{self.cert_directory}/admin.csr')
utilities.set_ownership_of_file(path=self.cert_directory, user='******', group='dynamite')
utilities.set_permissions_of_file(file_path=self.cert_directory, unix_permissions_integer=700)
utilities.set_permissions_of_file(file_path=f'{self.cert_directory}/{self.cert_name}',
unix_permissions_integer=600)
utilities.set_permissions_of_file(file_path=f'{self.cert_directory}/{self.key_name}',
unix_permissions_integer=600)
utilities.set_permissions_of_file(file_path=f'{self.cert_directory}/{self.trusted_ca_cert_name}',
unix_permissions_integer=600)
utilities.set_permissions_of_file(file_path=f'{self.cert_directory}/{self.trusted_ca_key_name}',
unix_permissions_integer=600)
es_main_config = config.ConfigManager(self.configuration_directory)
es_main_config.transport_pem_cert_file = f'security/auth/{self.cert_name}'
es_main_config.rest_api_pem_cert_file = es_main_config.transport_pem_cert_file
es_main_config.transport_pem_key_file = f'security/auth/{self.key_name}'
es_main_config.rest_api_pem_key_file = es_main_config.transport_pem_key_file
es_main_config.transport_trusted_cas_file = f'security/auth/{self.trusted_ca_cert_name}'
es_main_config.rest_api_trusted_cas_file = es_main_config.transport_trusted_cas_file
es_main_config.commit()
return results
def commit(self,
out_file_path: Optional[str] = None,
backup_directory: Optional[str] = None) -> None:
"""Write out an updated configuration file, and optionally backup the old one.
Args:
out_file_path: The path to the output file; if none given overwrites existing
backup_directory: The path to the backup directory
Returns:
None
"""
# Backup old configuration first
out_file_name = os.path.basename(out_file_path)
backup_file_name = out_file_name + '.backup'
if backup_directory:
utilities.backup_configuration_file(
out_file_path,
backup_directory,
destination_file_prefix=backup_file_name)
try:
with open(out_file_path, 'w') as config_raw_f:
config_raw_f.write(self.formatted_data)
except IOError:
raise general_exceptions.WriteConfigError(
'An error occurred while writing the configuration file to disk.'
)
utilities.set_permissions_of_file(out_file_path, 644)
self.logger.warning(
'Configuration updated. Restart this service to apply.')
def commit(self,
out_file_path: Optional[str] = None,
backup_directory: Optional[str] = None) -> None:
"""Write out an updated configuration file, and optionally backup the old one.
Args:
out_file_path: The path to the output file; if none given overwrites existing
backup_directory: The path to the backup directory
Returns:
None
"""
# Backup old configuration first
out_file_name = os.path.basename(out_file_path)
backup_file_name = out_file_name + '.backup'
self.formatted_data = f'-Xms{self.initial_memory}\n-Xmx{self.maximum_memory}\n' + \
'\n'.join(self._raw_extra_params)
if backup_directory:
utilities.backup_configuration_file(
out_file_path,
backup_directory,
destination_file_prefix=backup_file_name)
try:
with open(out_file_path, 'w') as config_raw_f:
config_raw_f.write(self.formatted_data)
except IOError:
raise general_exceptions.WriteConfigError(
'An error occurred while writing the configuration file to disk.'
)
utilities.set_permissions_of_file(out_file_path, 644)
def setup_filebeat(self):
"""
Creates necessary directory structure, and copies required files, generates a default configuration
"""
if self.stdout:
sys.stdout.write('[+] Creating Filebeat install directory.\n')
subprocess.call('mkdir -p {}'.format(self.install_directory),
shell=True)
if self.stdout:
sys.stdout.write('[+] Copying Filebeat to install directory.\n')
utilities.copytree(
os.path.join(const.INSTALL_CACHE, const.FILE_BEAT_DIRECTORY_NAME),
self.install_directory)
shutil.copy(
os.path.join(const.DEFAULT_CONFIGS, 'filebeat', 'filebeat.yml'),
self.install_directory)
if self.stdout:
sys.stdout.write(
'[+] Building configurations and setting up permissions.\n')
beats_config = FileBeatConfigurator(self.install_directory)
beats_config.set_monitor_target_paths(self.monitor_paths)
beats_config.write_config()
utilities.set_permissions_of_file(os.path.join(self.install_directory,
'filebeat.yml'),
unix_permissions_integer=501)
if 'FILEBEAT_HOME' not in open('/etc/dynamite/environment').read():
if self.stdout:
sys.stdout.write(
'[+] Updating FileBeat default script path [{}]\n'.format(
self.install_directory))
subprocess.call(
'echo FILEBEAT_HOME="{}" >> /etc/dynamite/environment'.format(
self.install_directory),
shell=True)
def invoke(self, shell: Optional[bool] = False, cwd: Optional[str] = None) -> List[
Tuple[List, bytes, bytes]]:
utilities.set_permissions_of_file(file_path=self.opendistro_security_admin, unix_permissions_integer='+x')
if not cwd:
cwd = self.security_conf_directory
attempts = 0
es_process_profile = profile.ProcessProfiler()
if not es_process_profile.is_listening():
process.ProcessManager().start()
while not es_process_profile.is_listening() and attempts < self.max_attempts:
attempts += 1
sleep(10)
results = super().invoke(shell, cwd)
if self.terminate_elasticsearch:
process.ProcessManager().stop()
return results
def write_config(self):
def update_dict_from_path(path, value):
"""
:param path: A tuple representing each level of a nested path in the yaml document
('vars', 'address-groups', 'HOME_NET') = /vars/address-groups/HOME_NET
:param value: The new value
:return: None
"""
partial_config_data = self.config_data
for i in range(0, len(path) - 1):
try:
partial_config_data = partial_config_data[path[i]]
except KeyError:
pass
partial_config_data.update({path[-1]: value})
# Backup old configuration first
source_configuration_file_path = os.path.join(self.install_directory, 'filebeat.yml')
if self.backup_configuration_directory:
destination_configuration_path = os.path.join(self.backup_configuration_directory, 'filebeat.yml.d')
try:
utilities.backup_configuration_file(source_configuration_file_path, destination_configuration_path,
destination_file_prefix='filebeat.yml.backup')
except general_exceptions.WriteConfigError:
raise filebeat_exceptions.WriteFilebeatConfigError(
'Suricata configuration failed to write [filebeat.yml].')
except general_exceptions.ReadConfigError:
raise filebeat_exceptions.ReadFilebeatConfigError(
'Suricata configuration failed to read [filebeat.yml].')
for k, v in vars(self).items():
if k not in self.tokens:
continue
token_path = self.tokens[k]
update_dict_from_path(token_path, v)
try:
with open(source_configuration_file_path, 'w') as configyaml:
dump(self.config_data, configyaml, default_flow_style=False)
utilities.set_permissions_of_file(source_configuration_file_path, 744)
except IOError:
raise filebeat_exceptions.WriteFilebeatConfigError("Could not locate {}".format(self.install_directory))
except Exception as e:
raise filebeat_exceptions.WriteFilebeatConfigError(
"General error while attempting to write new filebeat.yml file to {}; {}".format(
self.install_directory, e))
def commit(self,
out_file_path: Optional[str] = None,
backup_directory: Optional[str] = None,
top_text: Optional[str] = None) -> None:
"""Write out an updated configuration file, and optionally backup the old one.
Args:
out_file_path: The path to the output file; if none given overwrites existing
backup_directory: The path to the backup directory
top_text: The text to be appended at the top of the config file (typically used for YAML version header)
Returns:
None
"""
out_file_name = os.path.basename(out_file_path)
backup_file_name = out_file_name + '.backup'
def update_dict_from_path(path, value) -> None:
"""Update the internal YAML dictionary object with the new values from our config
Args:
path: A tuple representing each level of a nested path in the yaml document ('vars', 'address-groups', 'HOME_NET') = /vars/address-groups/HOME_NET
value: The new value
Returns:
None
"""
partial_config_data = self.config_data
if len(path) > 1:
for i in range(0, len(path) - 1):
k = path[i]
if isinstance(partial_config_data, dict):
partial_config_data = partial_config_data[k]
elif isinstance(partial_config_data, list):
for list_entry in partial_config_data:
if isinstance(list_entry, dict):
if k in list_entry.keys():
partial_config_data = list_entry[k]
else:
break
if value is None:
return
partial_config_data.update({path[-1]: value})
# Backup old configuration first
if backup_directory:
utilities.backup_configuration_file(
out_file_path,
backup_directory,
destination_file_prefix=backup_file_name)
for k, v in vars(self).items():
if k not in self.extract_tokens:
continue
token_path = self.extract_tokens[k]
update_dict_from_path(token_path, v)
try:
with open(out_file_path, 'w') as config_yaml_f:
if top_text:
config_yaml_f.write(f'{top_text}\n')
try:
dump(self.config_data,
config_yaml_f,
default_flow_style=False,
Dumper=NoAliasDumper)
except RecursionError:
dump(self.config_data,
config_yaml_f,
default_flow_style=False)
except IOError:
raise general_exceptions.WriteConfigError(
'An error occurred while writing the configuration file to disk.'
)
utilities.set_permissions_of_file(out_file_path, 644)
self.logger.warning(
'Configuration updated. Restart this service to apply.')
def setup_filebeat(self):
"""
Creates necessary directory structure, and copies required files, generates a default configuration
"""
env_file = os.path.join(const.CONFIG_PATH, 'environment')
self.logger.info('Creating FileBeat install directory.')
utilities.makedirs(self.install_directory, exist_ok=True)
self.logger.info('Copying FileBeat to install directory.')
try:
utilities.copytree(
os.path.join(const.INSTALL_CACHE,
const.FILE_BEAT_DIRECTORY_NAME),
self.install_directory)
shutil.copy(
os.path.join(const.DEFAULT_CONFIGS, 'filebeat',
'filebeat.yml'), self.install_directory)
except Exception as e:
self.logger.error(
"General error occurred while copying Filebeat configs.")
self.logger.debug(
"General error occurred while copying Filebeat configs; {}".
format(e))
raise filebeat_exceptions.InstallFilebeatError(
"General error occurred while copying Filebeat configs; {}".
format(e))
self.logger.info("Building configurations and setting up permissions.")
try:
beats_config = filebeat_configs.ConfigManager(
self.install_directory)
except filebeat_exceptions.ReadFilebeatConfigError:
self.logger.error("Failed to read Filebeat configuration.")
raise filebeat_exceptions.InstallFilebeatError(
"Failed to read Filebeat configuration.")
beats_config.set_monitor_target_paths(self.monitor_paths)
beats_config.set_agent_tag(self.agent_tag)
if (self.kafka_password
or self.kafka_username) and not self.kafka_topic:
self.logger.error(
"You have specified Kafka config options without specifying a Kafka topic."
)
raise filebeat_exceptions.InstallFilebeatError(
"You have specified Kafka config options without specifying a Kafka topic."
)
if self.kafka_topic:
self.logger.warning(
"You have enabled the Agent's Kafka output which does integrate natively with Dynamite "
"Monitor/LogStash component. You will have to bring your own broker. Happy Hacking!"
)
time.sleep(2)
beats_config.set_kafka_targets(target_hosts=self.targets,
topic=self.kafka_topic,
username=self.kafka_username,
password=self.kafka_password)
# setup example upstream LogStash example, just in case you want to configure later
beats_config.set_logstash_targets(target_hosts=['localhost:5601'])
beats_config.enable_kafka_output()
else:
# setup example upstream Kafka example, just in case you want to configure later
beats_config.set_kafka_targets(target_hosts=['localhost:9092'],
topic='dynamite-nsm-events')
beats_config.enable_logstash_output()
beats_config.set_logstash_targets(self.targets)
try:
beats_config.write_config()
except filebeat_exceptions.WriteFilebeatConfigError:
self.logger.error("Failed to write filebeat configuration.")
raise filebeat_exceptions.InstallFilebeatError(
"Failed to write filebeat configuration.")
try:
utilities.set_permissions_of_file(os.path.join(
self.install_directory, 'filebeat.yml'),
unix_permissions_integer=501)
except Exception as e:
self.logger.error(
"Failed to set permissions of filebeat.yml file.")
self.logger.debug(
"Failed to set permissions of filebeat.yml file; {}".format(e))
filebeat_exceptions.InstallFilebeatError(
"Failed to set permissions of filebeat.yml file; {}".format(e))
try:
with open(env_file) as env_f:
if 'FILEBEAT_HOME' not in env_f.read():
self.logger.info(
'Updating FileBeat default script path [{}]'.format(
self.install_directory))
subprocess.call('echo FILEBEAT_HOME="{}" >> {}'.format(
self.install_directory, env_file),
shell=True)
except Exception as e:
self.logger.error(
"General error occurred while attempting to install FileBeat.")
self.logger.debug(
"General error occurred while attempting to install FileBeat; {}"
.format(e))
raise filebeat_exceptions.InstallFilebeatError(
"General error occurred while attempting to install FileBeat; {}"
.format(e))
try:
sysctl = systemctl.SystemCtl()
except general_exceptions.CallProcessError:
raise filebeat_exceptions.InstallFilebeatError(
"Could not find systemctl.")
self.logger.info("Installing Filebeat systemd service.")
if not sysctl.install_and_enable(
os.path.join(const.DEFAULT_CONFIGS, 'systemd',
'filebeat.service')):
raise filebeat_exceptions.InstallFilebeatError(
"Failed to install Filebeat systemd service.")
sysctl.install_and_enable(
os.path.join(const.DEFAULT_CONFIGS, 'systemd',
'dynamite-agent.target'))
zeek_logs = None
zeek_home = self.environ.get('ZEEK_HOME')
suricata_logs = os.path.join(const.LOG_PATH, 'suricata')
if zeek_home:
zeek_logs = os.path.join(zeek_home, 'logs', 'current')
try:
self.logger.info('Patching Zeek/Suricata modules.')
beats_config.patch_modules(zeek_log_directory=zeek_logs,
suricata_log_directory=suricata_logs)
except filebeat_exceptions.WriteFilebeatModuleError:
self.logger.error('Could not patch Zeek/Suricata modules.')
raise filebeat_exceptions.InstallFilebeatError(
"Could not patch Zeek/Suricata modules.")
def setup(self, targets: List[str], target_type: Optional[str] = 'elasticsearch',
monitor_log_paths: Optional[List[str]] = None, agent_tag: Optional[str] = None) -> None:
"""Setup Filebeat
Args:
targets: A list of Elasticsearch/Kafka/Logstash targets to forward events to (E.G ["192.168.0.9 5044", ...])
target_type: The target type; current supported: elasticsearch (default), logstash, kafka, redis
monitor_log_paths: A tuple of log paths to monitor
agent_tag: A friendly name for the agent (defaults to the hostname with no spaces and _agt suffix)
Returns:
None
"""
from dynamite_nsm.services.zeek import profile as zeek_profile
from dynamite_nsm.services.suricata import profile as suricata_profile
sysctl = systemctl.SystemCtl()
zeek_log_root, suricata_log_root = None, None
# Directory setup
self.logger.debug(f'Creating directory: {self.install_directory}')
utilities.makedirs(self.install_directory)
utilities.makedirs(f'{self.install_directory}/logs')
self.logger.info('Installing files and directories.')
self.copy_filebeat_files_and_directories()
self.copy_file_or_directory_to_destination(f'{const.DEFAULT_CONFIGS}/filebeat/filebeat.yml',
self.install_directory)
# Overwrite with dynamite default configurations
self.copy_file_or_directory_to_destination(f'{const.DEFAULT_CONFIGS}/filebeat/module/', self.install_directory)
self.copy_file_or_directory_to_destination(f'{const.DEFAULT_CONFIGS}/filebeat/modules.d/',
self.install_directory)
filebeat_config = config.ConfigManager(self.install_directory, verbose=self.verbose, stdout=self.stdout)
if target_type == 'elasticsearch':
filebeat_config.switch_to_elasticsearch_target()
filebeat_config.elasticsearch_targets.target_strings = targets
self.logger.info(f'Enabling Elasticsearch connector: '
f'{filebeat_config.elasticsearch_targets.target_strings}')
elif target_type == 'logstash':
filebeat_config.switch_to_logstash_target()
filebeat_config.logstash_targets.target_strings = targets
elif target_type == 'kafka':
filebeat_config.switch_to_kafka_target()
filebeat_config.kafka_targets.target_strings = targets
elif target_type == 'redis':
filebeat_config.switch_to_redis_target()
filebeat_config.redis_targets.target_strings = targets
filebeat_config.input_logs = misc_filebeat_objs.InputLogs(
monitor_log_paths=[]
)
filebeat_config.field_processors.originating_agent_tag = agent_tag
if not monitor_log_paths:
environ = utilities.get_environment_file_dict()
zeek_log_root = f'{environ.get("ZEEK_HOME", "")}/logs/current/'
suricata_log_root = environ.get('SURICATA_LOGS', '')
zeek_profiler = zeek_profile.ProcessProfiler()
suricata_profiler = suricata_profile.ProcessProfiler()
if zeek_profiler.is_installed():
self.logger.info(f'Zeek installation found; monitoring: {zeek_log_root}*.log')
filebeat_config.input_logs.monitor_log_paths.append(f'{zeek_log_root}*.log')
if suricata_profiler.is_installed():
self.logger.info(f'Suricata installation found; monitoring: {suricata_log_root}/eve.json')
filebeat_config.input_logs.monitor_log_paths.append(f'{suricata_log_root}/eve.json')
else:
filebeat_config.input_logs = misc_filebeat_objs.InputLogs(
monitor_log_paths=monitor_log_paths
)
self.logger.info(f'Monitoring Paths = {filebeat_config.input_logs.monitor_log_paths}')
if not agent_tag:
filebeat_config.field_processors.originating_agent_tag = utilities.get_default_agent_tag()
self.logger.info(f'Agent Tag = {filebeat_config.field_processors.originating_agent_tag}')
self.logger.debug(filebeat_config.elasticsearch_targets.get_raw())
filebeat_config.commit()
self.logger.info('Applying configuration.')
# Fix Permissions
self.logger.info('Installing modules.')
filebeat_config.patch_modules(zeek_log_directory=zeek_log_root, suricata_log_directory=suricata_log_root)
# Setting up permissions
self.logger.info('Setting up file permissions.')
config_file = f'{self.install_directory}/filebeat.yml'
utilities.set_ownership_of_file(self.install_directory, user='******', group='dynamite')
utilities.set_permissions_of_file(f'{self.install_directory}/modules.d/',
unix_permissions_integer='go-w')
utilities.set_permissions_of_file(f'{self.install_directory}/module/', unix_permissions_integer='go-w')
utilities.set_ownership_of_file(config_file, user='******', group='dynamite')
utilities.set_permissions_of_file(config_file, unix_permissions_integer=644)
filebeat_config.enable_ecs_normalization()
# Install and enable service
self.logger.info(f'Installing service -> {const.DEFAULT_CONFIGS}/systemd/filebeat.service')
sysctl.install_and_enable(f'{const.DEFAULT_CONFIGS}/systemd/filebeat.service')
# Update environment file
self.create_update_filebeat_environment_variables()