def test_logstashyaml_update_pipeline_batch_delay(self):
self.config_manager.pipeline_batch_delay = 10
self.config_manager.write_logstash_config()
config_manager_read = config.ConfigManager(self.config_directory)
assert (config_manager_read.pipeline_batch_delay == 10)
def test_logstashyaml_update_path_data(self):
self.config_manager.path_data = '/var/log/dynamite/data/'
self.config_manager.write_logstash_config()
config_manager_read = config.ConfigManager(self.config_directory)
assert (config_manager_read.path_data == '/var/log/dynamite/data/')
def test_logstashyaml_update_node_name(self):
self.config_manager.node_name = 'logstash-1'
self.config_manager.write_logstash_config()
config_manager_read = config.ConfigManager(self.config_directory)
assert (config_manager_read.node_name == 'logstash-1')
def test_javaopts_update_heapsize(self):
self.config_manager.java_maximum_memory = 10
self.config_manager.java_initial_memory = 10
self.config_manager.write_jvm_config()
config_manager_read = config.ConfigManager(self.config_directory)
assert (config_manager_read.java_initial_memory == 10
and config_manager_read.java_maximum_memory == 10)
def setUp(self):
self.config_root = '/etc/dynamite/test'
self.config_directory = os.path.join(self.config_root, 'logstash')
# Setup Test Space
utilities.makedirs(self.config_directory, exist_ok=True)
create_dummy_logstashyaml(self.config_directory)
create_dummy_javaopts(self.config_directory)
self.config_manager = config.ConfigManager(
configuration_directory=self.config_directory)
def _setup_default_logstash_configs(self):
self.logger.info('Overwriting default configuration')
shutil.copy(
os.path.join(const.DEFAULT_CONFIGS, 'logstash', 'logstash.yml'),
self.configuration_directory)
ls_config = logstash_config.ConfigManager(
configuration_directory=self.configuration_directory)
self.logger.info('Setting up JVM default heap settings [{}GB]'.format(
self.heap_size_gigs))
ls_config.java_initial_memory = int(self.heap_size_gigs)
ls_config.java_maximum_memory = int(self.heap_size_gigs)
ls_config.write_configs()
def setup(self, node_name: Optional[str] = None, host: Optional[str] = None,
elasticsearch_host: Optional[str] = None, elasticsearch_port: Optional[int] = None,
pipeline_batch_size: Optional[int] = None, pipeline_batch_delay: Optional[int] = None,
heap_size_gigs: Optional[int] = None):
sysctl = systemctl.SystemCtl()
# System patching and directory setup
self.logger.debug('Patching sysctl.')
utilities.update_sysctl()
self.logger.debug('Patching file-handle limits.')
utilities.update_user_file_handle_limits()
utilities.makedirs(self.configuration_directory)
utilities.makedirs(self.install_directory)
utilities.makedirs(self.log_directory)
self.copy_logstash_fills_and_directories()
self.create_update_logstash_environment_variables()
# Overwrite with dynamite default configurations
self.copy_file_or_directory_to_destination(f'{const.DEFAULT_CONFIGS}/logstash/logstash.yml',
self.configuration_directory)
self.copy_file_or_directory_to_destination(f'{const.DEFAULT_CONFIGS}/logstash/jvm.options',
self.configuration_directory)
# Optimize Configurations
ls_main_config = config.ConfigManager(self.configuration_directory)
ls_java_config = config.JavaHeapOptionsConfigManager(self.configuration_directory)
ls_main_config.path_logs = self.log_directory
if not node_name:
node_name = utilities.get_default_es_node_name().replace('es_node', 'ls_node')
if not host:
host = utilities.get_primary_ip_address()
if not elasticsearch_host:
elasticsearch_host = utilities.get_primary_ip_address()
if not elasticsearch_port:
elasticsearch_port = 9200
if not pipeline_batch_size:
pipeline_batch_size = 125
if not pipeline_batch_delay:
pipeline_batch_delay = 50
if not heap_size_gigs:
heap_size_gigs = int((utilities.get_memory_available_bytes() / 10 ** 9) / 2)
self.logger.debug(f'Logstash will connect to Elasticsearch on {elasticsearch_host}:{elasticsearch_port}')
ls_main_config.node_name = node_name
ls_main_config.host = host
ls_main_config.pipeline_batch_size = pipeline_batch_size
ls_main_config.pipeline_batch_delay = pipeline_batch_delay
self.create_update_env_variable('LS_ES_HOST', elasticsearch_host)
self.create_update_env_variable('LS_ES_PORT', elasticsearch_port)
ls_java_config.initial_memory = f'{heap_size_gigs}g'
ls_java_config.maximum_memory = f'{heap_size_gigs}g'
self.logger.debug(f'Java Heap Initial & Max Memory = {heap_size_gigs} GB')
ls_main_config.commit()
ls_java_config.commit()
self.logger.info('Applying configuration.')
# Fix Permissions
self.logger.info('Setting up file permissions.')
utilities.set_ownership_of_file(self.configuration_directory, user='******', group='dynamite')
utilities.set_ownership_of_file(self.install_directory, user='******', group='dynamite')
utilities.set_ownership_of_file(self.log_directory, user='******', group='dynamite')
# Install and enable service
self.logger.info(f'Installing service -> {const.DEFAULT_CONFIGS}/systemd/logstash.service')
sysctl.install_and_enable(f'{const.DEFAULT_CONFIGS}/systemd/logstash.service')
from dynamite_nsm.services.logstash import config
from dynamite_nsm.utilities import get_environment_file_dict
from dynamite_nsm.cmd.service_interfaces import SimpleConfigManagerInterface
env_vars = get_environment_file_dict()
try:
interface = \
SimpleConfigManagerInterface(config.ConfigManager(env_vars['LS_PATH_CONF']),
interface_name='Logstash Main Configuration',
interface_description='Configure Logstash on this machine.',
defaults=dict(configuration_directory=env_vars['LS_PATH_CONF'], stdout=True)
)
except KeyError:
interface = None
except FileNotFoundError:
interface = None
"""
if not interface:
print(f'Skipping logstash.config.main as it was never retrieved successfully')
"""
def uninstall_logstash(prompt_user=True, stdout=True, verbose=False):
"""
Install Logstash with ElastiFlow & Synesis
:param prompt_user: Print a warning before continuing
:param stdout: Print the output to console
:param verbose: Include detailed debug messages
"""
log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
logger = get_logger('LOGSTASH', level=log_level, stdout=stdout)
env_file = os.path.join(const.CONFIG_PATH, 'environment')
environment_variables = utilities.get_environment_file_dict()
configuration_directory = environment_variables.get('LS_PATH_CONF')
ls_profiler = logstash_profile.ProcessProfiler()
ls_config = logstash_config.ConfigManager(
configuration_directory=configuration_directory)
if not ls_profiler.is_installed():
logger.error('LogStash is not installed.')
raise logstash_exceptions.UninstallLogstashError(
"LogStash is not installed.")
if prompt_user:
sys.stderr.write(
'\n\033[93m[-] WARNING! Removing Logstash Will Prevent ElasticSearch From Receiving Events.\033[0m\n'
)
resp = utilities.prompt_input(
'\n\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m '
)
while resp not in ['', 'no', 'yes']:
resp = utilities.prompt_input(
'\033[93m[?] Are you sure you wish to continue? ([no]|yes): \033[0m'
)
if resp != 'yes':
if stdout:
sys.stdout.write('\n[+] Exiting\n')
exit(0)
if ls_profiler.is_running():
logstash_process.ProcessManager().stop()
try:
shutil.rmtree(ls_config.ls_path_conf)
shutil.rmtree(ls_config.ls_home)
shutil.rmtree(ls_config.path_logs)
shutil.rmtree(const.INSTALL_CACHE, ignore_errors=True)
env_lines = ''
with open(env_file) as env_fr:
for line in env_fr.readlines():
if 'LS_PATH_CONF' in line:
continue
elif 'LS_HOME' in line:
continue
elif 'LS_LOGS' in line:
continue
elif 'ELASTIFLOW_' in line:
continue
elif 'SYNLITE_' in line:
continue
elif 'ES_PASSWD' in line:
continue
elif line.strip() == '':
continue
env_lines += line.strip() + '\n'
with open(env_file, 'w') as env_fw:
env_fw.write(env_lines)
except Exception as e:
logger.error(
"General error occurred while attempting to uninstall LogStash.".
format(e))
logger.debug(
"General error occurred while attempting to uninstall LogStash; {}"
.format(e))
raise logstash_exceptions.UninstallLogstashError(
"General error occurred while attempting to uninstall LogStash; {}"
.format(e))
try:
sysctl = systemctl.SystemCtl()
except general_exceptions.CallProcessError:
raise logstash_exceptions.UninstallLogstashError(
"Could not find systemctl.")
sysctl.uninstall_and_disable('logstash')