def is_running(self):
if self.elasticsearch_home:
try:
return elastic_process.ProcessManager().status()['running']
except KeyError:
return elastic_process.ProcessManager().status()['RUNNING']
return False
def invoke(self) -> Tuple[int, Dict]:
es_url = f'https://{self.network_host}:{self.http_port}'
es_cluster_data = {
'persistent': {
'script.max_compilations_rate': '1000/5m'
},
'transient': {
'script.max_compilations_rate': '1000/5m'
}
}
attempts = 0
es_process_profile = profile.ProcessProfiler()
if not es_process_profile.is_listening():
process.ProcessManager().start()
while not es_process_profile.is_listening(
) and attempts < self.max_attempts:
attempts += 1
sleep(10)
r = requests.put(url=f'{es_url}/_cluster/settings',
data=json.dumps(es_cluster_data),
auth=('admin', 'admin'),
headers={'content-type': 'application/json'},
verify=False)
if self.terminate_elasticsearch:
process.ProcessManager().stop()
if r.status_code != 200:
return r.status_code, {'error': r.text}
else:
return r.status_code, r.json()
def is_running(self) -> bool:
"""Check if Elasticsearch is running
Returns:
True, if running
"""
if self.elasticsearch_home:
try:
return elastic_process.ProcessManager().status()['running']
except KeyError:
return elastic_process.ProcessManager().status()['RUNNING']
return False
def invoke(self, shell: Optional[bool] = False, cwd: Optional[str] = None) -> List[
Tuple[List, bytes, bytes]]:
utilities.set_permissions_of_file(file_path=self.opendistro_security_admin, unix_permissions_integer='+x')
if not cwd:
cwd = self.security_conf_directory
attempts = 0
es_process_profile = profile.ProcessProfiler()
if not es_process_profile.is_listening():
process.ProcessManager().start()
while not es_process_profile.is_listening() and attempts < self.max_attempts:
attempts += 1
sleep(10)
results = super().invoke(shell, cwd)
if self.terminate_elasticsearch:
process.ProcessManager().stop()
return results
def change_elasticsearch_password(old_password, password='******', remote_host=None, remote_port=None,
prompt_user=True, stdout=True, verbose=False):
"""
Change the Elasticsearch password for all builtin users
:param old_password: The old Elasticsearch password
:param password: The new Elasticsearch password
:param prompt_user: If True, warning prompt is displayed before proceeding
:param stdout: Print status to stdout
:param verbose: Include detailed debug messages
"""
from dynamite_nsm.services.elasticsearch import process as elastic_process
from dynamite_nsm.services.elasticsearch import profile as elastic_profile
log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
logger = get_logger('ELASTICSEARCH', level=log_level, stdout=stdout)
if prompt_user:
resp = utilities.prompt_input(
'\n\033[93m[-] WARNING! Changing the ElasticSearch password may result in connected components losing '
'communication. Be sure to update Kibana/LogStash passwords.\n'
'[?] Are you sure you wish to continue? [no]|yes):\033[0m ')
while resp not in ['', 'no', 'yes']:
resp = utilities.prompt_input('\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m ')
if resp != 'yes':
if stdout:
sys.stdout.write('[+] Exiting\n')
exit(0)
if remote_host:
logger.info("Resetting ElasticSearch password on remote host: {}:{}".format(remote_host, remote_port))
else:
logger.info("Resetting ElasticSearch password on localhost.")
if elastic_profile.ProcessProfiler().is_installed():
# If ElasticSearch is installed Locally.
# Start the process, in order to perform a reset.
if not elastic_process.ProcessManager().start():
logger.error('Could not start ElasticSearch Process. Password reset failed.')
raise general_exceptions.ResetPasswordError(
"ElasticSearch process was not able to start, check your ElasticSearch logs.")
while not elastic_profile.ProcessProfiler().is_listening():
logger.info('Waiting for ElasticSearch API to become accessible.')
time.sleep(5)
logger.info('ElasticSearch API is up.')
logger.debug('Sleeping for 5 seconds, while ElasticSearch API finishes booting.')
time.sleep(5)
else:
logger.error("ElasticSearch is not installed, and no remote ElasticSearch host was specified.")
raise general_exceptions.ResetPasswordError(
"ElasticSearch is not installed, and no remote ElasticSearch host was specified.")
es_pw_config = PasswordConfigManager('elastic', current_password=old_password, remote_host=remote_host,
remote_http_port=remote_port)
logger.info("Attempting password reset.")
es_pw_config.set_all_passwords(password)
def _install_kibana_objects(self):
self.logger.info('Installing Kibana Dashboards')
self.logger.info('Waiting for ElasticSearch to become accessible.')
# Start ElasticSearch if it is installed locally and is not running
if self.elasticsearch_host in ['localhost', '127.0.0.1', '0.0.0.0', '::1', '::/128']:
self.logger.info('Starting ElasticSearch.')
elastic_process.ProcessManager().start()
while not elastic_profile.ProcessProfiler().is_listening():
self.logger.info('Waiting for ElasticSearch API to become accessible.')
time.sleep(5)
self.logger.info('ElasticSearch API is up.')
self.logger.info('Sleeping for 5 seconds, while ElasticSearch API finishes booting.')
time.sleep(5)
try:
kibana_proc = kibana_process.ProcessManager()
kibana_proc.optimize()
utilities.set_ownership_of_file(self.install_directory, user='******', group='dynamite')
utilities.set_ownership_of_file(self.configuration_directory, user='******', group='dynamite')
time.sleep(5)
self.logger.info('Starting Kibana.')
kibana_proc.start()
except Exception as e:
raise kibana_exceptions.InstallKibanaError("General error while starting Kibana process; {}".format(e))
kibana_api_start_attempts = 0
while not kibana_profile.ProcessProfiler().is_listening() and kibana_api_start_attempts != 5:
self.logger.info('Waiting for Kibana API to become accessible.')
kibana_api_start_attempts += 1
time.sleep(5)
if kibana_api_start_attempts == 5:
self.logger.error('Kibana API could not be started after {} attempts.'.format(kibana_api_start_attempts))
raise kibana_exceptions.InstallKibanaError(
"Kibana API could not be started after {} attempts.".format(kibana_api_start_attempts))
self.logger.info('Kibana API is up.')
self.logger.info('Sleeping for 10 seconds, while Kibana API finishes booting.')
time.sleep(10)
api_config = kibana_configs.ApiConfigManager(self.configuration_directory)
kibana_object_create_attempts = 1
while kibana_object_create_attempts != 5:
try:
self.logger.info('[Attempt {}] Attempting to install dashboards/visualizations.'.format(
kibana_object_create_attempts))
api_config.create_dynamite_kibana_objects()
break
except kibana_exceptions.CreateKibanaObjectsError:
kibana_object_create_attempts += 1
time.sleep(10)
if kibana_object_create_attempts == 5:
self.logger.error(
"Kibana objects could not be created after {} attempts".format(kibana_object_create_attempts))
raise kibana_exceptions.InstallKibanaError(
"Kibana objects could not be created after {} attempts".format(kibana_object_create_attempts))
self.logger.info('Successfully created dashboards/visualizations.')
kibana_proc.stop()
def stop(self) -> bool:
"""Stop the monitor services
Returns:
True, if successfully stopped
"""
kibana_res, logstash_res, elasticsearch_res = True, True, True
if not elasticsearch_profile.ProcessProfiler().is_installed():
self.logger.error('You must install kibana to run this command.')
return False
elasticsearch_res = elasticsearch_process.ProcessManager().stop()
if logstash_profile.ProcessProfiler().is_installed():
logstash_res = logstash_process.ProcessManager().stop()
if kibana_profile.ProcessProfiler().is_installed():
kibana_res = kibana_process.ProcessManager().stop()
return kibana_res and elasticsearch_res and logstash_res
def uninstall_elasticsearch(prompt_user=True, stdout=True, verbose=False):
"""
Uninstall ElasticSearch
:param prompt_user: Print a warning before continuing
:param stdout: Print the output to console
:param verbose: Include detailed debug messages
"""
log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
logger = get_logger('ELASTICSEARCH', level=log_level, stdout=stdout)
env_file = os.path.join(const.CONFIG_PATH, 'environment')
environment_variables = utilities.get_environment_file_dict()
es_profiler = elastic_profile.ProcessProfiler()
if not es_profiler.is_installed():
logger.error('ElasticSearch is not installed.')
raise elastic_exceptions.UninstallElasticsearchError(
"ElasticSearch is not installed.")
configuration_directory = environment_variables.get('ES_PATH_CONF')
es_config = elastic_configs.ConfigManager(
configuration_directory=configuration_directory)
if prompt_user:
sys.stderr.write(
'\n\033[93m[-] WARNING! Removing ElasticSearch Will Delete All Data.'
'\033[0m\n')
resp = utilities.prompt_input(
'\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m '
)
while resp not in ['', 'no', 'yes']:
resp = utilities.prompt_input(
'\n\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m '
)
if resp != 'yes':
if stdout:
sys.stdout.write('\n[+] Exiting\n')
exit(0)
if es_profiler.is_running():
elastic_process.ProcessManager().stop()
try:
shutil.rmtree(es_config.configuration_directory)
shutil.rmtree(es_config.es_home)
shutil.rmtree(es_config.path_logs)
shutil.rmtree(const.INSTALL_CACHE, ignore_errors=True)
env_lines = ''
with open(env_file) as env_fr:
for line in env_fr.readlines():
if 'ES_PATH_CONF' in line:
continue
elif 'ES_HOME' in line:
continue
elif 'ES_LOGS' in line:
continue
elif line.strip() == '':
continue
env_lines += line.strip() + '\n'
with open(env_file, 'w') as env_fw:
env_fw.write(env_lines)
except Exception as e:
logger.error(
"General error occurred while attempting to uninstall ElasticSearch."
.format(e))
logger.debug(
"General error occurred while attempting to uninstall ElasticSearch; {}"
.format(e))
raise elastic_exceptions.UninstallElasticsearchError(
"General error occurred while attempting to uninstall ElasticSearch; {}"
.format(e))
try:
sysctl = systemctl.SystemCtl()
except general_exceptions.CallProcessError:
raise elastic_exceptions.UninstallElasticsearchError(
"Could not find systemctl.")
sysctl.uninstall_and_disable('elasticsearch')
def setup_passwords(self):
env_dict = utilities.get_environment_file_dict()
def setup_from_bootstrap(s):
bootstrap_users_and_passwords = {}
for line in s.split('\n'):
if 'PASSWORD' in line:
_, user, _, password = line.split(' ')
if not isinstance(password, str):
password = password.decode()
bootstrap_users_and_passwords[user] = password
es_pass_config = elastic_configs.PasswordConfigManager(
auth_user='******',
current_password=bootstrap_users_and_passwords['elastic'],
stdout=self.stdout,
verbose=self.verbose)
es_pass_config.set_all_passwords(new_password=self.password)
if not elastic_profile.ProcessProfiler().is_installed():
self.logger.error(
'ElasticSearch must be installed and running to bootstrap passwords.'
)
raise elastic_exceptions.InstallElasticsearchError(
"ElasticSearch must be installed and running to bootstrap passwords."
)
self.logger.info('Creating certificate keystore.')
es_config_path = os.path.join(self.configuration_directory, 'config')
try:
utilities.makedirs(es_config_path, exist_ok=True)
except Exception as e:
self.logger.error(
"General error occurred while attempting to create {} directory."
.format(es_config_path))
self.logger.debug(
"General error occurred while attempting to create {} directory; {}"
.format(es_config_path, e))
raise elastic_exceptions.InstallElasticsearchError(
"General error occurred while attempting to create {} directory; {}"
.format(es_config_path, e))
es_cert_util = os.path.join(self.install_directory, 'bin',
'elasticsearch-certutil')
es_cert_keystore = os.path.join(self.configuration_directory, 'config',
'elastic-certificates.p12')
cert_p = subprocess.Popen(
[es_cert_util, 'cert', '-out', es_cert_keystore, '-pass', ''],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
stdin=subprocess.PIPE,
env=env_dict)
try:
cert_p_res = cert_p.communicate()
except Exception as e:
self.logger.error(
"General error occurred while attempting to install SSL keystores."
)
self.logger.debug(
"General error occurred while attempting to install SSL keystores; {}"
.format(e))
raise elastic_exceptions.InstallElasticsearchError(
"General error occurred while attempting to install SSL keystores; {}"
.format(e))
if not os.path.exists(es_cert_keystore):
self.logger.error(
'Failed to setup SSL certificate keystore: \noutput: {}\n\t'.
format(cert_p_res))
raise elastic_exceptions.InstallElasticsearchError(
"Failed to setup SSL keystore; {}".format(cert_p_res))
keystore_config_path = os.path.join(self.configuration_directory,
'config')
try:
utilities.set_ownership_of_file(keystore_config_path,
user='******',
group='dynamite')
except Exception as e:
self.logger.error(
'General error occurred while attempting to set permissions for {}.'
.format(keystore_config_path))
self.logger.debug(
"General error occurred while attempting to set permissions for {}; {}"
.format(keystore_config_path, e))
raise elastic_exceptions.InstallElasticsearchError(
"General error occurred while attempting to set permissions for {}; {}"
.format(keystore_config_path, e))
if not elastic_profile.ProcessProfiler().is_running():
elastic_process.ProcessManager().start()
attempts = 0
while not elastic_profile.ProcessProfiler().is_listening():
self.logger.info(
'Waiting for ElasticSearch API to become accessible.')
time.sleep(10)
attempts += 1
if attempts == 10:
self.logger.error(
"Failed to start ElasticSearch API after 10 attempts.")
raise elastic_exceptions.InstallElasticsearchError(
"Failed to start Elasticsearch API after 10 attempts.")
self.logger.info('ElasticSearch API is up.')
self.logger.debug(
'Sleeping for 5 seconds, while ElasticSearch API finishes booting.'
)
time.sleep(5)
self.logger.info('Bootstrapping passwords.')
es_password_util = os.path.join(self.install_directory, 'bin',
'elasticsearch-setup-passwords')
bootstrap_p = subprocess.Popen([es_password_util, 'auto'],
cwd=self.configuration_directory,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
stdin=subprocess.PIPE,
env=env_dict)
try:
bootstrap_p_res = bootstrap_p.communicate(input=b'y\n')
if not bootstrap_p_res:
self.logger.error('Failed to setup new passwords.')
raise elastic_exceptions.InstallElasticsearchError(
"Failed to bootstrap password.")
try:
if not isinstance(bootstrap_p_res[0], str):
setup_from_bootstrap(bootstrap_p_res[0].decode())
else:
setup_from_bootstrap(bootstrap_p_res[0])
except general_exceptions.ResetPasswordError:
self.logger.error("Failed to bootstrap password.")
raise elastic_exceptions.InstallElasticsearchError(
"Failed to bootstrap password.")
except Exception as e:
self.logger.error(
"General error occurred while attempting to bootstrap ElasticSearch passwords."
)
self.logger.debug(
"General error occurred while attempting to bootstrap ElasticSearch passwords {}"
.format(e))
raise elastic_exceptions.InstallElasticsearchError(
"General error occurred while attempting to bootstrap ElasticSearch passwords {}"
.format(e))
def status(self) -> Optional[Union[Dict, str]]:
"""Get the statuses of monitor services
Returns:
The statuses of monitor services
"""
agent_status = {}
kibana_status, elasticsearch_status, logstash_status = {}, {}, {}
if not elasticsearch_profile.ProcessProfiler().is_installed():
self.logger.error(
'You must install elasticsearch to run this command.')
return None
elasticsearch_status = elasticsearch_process.ProcessManager().status()
agent_status.update({
'elasticsearch': {
'running':
elasticsearch_status.get('running'),
'enabled_on_startup':
elasticsearch_status.get('enabled_on_startup')
}
})
if logstash_profile.ProcessProfiler().is_installed():
logstash_status = logstash_process.ProcessManager().status()
agent_status.update({
'logstash': {
'running': logstash_status.get('running'),
'enabled_on_startup':
logstash_status.get('enabled_on_startup')
}
})
if kibana_profile.ProcessProfiler().is_installed():
kibana_status = kibana_process.ProcessManager().status()
agent_status.update({
'kibana': {
'running': kibana_status.get('running'),
'enabled_on_startup':
kibana_status.get('enabled_on_startup')
}
})
if self.pretty_print_status:
colorize = utilities.PrintDecorations.colorize
child_services = [
['Service', 'Running', 'Enabled on Startup'],
[
'kibana',
colorize('yes', 'green')
if kibana_status.get('running') else colorize('no', 'red'),
colorize('yes', 'green')
if kibana_status.get('enabled_on_startup') else colorize(
'no', 'red')
]
]
if elasticsearch_status:
child_services.append([
'elasticsearch',
colorize('yes', 'green')
if elasticsearch_status.get('running') else colorize(
'no', 'red'),
colorize('yes', 'green')
if elasticsearch_status.get('enabled_on_startup') else
colorize('no', 'red')
])
if logstash_status:
child_services.append([
'logstash',
colorize('yes', 'green')
if elasticsearch_status.get('running') else colorize(
'no', 'red'),
colorize('yes', 'green')
if elasticsearch_status.get('enabled_on_startup') else
colorize('no', 'red')
])
return tabulate.tabulate(child_services, tablefmt='fancy_grid')
return agent_status
def setup(self):
"""Setup Elasticsearch, Logstash, and Kibana
Returns:
None
"""
if self.elasticsearch_install_directory or self.elasticsearch_configuration_directory or \
self.elasticsearch_log_directory:
if not (self.elasticsearch_install_directory
and self.elasticsearch_configuration_directory
and self.elasticsearch_log_directory):
self.logger.error(
'You must specify elasticsearch-configuration-directory, elasticsearch-install-directory, '
'and elasticsearch-log-directory.')
return None
elasticsearch_install.InstallManager(
configuration_directory=self.
elasticsearch_configuration_directory,
install_directory=self.elasticsearch_install_directory,
log_directory=self.elasticsearch_log_directory,
stdout=self.stdout,
verbose=self.verbose).setup(
node_name=utilities.get_default_es_node_name(),
network_host=utilities.get_primary_ip_address(),
port=9200)
if self.logstash_install_directory or self.logstash_configuration_directory or self.logstash_log_directory:
if not (self.logstash_install_directory
and self.logstash_configuration_directory
and self.logstash_log_directory):
self.logger.error(
'You must specify logstash-configuration-directory, logstash-install-directory, '
'and logstash-log-directory.')
return None
logstash_install.InstallManager(
configuration_directory=self.logstash_configuration_directory,
install_directory=self.logstash_install_directory,
log_directory=self.logstash_log_directory,
stdout=self.stdout,
verbose=self.verbose).setup(
node_name=utilities.get_default_es_node_name().replace(
'es', 'ls'),
elasticsearch_host=utilities.get_primary_ip_address(),
elasticsearch_port=9200)
if self.kibana_install_directory or self.kibana_configuration_directory or self.kibana_log_directory:
from dynamite_nsm.services.elasticsearch import process as elasticsearch_process
if not (self.kibana_install_directory
and self.kibana_configuration_directory
and self.kibana_log_directory):
self.logger.error(
'You must specify kibana-configuration-directory, kibana-install-directory, '
'and kibana-log-directory.')
return None
self.logger.info('Starting Elasticsearch.')
elasticsearch_process.ProcessManager().start()
kibana_install.InstallManager(
configuration_directory=self.kibana_configuration_directory,
install_directory=self.kibana_install_directory,
log_directory=self.kibana_log_directory,
stdout=self.stdout,
verbose=self.verbose).setup(
host=utilities.get_primary_ip_address(),
port=5601,
elasticsearch_targets=[
f'https://{utilities.get_primary_ip_address()}:9200'
])