def change_elasticsearch_password(old_password, password='******', remote_host=None, remote_port=None,
prompt_user=True, stdout=True, verbose=False):
"""
Change the Elasticsearch password for all builtin users
:param old_password: The old Elasticsearch password
:param password: The new Elasticsearch password
:param prompt_user: If True, warning prompt is displayed before proceeding
:param stdout: Print status to stdout
:param verbose: Include detailed debug messages
"""
from dynamite_nsm.services.elasticsearch import process as elastic_process
from dynamite_nsm.services.elasticsearch import profile as elastic_profile
log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
logger = get_logger('ELASTICSEARCH', level=log_level, stdout=stdout)
if prompt_user:
resp = utilities.prompt_input(
'\n\033[93m[-] WARNING! Changing the ElasticSearch password may result in connected components losing '
'communication. Be sure to update Kibana/LogStash passwords.\n'
'[?] Are you sure you wish to continue? [no]|yes):\033[0m ')
while resp not in ['', 'no', 'yes']:
resp = utilities.prompt_input('\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m ')
if resp != 'yes':
if stdout:
sys.stdout.write('[+] Exiting\n')
exit(0)
if remote_host:
logger.info("Resetting ElasticSearch password on remote host: {}:{}".format(remote_host, remote_port))
else:
logger.info("Resetting ElasticSearch password on localhost.")
if elastic_profile.ProcessProfiler().is_installed():
# If ElasticSearch is installed Locally.
# Start the process, in order to perform a reset.
if not elastic_process.ProcessManager().start():
logger.error('Could not start ElasticSearch Process. Password reset failed.')
raise general_exceptions.ResetPasswordError(
"ElasticSearch process was not able to start, check your ElasticSearch logs.")
while not elastic_profile.ProcessProfiler().is_listening():
logger.info('Waiting for ElasticSearch API to become accessible.')
time.sleep(5)
logger.info('ElasticSearch API is up.')
logger.debug('Sleeping for 5 seconds, while ElasticSearch API finishes booting.')
time.sleep(5)
else:
logger.error("ElasticSearch is not installed, and no remote ElasticSearch host was specified.")
raise general_exceptions.ResetPasswordError(
"ElasticSearch is not installed, and no remote ElasticSearch host was specified.")
es_pw_config = PasswordConfigManager('elastic', current_password=old_password, remote_host=remote_host,
remote_http_port=remote_port)
logger.info("Attempting password reset.")
es_pw_config.set_all_passwords(password)
def invoke(self) -> Tuple[int, Dict]:
es_url = f'https://{self.network_host}:{self.http_port}'
es_cluster_data = {
'persistent': {
'script.max_compilations_rate': '1000/5m'
},
'transient': {
'script.max_compilations_rate': '1000/5m'
}
}
attempts = 0
es_process_profile = profile.ProcessProfiler()
if not es_process_profile.is_listening():
process.ProcessManager().start()
while not es_process_profile.is_listening(
) and attempts < self.max_attempts:
attempts += 1
sleep(10)
r = requests.put(url=f'{es_url}/_cluster/settings',
data=json.dumps(es_cluster_data),
auth=('admin', 'admin'),
headers={'content-type': 'application/json'},
verify=False)
if self.terminate_elasticsearch:
process.ProcessManager().stop()
if r.status_code != 200:
return r.status_code, {'error': r.text}
else:
return r.status_code, r.json()
def __init__(self,
stdout: Optional[bool] = True,
verbose: Optional[bool] = False,
pretty_print_status: Optional[bool] = False):
"""Manage Elasticsearch Process
Args:
stdout: Print output to console
verbose: Include detailed debug messages
pretty_print_status: If enabled, status will be printed in a tabulated style
Returns:
None
"""
environ = utilities.get_environment_file_dict()
process.BaseProcessManager.__init__(
self,
'elasticsearch.service',
'elasticsearch.process',
log_path=environ.get('ES_LOGS'),
create_pid_file=True,
stdout=stdout,
verbose=verbose,
pretty_print_status=pretty_print_status)
if not elasticsearch_profile.ProcessProfiler().is_installed():
self.logger.error(
"Elasticsearch is not installed. Install it with 'dynamite elasticsearch install -h'"
)
raise CallElasticProcessError("Elasticsearch is not installed.")
def _install_kibana_objects(self):
self.logger.info('Installing Kibana Dashboards')
self.logger.info('Waiting for ElasticSearch to become accessible.')
# Start ElasticSearch if it is installed locally and is not running
if self.elasticsearch_host in ['localhost', '127.0.0.1', '0.0.0.0', '::1', '::/128']:
self.logger.info('Starting ElasticSearch.')
elastic_process.ProcessManager().start()
while not elastic_profile.ProcessProfiler().is_listening():
self.logger.info('Waiting for ElasticSearch API to become accessible.')
time.sleep(5)
self.logger.info('ElasticSearch API is up.')
self.logger.info('Sleeping for 5 seconds, while ElasticSearch API finishes booting.')
time.sleep(5)
try:
kibana_proc = kibana_process.ProcessManager()
kibana_proc.optimize()
utilities.set_ownership_of_file(self.install_directory, user='******', group='dynamite')
utilities.set_ownership_of_file(self.configuration_directory, user='******', group='dynamite')
time.sleep(5)
self.logger.info('Starting Kibana.')
kibana_proc.start()
except Exception as e:
raise kibana_exceptions.InstallKibanaError("General error while starting Kibana process; {}".format(e))
kibana_api_start_attempts = 0
while not kibana_profile.ProcessProfiler().is_listening() and kibana_api_start_attempts != 5:
self.logger.info('Waiting for Kibana API to become accessible.')
kibana_api_start_attempts += 1
time.sleep(5)
if kibana_api_start_attempts == 5:
self.logger.error('Kibana API could not be started after {} attempts.'.format(kibana_api_start_attempts))
raise kibana_exceptions.InstallKibanaError(
"Kibana API could not be started after {} attempts.".format(kibana_api_start_attempts))
self.logger.info('Kibana API is up.')
self.logger.info('Sleeping for 10 seconds, while Kibana API finishes booting.')
time.sleep(10)
api_config = kibana_configs.ApiConfigManager(self.configuration_directory)
kibana_object_create_attempts = 1
while kibana_object_create_attempts != 5:
try:
self.logger.info('[Attempt {}] Attempting to install dashboards/visualizations.'.format(
kibana_object_create_attempts))
api_config.create_dynamite_kibana_objects()
break
except kibana_exceptions.CreateKibanaObjectsError:
kibana_object_create_attempts += 1
time.sleep(10)
if kibana_object_create_attempts == 5:
self.logger.error(
"Kibana objects could not be created after {} attempts".format(kibana_object_create_attempts))
raise kibana_exceptions.InstallKibanaError(
"Kibana objects could not be created after {} attempts".format(kibana_object_create_attempts))
self.logger.info('Successfully created dashboards/visualizations.')
kibana_proc.stop()
def __init__(self, prompt_user, stdout, verbose):
execution_strategy.BaseExecStrategy.__init__(
self,
strategy_name="monitor_uninstall",
strategy_description="Uninstall Monitor.",
functions=(
utilities.create_dynamite_environment_file,
prompt_monitor_uninstall,
),
arguments=(
# utilities.create_dynamite_environment_file
{},
# prompt_user
{
"prompt_user": bool(prompt_user),
"stdout": bool(stdout)
},
),
return_formats=(None, None))
if kb_profile.ProcessProfiler().is_installed():
self.add_function(func=kb_install.uninstall_kibana,
argument_dict={
'prompt_user': False,
'stdout': bool(stdout),
'verbose': bool(verbose)
})
if ls_profile.ProcessProfiler().is_installed():
self.add_function(func=ls_install.uninstall_logstash,
argument_dict={
'prompt_user': False,
'stdout': bool(stdout),
'verbose': bool(verbose)
})
if es_profile.ProcessProfiler().is_installed():
self.add_function(func=es_install.uninstall_elasticsearch,
argument_dict={
'prompt_user': False,
'stdout': bool(stdout),
'verbose': bool(verbose)
})
self.add_function(func=log_message,
argument_dict={
"msg":
'*** Monitor uninstalled successfully. ***',
'stdout': bool(stdout),
'verbose': bool(verbose)
})
def stop(self) -> bool:
"""Stop the monitor services
Returns:
True, if successfully stopped
"""
kibana_res, logstash_res, elasticsearch_res = True, True, True
if not elasticsearch_profile.ProcessProfiler().is_installed():
self.logger.error('You must install kibana to run this command.')
return False
elasticsearch_res = elasticsearch_process.ProcessManager().stop()
if logstash_profile.ProcessProfiler().is_installed():
logstash_res = logstash_process.ProcessManager().stop()
if kibana_profile.ProcessProfiler().is_installed():
kibana_res = kibana_process.ProcessManager().stop()
return kibana_res and elasticsearch_res and logstash_res
def invoke(self, shell: Optional[bool] = False, cwd: Optional[str] = None) -> List[
Tuple[List, bytes, bytes]]:
utilities.set_permissions_of_file(file_path=self.opendistro_security_admin, unix_permissions_integer='+x')
if not cwd:
cwd = self.security_conf_directory
attempts = 0
es_process_profile = profile.ProcessProfiler()
if not es_process_profile.is_listening():
process.ProcessManager().start()
while not es_process_profile.is_listening() and attempts < self.max_attempts:
attempts += 1
sleep(10)
results = super().invoke(shell, cwd)
if self.terminate_elasticsearch:
process.ProcessManager().stop()
return results
def __init__(self, install_directory, configuration_directory, log_directory, host='0.0.0.0', port=5601,
elasticsearch_host=None, elasticsearch_port=None, elasticsearch_password='******',
download_kibana_archive=True, stdout=True, verbose=False):
"""
:param install_directory: Path to the install directory (E.G /opt/dynamite/kibana/)
:param configuration_directory: Path to the configuration directory (E.G /etc/dynamite/kibana/)
:param log_directory: Path to the log directory (E.G /var/log/dynamite/kibana/)
:param host: The IP address to listen on (E.G "0.0.0.0")
:param port: The port that the Kibana UI/API is bound to (E.G 5601)
:param elasticsearch_host: A hostname/IP of the target elasticsearch instance
:param elasticsearch_port: A port number for the target elasticsearch instance
:param elasticsearch_password: The password used for authentication across all builtin ES users
:param download_kibana_archive: If True, download the Kibana archive from a mirror
:param stdout: Print the output to console
:param verbose: Include detailed debug messages
"""
self.host = host
self.port = port
self.elasticsearch_host = elasticsearch_host
self.elasticsearch_port = elasticsearch_port
self.elasticsearch_password = elasticsearch_password
if not elasticsearch_host:
if elastic_profile.ProcessProfiler().is_installed():
self.elasticsearch_host = 'localhost'
else:
raise kibana_exceptions.InstallKibanaError(
"ElasticSearch must either be installed locally, or a remote host must be specified.")
self.install_directory = install_directory
self.configuration_directory = configuration_directory
self.log_directory = log_directory
self.stdout = stdout
self.verbose = verbose
utilities.create_dynamite_environment_file()
if download_kibana_archive:
try:
self.download_from_mirror(const.KIBANA_MIRRORS, const.KIBANA_ARCHIVE_NAME, stdout=stdout,
verbose=verbose)
except (general_exceptions.ArchiveExtractionError, general_exceptions.DownloadError):
self.logger.error("Failed to download Kibana archive.")
raise kibana_exceptions.InstallKibanaError("Failed to download Kibana archive.")
try:
self.extract_archive(os.path.join(const.INSTALL_CACHE, const.KIBANA_ARCHIVE_NAME))
except general_exceptions.ArchiveExtractionError:
self.logger.error("Failed to extract Kibana archive.")
raise kibana_exceptions.InstallKibanaError("Failed to extract Kibana archive.")
install.BaseInstallManager.__init__(self, 'kibana', verbose=self.verbose, stdout=self.stdout)
def uninstall(self):
from dynamite_nsm.services.elasticsearch import profile as elasticsearch_profile
from dynamite_nsm.services.logstash import profile as logstash_profile
from dynamite_nsm.services.kibana import profile as kibana_profile
if elasticsearch_profile.ProcessProfiler().is_installed():
elasticsearch_install.UninstallManager(
purge_config=True, stdout=self.stdout,
verbose=self.verbose).uninstall()
if logstash_profile.ProcessProfiler().is_installed():
logstash_install.UninstallManager(
purge_config=True, stdout=self.stdout,
verbose=self.verbose).uninstall()
if kibana_profile.ProcessProfiler().is_installed():
kibana_install.UninstallManager(purge_config=True,
stdout=self.stdout,
verbose=self.verbose).uninstall()
def __init__(self, old_password, new_password, stdout, verbose):
execution_strategy.BaseExecStrategy.__init__(
self,
strategy_name="monitor_change_password",
strategy_description=
"Change the password for all monitor components.",
)
if es_profile.ProcessProfiler().is_installed():
self.add_function(func=es_config.change_elasticsearch_password,
argument_dict={
'old_password': str(old_password),
'password': str(new_password),
'prompt_user': False,
'stdout': bool(stdout),
'verbose': bool(verbose)
})
if ls_profile.ProcessProfiler().is_installed():
self.add_function(
func=ls_config.change_logstash_elasticsearch_password,
argument_dict={
'password': str(new_password),
'prompt_user': False,
'stdout': bool(stdout),
'verbose': bool(verbose)
})
if kb_profile.ProcessProfiler().is_installed():
self.add_function(
func=kb_config.change_kibana_elasticsearch_password,
argument_dict={
'password': str(new_password),
'prompt_user': False,
'stdout': bool(stdout),
'verbose': bool(verbose)
})
self.add_function(func=log_message,
argument_dict={
'msg': 'Monitor passwords changed successfully!',
'stdout': bool(stdout),
'verbose': bool(verbose)
})
def __init__(self, stdout=True, verbose=False, pretty_print_status=False):
environ = utilities.get_environment_file_dict()
try:
process.BaseProcessManager.__init__(
self,
'elasticsearch.service',
'elasticsearch',
log_path=environ.get('ES_LOGS'),
pid_file=os.path.join(PID_DIRECTORY, 'elasticsearch.pid'),
stdout=stdout,
verbose=verbose,
pretty_print_status=pretty_print_status)
except general_exceptions.CallProcessError:
self.logger.error("Could not find systemctl on this system.")
raise elasticsearch_exceptions.CallElasticProcessError(
"Could not find systemctl.")
if not elasticsearch_profile.ProcessProfiler().is_installed():
self.logger.error(
"ElasticSearch is not installed. Install it with 'dynamite elasticsearch install -h'"
)
raise elasticsearch_exceptions.CallElasticProcessError(
"ElasticSearch is not installed.")
def __init__(self,
configuration_directory,
install_directory,
log_directory,
host='0.0.0.0',
elasticsearch_host='localhost',
elasticsearch_port=9200,
elasticsearch_password='******',
heap_size_gigs=4,
download_logstash_archive=True,
stdout=True,
verbose=False):
"""
:param configuration_directory: Path to the configuration directory (E.G /etc/dynamite/logstash/)
:param install_directory: Path to the install directory (E.G /opt/dynamite/logstash/)
:param log_directory: Path to the log directory (E.G /var/log/dynamite/logstash/)
:param host: The IP address to listen on (E.G "0.0.0.0")
:param elasticsearch_host: A hostname/IP of the target elasticsearch instance
:param elasticsearch_port: A port number for the target elasticsearch instance
:param elasticsearch_password: The password used for authentication across all builtin ES users
:param heap_size_gigs: The initial/max java heap space to allocate
:param download_logstash_archive: If True, download the LogStash archive from a mirror
:param stdout: Print output to console
:param verbose: Include output from system utilities
"""
self.host = host
if not elasticsearch_host:
if elastic_profile.ProcessProfiler().is_installed():
self.elasticsearch_host = 'localhost'
self.logger.info(
"Assuming LogStash will connect to local ElasticSearch instance, "
"as ElasticSearch is installed on this host.")
else:
self.logger.error(
"ElasticSearch must either be installed locally, or a remote host must be specified."
)
raise logstash_exceptions.InstallLogstashError(
"ElasticSearch must either be installed locally, or a remote host must be specified."
)
else:
self.elasticsearch_host = elasticsearch_host
self.elasticsearch_port = elasticsearch_port
self.configuration_directory = configuration_directory
self.install_directory = install_directory
self.elasticsearch_password = elasticsearch_password
self.heap_size_gigs = heap_size_gigs
self.log_directory = log_directory
self.stdout = stdout
self.verbose = verbose
utilities.create_dynamite_environment_file()
if download_logstash_archive:
try:
self.download_from_mirror(const.LOGSTASH_MIRRORS,
const.LOGSTASH_ARCHIVE_NAME,
stdout=stdout,
verbose=verbose)
except (general_exceptions.ArchiveExtractionError,
general_exceptions.DownloadError):
self.logger.error("Failed to download LogStash archive.")
raise logstash_exceptions.InstallLogstashError(
"Failed to download LogStash archive.")
try:
self.extract_archive(
os.path.join(const.INSTALL_CACHE, const.LOGSTASH_ARCHIVE_NAME))
except general_exceptions.ArchiveExtractionError:
self.logger.error("Failed to extract LogStash archive.")
raise logstash_exceptions.InstallLogstashError(
"Failed to extract LogStash archive.")
install.BaseInstallManager.__init__(self,
'logstash',
verbose=self.verbose,
stdout=stdout)
def uninstall_elasticsearch(prompt_user=True, stdout=True, verbose=False):
"""
Uninstall ElasticSearch
:param prompt_user: Print a warning before continuing
:param stdout: Print the output to console
:param verbose: Include detailed debug messages
"""
log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
logger = get_logger('ELASTICSEARCH', level=log_level, stdout=stdout)
env_file = os.path.join(const.CONFIG_PATH, 'environment')
environment_variables = utilities.get_environment_file_dict()
es_profiler = elastic_profile.ProcessProfiler()
if not es_profiler.is_installed():
logger.error('ElasticSearch is not installed.')
raise elastic_exceptions.UninstallElasticsearchError(
"ElasticSearch is not installed.")
configuration_directory = environment_variables.get('ES_PATH_CONF')
es_config = elastic_configs.ConfigManager(
configuration_directory=configuration_directory)
if prompt_user:
sys.stderr.write(
'\n\033[93m[-] WARNING! Removing ElasticSearch Will Delete All Data.'
'\033[0m\n')
resp = utilities.prompt_input(
'\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m '
)
while resp not in ['', 'no', 'yes']:
resp = utilities.prompt_input(
'\n\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m '
)
if resp != 'yes':
if stdout:
sys.stdout.write('\n[+] Exiting\n')
exit(0)
if es_profiler.is_running():
elastic_process.ProcessManager().stop()
try:
shutil.rmtree(es_config.configuration_directory)
shutil.rmtree(es_config.es_home)
shutil.rmtree(es_config.path_logs)
shutil.rmtree(const.INSTALL_CACHE, ignore_errors=True)
env_lines = ''
with open(env_file) as env_fr:
for line in env_fr.readlines():
if 'ES_PATH_CONF' in line:
continue
elif 'ES_HOME' in line:
continue
elif 'ES_LOGS' in line:
continue
elif line.strip() == '':
continue
env_lines += line.strip() + '\n'
with open(env_file, 'w') as env_fw:
env_fw.write(env_lines)
except Exception as e:
logger.error(
"General error occurred while attempting to uninstall ElasticSearch."
.format(e))
logger.debug(
"General error occurred while attempting to uninstall ElasticSearch; {}"
.format(e))
raise elastic_exceptions.UninstallElasticsearchError(
"General error occurred while attempting to uninstall ElasticSearch; {}"
.format(e))
try:
sysctl = systemctl.SystemCtl()
except general_exceptions.CallProcessError:
raise elastic_exceptions.UninstallElasticsearchError(
"Could not find systemctl.")
sysctl.uninstall_and_disable('elasticsearch')
def install_elasticsearch(configuration_directory,
install_directory,
log_directory,
password='******',
heap_size_gigs=4,
install_jdk=True,
create_dynamite_user=True,
stdout=True,
verbose=False):
"""
Install ElasticSearch
:param configuration_directory: Path to the configuration directory (E.G /etc/dynamite/elasticsearch/)
:param install_directory: Path to the install directory (E.G /opt/dynamite/elasticsearch/)
:param log_directory: Path to the log directory (E.G /var/log/dynamite/elasticsearch/)
:param password: The password used for authentication across all builtin users
:param heap_size_gigs: The initial/max java heap space to allocate
:param install_jdk: Install the latest OpenJDK that will be used by Logstash/ElasticSearch
:param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run Logstash/ElasticSearch
:param stdout: Print the output to console
:param verbose: Include detailed debug messages
"""
log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
logger = get_logger('ELASTICSEARCH', level=log_level, stdout=stdout)
es_profiler = elastic_profile.ProcessProfiler()
if es_profiler.is_installed():
logger.error('ElasticSearch is already installed.')
raise elastic_exceptions.AlreadyInstalledElasticsearchError()
if utilities.get_memory_available_bytes() < 6 * (1000**3):
sys.stderr.write(
'\n\033[93m[-] WARNING! ElasticSearch should have at-least 6GB to run '
'currently available [{} GB]\033[0m\n'.format(
utilities.get_memory_available_bytes() / (1000**3)))
if str(utilities.prompt_input(
'\033[93m[?] Continue? [y|N]:\033[0m ')).lower() != 'y':
sys.stdout.write('\n[+] Exiting\n')
exit(0)
es_installer = InstallManager(
configuration_directory=configuration_directory,
install_directory=install_directory,
log_directory=log_directory,
password=password,
heap_size_gigs=heap_size_gigs,
download_elasticsearch_archive=not es_profiler.is_downloaded(),
stdout=stdout,
verbose=verbose)
if install_jdk:
try:
utilities.download_java(stdout=stdout)
utilities.extract_java()
utilities.setup_java()
except Exception as e:
logger.error(
'General error occurred while attempting to setup Java.')
logger.debug(
"General error occurred while attempting to setup Java; {}".
format(e))
raise elastic_exceptions.InstallElasticsearchError(
"General error occurred while attempting to setup Java; {}".
format(e))
if create_dynamite_user:
utilities.create_dynamite_user(utilities.generate_random_password(50))
es_installer.setup_elasticsearch()
def setup_passwords(self):
env_dict = utilities.get_environment_file_dict()
def setup_from_bootstrap(s):
bootstrap_users_and_passwords = {}
for line in s.split('\n'):
if 'PASSWORD' in line:
_, user, _, password = line.split(' ')
if not isinstance(password, str):
password = password.decode()
bootstrap_users_and_passwords[user] = password
es_pass_config = elastic_configs.PasswordConfigManager(
auth_user='******',
current_password=bootstrap_users_and_passwords['elastic'],
stdout=self.stdout,
verbose=self.verbose)
es_pass_config.set_all_passwords(new_password=self.password)
if not elastic_profile.ProcessProfiler().is_installed():
self.logger.error(
'ElasticSearch must be installed and running to bootstrap passwords.'
)
raise elastic_exceptions.InstallElasticsearchError(
"ElasticSearch must be installed and running to bootstrap passwords."
)
self.logger.info('Creating certificate keystore.')
es_config_path = os.path.join(self.configuration_directory, 'config')
try:
utilities.makedirs(es_config_path, exist_ok=True)
except Exception as e:
self.logger.error(
"General error occurred while attempting to create {} directory."
.format(es_config_path))
self.logger.debug(
"General error occurred while attempting to create {} directory; {}"
.format(es_config_path, e))
raise elastic_exceptions.InstallElasticsearchError(
"General error occurred while attempting to create {} directory; {}"
.format(es_config_path, e))
es_cert_util = os.path.join(self.install_directory, 'bin',
'elasticsearch-certutil')
es_cert_keystore = os.path.join(self.configuration_directory, 'config',
'elastic-certificates.p12')
cert_p = subprocess.Popen(
[es_cert_util, 'cert', '-out', es_cert_keystore, '-pass', ''],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
stdin=subprocess.PIPE,
env=env_dict)
try:
cert_p_res = cert_p.communicate()
except Exception as e:
self.logger.error(
"General error occurred while attempting to install SSL keystores."
)
self.logger.debug(
"General error occurred while attempting to install SSL keystores; {}"
.format(e))
raise elastic_exceptions.InstallElasticsearchError(
"General error occurred while attempting to install SSL keystores; {}"
.format(e))
if not os.path.exists(es_cert_keystore):
self.logger.error(
'Failed to setup SSL certificate keystore: \noutput: {}\n\t'.
format(cert_p_res))
raise elastic_exceptions.InstallElasticsearchError(
"Failed to setup SSL keystore; {}".format(cert_p_res))
keystore_config_path = os.path.join(self.configuration_directory,
'config')
try:
utilities.set_ownership_of_file(keystore_config_path,
user='******',
group='dynamite')
except Exception as e:
self.logger.error(
'General error occurred while attempting to set permissions for {}.'
.format(keystore_config_path))
self.logger.debug(
"General error occurred while attempting to set permissions for {}; {}"
.format(keystore_config_path, e))
raise elastic_exceptions.InstallElasticsearchError(
"General error occurred while attempting to set permissions for {}; {}"
.format(keystore_config_path, e))
if not elastic_profile.ProcessProfiler().is_running():
elastic_process.ProcessManager().start()
attempts = 0
while not elastic_profile.ProcessProfiler().is_listening():
self.logger.info(
'Waiting for ElasticSearch API to become accessible.')
time.sleep(10)
attempts += 1
if attempts == 10:
self.logger.error(
"Failed to start ElasticSearch API after 10 attempts.")
raise elastic_exceptions.InstallElasticsearchError(
"Failed to start Elasticsearch API after 10 attempts.")
self.logger.info('ElasticSearch API is up.')
self.logger.debug(
'Sleeping for 5 seconds, while ElasticSearch API finishes booting.'
)
time.sleep(5)
self.logger.info('Bootstrapping passwords.')
es_password_util = os.path.join(self.install_directory, 'bin',
'elasticsearch-setup-passwords')
bootstrap_p = subprocess.Popen([es_password_util, 'auto'],
cwd=self.configuration_directory,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
stdin=subprocess.PIPE,
env=env_dict)
try:
bootstrap_p_res = bootstrap_p.communicate(input=b'y\n')
if not bootstrap_p_res:
self.logger.error('Failed to setup new passwords.')
raise elastic_exceptions.InstallElasticsearchError(
"Failed to bootstrap password.")
try:
if not isinstance(bootstrap_p_res[0], str):
setup_from_bootstrap(bootstrap_p_res[0].decode())
else:
setup_from_bootstrap(bootstrap_p_res[0])
except general_exceptions.ResetPasswordError:
self.logger.error("Failed to bootstrap password.")
raise elastic_exceptions.InstallElasticsearchError(
"Failed to bootstrap password.")
except Exception as e:
self.logger.error(
"General error occurred while attempting to bootstrap ElasticSearch passwords."
)
self.logger.debug(
"General error occurred while attempting to bootstrap ElasticSearch passwords {}"
.format(e))
raise elastic_exceptions.InstallElasticsearchError(
"General error occurred while attempting to bootstrap ElasticSearch passwords {}"
.format(e))
def __init__(self,
configuration_directory,
notebook_home,
elasticsearch_host=None,
elasticsearch_port=None,
elasticsearch_password='******',
jupyterhub_host=None,
jupyterhub_password='******',
download_dynamite_sdk_archive=True,
extract_dynamite_sdk_archive=True,
stdout=True,
verbose=False):
"""
:param configuration_directory: Path to the configuration directory (E.G /etc/dynamite/dynamite_sdk/)
:param notebook_home: The path where Jupyter notebooks are stored
:param elasticsearch_host: A hostname/IP of the target elasticsearch instance
:param elasticsearch_port: A port number for the target elasticsearch instance
:param elasticsearch_password: The password used for authentication across all builtin ES users
:param jupyterhub_host: The host by which users can access this instance;
(Used for creating kibana -> Jupyter hyperlinks)
:param jupyterhub_password: The password used for authenticating to jupyterhub (via jupyter user)
:param download_dynamite_sdk_archive: If True, download the DynamiteSDK archive from a mirror
:param extract_dynamite_sdk_archive: If True, extracts the DynamiteSDK archive
:param stdout: Print output to console
:param verbose: Include detailed debug messages
"""
log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
self.logger = get_logger('LAB', level=log_level, stdout=stdout)
self.elasticsearch_host = elasticsearch_host
self.elasticsearch_port = elasticsearch_port
self.elasticsearch_password = elasticsearch_password
self.jupyterhub_host = jupyterhub_host
self.jupyterhub_password = jupyterhub_password
self.configuration_directory = configuration_directory
self.notebook_home = notebook_home
if download_dynamite_sdk_archive:
try:
self.download_dynamite_sdk(stdout=stdout)
except general_exceptions.DownloadError:
self.logger.error('Failed to download DynamiteSDK archive.')
raise lab_exceptions.InstallLabError(
"Failed to download DynamiteSDK archive.")
if extract_dynamite_sdk_archive:
try:
self.extract_dynamite_sdk()
except general_exceptions.ArchiveExtractionError:
self.logger.error('Failed to extract DynamiteSDK archive.')
raise lab_exceptions.InstallLabError(
"Failed to extract DynamiteSDK archive.")
try:
self.install_dependencies(stdout=stdout, verbose=verbose)
self.install_jupyterhub(stdout=stdout)
except (general_exceptions.InvalidOsPackageManagerDetectedError,
general_exceptions.OsPackageManagerInstallError,
general_exceptions.OsPackageManagerRefreshError):
self.logger.error('One or more OS dependencies failed to install.')
raise lab_exceptions.InstallLabError(
"One or more OS dependencies failed to install.")
self.logger.info('Creating jupyter user in dynamite group.')
utilities.create_jupyter_user(password=self.jupyterhub_password)
self.stdout = stdout
self.verbose = verbose
if not elasticsearch_host:
if elastic_profile.ProcessProfiler().is_installed():
self.elasticsearch_host = 'localhost'
else:
raise lab_exceptions.InstallLabError(
"Elasticsearch must either be installed locally, or a remote host must be specified."
)
def __init__(self, logstash_listen_address, kibana_listen_address,
kibana_listen_port, elasticsearch_host, elasticsearch_port,
elasticsearch_password, elasticsearch_heap_size_gigs,
logstash_heap_size_gigs, install_jdk, stdout, verbose):
execution_strategy.BaseExecStrategy.__init__(
self,
strategy_name="monitor_install",
strategy_description=
"Install ElasticSearch, LogStash, and Kibana on the same instance.",
functions=(utilities.create_dynamite_environment_file, ),
arguments=({}, ),
return_formats=(None, ))
self.add_function(func=remove_elasticsearch_tar_archive,
argument_dict={},
return_format=None)
self.add_function(func=remove_logstash_tar_archive,
argument_dict={},
return_format=None)
self.add_function(func=remove_kibana_tar_archive,
argument_dict={},
return_format=None)
if not es_profile.ProcessProfiler().is_installed():
self.add_function(func=es_install.install_elasticsearch,
argument_dict={
"configuration_directory":
"/etc/dynamite/elasticsearch/",
"install_directory":
"/opt/dynamite/elasticsearch/",
"log_directory":
"/var/log/dynamite/elasticsearch/",
"password":
str(elasticsearch_password),
"heap_size_gigs":
int(elasticsearch_heap_size_gigs),
"install_jdk":
bool(install_jdk),
"create_dynamite_user":
True,
"stdout":
bool(stdout),
"verbose":
bool(verbose)
},
return_format=None)
else:
self.add_function(
func=log_message,
argument_dict={
"msg":
'Skipping ElasticSearch installation; already installed.',
'stdout': bool(stdout),
'verbose': bool(verbose)
},
return_format=None)
self.add_function(func=es_process.start,
argument_dict={"stdout": False},
return_format=None)
if not ls_profile.ProcessProfiler().is_installed():
self.add_function(func=ls_install.install_logstash,
argument_dict={
"configuration_directory":
"/etc/dynamite/logstash/",
"install_directory":
"/opt/dynamite/logstash/",
"log_directory":
"/var/log/dynamite/logstash/",
"host":
str(logstash_listen_address),
"elasticsearch_host":
str(elasticsearch_host),
"elasticsearch_port":
int(elasticsearch_port),
"elasticsearch_password":
str(elasticsearch_password),
"heap_size_gigs":
int(logstash_heap_size_gigs),
"install_jdk":
False,
"create_dynamite_user":
False,
"stdout":
bool(stdout),
"verbose":
bool(verbose)
},
return_format=None)
else:
self.add_function(
func=log_message,
argument_dict={
"msg":
'Skipping LogStash installation; already installed.',
'stdout': bool(stdout),
'verbose': bool(verbose)
},
return_format=None)
if not kb_profile.ProcessProfiler().is_installed():
self.add_function(func=kb_install.install_kibana,
argument_dict={
"configuration_directory":
"/etc/dynamite/kibana/",
"install_directory":
"/opt/dynamite/kibana/",
"log_directory":
"/var/log/dynamite/kibana/",
"host":
str(kibana_listen_address),
"port":
int(kibana_listen_port),
"elasticsearch_host":
str(elasticsearch_host),
"elasticsearch_port":
int(elasticsearch_port),
"elasticsearch_password":
str(elasticsearch_password),
"create_dynamite_user":
True,
"stdout":
bool(stdout),
"verbose":
bool(verbose)
},
return_format=None)
else:
self.add_function(
func=log_message,
argument_dict={
"msg": 'Skipping Kibana installation; already installed.',
'stdout': bool(stdout),
'verbose': bool(verbose)
},
return_format=None)
self.add_function(func=kb_process.stop,
argument_dict={"stdout": False},
return_format=None)
self.add_function(func=es_process.stop,
argument_dict={"stdout": False},
return_format=None)
self.add_function(func=log_message,
argument_dict={
"msg": '*** Monitor installed successfully. ***',
'stdout': bool(stdout),
'verbose': bool(verbose)
},
return_format=None)
self.add_function(
func=log_message,
argument_dict={
"msg":
'Next, Start your monitor: '
'\'dynamite monitor start\'. It will be available at: \033[4m{}:{}\033[0m once started.'
''.format(kibana_listen_address, kibana_listen_port),
'stdout':
bool(stdout),
'verbose':
bool(verbose)
},
return_format=None)
def status(self) -> Optional[Union[Dict, str]]:
"""Get the statuses of monitor services
Returns:
The statuses of monitor services
"""
agent_status = {}
kibana_status, elasticsearch_status, logstash_status = {}, {}, {}
if not elasticsearch_profile.ProcessProfiler().is_installed():
self.logger.error(
'You must install elasticsearch to run this command.')
return None
elasticsearch_status = elasticsearch_process.ProcessManager().status()
agent_status.update({
'elasticsearch': {
'running':
elasticsearch_status.get('running'),
'enabled_on_startup':
elasticsearch_status.get('enabled_on_startup')
}
})
if logstash_profile.ProcessProfiler().is_installed():
logstash_status = logstash_process.ProcessManager().status()
agent_status.update({
'logstash': {
'running': logstash_status.get('running'),
'enabled_on_startup':
logstash_status.get('enabled_on_startup')
}
})
if kibana_profile.ProcessProfiler().is_installed():
kibana_status = kibana_process.ProcessManager().status()
agent_status.update({
'kibana': {
'running': kibana_status.get('running'),
'enabled_on_startup':
kibana_status.get('enabled_on_startup')
}
})
if self.pretty_print_status:
colorize = utilities.PrintDecorations.colorize
child_services = [
['Service', 'Running', 'Enabled on Startup'],
[
'kibana',
colorize('yes', 'green')
if kibana_status.get('running') else colorize('no', 'red'),
colorize('yes', 'green')
if kibana_status.get('enabled_on_startup') else colorize(
'no', 'red')
]
]
if elasticsearch_status:
child_services.append([
'elasticsearch',
colorize('yes', 'green')
if elasticsearch_status.get('running') else colorize(
'no', 'red'),
colorize('yes', 'green')
if elasticsearch_status.get('enabled_on_startup') else
colorize('no', 'red')
])
if logstash_status:
child_services.append([
'logstash',
colorize('yes', 'green')
if elasticsearch_status.get('running') else colorize(
'no', 'red'),
colorize('yes', 'green')
if elasticsearch_status.get('enabled_on_startup') else
colorize('no', 'red')
])
return tabulate.tabulate(child_services, tablefmt='fancy_grid')
return agent_status