def invoke(self, shell: Optional[bool] = False, cwd: Optional[str] = None) -> \
List[Tuple[List, bytes, bytes]]:
utilities.makedirs(f'{self.cert_directory}')
if not cwd:
cwd = self.cert_directory
results = super().invoke(shell=shell, cwd=cwd)
utilities.safely_remove_file(f'{self.cert_directory}/admin-key-temp.pem')
utilities.safely_remove_file(f'{self.cert_directory}/admin.csr')
utilities.set_ownership_of_file(path=self.cert_directory, user='******', group='dynamite')
utilities.set_permissions_of_file(file_path=self.cert_directory, unix_permissions_integer=700)
utilities.set_permissions_of_file(file_path=f'{self.cert_directory}/{self.cert_name}',
unix_permissions_integer=600)
utilities.set_permissions_of_file(file_path=f'{self.cert_directory}/{self.key_name}',
unix_permissions_integer=600)
utilities.set_permissions_of_file(file_path=f'{self.cert_directory}/{self.trusted_ca_cert_name}',
unix_permissions_integer=600)
utilities.set_permissions_of_file(file_path=f'{self.cert_directory}/{self.trusted_ca_key_name}',
unix_permissions_integer=600)
es_main_config = config.ConfigManager(self.configuration_directory)
es_main_config.transport_pem_cert_file = f'security/auth/{self.cert_name}'
es_main_config.rest_api_pem_cert_file = es_main_config.transport_pem_cert_file
es_main_config.transport_pem_key_file = f'security/auth/{self.key_name}'
es_main_config.rest_api_pem_key_file = es_main_config.transport_pem_key_file
es_main_config.transport_trusted_cas_file = f'security/auth/{self.trusted_ca_cert_name}'
es_main_config.rest_api_trusted_cas_file = es_main_config.transport_trusted_cas_file
es_main_config.commit()
return results
def test_elasticyaml_update_network_host(self):
self.config_manager.network_host = 'myhost.local'
self.config_manager.write_elasticsearch_config()
config_manager_read = config.ConfigManager(self.config_directory)
assert (config_manager_read.network_host == 'myhost.local')
def test_elasticyaml_update_path_logs(self):
self.config_manager.path_logs = '/var/log/dynamite/test/logs'
self.config_manager.write_elasticsearch_config()
config_manager_read = config.ConfigManager(self.config_directory)
assert (config_manager_read.path_logs == '/var/log/dynamite/test/logs')
def test_elasticyaml_update_cluster_name(self):
self.config_manager.cluster_name = 'my-new-cluster'
self.config_manager.write_elasticsearch_config()
config_manager_read = config.ConfigManager(self.config_directory)
assert (config_manager_read.cluster_name == 'my-new-cluster')
def test_javaopts_update_heapsize(self):
self.config_manager.java_maximum_memory = 10
self.config_manager.java_initial_memory = 10
self.config_manager.write_jvm_config()
config_manager_read = config.ConfigManager(self.config_directory)
assert (config_manager_read.java_initial_memory == 10 and config_manager_read.java_maximum_memory == 10)
def setUp(self):
self.config_root = '/etc/dynamite/test'
self.config_directory = os.path.join(self.config_root, 'elasticsearch')
# Setup Test Space
os.makedirs(self.config_directory, exist_ok=True)
create_dummy_elasticyaml(self.config_directory)
create_dummy_javaopts(self.config_directory)
self.config_manager = config.ConfigManager(configuration_directory=self.config_directory)
def is_listening(self):
if not self.elasticsearch_config:
return False
if not os.path.exists(self.elasticsearch_config):
return False
es_config_obj = elastic_configs.ConfigManager(
configuration_directory=self.elasticsearch_config)
host = es_config_obj.network_host
port = es_config_obj.http_port
if host.strip() == '0.0.0.0':
host = 'localhost'
return utilities.check_socket(host, port)
def is_listening(self) -> bool:
""" Check if Elasticsearch is listening
Returns:
True, if Elasticsearch HTTP service is listening
"""
if not self.elasticsearch_config:
return False
if not os.path.exists(self.elasticsearch_config):
return False
es_config_obj = elastic_configs.ConfigManager(configuration_directory=self.elasticsearch_config)
host = es_config_obj.network_host
port = es_config_obj.http_port
return utilities.check_socket(host, port)
def _setup_default_elasticsearch_configs(self):
self.logger.info('Overwriting default configuration.')
shutil.copy(
os.path.join(const.DEFAULT_CONFIGS, 'elasticsearch',
'elasticsearch.yml'), self.configuration_directory)
try:
es_config = elastic_configs.ConfigManager(
configuration_directory=self.configuration_directory)
except elastic_exceptions.ReadElasticConfigError:
self.logger.error('Failed to read ElasticSearch config.')
raise elastic_exceptions.InstallElasticsearchError(
"Failed to read ElasticSearch config.")
except Exception as e:
self.logger.error(
"General error occurred while reading ElasticSearch configs.")
self.logger.debug(
"General error occurred while reading ElasticSearch configs; {}"
.format(e))
raise elastic_exceptions.InstallElasticsearchError(
"General error occurred while reading elasticsearch configs; {}"
.format(e))
self.logger.info('Setting up JVM default heap settings [{}GB]'.format(
self.heap_size_gigs))
es_config.java_initial_memory = int(self.heap_size_gigs)
es_config.java_maximum_memory = int(self.heap_size_gigs)
es_config.network_host = self.host
es_config.http_port = self.port
try:
es_config.write_configs()
except elastic_exceptions.WriteElasticConfigError:
self.logger.error('Failed to write ElasticSearch config.')
raise elastic_exceptions.InstallElasticsearchError(
"Failed to write ElasticSearch config.")
except Exception as e:
self.logger.error(
"General error occurred while writing ElasticSearch configs.")
self.logger.debug(
"General error occurred while writing ElasticSearch configs; {}"
.format(e))
raise elastic_exceptions.InstallElasticsearchError(
"General error occurred while writing ElasticSearch configs; {}"
.format(e))
def uninstall_elasticsearch(prompt_user=True, stdout=True, verbose=False):
"""
Uninstall ElasticSearch
:param prompt_user: Print a warning before continuing
:param stdout: Print the output to console
:param verbose: Include detailed debug messages
"""
log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
logger = get_logger('ELASTICSEARCH', level=log_level, stdout=stdout)
env_file = os.path.join(const.CONFIG_PATH, 'environment')
environment_variables = utilities.get_environment_file_dict()
es_profiler = elastic_profile.ProcessProfiler()
if not es_profiler.is_installed():
logger.error('ElasticSearch is not installed.')
raise elastic_exceptions.UninstallElasticsearchError(
"ElasticSearch is not installed.")
configuration_directory = environment_variables.get('ES_PATH_CONF')
es_config = elastic_configs.ConfigManager(
configuration_directory=configuration_directory)
if prompt_user:
sys.stderr.write(
'\n\033[93m[-] WARNING! Removing ElasticSearch Will Delete All Data.'
'\033[0m\n')
resp = utilities.prompt_input(
'\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m '
)
while resp not in ['', 'no', 'yes']:
resp = utilities.prompt_input(
'\n\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m '
)
if resp != 'yes':
if stdout:
sys.stdout.write('\n[+] Exiting\n')
exit(0)
if es_profiler.is_running():
elastic_process.ProcessManager().stop()
try:
shutil.rmtree(es_config.configuration_directory)
shutil.rmtree(es_config.es_home)
shutil.rmtree(es_config.path_logs)
shutil.rmtree(const.INSTALL_CACHE, ignore_errors=True)
env_lines = ''
with open(env_file) as env_fr:
for line in env_fr.readlines():
if 'ES_PATH_CONF' in line:
continue
elif 'ES_HOME' in line:
continue
elif 'ES_LOGS' in line:
continue
elif line.strip() == '':
continue
env_lines += line.strip() + '\n'
with open(env_file, 'w') as env_fw:
env_fw.write(env_lines)
except Exception as e:
logger.error(
"General error occurred while attempting to uninstall ElasticSearch."
.format(e))
logger.debug(
"General error occurred while attempting to uninstall ElasticSearch; {}"
.format(e))
raise elastic_exceptions.UninstallElasticsearchError(
"General error occurred while attempting to uninstall ElasticSearch; {}"
.format(e))
try:
sysctl = systemctl.SystemCtl()
except general_exceptions.CallProcessError:
raise elastic_exceptions.UninstallElasticsearchError(
"Could not find systemctl.")
sysctl.uninstall_and_disable('elasticsearch')
def setup(self,
node_name: Optional[str] = None,
network_host: Optional[str] = None,
port: Optional[int] = None,
initial_master_nodes: Optional[List[str]] = None,
discover_seed_hosts: Optional[List[str]] = None,
tls_cert_subject: Optional[str] = None,
heap_size_gigs: Optional[int] = None) -> None:
"""Setup Elasticsearch
Args:
node_name: The name of this elasticsearch node
network_host: The IP address to listen on (E.G "0.0.0.0")
port: The port that the ES API is bound to (E.G 9200)
initial_master_nodes: A list of nodes representing master (and master-eligible) nodes in this cluster
discover_seed_hosts: A list of IPs on other hosts you wish to form a cluster with
tls_cert_subject: Denotes the thing being secured; E.G (/C=US/ST=GA/L=Atlanta/O=Dynamite Analytics/OU=R&D/CN=dynamite.ai)
heap_size_gigs: The initial/max java heap space to allocate
Returns:
None
"""
sysctl = systemctl.SystemCtl()
# System patching and directory setup
self.logger.debug('Patching sysctl.')
utilities.update_sysctl()
self.logger.debug('Patching file-handle limits.')
utilities.update_user_file_handle_limits()
self.logger.debug(
f'Creating directory: {self.configuration_directory}')
utilities.makedirs(self.configuration_directory)
self.logger.debug(f'Creating directory: {self.install_directory}')
utilities.makedirs(self.install_directory)
self.logger.debug(f'Creating directory: {self.log_directory}')
utilities.makedirs(self.log_directory)
self.copy_elasticsearch_files_and_directories()
self.create_update_elasticsearch_environment_variables()
# Overwrite with dynamite default configurations
self.copy_file_or_directory_to_destination(
f'{const.DEFAULT_CONFIGS}/elasticsearch/elasticsearch.yml',
self.configuration_directory)
self.copy_file_or_directory_to_destination(
f'{const.DEFAULT_CONFIGS}/elasticsearch/jvm.options',
self.configuration_directory)
self.copy_file_or_directory_to_destination(
f'{const.DEFAULT_CONFIGS}/elasticsearch/security',
self.configuration_directory)
# Optimize Configurations
es_main_config = config.ConfigManager(self.configuration_directory,
verbose=self.verbose,
stdout=self.stdout)
es_java_config = config.JavaHeapOptionsConfigManager(
self.configuration_directory,
verbose=self.verbose,
stdout=self.stdout)
es_main_config.path_logs = self.log_directory
if not node_name:
node_name = utilities.get_default_es_node_name()
if not network_host:
network_host = utilities.get_primary_ip_address()
if not port:
port = 9200
if not initial_master_nodes:
initial_master_nodes = [node_name]
if not discover_seed_hosts:
discover_seed_hosts = [network_host]
if not tls_cert_subject:
tls_cert_subject = '/C=US/ST=GA/L=Atlanta/O=Dynamite/OU=R&D/CN=dynamite.ai'
else:
tls_cert_subject = tls_cert_subject
if not heap_size_gigs:
heap_size_gigs = int(
(utilities.get_memory_available_bytes() / 10**9) / 2)
formatted_subj = tls_cert_subject.lstrip("/").replace("/", ",")
formatted_subj_2 = ','.join(reversed(formatted_subj.split(',')))
es_main_config.node_name = node_name
es_main_config.network_host = network_host
es_main_config.http_port = port
es_main_config.initial_master_nodes = initial_master_nodes
es_main_config.seed_hosts = discover_seed_hosts
es_main_config.authcz_admin_distinguished_names = [
formatted_subj, formatted_subj_2
]
es_java_config.initial_memory = f'{heap_size_gigs}g'
es_java_config.maximum_memory = f'{heap_size_gigs}g'
self.logger.debug(
f'Java Heap Initial & Max Memory = {heap_size_gigs} GB')
es_main_config.commit()
es_java_config.commit()
self.logger.info('Applying configuration.')
# Fix Permissions
self.logger.info('Setting up file permissions.')
utilities.set_ownership_of_file(self.configuration_directory,
user='******',
group='dynamite')
utilities.set_ownership_of_file(self.install_directory,
user='******',
group='dynamite')
utilities.set_ownership_of_file(self.log_directory,
user='******',
group='dynamite')
# Install and enable service
self.logger.info(
f'Installing service -> {const.DEFAULT_CONFIGS}/systemd/elasticsearch.service'
)
sysctl.install_and_enable(
f'{const.DEFAULT_CONFIGS}/systemd/elasticsearch.service')
self.logger.info('Generating SSL Certificates and Keys.')
ssl_gen_task_results = setup_security.GenerateElasticsearchSSLCertificates(
subj=tls_cert_subject).invoke()
for res in ssl_gen_task_results:
cmd, out, err = res
self.logger.debug(f'{" ".join(cmd)} -> out: {out} err: {err}')
self.logger.info('Installing SSL Certificates and Keys')
install_ssl_task_results = setup_security.InstallElasticsearchCertificates(
network_host=network_host).invoke()
for res in install_ssl_task_results:
cmd, out, err = res
self.logger.debug(f'{" ".join(cmd)} -> out: {out} err: {err}')
self.logger.info('Setting up persistent cluster settings.')
configure_cluster_task_results = configure_cluster.UpdateClusterSettings(
network_host=network_host, http_port=port).invoke()
rcode, msg = configure_cluster_task_results
self.logger.debug(f'rcode: {rcode}-> msg: {msg}')
"""
from dynamite_nsm.services.elasticsearch import config
from dynamite_nsm.utilities import get_environment_file_dict
from dynamite_nsm.cmd.service_interfaces import SimpleConfigManagerInterface
env_vars = get_environment_file_dict()
try:
interface = \
SimpleConfigManagerInterface(config.ConfigManager(env_vars['ES_PATH_CONF']),
interface_name='Elasticsearch Configuration',
interface_description='Configure Elasticsearch on this machine.',
defaults=dict(configuration_directory=env_vars['ES_PATH_CONF'], stdout=True)
)
except KeyError:
interface = None
except FileNotFoundError:
interface = None
"""
if not interface:
print(f'Skipping elastic.config.main as it was never retrieved successfully')
"""