def _update_sysctl(stdout=False):
if stdout:
sys.stdout.write(
'[+] Setting up Max File Handles [65535] VM Max Map Count [262144] \n'
)
utilities.update_user_file_handle_limits()
utilities.update_sysctl()
def _update_sysctl(self):
self.logger.info(
'Setting up Max File Handles [65535] VM Max Map Count [262144]')
try:
utilities.update_user_file_handle_limits()
except Exception as e:
self.logger.error(
'General error while setting user file-handle limits.')
self.logger.debug(
"General error while setting user file-handle limits; {}".
format(e))
raise logstash_exceptions.InstallLogstashError(
"General error while setting user file-handle limits; {}".
format(e))
try:
utilities.update_sysctl(verbose=self.verbose)
except Exception as e:
self.logger.error('General error while setting VM Max Map Count.')
self.logger.debug(
"General error while setting VM Max Map Count; {}".format(e))
raise logstash_exceptions.InstallLogstashError(
"General error while setting VM Max Map Count; {}".format(e))
def setup(self, node_name: Optional[str] = None, host: Optional[str] = None,
elasticsearch_host: Optional[str] = None, elasticsearch_port: Optional[int] = None,
pipeline_batch_size: Optional[int] = None, pipeline_batch_delay: Optional[int] = None,
heap_size_gigs: Optional[int] = None):
sysctl = systemctl.SystemCtl()
# System patching and directory setup
self.logger.debug('Patching sysctl.')
utilities.update_sysctl()
self.logger.debug('Patching file-handle limits.')
utilities.update_user_file_handle_limits()
utilities.makedirs(self.configuration_directory)
utilities.makedirs(self.install_directory)
utilities.makedirs(self.log_directory)
self.copy_logstash_fills_and_directories()
self.create_update_logstash_environment_variables()
# Overwrite with dynamite default configurations
self.copy_file_or_directory_to_destination(f'{const.DEFAULT_CONFIGS}/logstash/logstash.yml',
self.configuration_directory)
self.copy_file_or_directory_to_destination(f'{const.DEFAULT_CONFIGS}/logstash/jvm.options',
self.configuration_directory)
# Optimize Configurations
ls_main_config = config.ConfigManager(self.configuration_directory)
ls_java_config = config.JavaHeapOptionsConfigManager(self.configuration_directory)
ls_main_config.path_logs = self.log_directory
if not node_name:
node_name = utilities.get_default_es_node_name().replace('es_node', 'ls_node')
if not host:
host = utilities.get_primary_ip_address()
if not elasticsearch_host:
elasticsearch_host = utilities.get_primary_ip_address()
if not elasticsearch_port:
elasticsearch_port = 9200
if not pipeline_batch_size:
pipeline_batch_size = 125
if not pipeline_batch_delay:
pipeline_batch_delay = 50
if not heap_size_gigs:
heap_size_gigs = int((utilities.get_memory_available_bytes() / 10 ** 9) / 2)
self.logger.debug(f'Logstash will connect to Elasticsearch on {elasticsearch_host}:{elasticsearch_port}')
ls_main_config.node_name = node_name
ls_main_config.host = host
ls_main_config.pipeline_batch_size = pipeline_batch_size
ls_main_config.pipeline_batch_delay = pipeline_batch_delay
self.create_update_env_variable('LS_ES_HOST', elasticsearch_host)
self.create_update_env_variable('LS_ES_PORT', elasticsearch_port)
ls_java_config.initial_memory = f'{heap_size_gigs}g'
ls_java_config.maximum_memory = f'{heap_size_gigs}g'
self.logger.debug(f'Java Heap Initial & Max Memory = {heap_size_gigs} GB')
ls_main_config.commit()
ls_java_config.commit()
self.logger.info('Applying configuration.')
# Fix Permissions
self.logger.info('Setting up file permissions.')
utilities.set_ownership_of_file(self.configuration_directory, user='******', group='dynamite')
utilities.set_ownership_of_file(self.install_directory, user='******', group='dynamite')
utilities.set_ownership_of_file(self.log_directory, user='******', group='dynamite')
# Install and enable service
self.logger.info(f'Installing service -> {const.DEFAULT_CONFIGS}/systemd/logstash.service')
sysctl.install_and_enable(f'{const.DEFAULT_CONFIGS}/systemd/logstash.service')
def setup(self,
node_name: Optional[str] = None,
network_host: Optional[str] = None,
port: Optional[int] = None,
initial_master_nodes: Optional[List[str]] = None,
discover_seed_hosts: Optional[List[str]] = None,
tls_cert_subject: Optional[str] = None,
heap_size_gigs: Optional[int] = None) -> None:
"""Setup Elasticsearch
Args:
node_name: The name of this elasticsearch node
network_host: The IP address to listen on (E.G "0.0.0.0")
port: The port that the ES API is bound to (E.G 9200)
initial_master_nodes: A list of nodes representing master (and master-eligible) nodes in this cluster
discover_seed_hosts: A list of IPs on other hosts you wish to form a cluster with
tls_cert_subject: Denotes the thing being secured; E.G (/C=US/ST=GA/L=Atlanta/O=Dynamite Analytics/OU=R&D/CN=dynamite.ai)
heap_size_gigs: The initial/max java heap space to allocate
Returns:
None
"""
sysctl = systemctl.SystemCtl()
# System patching and directory setup
self.logger.debug('Patching sysctl.')
utilities.update_sysctl()
self.logger.debug('Patching file-handle limits.')
utilities.update_user_file_handle_limits()
self.logger.debug(
f'Creating directory: {self.configuration_directory}')
utilities.makedirs(self.configuration_directory)
self.logger.debug(f'Creating directory: {self.install_directory}')
utilities.makedirs(self.install_directory)
self.logger.debug(f'Creating directory: {self.log_directory}')
utilities.makedirs(self.log_directory)
self.copy_elasticsearch_files_and_directories()
self.create_update_elasticsearch_environment_variables()
# Overwrite with dynamite default configurations
self.copy_file_or_directory_to_destination(
f'{const.DEFAULT_CONFIGS}/elasticsearch/elasticsearch.yml',
self.configuration_directory)
self.copy_file_or_directory_to_destination(
f'{const.DEFAULT_CONFIGS}/elasticsearch/jvm.options',
self.configuration_directory)
self.copy_file_or_directory_to_destination(
f'{const.DEFAULT_CONFIGS}/elasticsearch/security',
self.configuration_directory)
# Optimize Configurations
es_main_config = config.ConfigManager(self.configuration_directory,
verbose=self.verbose,
stdout=self.stdout)
es_java_config = config.JavaHeapOptionsConfigManager(
self.configuration_directory,
verbose=self.verbose,
stdout=self.stdout)
es_main_config.path_logs = self.log_directory
if not node_name:
node_name = utilities.get_default_es_node_name()
if not network_host:
network_host = utilities.get_primary_ip_address()
if not port:
port = 9200
if not initial_master_nodes:
initial_master_nodes = [node_name]
if not discover_seed_hosts:
discover_seed_hosts = [network_host]
if not tls_cert_subject:
tls_cert_subject = '/C=US/ST=GA/L=Atlanta/O=Dynamite/OU=R&D/CN=dynamite.ai'
else:
tls_cert_subject = tls_cert_subject
if not heap_size_gigs:
heap_size_gigs = int(
(utilities.get_memory_available_bytes() / 10**9) / 2)
formatted_subj = tls_cert_subject.lstrip("/").replace("/", ",")
formatted_subj_2 = ','.join(reversed(formatted_subj.split(',')))
es_main_config.node_name = node_name
es_main_config.network_host = network_host
es_main_config.http_port = port
es_main_config.initial_master_nodes = initial_master_nodes
es_main_config.seed_hosts = discover_seed_hosts
es_main_config.authcz_admin_distinguished_names = [
formatted_subj, formatted_subj_2
]
es_java_config.initial_memory = f'{heap_size_gigs}g'
es_java_config.maximum_memory = f'{heap_size_gigs}g'
self.logger.debug(
f'Java Heap Initial & Max Memory = {heap_size_gigs} GB')
es_main_config.commit()
es_java_config.commit()
self.logger.info('Applying configuration.')
# Fix Permissions
self.logger.info('Setting up file permissions.')
utilities.set_ownership_of_file(self.configuration_directory,
user='******',
group='dynamite')
utilities.set_ownership_of_file(self.install_directory,
user='******',
group='dynamite')
utilities.set_ownership_of_file(self.log_directory,
user='******',
group='dynamite')
# Install and enable service
self.logger.info(
f'Installing service -> {const.DEFAULT_CONFIGS}/systemd/elasticsearch.service'
)
sysctl.install_and_enable(
f'{const.DEFAULT_CONFIGS}/systemd/elasticsearch.service')
self.logger.info('Generating SSL Certificates and Keys.')
ssl_gen_task_results = setup_security.GenerateElasticsearchSSLCertificates(
subj=tls_cert_subject).invoke()
for res in ssl_gen_task_results:
cmd, out, err = res
self.logger.debug(f'{" ".join(cmd)} -> out: {out} err: {err}')
self.logger.info('Installing SSL Certificates and Keys')
install_ssl_task_results = setup_security.InstallElasticsearchCertificates(
network_host=network_host).invoke()
for res in install_ssl_task_results:
cmd, out, err = res
self.logger.debug(f'{" ".join(cmd)} -> out: {out} err: {err}')
self.logger.info('Setting up persistent cluster settings.')
configure_cluster_task_results = configure_cluster.UpdateClusterSettings(
network_host=network_host, http_port=port).invoke()
rcode, msg = configure_cluster_task_results
self.logger.debug(f'rcode: {rcode}-> msg: {msg}')
"""