def install_logstash(host='0.0.0.0',
elasticsearch_host='localhost',
elasticsearch_port=9200,
elasticsearch_password='******',
install_jdk=True,
create_dynamite_user=True,
stdout=False):
"""
Install Logstash/ElastiFlow
:param host: The IP address to bind LogStash listeners too
:param elasticsearch_password: The password used for authentication across all builtin ES users
:param elasticsearch_host: A hostname/IP of the target elasticsearch instance
:param elasticsearch_port: A port number for the target elasticsearch instance
:param elasticsearch_password: The password used for authentication across all builtin ES users
:param install_jdk: Install the latest OpenJDK that will be used by Logstash/ElasticSearch
:param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run Logstash/ElasticSearch
:param stdout: Print the output to console
:return: True, if installation succeeded
"""
ls_profiler = LogstashProfiler()
if ls_profiler.is_installed:
sys.stderr.write(
'[-] LogStash is already installed. If you wish to re-install, first uninstall.\n'
)
return False
if utilities.get_memory_available_bytes() < 6 * (1000**3):
sys.stderr.write(
'[-] Dynamite Logstash requires at-least 6GB to run currently available [{} GB]\n'
.format(utilities.get_memory_available_bytes() / (1000**3)))
return False
try:
ls_installer = LogstashInstaller(
host=host,
elasticsearch_host=elasticsearch_host,
elasticsearch_port=elasticsearch_port,
elasticsearch_password=elasticsearch_password)
if install_jdk:
utilities.download_java(stdout=True)
utilities.extract_java(stdout=True)
utilities.setup_java()
if create_dynamite_user:
utilities.create_dynamite_user(
utilities.generate_random_password(50))
ls_installer.download_logstash(stdout=True)
ls_installer.extract_logstash(stdout=True)
ls_installer.setup_logstash(stdout=True)
except Exception:
sys.stderr.write(
'[-] A fatal error occurred while attempting to install LogStash: '
)
traceback.print_exc(file=sys.stderr)
return False
if stdout:
sys.stdout.write(
'[+] *** LogStash + ElastiFlow (w/ Zeek Support) installed successfully. ***\n\n'
)
sys.stdout.write(
'[+] Next, Start your collector: \'dynamite start logstash\'.\n')
sys.stdout.flush()
return True
def install_kibana(elasticsearch_host='localhost',
elasticsearch_port=9200,
elasticsearch_password='******',
install_jdk=True,
create_dynamite_user=True,
stdout=False,
verbose=False):
"""
Install Kibana/ElastiFlow Dashboards
:param elasticsearch_host: [Optional] A hostname/IP of the target elasticsearch instance
:param elasticsearch_port: [Optional] A port number for the target elasticsearch instance
:param elasticsearch_password: The password used for authentication across all builtin ES users
:param install_jdk: Install the latest OpenJDK that will be used by Logstash/ElasticSearch
:param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run
Logstash/ElasticSearch/Kibana
:param stdout: Print the output to console
:param verbose: Include output from system utilities
:return: True, if installation succeeded
"""
kb_profiler = KibanaProfiler()
if kb_profiler.is_installed:
sys.stderr.write(
'[-] Kibana is already installed. If you wish to re-install, first uninstall.\n'
)
return False
if utilities.get_memory_available_bytes() < 2 * (1000**3):
sys.stderr.write(
'[-] Dynamite Kibana requires at-least 2GB to run currently available [{} GB]\n'
.format(utilities.get_memory_available_bytes() / (1000**3)))
return False
try:
kb_installer = KibanaInstaller(
elasticsearch_host=elasticsearch_host,
elasticsearch_port=elasticsearch_port,
elasticsearch_password=elasticsearch_password,
download_kibana_archive=not kb_profiler.is_downloaded,
stdout=stdout,
verbose=verbose)
if install_jdk:
utilities.download_java(stdout=stdout)
utilities.extract_java(stdout=stdout)
utilities.setup_java()
if create_dynamite_user:
utilities.create_dynamite_user(
utilities.generate_random_password(50))
kb_installer.setup_kibana()
except Exception:
sys.stderr.write(
'[-] A fatal error occurred while attempting to install Kibana: ')
traceback.print_exc(file=sys.stderr)
return False
if stdout:
sys.stdout.write(
'[+] *** Kibana + Dashboards installed successfully. ***\n\n')
sys.stdout.write(
'[+] Next, Start your collector: \'dynamite start kibana\'.\n')
sys.stdout.flush()
return KibanaProfiler(stderr=False).is_installed
def install_elasticsearch(password='******',
install_jdk=True,
create_dynamite_user=True,
stdout=True,
verbose=False):
"""
Install ElasticSearch
:param password: The password used for authentication across all builtin users
:param install_jdk: Install the latest OpenJDK that will be used by Logstash/ElasticSearch
:param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run Logstash/ElasticSearch
:param stdout: Print the output to console
:param verbose: Include output from system utilities
:return: True, if installation succeeded
"""
es_profiler = ElasticProfiler()
if es_profiler.is_installed:
sys.stderr.write(
'[-] ElasticSearch is already installed. If you wish to re-install, first uninstall.\n'
)
return False
if utilities.get_memory_available_bytes() < 6 * (1000**3):
sys.stderr.write(
'[-] Dynamite ElasticSearch requires at-least 6GB to run currently available [{} GB]\n'
.format(utilities.get_memory_available_bytes() / (1000**3)))
return False
try:
es_installer = ElasticInstaller(
password=password,
download_elasticsearch_archive=not es_profiler.is_downloaded,
stdout=stdout,
verbose=verbose)
if install_jdk:
utilities.download_java(stdout=stdout)
utilities.extract_java(stdout=stdout)
utilities.setup_java()
if create_dynamite_user:
utilities.create_dynamite_user(
utilities.generate_random_password(50))
es_installer.setup_elasticsearch()
except Exception:
sys.stderr.write(
'[-] A fatal error occurred while attempting to install ElasticSearch: '
)
traceback.print_exc(file=sys.stderr)
return False
if stdout:
sys.stdout.write(
'[+] *** ElasticSearch installed successfully. ***\n\n')
sys.stdout.write(
'[+] Next, Start your cluster: \'dynamite start elasticsearch\'.\n'
)
sys.stdout.flush()
return ElasticProfiler(stderr=False).is_installed
def install_monitor(elasticsearch_password='******', verbose=False):
"""
Installs Logstash (with ElastiFlow templates modified to work with Zeek), ElasticSearch, and Kibana.
:param elasticsearch_password: The password used for authentication across all builtin ES users
:param verbose: Include output from system utilities
:return: True, if installation succeeded
"""
es_pre_profiler = elasticsearch.ElasticProfiler()
ls_pre_profiler = logstash.LogstashProfiler()
kb_pre_profiler = kibana.KibanaProfiler()
if ls_pre_profiler.is_installed and es_pre_profiler.is_installed and kb_pre_profiler.is_installed:
sys.stderr.write(
'[-] Monitor is already installed. If you wish to re-install, first uninstall.\n'
)
return False
if utilities.get_memory_available_bytes() < 14 * (1000**3):
sys.stderr.write(
'[-] WARNING Dynamite standalone monitor requires '
'at-least 14GB to run currently available [{} GB]\n'.format(
utilities.get_memory_available_bytes() / (1024**3)))
if str(utilities.prompt_input('Continue? [y|N]: ')).lower() != 'y':
return False
utilities.create_dynamite_user(utilities.generate_random_password(50))
utilities.download_java(stdout=True)
utilities.extract_java(stdout=True)
utilities.setup_java()
if not es_pre_profiler.is_installed:
sys.stdout.write('[+] Installing Elasticsearch on localhost.\n')
es_installer = elasticsearch.ElasticInstaller(
host='0.0.0.0',
port=9200,
download_elasticsearch_archive=not ls_pre_profiler.is_downloaded,
password=elasticsearch_password,
stdout=True,
verbose=verbose)
es_installer.setup_elasticsearch()
if not elasticsearch.ElasticProfiler().is_installed:
sys.stderr.write(
'[-] ElasticSearch failed to install on localhost.\n')
return False
sys.stdout.write('[+] Starting ElasticSearch on localhost.\n')
es_process = elasticsearch.ElasticProcess()
es_process.start()
if not ls_pre_profiler.is_installed:
ls_installer = logstash.LogstashInstaller(
host='0.0.0.0',
elasticsearch_password=elasticsearch_password,
download_logstash_archive=not es_pre_profiler.is_downloaded,
stdout=True,
verbose=verbose)
ls_installer.setup_logstash()
if not logstash.LogstashProfiler().is_installed:
sys.stderr.write('[-] LogStash failed to install on localhost.\n')
return False
if not kb_pre_profiler.is_installed and elasticsearch.ElasticProfiler(
).is_installed:
sys.stdout.write('[+] Installing Kibana on localhost.\n')
kb_installer = kibana.KibanaInstaller(
host='0.0.0.0',
port=5601,
elasticsearch_host='localhost',
elasticsearch_port=9200,
elasticsearch_password=elasticsearch_password,
download_kibana_archive=not kb_pre_profiler.is_downloaded,
stdout=True,
verbose=verbose)
if not kb_pre_profiler.is_downloaded:
kb_installer.download_kibana(stdout=True)
kb_installer.extract_kibana(stdout=True)
kb_installer.setup_kibana()
if not kibana.KibanaProfiler().is_installed:
sys.stderr.write('[-] Kibana failed to install on localhost.\n')
return False
sys.stdout.write(
'[+] Monitor installation complete. Start the monitor: \'dynamite start monitor\'.\n'
)
sys.stdout.flush()
return True
def install_monitor(elasticsearch_password='******'):
"""
Installs Logstash (with ElastiFlow templates modified to work with Zeek), ElasticSearch, and Kibana.
:return: True, if installation succeeded
"""
if utilities.get_memory_available_bytes() < 14 * (1000**3):
sys.stderr.write(
'[-] Dynamite standalone monitor requires '
'at-least 14GB to run currently available [{} GB]\n'.format(
utilities.get_memory_available_bytes() / (1024**3)))
return False
utilities.create_dynamite_user(utilities.generate_random_password(50))
utilities.download_java(stdout=True)
utilities.extract_java(stdout=True)
utilities.setup_java()
es_installer = elasticsearch.ElasticInstaller(
host='0.0.0.0', port=9200, password=elasticsearch_password)
es_pre_profiler = elasticsearch.ElasticProfiler()
es_process = elasticsearch.ElasticProcess()
ls_installer = logstash.LogstashInstaller(
host='0.0.0.0', elasticsearch_password=elasticsearch_password)
ls_pre_profiler = logstash.LogstashProfiler()
kb_installer = kibana.KibanaInstaller(
host='0.0.0.0',
port=5601,
elasticsearch_host='localhost',
elasticsearch_port=9200,
elasticsearch_password=elasticsearch_password)
kb_pre_profiler = kibana.KibanaProfiler()
if not es_pre_profiler.is_installed:
sys.stdout.write('[+] Installing Elasticsearch on localhost.\n')
if not es_pre_profiler.is_downloaded:
es_installer.download_elasticsearch(stdout=True)
es_installer.extract_elasticsearch(stdout=True)
es_installer.setup_elasticsearch(stdout=True)
if not elasticsearch.ElasticProfiler().is_installed:
sys.stderr.write(
'[-] ElasticSearch failed to install on localhost.\n')
return False
sys.stdout.write('[+] Starting ElasticSearch on localhost.\n')
es_process.start()
if not ls_pre_profiler.is_installed:
if not ls_pre_profiler.is_downloaded:
ls_installer.download_logstash(stdout=True)
ls_installer.extract_logstash(stdout=True)
ls_installer.setup_logstash(stdout=True)
if not logstash.LogstashProfiler().is_installed:
sys.stderr.write('[-] LogStash failed to install on localhost.\n')
return False
if not kb_pre_profiler.is_installed and elasticsearch.ElasticProfiler(
).is_installed:
sys.stdout.write('[+] Installing Kibana on localhost.\n')
if not kb_pre_profiler.is_downloaded:
kb_installer.download_kibana(stdout=True)
kb_installer.extract_kibana(stdout=True)
kb_installer.setup_kibana(stdout=True)
if not kibana.KibanaProfiler().is_installed:
sys.stderr.write('[-] Kibana failed to install on localhost.\n')
return False
sys.stdout.write(
'[+] Monitor installation complete. Start the monitor: \'dynamite start monitor\'.\n'
)
sys.stdout.flush()
return True
def install_logstash(configuration_directory,
install_directory,
log_directory,
host='0.0.0.0',
elasticsearch_host='localhost',
elasticsearch_port=9200,
elasticsearch_password='******',
heap_size_gigs=4,
install_jdk=True,
create_dynamite_user=True,
stdout=False,
verbose=False):
"""
Install Logstash with ElastiFlow & Synesis
:param configuration_directory: Path to the configuration directory (E.G /etc/dynamite/logstash/)
:param install_directory: Path to the install directory (E.G /opt/dynamite/logstash/)
:param log_directory: Path to the log directory (E.G /var/log/dynamite/logstash/)
:param host: The IP address to bind LogStash listeners too
:param elasticsearch_password: The password used for authentication across all builtin ES users
:param elasticsearch_host: A hostname/IP of the target elasticsearch instance
:param elasticsearch_port: A port number for the target elasticsearch instance
:param elasticsearch_password: The password used for authentication across all builtin ES users
:param heap_size_gigs: The initial/max java heap space to allocate
:param install_jdk: Install the latest OpenJDK that will be used by Logstash/ElasticSearch
:param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run Logstash/ElasticSearch
:param stdout: Print the output to console
:param verbose: Include output from system utilities
:return: True, if installation succeeded
"""
log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
logger = get_logger('LOGSTASH', level=log_level, stdout=stdout)
ls_profiler = logstash_profile.ProcessProfiler()
if ls_profiler.is_installed():
logger.error('LogStash is already installed.')
raise logstash_exceptions.AlreadyInstalledLogstashError()
if utilities.get_memory_available_bytes() < 6 * (1000**3):
sys.stderr.write(
'\n\033[93m[-] WARNING! LogStash should have at-least 6GB to run '
'currently available [{} GB]\033[0m\n'.format(
utilities.get_memory_available_bytes() / (1000**3)))
if str(utilities.prompt_input(
'\033[93m[?] Continue? [y|N]:\033[0m ')).lower() != 'y':
sys.stdout.write('\n[+] Exiting\n')
exit(0)
ls_installer = InstallManager(
configuration_directory,
install_directory,
log_directory,
host=host,
elasticsearch_host=elasticsearch_host,
elasticsearch_port=elasticsearch_port,
elasticsearch_password=elasticsearch_password,
heap_size_gigs=heap_size_gigs,
download_logstash_archive=not ls_profiler.is_downloaded(),
stdout=stdout,
verbose=verbose)
if install_jdk:
try:
utilities.download_java(stdout=stdout)
utilities.extract_java()
utilities.setup_java()
except Exception as e:
logger.error(
'General error occurred while attempting to setup Java.')
logger.debug(
"General error occurred while attempting to setup Java; {}".
format(e))
raise logstash_exceptions.InstallLogstashError(
"General error occurred while attempting to setup Java; {}".
format(e))
if create_dynamite_user:
utilities.create_dynamite_user(utilities.generate_random_password(50))
ls_installer.setup_logstash()
def install_logstash(host='0.0.0.0',
elasticsearch_host='localhost',
elasticsearch_port=9200,
elasticsearch_password='******',
install_jdk=True,
create_dynamite_user=True,
stdout=False,
verbose=False):
"""
Install Logstash/ElastiFlow
:param host: The IP address to bind LogStash listeners too
:param elasticsearch_password: The password used for authentication across all builtin ES users
:param elasticsearch_host: A hostname/IP of the target elasticsearch instance
:param elasticsearch_port: A port number for the target elasticsearch instance
:param elasticsearch_password: The password used for authentication across all builtin ES users
:param install_jdk: Install the latest OpenJDK that will be used by Logstash/ElasticSearch
:param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run Logstash/ElasticSearch
:param stdout: Print the output to console
:param verbose: Include output from system utilities
:return: True, if installation succeeded
"""
ls_profiler = LogstashProfiler()
if ls_profiler.is_installed:
sys.stderr.write(
'[-] LogStash is already installed. If you wish to re-install, first uninstall.\n'
)
return False
if utilities.get_memory_available_bytes() < 6 * (1000**3):
sys.stderr.write(
'[-] WARNING! Dynamite Logstash should have at-least 6GB to run '
'currently available [{} GB]\n'.format(
utilities.get_memory_available_bytes() / (1000**3)))
if str(utilities.prompt_input('Continue? [y|N]: ')).lower() != 'y':
return False
try:
ls_installer = LogstashInstaller(
host=host,
elasticsearch_host=elasticsearch_host,
elasticsearch_port=elasticsearch_port,
elasticsearch_password=elasticsearch_password,
download_logstash_archive=not ls_profiler.is_downloaded,
stdout=stdout,
verbose=verbose)
if install_jdk:
utilities.download_java(stdout=stdout)
utilities.extract_java(stdout=stdout)
utilities.setup_java()
if create_dynamite_user:
utilities.create_dynamite_user(
utilities.generate_random_password(50))
ls_installer.setup_logstash()
except Exception:
sys.stderr.write(
'[-] A fatal error occurred while attempting to install LogStash: '
)
traceback.print_exc(file=sys.stderr)
return False
if stdout:
sys.stdout.write(
'[+] *** LogStash installed event/alert pipelines. ***\n\n')
sys.stdout.write(
'[+] Next, Start your collector: \'dynamite start logstash\'.\n')
sys.stdout.flush()
return LogstashProfiler(stderr=False).is_installed
def install_elasticsearch(configuration_directory,
install_directory,
log_directory,
password='******',
heap_size_gigs=4,
install_jdk=True,
create_dynamite_user=True,
stdout=True,
verbose=False):
"""
Install ElasticSearch
:param configuration_directory: Path to the configuration directory (E.G /etc/dynamite/elasticsearch/)
:param install_directory: Path to the install directory (E.G /opt/dynamite/elasticsearch/)
:param log_directory: Path to the log directory (E.G /var/log/dynamite/elasticsearch/)
:param password: The password used for authentication across all builtin users
:param heap_size_gigs: The initial/max java heap space to allocate
:param install_jdk: Install the latest OpenJDK that will be used by Logstash/ElasticSearch
:param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run Logstash/ElasticSearch
:param stdout: Print the output to console
:param verbose: Include detailed debug messages
"""
log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
logger = get_logger('ELASTICSEARCH', level=log_level, stdout=stdout)
es_profiler = elastic_profile.ProcessProfiler()
if es_profiler.is_installed():
logger.error('ElasticSearch is already installed.')
raise elastic_exceptions.AlreadyInstalledElasticsearchError()
if utilities.get_memory_available_bytes() < 6 * (1000**3):
sys.stderr.write(
'\n\033[93m[-] WARNING! ElasticSearch should have at-least 6GB to run '
'currently available [{} GB]\033[0m\n'.format(
utilities.get_memory_available_bytes() / (1000**3)))
if str(utilities.prompt_input(
'\033[93m[?] Continue? [y|N]:\033[0m ')).lower() != 'y':
sys.stdout.write('\n[+] Exiting\n')
exit(0)
es_installer = InstallManager(
configuration_directory=configuration_directory,
install_directory=install_directory,
log_directory=log_directory,
password=password,
heap_size_gigs=heap_size_gigs,
download_elasticsearch_archive=not es_profiler.is_downloaded(),
stdout=stdout,
verbose=verbose)
if install_jdk:
try:
utilities.download_java(stdout=stdout)
utilities.extract_java()
utilities.setup_java()
except Exception as e:
logger.error(
'General error occurred while attempting to setup Java.')
logger.debug(
"General error occurred while attempting to setup Java; {}".
format(e))
raise elastic_exceptions.InstallElasticsearchError(
"General error occurred while attempting to setup Java; {}".
format(e))
if create_dynamite_user:
utilities.create_dynamite_user(utilities.generate_random_password(50))
es_installer.setup_elasticsearch()