def install_logstash(host='0.0.0.0', elasticsearch_host='localhost', elasticsearch_port=9200, elasticsearch_password='******', install_jdk=True, create_dynamite_user=True, stdout=False): """ Install Logstash/ElastiFlow :param host: The IP address to bind LogStash listeners too :param elasticsearch_password: The password used for authentication across all builtin ES users :param elasticsearch_host: A hostname/IP of the target elasticsearch instance :param elasticsearch_port: A port number for the target elasticsearch instance :param elasticsearch_password: The password used for authentication across all builtin ES users :param install_jdk: Install the latest OpenJDK that will be used by Logstash/ElasticSearch :param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run Logstash/ElasticSearch :param stdout: Print the output to console :return: True, if installation succeeded """ ls_profiler = LogstashProfiler() if ls_profiler.is_installed: sys.stderr.write( '[-] LogStash is already installed. If you wish to re-install, first uninstall.\n' ) return False if utilities.get_memory_available_bytes() < 6 * (1000**3): sys.stderr.write( '[-] Dynamite Logstash requires at-least 6GB to run currently available [{} GB]\n' .format(utilities.get_memory_available_bytes() / (1000**3))) return False try: ls_installer = LogstashInstaller( host=host, elasticsearch_host=elasticsearch_host, elasticsearch_port=elasticsearch_port, elasticsearch_password=elasticsearch_password) if install_jdk: utilities.download_java(stdout=True) utilities.extract_java(stdout=True) utilities.setup_java() if create_dynamite_user: utilities.create_dynamite_user( utilities.generate_random_password(50)) ls_installer.download_logstash(stdout=True) ls_installer.extract_logstash(stdout=True) ls_installer.setup_logstash(stdout=True) except Exception: sys.stderr.write( '[-] A fatal error occurred while attempting to install LogStash: ' ) traceback.print_exc(file=sys.stderr) return False if stdout: sys.stdout.write( '[+] *** LogStash + ElastiFlow (w/ Zeek Support) installed successfully. ***\n\n' ) sys.stdout.write( '[+] Next, Start your collector: \'dynamite start logstash\'.\n') sys.stdout.flush() return True
def install_kibana(elasticsearch_host='localhost', elasticsearch_port=9200, elasticsearch_password='******', install_jdk=True, create_dynamite_user=True, stdout=False, verbose=False): """ Install Kibana/ElastiFlow Dashboards :param elasticsearch_host: [Optional] A hostname/IP of the target elasticsearch instance :param elasticsearch_port: [Optional] A port number for the target elasticsearch instance :param elasticsearch_password: The password used for authentication across all builtin ES users :param install_jdk: Install the latest OpenJDK that will be used by Logstash/ElasticSearch :param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run Logstash/ElasticSearch/Kibana :param stdout: Print the output to console :param verbose: Include output from system utilities :return: True, if installation succeeded """ kb_profiler = KibanaProfiler() if kb_profiler.is_installed: sys.stderr.write( '[-] Kibana is already installed. If you wish to re-install, first uninstall.\n' ) return False if utilities.get_memory_available_bytes() < 2 * (1000**3): sys.stderr.write( '[-] Dynamite Kibana requires at-least 2GB to run currently available [{} GB]\n' .format(utilities.get_memory_available_bytes() / (1000**3))) return False try: kb_installer = KibanaInstaller( elasticsearch_host=elasticsearch_host, elasticsearch_port=elasticsearch_port, elasticsearch_password=elasticsearch_password, download_kibana_archive=not kb_profiler.is_downloaded, stdout=stdout, verbose=verbose) if install_jdk: utilities.download_java(stdout=stdout) utilities.extract_java(stdout=stdout) utilities.setup_java() if create_dynamite_user: utilities.create_dynamite_user( utilities.generate_random_password(50)) kb_installer.setup_kibana() except Exception: sys.stderr.write( '[-] A fatal error occurred while attempting to install Kibana: ') traceback.print_exc(file=sys.stderr) return False if stdout: sys.stdout.write( '[+] *** Kibana + Dashboards installed successfully. ***\n\n') sys.stdout.write( '[+] Next, Start your collector: \'dynamite start kibana\'.\n') sys.stdout.flush() return KibanaProfiler(stderr=False).is_installed
def install_elasticsearch(password='******', install_jdk=True, create_dynamite_user=True, stdout=True, verbose=False): """ Install ElasticSearch :param password: The password used for authentication across all builtin users :param install_jdk: Install the latest OpenJDK that will be used by Logstash/ElasticSearch :param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run Logstash/ElasticSearch :param stdout: Print the output to console :param verbose: Include output from system utilities :return: True, if installation succeeded """ es_profiler = ElasticProfiler() if es_profiler.is_installed: sys.stderr.write( '[-] ElasticSearch is already installed. If you wish to re-install, first uninstall.\n' ) return False if utilities.get_memory_available_bytes() < 6 * (1000**3): sys.stderr.write( '[-] Dynamite ElasticSearch requires at-least 6GB to run currently available [{} GB]\n' .format(utilities.get_memory_available_bytes() / (1000**3))) return False try: es_installer = ElasticInstaller( password=password, download_elasticsearch_archive=not es_profiler.is_downloaded, stdout=stdout, verbose=verbose) if install_jdk: utilities.download_java(stdout=stdout) utilities.extract_java(stdout=stdout) utilities.setup_java() if create_dynamite_user: utilities.create_dynamite_user( utilities.generate_random_password(50)) es_installer.setup_elasticsearch() except Exception: sys.stderr.write( '[-] A fatal error occurred while attempting to install ElasticSearch: ' ) traceback.print_exc(file=sys.stderr) return False if stdout: sys.stdout.write( '[+] *** ElasticSearch installed successfully. ***\n\n') sys.stdout.write( '[+] Next, Start your cluster: \'dynamite start elasticsearch\'.\n' ) sys.stdout.flush() return ElasticProfiler(stderr=False).is_installed
def install_monitor(elasticsearch_password='******', verbose=False): """ Installs Logstash (with ElastiFlow templates modified to work with Zeek), ElasticSearch, and Kibana. :param elasticsearch_password: The password used for authentication across all builtin ES users :param verbose: Include output from system utilities :return: True, if installation succeeded """ es_pre_profiler = elasticsearch.ElasticProfiler() ls_pre_profiler = logstash.LogstashProfiler() kb_pre_profiler = kibana.KibanaProfiler() if ls_pre_profiler.is_installed and es_pre_profiler.is_installed and kb_pre_profiler.is_installed: sys.stderr.write( '[-] Monitor is already installed. If you wish to re-install, first uninstall.\n' ) return False if utilities.get_memory_available_bytes() < 14 * (1000**3): sys.stderr.write( '[-] WARNING Dynamite standalone monitor requires ' 'at-least 14GB to run currently available [{} GB]\n'.format( utilities.get_memory_available_bytes() / (1024**3))) if str(utilities.prompt_input('Continue? [y|N]: ')).lower() != 'y': return False utilities.create_dynamite_user(utilities.generate_random_password(50)) utilities.download_java(stdout=True) utilities.extract_java(stdout=True) utilities.setup_java() if not es_pre_profiler.is_installed: sys.stdout.write('[+] Installing Elasticsearch on localhost.\n') es_installer = elasticsearch.ElasticInstaller( host='0.0.0.0', port=9200, download_elasticsearch_archive=not ls_pre_profiler.is_downloaded, password=elasticsearch_password, stdout=True, verbose=verbose) es_installer.setup_elasticsearch() if not elasticsearch.ElasticProfiler().is_installed: sys.stderr.write( '[-] ElasticSearch failed to install on localhost.\n') return False sys.stdout.write('[+] Starting ElasticSearch on localhost.\n') es_process = elasticsearch.ElasticProcess() es_process.start() if not ls_pre_profiler.is_installed: ls_installer = logstash.LogstashInstaller( host='0.0.0.0', elasticsearch_password=elasticsearch_password, download_logstash_archive=not es_pre_profiler.is_downloaded, stdout=True, verbose=verbose) ls_installer.setup_logstash() if not logstash.LogstashProfiler().is_installed: sys.stderr.write('[-] LogStash failed to install on localhost.\n') return False if not kb_pre_profiler.is_installed and elasticsearch.ElasticProfiler( ).is_installed: sys.stdout.write('[+] Installing Kibana on localhost.\n') kb_installer = kibana.KibanaInstaller( host='0.0.0.0', port=5601, elasticsearch_host='localhost', elasticsearch_port=9200, elasticsearch_password=elasticsearch_password, download_kibana_archive=not kb_pre_profiler.is_downloaded, stdout=True, verbose=verbose) if not kb_pre_profiler.is_downloaded: kb_installer.download_kibana(stdout=True) kb_installer.extract_kibana(stdout=True) kb_installer.setup_kibana() if not kibana.KibanaProfiler().is_installed: sys.stderr.write('[-] Kibana failed to install on localhost.\n') return False sys.stdout.write( '[+] Monitor installation complete. Start the monitor: \'dynamite start monitor\'.\n' ) sys.stdout.flush() return True
def install_monitor(elasticsearch_password='******'): """ Installs Logstash (with ElastiFlow templates modified to work with Zeek), ElasticSearch, and Kibana. :return: True, if installation succeeded """ if utilities.get_memory_available_bytes() < 14 * (1000**3): sys.stderr.write( '[-] Dynamite standalone monitor requires ' 'at-least 14GB to run currently available [{} GB]\n'.format( utilities.get_memory_available_bytes() / (1024**3))) return False utilities.create_dynamite_user(utilities.generate_random_password(50)) utilities.download_java(stdout=True) utilities.extract_java(stdout=True) utilities.setup_java() es_installer = elasticsearch.ElasticInstaller( host='0.0.0.0', port=9200, password=elasticsearch_password) es_pre_profiler = elasticsearch.ElasticProfiler() es_process = elasticsearch.ElasticProcess() ls_installer = logstash.LogstashInstaller( host='0.0.0.0', elasticsearch_password=elasticsearch_password) ls_pre_profiler = logstash.LogstashProfiler() kb_installer = kibana.KibanaInstaller( host='0.0.0.0', port=5601, elasticsearch_host='localhost', elasticsearch_port=9200, elasticsearch_password=elasticsearch_password) kb_pre_profiler = kibana.KibanaProfiler() if not es_pre_profiler.is_installed: sys.stdout.write('[+] Installing Elasticsearch on localhost.\n') if not es_pre_profiler.is_downloaded: es_installer.download_elasticsearch(stdout=True) es_installer.extract_elasticsearch(stdout=True) es_installer.setup_elasticsearch(stdout=True) if not elasticsearch.ElasticProfiler().is_installed: sys.stderr.write( '[-] ElasticSearch failed to install on localhost.\n') return False sys.stdout.write('[+] Starting ElasticSearch on localhost.\n') es_process.start() if not ls_pre_profiler.is_installed: if not ls_pre_profiler.is_downloaded: ls_installer.download_logstash(stdout=True) ls_installer.extract_logstash(stdout=True) ls_installer.setup_logstash(stdout=True) if not logstash.LogstashProfiler().is_installed: sys.stderr.write('[-] LogStash failed to install on localhost.\n') return False if not kb_pre_profiler.is_installed and elasticsearch.ElasticProfiler( ).is_installed: sys.stdout.write('[+] Installing Kibana on localhost.\n') if not kb_pre_profiler.is_downloaded: kb_installer.download_kibana(stdout=True) kb_installer.extract_kibana(stdout=True) kb_installer.setup_kibana(stdout=True) if not kibana.KibanaProfiler().is_installed: sys.stderr.write('[-] Kibana failed to install on localhost.\n') return False sys.stdout.write( '[+] Monitor installation complete. Start the monitor: \'dynamite start monitor\'.\n' ) sys.stdout.flush() return True
def install_logstash(configuration_directory, install_directory, log_directory, host='0.0.0.0', elasticsearch_host='localhost', elasticsearch_port=9200, elasticsearch_password='******', heap_size_gigs=4, install_jdk=True, create_dynamite_user=True, stdout=False, verbose=False): """ Install Logstash with ElastiFlow & Synesis :param configuration_directory: Path to the configuration directory (E.G /etc/dynamite/logstash/) :param install_directory: Path to the install directory (E.G /opt/dynamite/logstash/) :param log_directory: Path to the log directory (E.G /var/log/dynamite/logstash/) :param host: The IP address to bind LogStash listeners too :param elasticsearch_password: The password used for authentication across all builtin ES users :param elasticsearch_host: A hostname/IP of the target elasticsearch instance :param elasticsearch_port: A port number for the target elasticsearch instance :param elasticsearch_password: The password used for authentication across all builtin ES users :param heap_size_gigs: The initial/max java heap space to allocate :param install_jdk: Install the latest OpenJDK that will be used by Logstash/ElasticSearch :param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run Logstash/ElasticSearch :param stdout: Print the output to console :param verbose: Include output from system utilities :return: True, if installation succeeded """ log_level = logging.INFO if verbose: log_level = logging.DEBUG logger = get_logger('LOGSTASH', level=log_level, stdout=stdout) ls_profiler = logstash_profile.ProcessProfiler() if ls_profiler.is_installed(): logger.error('LogStash is already installed.') raise logstash_exceptions.AlreadyInstalledLogstashError() if utilities.get_memory_available_bytes() < 6 * (1000**3): sys.stderr.write( '\n\033[93m[-] WARNING! LogStash should have at-least 6GB to run ' 'currently available [{} GB]\033[0m\n'.format( utilities.get_memory_available_bytes() / (1000**3))) if str(utilities.prompt_input( '\033[93m[?] Continue? [y|N]:\033[0m ')).lower() != 'y': sys.stdout.write('\n[+] Exiting\n') exit(0) ls_installer = InstallManager( configuration_directory, install_directory, log_directory, host=host, elasticsearch_host=elasticsearch_host, elasticsearch_port=elasticsearch_port, elasticsearch_password=elasticsearch_password, heap_size_gigs=heap_size_gigs, download_logstash_archive=not ls_profiler.is_downloaded(), stdout=stdout, verbose=verbose) if install_jdk: try: utilities.download_java(stdout=stdout) utilities.extract_java() utilities.setup_java() except Exception as e: logger.error( 'General error occurred while attempting to setup Java.') logger.debug( "General error occurred while attempting to setup Java; {}". format(e)) raise logstash_exceptions.InstallLogstashError( "General error occurred while attempting to setup Java; {}". format(e)) if create_dynamite_user: utilities.create_dynamite_user(utilities.generate_random_password(50)) ls_installer.setup_logstash()
def install_logstash(host='0.0.0.0', elasticsearch_host='localhost', elasticsearch_port=9200, elasticsearch_password='******', install_jdk=True, create_dynamite_user=True, stdout=False, verbose=False): """ Install Logstash/ElastiFlow :param host: The IP address to bind LogStash listeners too :param elasticsearch_password: The password used for authentication across all builtin ES users :param elasticsearch_host: A hostname/IP of the target elasticsearch instance :param elasticsearch_port: A port number for the target elasticsearch instance :param elasticsearch_password: The password used for authentication across all builtin ES users :param install_jdk: Install the latest OpenJDK that will be used by Logstash/ElasticSearch :param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run Logstash/ElasticSearch :param stdout: Print the output to console :param verbose: Include output from system utilities :return: True, if installation succeeded """ ls_profiler = LogstashProfiler() if ls_profiler.is_installed: sys.stderr.write( '[-] LogStash is already installed. If you wish to re-install, first uninstall.\n' ) return False if utilities.get_memory_available_bytes() < 6 * (1000**3): sys.stderr.write( '[-] WARNING! Dynamite Logstash should have at-least 6GB to run ' 'currently available [{} GB]\n'.format( utilities.get_memory_available_bytes() / (1000**3))) if str(utilities.prompt_input('Continue? [y|N]: ')).lower() != 'y': return False try: ls_installer = LogstashInstaller( host=host, elasticsearch_host=elasticsearch_host, elasticsearch_port=elasticsearch_port, elasticsearch_password=elasticsearch_password, download_logstash_archive=not ls_profiler.is_downloaded, stdout=stdout, verbose=verbose) if install_jdk: utilities.download_java(stdout=stdout) utilities.extract_java(stdout=stdout) utilities.setup_java() if create_dynamite_user: utilities.create_dynamite_user( utilities.generate_random_password(50)) ls_installer.setup_logstash() except Exception: sys.stderr.write( '[-] A fatal error occurred while attempting to install LogStash: ' ) traceback.print_exc(file=sys.stderr) return False if stdout: sys.stdout.write( '[+] *** LogStash installed event/alert pipelines. ***\n\n') sys.stdout.write( '[+] Next, Start your collector: \'dynamite start logstash\'.\n') sys.stdout.flush() return LogstashProfiler(stderr=False).is_installed
def install_elasticsearch(configuration_directory, install_directory, log_directory, password='******', heap_size_gigs=4, install_jdk=True, create_dynamite_user=True, stdout=True, verbose=False): """ Install ElasticSearch :param configuration_directory: Path to the configuration directory (E.G /etc/dynamite/elasticsearch/) :param install_directory: Path to the install directory (E.G /opt/dynamite/elasticsearch/) :param log_directory: Path to the log directory (E.G /var/log/dynamite/elasticsearch/) :param password: The password used for authentication across all builtin users :param heap_size_gigs: The initial/max java heap space to allocate :param install_jdk: Install the latest OpenJDK that will be used by Logstash/ElasticSearch :param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run Logstash/ElasticSearch :param stdout: Print the output to console :param verbose: Include detailed debug messages """ log_level = logging.INFO if verbose: log_level = logging.DEBUG logger = get_logger('ELASTICSEARCH', level=log_level, stdout=stdout) es_profiler = elastic_profile.ProcessProfiler() if es_profiler.is_installed(): logger.error('ElasticSearch is already installed.') raise elastic_exceptions.AlreadyInstalledElasticsearchError() if utilities.get_memory_available_bytes() < 6 * (1000**3): sys.stderr.write( '\n\033[93m[-] WARNING! ElasticSearch should have at-least 6GB to run ' 'currently available [{} GB]\033[0m\n'.format( utilities.get_memory_available_bytes() / (1000**3))) if str(utilities.prompt_input( '\033[93m[?] Continue? [y|N]:\033[0m ')).lower() != 'y': sys.stdout.write('\n[+] Exiting\n') exit(0) es_installer = InstallManager( configuration_directory=configuration_directory, install_directory=install_directory, log_directory=log_directory, password=password, heap_size_gigs=heap_size_gigs, download_elasticsearch_archive=not es_profiler.is_downloaded(), stdout=stdout, verbose=verbose) if install_jdk: try: utilities.download_java(stdout=stdout) utilities.extract_java() utilities.setup_java() except Exception as e: logger.error( 'General error occurred while attempting to setup Java.') logger.debug( "General error occurred while attempting to setup Java; {}". format(e)) raise elastic_exceptions.InstallElasticsearchError( "General error occurred while attempting to setup Java; {}". format(e)) if create_dynamite_user: utilities.create_dynamite_user(utilities.generate_random_password(50)) es_installer.setup_elasticsearch()