예제 #1
0
def install_logstash(host='0.0.0.0',
                     elasticsearch_host='localhost',
                     elasticsearch_port=9200,
                     elasticsearch_password='******',
                     install_jdk=True,
                     create_dynamite_user=True,
                     stdout=False):
    """
    Install Logstash/ElastiFlow
    :param host: The IP address to bind LogStash listeners too
    :param elasticsearch_password: The password used for authentication across all builtin ES users
    :param elasticsearch_host: A hostname/IP of the target elasticsearch instance
    :param elasticsearch_port: A port number for the target elasticsearch instance
    :param elasticsearch_password: The password used for authentication across all builtin ES users
    :param install_jdk: Install the latest OpenJDK that will be used by Logstash/ElasticSearch
    :param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run Logstash/ElasticSearch
    :param stdout: Print the output to console
    :return: True, if installation succeeded
    """
    ls_profiler = LogstashProfiler()
    if ls_profiler.is_installed:
        sys.stderr.write(
            '[-] LogStash is already installed. If you wish to re-install, first uninstall.\n'
        )
        return False
    if utilities.get_memory_available_bytes() < 6 * (1000**3):
        sys.stderr.write(
            '[-] Dynamite Logstash requires at-least 6GB to run currently available [{} GB]\n'
            .format(utilities.get_memory_available_bytes() / (1000**3)))
        return False
    try:
        ls_installer = LogstashInstaller(
            host=host,
            elasticsearch_host=elasticsearch_host,
            elasticsearch_port=elasticsearch_port,
            elasticsearch_password=elasticsearch_password)
        if install_jdk:
            utilities.download_java(stdout=True)
            utilities.extract_java(stdout=True)
            utilities.setup_java()
        if create_dynamite_user:
            utilities.create_dynamite_user(
                utilities.generate_random_password(50))
        ls_installer.download_logstash(stdout=True)
        ls_installer.extract_logstash(stdout=True)
        ls_installer.setup_logstash(stdout=True)
    except Exception:
        sys.stderr.write(
            '[-] A fatal error occurred while attempting to install LogStash: '
        )
        traceback.print_exc(file=sys.stderr)
        return False
    if stdout:
        sys.stdout.write(
            '[+] *** LogStash + ElastiFlow (w/ Zeek Support) installed successfully. ***\n\n'
        )
        sys.stdout.write(
            '[+] Next, Start your collector: \'dynamite start logstash\'.\n')
        sys.stdout.flush()
    return True
예제 #2
0
def install_kibana(elasticsearch_host='localhost',
                   elasticsearch_port=9200,
                   elasticsearch_password='******',
                   install_jdk=True,
                   create_dynamite_user=True,
                   stdout=False,
                   verbose=False):
    """
    Install Kibana/ElastiFlow Dashboards

    :param elasticsearch_host: [Optional] A hostname/IP of the target elasticsearch instance
    :param elasticsearch_port: [Optional] A port number for the target elasticsearch instance
    :param elasticsearch_password: The password used for authentication across all builtin ES users
    :param install_jdk: Install the latest OpenJDK that will be used by Logstash/ElasticSearch
    :param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run
    Logstash/ElasticSearch/Kibana
    :param stdout: Print the output to console
    :param verbose: Include output from system utilities
    :return: True, if installation succeeded
    """
    kb_profiler = KibanaProfiler()
    if kb_profiler.is_installed:
        sys.stderr.write(
            '[-] Kibana is already installed. If you wish to re-install, first uninstall.\n'
        )
        return False
    if utilities.get_memory_available_bytes() < 2 * (1000**3):
        sys.stderr.write(
            '[-] Dynamite Kibana requires at-least 2GB to run currently available [{} GB]\n'
            .format(utilities.get_memory_available_bytes() / (1000**3)))
        return False
    try:
        kb_installer = KibanaInstaller(
            elasticsearch_host=elasticsearch_host,
            elasticsearch_port=elasticsearch_port,
            elasticsearch_password=elasticsearch_password,
            download_kibana_archive=not kb_profiler.is_downloaded,
            stdout=stdout,
            verbose=verbose)
        if install_jdk:
            utilities.download_java(stdout=stdout)
            utilities.extract_java(stdout=stdout)
            utilities.setup_java()
        if create_dynamite_user:
            utilities.create_dynamite_user(
                utilities.generate_random_password(50))
        kb_installer.setup_kibana()
    except Exception:
        sys.stderr.write(
            '[-] A fatal error occurred while attempting to install Kibana: ')
        traceback.print_exc(file=sys.stderr)
        return False
    if stdout:
        sys.stdout.write(
            '[+] *** Kibana + Dashboards installed successfully. ***\n\n')
        sys.stdout.write(
            '[+] Next, Start your collector: \'dynamite start kibana\'.\n')
        sys.stdout.flush()
    return KibanaProfiler(stderr=False).is_installed
예제 #3
0
def install_elasticsearch(password='******',
                          install_jdk=True,
                          create_dynamite_user=True,
                          stdout=True,
                          verbose=False):
    """
    Install ElasticSearch

    :param password: The password used for authentication across all builtin users
    :param install_jdk: Install the latest OpenJDK that will be used by Logstash/ElasticSearch
    :param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run Logstash/ElasticSearch
    :param stdout: Print the output to console
    :param verbose: Include output from system utilities
    :return: True, if installation succeeded
    """
    es_profiler = ElasticProfiler()
    if es_profiler.is_installed:
        sys.stderr.write(
            '[-] ElasticSearch is already installed. If you wish to re-install, first uninstall.\n'
        )
        return False
    if utilities.get_memory_available_bytes() < 6 * (1000**3):
        sys.stderr.write(
            '[-] Dynamite ElasticSearch requires at-least 6GB to run currently available [{} GB]\n'
            .format(utilities.get_memory_available_bytes() / (1000**3)))
        return False
    try:
        es_installer = ElasticInstaller(
            password=password,
            download_elasticsearch_archive=not es_profiler.is_downloaded,
            stdout=stdout,
            verbose=verbose)
        if install_jdk:
            utilities.download_java(stdout=stdout)
            utilities.extract_java(stdout=stdout)
            utilities.setup_java()
        if create_dynamite_user:
            utilities.create_dynamite_user(
                utilities.generate_random_password(50))
        es_installer.setup_elasticsearch()
    except Exception:
        sys.stderr.write(
            '[-] A fatal error occurred while attempting to install ElasticSearch: '
        )
        traceback.print_exc(file=sys.stderr)
        return False
    if stdout:
        sys.stdout.write(
            '[+] *** ElasticSearch installed successfully. ***\n\n')
        sys.stdout.write(
            '[+] Next, Start your cluster: \'dynamite start elasticsearch\'.\n'
        )
    sys.stdout.flush()
    return ElasticProfiler(stderr=False).is_installed
예제 #4
0
def install_monitor(elasticsearch_password='******', verbose=False):
    """
    Installs Logstash (with ElastiFlow templates modified to work with Zeek), ElasticSearch, and Kibana.

    :param elasticsearch_password: The password used for authentication across all builtin ES users
    :param verbose: Include output from system utilities
    :return: True, if installation succeeded
    """
    es_pre_profiler = elasticsearch.ElasticProfiler()
    ls_pre_profiler = logstash.LogstashProfiler()
    kb_pre_profiler = kibana.KibanaProfiler()
    if ls_pre_profiler.is_installed and es_pre_profiler.is_installed and kb_pre_profiler.is_installed:
        sys.stderr.write(
            '[-] Monitor is already installed. If you wish to re-install, first uninstall.\n'
        )
        return False
    if utilities.get_memory_available_bytes() < 14 * (1000**3):
        sys.stderr.write(
            '[-] WARNING Dynamite standalone monitor requires '
            'at-least 14GB to run currently available [{} GB]\n'.format(
                utilities.get_memory_available_bytes() / (1024**3)))
        if str(utilities.prompt_input('Continue? [y|N]: ')).lower() != 'y':
            return False
    utilities.create_dynamite_user(utilities.generate_random_password(50))
    utilities.download_java(stdout=True)
    utilities.extract_java(stdout=True)
    utilities.setup_java()
    if not es_pre_profiler.is_installed:
        sys.stdout.write('[+] Installing Elasticsearch on localhost.\n')
        es_installer = elasticsearch.ElasticInstaller(
            host='0.0.0.0',
            port=9200,
            download_elasticsearch_archive=not ls_pre_profiler.is_downloaded,
            password=elasticsearch_password,
            stdout=True,
            verbose=verbose)
        es_installer.setup_elasticsearch()
        if not elasticsearch.ElasticProfiler().is_installed:
            sys.stderr.write(
                '[-] ElasticSearch failed to install on localhost.\n')
            return False
    sys.stdout.write('[+] Starting ElasticSearch on localhost.\n')
    es_process = elasticsearch.ElasticProcess()
    es_process.start()
    if not ls_pre_profiler.is_installed:
        ls_installer = logstash.LogstashInstaller(
            host='0.0.0.0',
            elasticsearch_password=elasticsearch_password,
            download_logstash_archive=not es_pre_profiler.is_downloaded,
            stdout=True,
            verbose=verbose)
        ls_installer.setup_logstash()
        if not logstash.LogstashProfiler().is_installed:
            sys.stderr.write('[-] LogStash failed to install on localhost.\n')
            return False
    if not kb_pre_profiler.is_installed and elasticsearch.ElasticProfiler(
    ).is_installed:
        sys.stdout.write('[+] Installing Kibana on localhost.\n')
        kb_installer = kibana.KibanaInstaller(
            host='0.0.0.0',
            port=5601,
            elasticsearch_host='localhost',
            elasticsearch_port=9200,
            elasticsearch_password=elasticsearch_password,
            download_kibana_archive=not kb_pre_profiler.is_downloaded,
            stdout=True,
            verbose=verbose)
        if not kb_pre_profiler.is_downloaded:
            kb_installer.download_kibana(stdout=True)
            kb_installer.extract_kibana(stdout=True)
        kb_installer.setup_kibana()
        if not kibana.KibanaProfiler().is_installed:
            sys.stderr.write('[-] Kibana failed to install on localhost.\n')
            return False
        sys.stdout.write(
            '[+] Monitor installation complete. Start the monitor: \'dynamite start monitor\'.\n'
        )
        sys.stdout.flush()
    return True
예제 #5
0
def install_monitor(elasticsearch_password='******'):
    """
    Installs Logstash (with ElastiFlow templates modified to work with Zeek), ElasticSearch, and Kibana.

    :return: True, if installation succeeded
    """
    if utilities.get_memory_available_bytes() < 14 * (1000**3):
        sys.stderr.write(
            '[-] Dynamite standalone monitor requires '
            'at-least 14GB to run currently available [{} GB]\n'.format(
                utilities.get_memory_available_bytes() / (1024**3)))
        return False
    utilities.create_dynamite_user(utilities.generate_random_password(50))
    utilities.download_java(stdout=True)
    utilities.extract_java(stdout=True)
    utilities.setup_java()
    es_installer = elasticsearch.ElasticInstaller(
        host='0.0.0.0', port=9200, password=elasticsearch_password)
    es_pre_profiler = elasticsearch.ElasticProfiler()
    es_process = elasticsearch.ElasticProcess()
    ls_installer = logstash.LogstashInstaller(
        host='0.0.0.0', elasticsearch_password=elasticsearch_password)
    ls_pre_profiler = logstash.LogstashProfiler()
    kb_installer = kibana.KibanaInstaller(
        host='0.0.0.0',
        port=5601,
        elasticsearch_host='localhost',
        elasticsearch_port=9200,
        elasticsearch_password=elasticsearch_password)
    kb_pre_profiler = kibana.KibanaProfiler()
    if not es_pre_profiler.is_installed:
        sys.stdout.write('[+] Installing Elasticsearch on localhost.\n')
        if not es_pre_profiler.is_downloaded:
            es_installer.download_elasticsearch(stdout=True)
            es_installer.extract_elasticsearch(stdout=True)
        es_installer.setup_elasticsearch(stdout=True)
        if not elasticsearch.ElasticProfiler().is_installed:
            sys.stderr.write(
                '[-] ElasticSearch failed to install on localhost.\n')
            return False
    sys.stdout.write('[+] Starting ElasticSearch on localhost.\n')
    es_process.start()
    if not ls_pre_profiler.is_installed:
        if not ls_pre_profiler.is_downloaded:
            ls_installer.download_logstash(stdout=True)
            ls_installer.extract_logstash(stdout=True)
        ls_installer.setup_logstash(stdout=True)
        if not logstash.LogstashProfiler().is_installed:
            sys.stderr.write('[-] LogStash failed to install on localhost.\n')
            return False
    if not kb_pre_profiler.is_installed and elasticsearch.ElasticProfiler(
    ).is_installed:
        sys.stdout.write('[+] Installing Kibana on localhost.\n')
        if not kb_pre_profiler.is_downloaded:
            kb_installer.download_kibana(stdout=True)
            kb_installer.extract_kibana(stdout=True)
        kb_installer.setup_kibana(stdout=True)
        if not kibana.KibanaProfiler().is_installed:
            sys.stderr.write('[-] Kibana failed to install on localhost.\n')
            return False
        sys.stdout.write(
            '[+] Monitor installation complete. Start the monitor: \'dynamite start monitor\'.\n'
        )
        sys.stdout.flush()
    return True
예제 #6
0
def install_logstash(configuration_directory,
                     install_directory,
                     log_directory,
                     host='0.0.0.0',
                     elasticsearch_host='localhost',
                     elasticsearch_port=9200,
                     elasticsearch_password='******',
                     heap_size_gigs=4,
                     install_jdk=True,
                     create_dynamite_user=True,
                     stdout=False,
                     verbose=False):
    """
    Install Logstash with ElastiFlow & Synesis

    :param configuration_directory: Path to the configuration directory (E.G /etc/dynamite/logstash/)
    :param install_directory: Path to the install directory (E.G /opt/dynamite/logstash/)
    :param log_directory: Path to the log directory (E.G /var/log/dynamite/logstash/)
    :param host: The IP address to bind LogStash listeners too
    :param elasticsearch_password: The password used for authentication across all builtin ES users
    :param elasticsearch_host: A hostname/IP of the target elasticsearch instance
    :param elasticsearch_port: A port number for the target elasticsearch instance
    :param elasticsearch_password: The password used for authentication across all builtin ES users
    :param heap_size_gigs: The initial/max java heap space to allocate
    :param install_jdk: Install the latest OpenJDK that will be used by Logstash/ElasticSearch
    :param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run Logstash/ElasticSearch
    :param stdout: Print the output to console
    :param verbose: Include output from system utilities
    :return: True, if installation succeeded
    """
    log_level = logging.INFO
    if verbose:
        log_level = logging.DEBUG
    logger = get_logger('LOGSTASH', level=log_level, stdout=stdout)

    ls_profiler = logstash_profile.ProcessProfiler()
    if ls_profiler.is_installed():
        logger.error('LogStash is already installed.')
        raise logstash_exceptions.AlreadyInstalledLogstashError()
    if utilities.get_memory_available_bytes() < 6 * (1000**3):
        sys.stderr.write(
            '\n\033[93m[-] WARNING! LogStash should have at-least 6GB to run '
            'currently available [{} GB]\033[0m\n'.format(
                utilities.get_memory_available_bytes() / (1000**3)))
        if str(utilities.prompt_input(
                '\033[93m[?] Continue? [y|N]:\033[0m ')).lower() != 'y':
            sys.stdout.write('\n[+] Exiting\n')
            exit(0)
    ls_installer = InstallManager(
        configuration_directory,
        install_directory,
        log_directory,
        host=host,
        elasticsearch_host=elasticsearch_host,
        elasticsearch_port=elasticsearch_port,
        elasticsearch_password=elasticsearch_password,
        heap_size_gigs=heap_size_gigs,
        download_logstash_archive=not ls_profiler.is_downloaded(),
        stdout=stdout,
        verbose=verbose)
    if install_jdk:
        try:
            utilities.download_java(stdout=stdout)
            utilities.extract_java()
            utilities.setup_java()
        except Exception as e:
            logger.error(
                'General error occurred while attempting to setup Java.')
            logger.debug(
                "General error occurred while attempting to setup Java; {}".
                format(e))
            raise logstash_exceptions.InstallLogstashError(
                "General error occurred while attempting to setup Java; {}".
                format(e))
    if create_dynamite_user:
        utilities.create_dynamite_user(utilities.generate_random_password(50))
    ls_installer.setup_logstash()
예제 #7
0
def install_logstash(host='0.0.0.0',
                     elasticsearch_host='localhost',
                     elasticsearch_port=9200,
                     elasticsearch_password='******',
                     install_jdk=True,
                     create_dynamite_user=True,
                     stdout=False,
                     verbose=False):
    """
    Install Logstash/ElastiFlow
    :param host: The IP address to bind LogStash listeners too
    :param elasticsearch_password: The password used for authentication across all builtin ES users
    :param elasticsearch_host: A hostname/IP of the target elasticsearch instance
    :param elasticsearch_port: A port number for the target elasticsearch instance
    :param elasticsearch_password: The password used for authentication across all builtin ES users
    :param install_jdk: Install the latest OpenJDK that will be used by Logstash/ElasticSearch
    :param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run Logstash/ElasticSearch
    :param stdout: Print the output to console
    :param verbose: Include output from system utilities
    :return: True, if installation succeeded
    """
    ls_profiler = LogstashProfiler()
    if ls_profiler.is_installed:
        sys.stderr.write(
            '[-] LogStash is already installed. If you wish to re-install, first uninstall.\n'
        )
        return False
    if utilities.get_memory_available_bytes() < 6 * (1000**3):
        sys.stderr.write(
            '[-] WARNING! Dynamite Logstash should have at-least 6GB to run '
            'currently available [{} GB]\n'.format(
                utilities.get_memory_available_bytes() / (1000**3)))
        if str(utilities.prompt_input('Continue? [y|N]: ')).lower() != 'y':
            return False
    try:
        ls_installer = LogstashInstaller(
            host=host,
            elasticsearch_host=elasticsearch_host,
            elasticsearch_port=elasticsearch_port,
            elasticsearch_password=elasticsearch_password,
            download_logstash_archive=not ls_profiler.is_downloaded,
            stdout=stdout,
            verbose=verbose)
        if install_jdk:
            utilities.download_java(stdout=stdout)
            utilities.extract_java(stdout=stdout)
            utilities.setup_java()
        if create_dynamite_user:
            utilities.create_dynamite_user(
                utilities.generate_random_password(50))
        ls_installer.setup_logstash()
    except Exception:
        sys.stderr.write(
            '[-] A fatal error occurred while attempting to install LogStash: '
        )
        traceback.print_exc(file=sys.stderr)
        return False
    if stdout:
        sys.stdout.write(
            '[+] *** LogStash installed event/alert pipelines. ***\n\n')
        sys.stdout.write(
            '[+] Next, Start your collector: \'dynamite start logstash\'.\n')
        sys.stdout.flush()
    return LogstashProfiler(stderr=False).is_installed
예제 #8
0
def install_elasticsearch(configuration_directory,
                          install_directory,
                          log_directory,
                          password='******',
                          heap_size_gigs=4,
                          install_jdk=True,
                          create_dynamite_user=True,
                          stdout=True,
                          verbose=False):
    """
    Install ElasticSearch

    :param configuration_directory: Path to the configuration directory (E.G /etc/dynamite/elasticsearch/)
    :param install_directory: Path to the install directory (E.G /opt/dynamite/elasticsearch/)
    :param log_directory: Path to the log directory (E.G /var/log/dynamite/elasticsearch/)
    :param password: The password used for authentication across all builtin users
    :param heap_size_gigs: The initial/max java heap space to allocate
    :param install_jdk: Install the latest OpenJDK that will be used by Logstash/ElasticSearch
    :param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run Logstash/ElasticSearch
    :param stdout: Print the output to console
    :param verbose: Include detailed debug messages
    """

    log_level = logging.INFO
    if verbose:
        log_level = logging.DEBUG
    logger = get_logger('ELASTICSEARCH', level=log_level, stdout=stdout)

    es_profiler = elastic_profile.ProcessProfiler()
    if es_profiler.is_installed():
        logger.error('ElasticSearch is already installed.')
        raise elastic_exceptions.AlreadyInstalledElasticsearchError()
    if utilities.get_memory_available_bytes() < 6 * (1000**3):
        sys.stderr.write(
            '\n\033[93m[-] WARNING! ElasticSearch should have at-least 6GB to run '
            'currently available [{} GB]\033[0m\n'.format(
                utilities.get_memory_available_bytes() / (1000**3)))
        if str(utilities.prompt_input(
                '\033[93m[?] Continue? [y|N]:\033[0m ')).lower() != 'y':
            sys.stdout.write('\n[+] Exiting\n')
            exit(0)
    es_installer = InstallManager(
        configuration_directory=configuration_directory,
        install_directory=install_directory,
        log_directory=log_directory,
        password=password,
        heap_size_gigs=heap_size_gigs,
        download_elasticsearch_archive=not es_profiler.is_downloaded(),
        stdout=stdout,
        verbose=verbose)
    if install_jdk:
        try:
            utilities.download_java(stdout=stdout)
            utilities.extract_java()
            utilities.setup_java()
        except Exception as e:
            logger.error(
                'General error occurred while attempting to setup Java.')
            logger.debug(
                "General error occurred while attempting to setup Java; {}".
                format(e))
            raise elastic_exceptions.InstallElasticsearchError(
                "General error occurred while attempting to setup Java; {}".
                format(e))
    if create_dynamite_user:
        utilities.create_dynamite_user(utilities.generate_random_password(50))
    es_installer.setup_elasticsearch()