def change_sdk_elasticsearch_password(password='******', prompt_user=True, stdout=False): """ Change the DynamiteSDK to ElasticSearch password :param password: The password that the SDK will use to connect to ElasticSearch :param prompt_user: Whether or not to warn the user :param stdout: Print output to console """ environment_variables = utilities.get_environment_file_dict() configuration_directory = environment_variables.get('DYNAMITE_LAB_CONFIG') if prompt_user: resp = utilities.prompt_input( '\033[93m[-] Changing the SDK password can cause your notebooks to lose communication with ElasticSearch.\n' '[?] Are you sure you wish to continue? [no]|yes):\033[0m ') while resp not in ['', 'no', 'yes']: resp = utilities.prompt_input( '\033[93m[?] Are you sure you wish to continue? ([no]|yes): \033[0m' ) if resp != 'yes': if stdout: sys.stdout.write('\n[+] Exiting\n') return dynamite_lab_config = ConfigManager( configuration_directory=configuration_directory) try: dynamite_lab_config.elasticsearch_password = password dynamite_lab_config.write_config() except lab_exceptions.WriteLabConfigError: raise general_exceptions.ResetPasswordError( "Could not write new password to DynamiteSDK config.cfg.")
def change_sdk_elasticsearch_password(password='******', prompt_user=True, stdout=False): """ Change the DynamiteSDK to ElasticSearch password :param password: The password that the SDK will use to connect to ElasticSearch :param prompt_user: Whether or not to warn the user :param stdout: Print output to console :return: True if changed successfully """ environment_variables = utilities.get_environment_file_dict() configuration_directory = environment_variables.get('DYNAMITE_LAB_CONFIG') if prompt_user: resp = utilities.prompt_input( 'Changing the SDK password can cause your notebooks to lose communication with ElasticSearch. ' 'Are you sure you wish to continue? [no]|yes): ') while resp not in ['', 'no', 'yes']: resp = utilities.prompt_input( 'Are you sure you wish to continue? ([no]|yes): ') if resp != 'yes': if stdout: sys.stdout.write('[+] Exiting\n') return False dynamite_lab_config = DynamiteLabConfigurator( configuration_directory=configuration_directory) dynamite_lab_config.elasticsearch_password = password dynamite_lab_config.write_config() return True
def uninstall_logstash(stdout=False, prompt_user=True): """ Uninstall Logstash/ElastiFlow :param stdout: Print the output to console :param prompt_user: Print a warning before continuing :return: True, if uninstall succeeded """ environment_variables = utilities.get_environment_file_dict() configuration_directory = environment_variables.get('LS_PATH_CONF') ls_profiler = LogstashProfiler() ls_config = LogstashConfigurator( configuration_directory=configuration_directory) if not ls_profiler.is_installed: sys.stderr.write('[-] LogStash is not installed.\n') return False if prompt_user: sys.stderr.write( '[-] WARNING! REMOVING LOGSTASH WILL PREVENT ELASTICSEARCH FROM RECEIVING EVENTS.\n' ) resp = utilities.prompt_input( 'Are you sure you wish to continue? ([no]|yes): ') while resp not in ['', 'no', 'yes']: resp = utilities.prompt_input( 'Are you sure you wish to continue? ([no]|yes): ') if resp != 'yes': if stdout: sys.stdout.write('[+] Exiting\n') return False if ls_profiler.is_running: LogstashProcess().stop(stdout=stdout) try: shutil.rmtree(ls_config.ls_path_conf) shutil.rmtree(ls_config.ls_home) shutil.rmtree(ls_config.get_log_path()) shutil.rmtree('/tmp/dynamite/install_cache/', ignore_errors=True) env_lines = '' for line in open('/etc/dynamite/environment').readlines(): if 'LS_PATH_CONF' in line: continue elif 'LS_HOME' in line: continue elif 'ELASTIFLOW_' in line: continue elif 'SYNLITE_' in line: continue elif line.strip() == '': continue env_lines += line.strip() + '\n' open('/etc/dynamite/environment', 'w').write(env_lines) if stdout: sys.stdout.write('[+] LogStash uninstalled successfully.\n') except Exception: sys.stderr.write( '[-] A fatal error occurred while attempting to uninstall LogStash: ' ) traceback.print_exc(file=sys.stderr) return False return True
def change_elasticsearch_password(old_password, password='******', remote_host=None, remote_port=None, prompt_user=True, stdout=True, verbose=False): """ Change the Elasticsearch password for all builtin users :param old_password: The old Elasticsearch password :param password: The new Elasticsearch password :param prompt_user: If True, warning prompt is displayed before proceeding :param stdout: Print status to stdout :param verbose: Include detailed debug messages """ from dynamite_nsm.services.elasticsearch import process as elastic_process from dynamite_nsm.services.elasticsearch import profile as elastic_profile log_level = logging.INFO if verbose: log_level = logging.DEBUG logger = get_logger('ELASTICSEARCH', level=log_level, stdout=stdout) if prompt_user: resp = utilities.prompt_input( '\n\033[93m[-] WARNING! Changing the ElasticSearch password may result in connected components losing ' 'communication. Be sure to update Kibana/LogStash passwords.\n' '[?] Are you sure you wish to continue? [no]|yes):\033[0m ') while resp not in ['', 'no', 'yes']: resp = utilities.prompt_input('\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m ') if resp != 'yes': if stdout: sys.stdout.write('[+] Exiting\n') exit(0) if remote_host: logger.info("Resetting ElasticSearch password on remote host: {}:{}".format(remote_host, remote_port)) else: logger.info("Resetting ElasticSearch password on localhost.") if elastic_profile.ProcessProfiler().is_installed(): # If ElasticSearch is installed Locally. # Start the process, in order to perform a reset. if not elastic_process.ProcessManager().start(): logger.error('Could not start ElasticSearch Process. Password reset failed.') raise general_exceptions.ResetPasswordError( "ElasticSearch process was not able to start, check your ElasticSearch logs.") while not elastic_profile.ProcessProfiler().is_listening(): logger.info('Waiting for ElasticSearch API to become accessible.') time.sleep(5) logger.info('ElasticSearch API is up.') logger.debug('Sleeping for 5 seconds, while ElasticSearch API finishes booting.') time.sleep(5) else: logger.error("ElasticSearch is not installed, and no remote ElasticSearch host was specified.") raise general_exceptions.ResetPasswordError( "ElasticSearch is not installed, and no remote ElasticSearch host was specified.") es_pw_config = PasswordConfigManager('elastic', current_password=old_password, remote_host=remote_host, remote_http_port=remote_port) logger.info("Attempting password reset.") es_pw_config.set_all_passwords(password)
def uninstall_elasticsearch(stdout=False, prompt_user=True): """ Uninstall ElasticSearch :param stdout: Print the output to console :param prompt_user: Print a warning before continuing :return: True, if uninstall succeeded """ environment_variables = utilities.get_environment_file_dict() configuration_directory = environment_variables.get('ES_PATH_CONF') es_profiler = ElasticProfiler() es_config = ElasticConfigurator( configuration_directory=configuration_directory) if not es_profiler.is_installed: sys.stderr.write('[-] ElasticSearch is not installed.\n') return False if prompt_user: sys.stderr.write( '[-] WARNING! REMOVING ELASTICSEARCH WILL LIKELY RESULT IN ALL DATA BEING LOST.\n' ) resp = utilities.prompt_input( 'Are you sure you wish to continue? ([no]|yes): ') while resp not in ['', 'no', 'yes']: resp = utilities.prompt_input( 'Are you sure you wish to continue? ([no]|yes): ') if resp != 'yes': if stdout: sys.stdout.write('[+] Exiting\n') return False if es_profiler.is_running: ElasticProcess().stop(stdout=stdout) try: shutil.rmtree(es_config.configuration_directory) shutil.rmtree(es_config.es_home) shutil.rmtree(es_config.get_log_path()) shutil.rmtree('/tmp/dynamite/install_cache/', ignore_errors=True) env_lines = '' for line in open('/etc/dynamite/environment').readlines(): if 'ES_PATH_CONF' in line: continue elif 'ES_HOME' in line: continue elif line.strip() == '': continue env_lines += line.strip() + '\n' open('/etc/dynamite/environment', 'w').write(env_lines) if stdout: sys.stdout.write('[+] ElasticSearch uninstalled successfully.\n') except Exception: sys.stderr.write( '[-] A fatal error occurred while attempting to uninstall ElasticSearch: ' ) traceback.print_exc(file=sys.stderr) return False return True
def uninstall_kibana(stdout=False, prompt_user=True): """ Uninstall Kibana/ElastiFlow Dashboards :param stdout: Print the output to console :param prompt_user: Print a warning before continuing :return: True, if uninstall succeeded """ kb_profiler = KibanaProfiler() kb_config = KibanaConfigurator( configuration_directory=CONFIGURATION_DIRECTORY) if not kb_profiler.is_installed: sys.stderr.write('[-] Kibana is not installed.\n') return False if prompt_user: sys.stderr.write( '[-] WARNING! REMOVING KIBANA WILL PREVENT YOU FROM VIEWING NETWORK EVENTS.\n' ) resp = utilities.prompt_input( 'Are you sure you wish to continue? ([no]|yes): ') while resp not in ['', 'no', 'yes']: resp = utilities.prompt_input( 'Are you sure you wish to continue? ([no]|yes): ') if resp != 'yes': if stdout: sys.stdout.write('[+] Exiting\n') return False if kb_profiler.is_running: KibanaProcess().stop(stdout=stdout) try: shutil.rmtree(kb_config.kibana_path_conf) shutil.rmtree(kb_config.kibana_home) shutil.rmtree(kb_config.kibana_logs) shutil.rmtree('/tmp/dynamite/install_cache/', ignore_errors=True) env_lines = '' for line in open('/etc/dynamite/environment').readlines(): if 'KIBANA_PATH_CONF' in line: continue elif 'KIBANA_HOME' in line: continue elif 'KIBANA_LOGS' in line: continue elif line.strip() == '': continue env_lines += line.strip() + '\n' open('/etc/dynamite/environment', 'w').write(env_lines) if stdout: sys.stdout.write('[+] Kibana uninstalled successfully.\n') except Exception: sys.stderr.write( '[-] A fatal error occurred while attempting to uninstall Kibana: ' ) traceback.print_exc(file=sys.stderr) return False return True
def change_logstash_elasticsearch_password(password='******', prompt_user=True, stdout=False): if prompt_user: resp = utilities.prompt_input( 'Changing the LogStash password can cause LogStash to lose communication with ElasticSearch. ' 'Are you sure you wish to continue? [no]|yes): ') while resp not in ['', 'no', 'yes']: resp = utilities.prompt_input('Are you sure you wish to continue? ([no]|yes): ') if resp != 'yes': if stdout: sys.stdout.write('[+] Exiting\n') return False LogstashConfigurator(configuration_directory=CONFIGURATION_DIRECTORY).set_elasticsearch_password(password=password) return LogstashProcess().restart(stdout=True)
def change_kibana_elasticsearch_password(password='******', prompt_user=True, stdout=True, verbose=False): """ Change the password used by Kibana to authenticate to ElasticSearch :param password: The new Elasticsearch password :param prompt_user: If True, warning prompt is displayed before proceeding :param stdout: Print status to stdout :param verbose: Include detailed debug messages """ from dynamite_nsm.services.kibana import process as kibana_process from dynamite_nsm.services.kibana import profile as kibana_profile log_level = logging.INFO if verbose: log_level = logging.DEBUG logger = get_logger('KIBANA', level=log_level, stdout=stdout) environment_variables = utilities.get_environment_file_dict() if not kibana_profile.ProcessProfiler().is_installed(): logger.error( "Password reset failed. Kibana is not installed on this host.") if prompt_user: resp = utilities.prompt_input( '\033[93m[-] WARNING! Changing the Kibana password can cause Kibana to lose communication with ' 'ElasticSearch.\n[?] Are you sure you wish to continue? [no]|yes):\033[0m ' ) while resp not in ['', 'no', 'yes']: resp = utilities.prompt_input( '\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m ' ) if resp != 'yes': if stdout: sys.stdout.write('\n[+] Exiting\n') exit(0) try: kb_config = ConfigManager( environment_variables.get('KIBANA_PATH_CONF')) kb_config.elasticsearch_password = password kb_config.write_config() except (kibana_exceptions.ReadKibanaConfigError, kibana_exceptions.WriteKibanaConfigError): logger.error("Could not read/write Kibana configuration.") raise general_exceptions.ResetPasswordError( "Could not read/write Kibana configuration.") kibana_process.ProcessManager().restart()
def prompt_agent_uninstall(prompt_user=True, stdout=True): if prompt_user: sys.stderr.write( '\n\033[93m[-] WARNING! Removing Agent Will Remove the Agent and all of it\'s installed components: {}.' '\033[0m\n'.format(get_installed_agent_analyzers())) resp = prompt_input( '\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m ' ) while resp not in ['', 'no', 'yes']: resp = prompt_input( '\033[93m[?] Are you sure you wish to continue? ([no]|yes): \033[0m' ) if resp != 'yes': if stdout: sys.stdout.write('\n[+] Exiting\n') exit(0)
def prompt_monitor_uninstall(prompt_user=True, stdout=True): if prompt_user: sys.stderr.write( '\n\033[93m[-] WARNING! Removing Monitor Will Delete All Saved Network Events and Corresponding ' 'Visualisations.\033[0m\n') resp = prompt_input( '\033[93m[?] Are you sure you wish to continue? ([no]|yes): \033[0m' ) while resp not in ['', 'no', 'yes']: resp = prompt_input( '\033[93m[?] Are you sure you wish to continue? ([no]|yes): \033[0m' ) if resp != 'yes': if stdout: sys.stdout.write('\n[+] Exiting\n') exit(0)
def change_logstash_elasticsearch_password(password='******', prompt_user=True, stdout=True, verbose=False): """ Change the password used by Logstash to authenticate to Elasticsearch :param password: The new Elasticsearch password :param prompt_user: If True, warning prompt is displayed before proceeding :param stdout: Print status to stdout :param verbose: Include detailed debug messages :return: True, if successful """ from dynamite_nsm.services.logstash import process as logstash_process from dynamite_nsm.services.logstash import profile as logstash_profile log_level = logging.INFO if verbose: log_level = logging.DEBUG logger = get_logger('LOGSTASH', level=log_level, stdout=stdout) environment_variables = utilities.get_environment_file_dict() if not logstash_profile.ProcessProfiler().is_installed(): logger.error( "Password reset failed. LogStash is not installed on this host.") raise general_exceptions.ResetPasswordError( "Password reset failed. LogStash is not installed on this host.") if prompt_user: resp = utilities.prompt_input( '\n\033[93m[-] WARNING! Changing the LogStash password can cause LogStash to lose communication with ' 'ElasticSearch. \n[?] Are you sure you wish to continue? [no]|yes):\033[0m ' ) while resp not in ['', 'no', 'yes']: resp = utilities.prompt_input( '\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m ' ) if resp != 'yes': if stdout: sys.stdout.write('\n[+] Exiting\n') exit(0) ConfigManager( environment_variables.get('LS_PATH_CONF')).set_elasticsearch_password( password=password) logstash_process.ProcessManager().restart()
def prompt_password_change_options(): """ Provide the user with a choice between changing the jupyter user password (logging into jupyterhub) or changing the password that the SDK uses to connect to ElasticSearch. """ resp = utilities.prompt_input( '[+] 1. Change the password the SDK uses to connect to Elasticsearch.\n' '[+] 2. Change the password for logging into Jupyterhub (jupyter user).\n\n' '[?] Select an option [1, 2]: ') while str(resp) not in ['', '1', '2']: resp = utilities.prompt_input('Select an option [1, 2]: ') if str(resp) == '1': return change_sdk_elasticsearch_password(utilities.prompt_password( 'Enter the new Elasticsearch password: '******'passwd', 'jupyter'])
def check_elasticsearch_target(host, port, perform_check=True): if not perform_check: return if not check_socket(host, port): print( "\n\033[93m[-] ElasticSearch does not appear to be started on: {}:{}.\033[0m" .format(host, port)) if str(prompt_input( '\033[93m[?] Continue? [y|N]:\033[0m ')).lower() != 'y': exit(0) return
def uninstall_monitor(prompt_user=True): """ Uninstall standalone monitor components (ElasticSearch, Logstash, and Kibana) :return: True, if uninstall successful """ es_profiler = elasticsearch.ElasticProfiler() ls_profiler = logstash.LogstashProfiler() kb_profiler = kibana.KibanaProfiler() if not (es_profiler.is_installed and ls_profiler.is_installed and kb_profiler.is_installed): sys.stderr.write( '[-] A standalone monitor installation was not detected on this system. Please uninstall ' 'ElasticSearch, Logstash, or Kibana individually.\n') return False if prompt_user: sys.stderr.write( '[-] WARNING! UNINSTALLING THE MONITOR WILL PREVENT EVENTS FROM BEING PROCESSED/VISUALIZED.\n' ) resp = utilities.prompt_input( 'Are you sure you wish to continue? ([no]|yes): ') while resp not in ['', 'no', 'yes']: resp = utilities.prompt_input( 'Are you sure you wish to continue? ([no]|yes): ') if resp != 'yes': sys.stdout.write('[+] Exiting\n') return False es_uninstall = elasticsearch.uninstall_elasticsearch(stdout=True, prompt_user=False) ls_uninstall = logstash.uninstall_logstash(stdout=True, prompt_user=False) kb_uninstall = kibana.uninstall_kibana(stdout=True, prompt_user=False) res = es_uninstall and ls_uninstall and kb_uninstall if res: sys.stdout.write('[+] Monitor uninstalled successfully.\n') else: sys.stderr.write( '[-] An error occurred while uninstalling one or more monitor components.\n' ) return res
def install_kibana(install_directory, configuration_directory, log_directory, host='0.0.0.0', port=5601, elasticsearch_host='localhost', elasticsearch_port=9200, elasticsearch_password='******', create_dynamite_user=True, stdout=False, verbose=False): """ Install Kibana/ElastiFlow/Synesis Dashboards :param install_directory: Path to the install directory (E.G /opt/dynamite/kibana/) :param configuration_directory: Path to the configuration directory (E.G /etc/dynamite/kibana/) :param log_directory: Path to the log directory (E.G /var/log/dynamite/kibana/) :param host: The IP address to listen on (E.G "0.0.0.0") :param port: The port that the Kibana UI/API is bound to (E.G 5601) :param elasticsearch_host: A hostname/IP of the target ElasticSearch instance :param elasticsearch_port: A port number for the target ElasticSearch instance :param elasticsearch_password: The password used for authentication across all builtin ES users :param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run Logstash/ElasticSearch/Kibana :param stdout: Print the output to console :param verbose: Include output from system utilities """ log_level = logging.INFO if verbose: log_level = logging.DEBUG logger = get_logger('KIBANA', level=log_level, stdout=stdout) kb_profiler = kibana_profile.ProcessProfiler() if kb_profiler.is_installed(): logger.error('Kibana is already installed. If you wish to re-install, first uninstall.') raise kibana_exceptions.AlreadyInstalledKibanaError() if utilities.get_memory_available_bytes() < 2 * (1000 ** 3): sys.stderr.write('\n\033[93m[-] WARNING! Kibana should have at-least 2GB to run ' 'currently available [{} GB]\033[0m\n'.format( utilities.get_memory_available_bytes() / (1000 ** 3))) if str(utilities.prompt_input('\033[93m[?] Continue? [y|N]:\033[0m ')).lower() != 'y': sys.stdout.write('\n[+] Exiting\n') exit(0) kb_installer = InstallManager(install_directory, configuration_directory, log_directory, host=host, port=port, elasticsearch_host=elasticsearch_host, elasticsearch_port=elasticsearch_port, elasticsearch_password=elasticsearch_password, download_kibana_archive=not kb_profiler.is_downloaded(), stdout=stdout, verbose=verbose) if create_dynamite_user: utilities.create_dynamite_user(utilities.generate_random_password(50)) kb_installer.setup_kibana()
def uninstall_zeek(prompt_user=True, stdout=True, verbose=False): """ Uninstall Zeek :param prompt_user: Print a warning before continuing :param stdout: Print the output to console :param verbose: Include detailed debug messages """ log_level = logging.INFO if verbose: log_level = logging.DEBUG logger = get_logger('ZEEK', level=log_level, stdout=stdout) logger.info("Uninstalling Zeek.") env_file = os.path.join(const.CONFIG_PATH, 'environment') environment_variables = utilities.get_environment_file_dict() zeek_profiler = zeek_profile.ProcessProfiler() if not zeek_profiler.is_installed(): logger.error("Zeek is not installed. Cannot uninstall.") raise zeek_exceptions.UninstallZeekError("Zeek is not installed.") if prompt_user: sys.stderr.write('\n\033[93m[-] WARNING! Removing Zeek Will Remove Critical Agent Functionality.\033[0m\n') resp = utilities.prompt_input('\033[93m[?] Are you sure you wish to continue? ([no]|yes): \033[0m') while resp not in ['', 'no', 'yes']: resp = utilities.prompt_input('\033[93m[?] Are you sure you wish to continue? ([no]|yes): \033[0m') if resp != 'yes': if stdout: sys.stdout.write('\n[+] Exiting\n') exit(0) if zeek_profiler.is_running(): try: zeek_process.ProcessManager().stop() except zeek_exceptions.CallZeekProcessError as e: logger.error("Could not kill Zeek process. Cannot uninstall.") logger.debug("Could not kill Zeek process. Cannot uninstall; {}".format(e)) raise zeek_exceptions.UninstallZeekError("Could not kill Zeek process; {}".format(e)) install_directory = environment_variables.get('ZEEK_HOME') config_directory = environment_variables.get('ZEEK_SCRIPTS') try: with open(env_file) as env_fr: env_lines = '' for line in env_fr.readlines(): if 'ZEEK_HOME' in line: continue elif 'ZEEK_SCRIPTS' in line: continue elif 'PF_RING_HOME' in line: continue elif line.strip() == '': continue env_lines += line.strip() + '\n' with open(env_file, 'w') as env_fw: env_fw.write(env_lines) if zeek_profiler.is_installed(): shutil.rmtree(install_directory, ignore_errors=True) shutil.rmtree(config_directory, ignore_errors=True) except Exception as e: logger.error("General error occurred while attempting to uninstall Zeek.") logger.debug("General error occurred while attempting to uninstall Zeek; {}".format(e)) raise zeek_exceptions.UninstallZeekError( "General error occurred while attempting to uninstall Zeek; {}".format(e)) try: sysctl = systemctl.SystemCtl() except general_exceptions.CallProcessError: raise zeek_exceptions.UninstallZeekError("Could not find systemctl.") sysctl.uninstall_and_disable('zeek')
def uninstall_agent(prompt_user=True): """ Uninstall the agent :param prompt_user: Print a warning before continuing :return: True, if uninstall succeeded """ environment_variables = utilities.get_environment_file_dict() filebeat_profiler = filebeat.FileBeatProfiler() pf_profiler = pf_ring.PFRingProfiler() zeek_profiler = zeek.ZeekProfiler() suricata_profiler = suricata.SuricataProfiler() if not (filebeat_profiler.is_installed or zeek_profiler.is_installed or suricata_profiler.is_installed): sys.stderr.write('[-] No agent installation detected.\n') return False if filebeat_profiler.is_installed: filebeat_config = filebeat.FileBeatConfigurator( install_directory=environment_variables.get('FILEBEAT_HOME')) if prompt_user: sys.stderr.write( '[-] WARNING! REMOVING THE AGENT WILL RESULT IN EVENTS NO LONGER BEING SENT TO {}.\n' .format(filebeat_config.get_logstash_targets())) resp = utilities.prompt_input( 'Are you sure you wish to continue? ([no]|yes): ') while resp not in ['', 'no', 'yes']: resp = utilities.prompt_input( 'Are you sure you wish to continue? ([no]|yes): ') if resp != 'yes': sys.stdout.write('[+] Exiting\n') return False if filebeat_profiler.is_running: filebeat.FileBeatProcess().stop(stdout=True) if zeek_profiler.is_running: zeek.ZeekProcess().stop() if pf_profiler.is_installed: shutil.rmtree(environment_variables.get('PF_RING_HOME')) os.remove('/opt/dynamite/.agent_environment_prepared') if filebeat_profiler.is_installed: shutil.rmtree(environment_variables.get('FILEBEAT_HOME'), ignore_errors=True) if zeek_profiler.is_installed: shutil.rmtree(environment_variables.get('ZEEK_HOME'), ignore_errors=True) shutil.rmtree(environment_variables.get('ZEEK_SCRIPTS'), ignore_errors=True) if suricata_profiler.is_installed: shutil.rmtree(environment_variables.get('SURICATA_HOME'), ignore_errors=True) shutil.rmtree(environment_variables.get('SURICATA_CONFIG'), ignore_errors=True) shutil.rmtree(environment_variables.get('OINKMASTER_HOME'), ignore_errors=True) shutil.rmtree(const.INSTALL_CACHE, ignore_errors=True) env_lines = '' for line in open('/etc/dynamite/environment').readlines(): if 'FILEBEAT_HOME' in line: continue elif 'ZEEK_HOME' in line: continue elif 'ZEEK_SCRIPTS' in line: continue elif 'SURICATA_HOME' in line: continue elif 'SURICATA_CONFIG' in line: continue elif 'PF_RING_HOME' in line: continue elif 'OINKMASTER_HOME' in line: continue elif line.strip() == '': continue env_lines += line.strip() + '\n' with open('/etc/dynamite/environment', 'w') as f: f.write(env_lines) return True
def uninstall_kibana(prompt_user=True, stdout=True, verbose=False): """ Uninstall Kibana/ElastiFlow/Synesis Dashboards :param prompt_user: Print a warning before continuing :param stdout: Print the output to console :param verbose: Include detailed debug messages """ log_level = logging.INFO if verbose: log_level = logging.DEBUG logger = get_logger('KIBANA', level=log_level, stdout=stdout) env_file = os.path.join(const.CONFIG_PATH, 'environment') environment_variables = utilities.get_environment_file_dict() kb_profiler = kibana_profile.ProcessProfiler() if not kb_profiler.is_installed(): raise kibana_exceptions.UninstallKibanaError("Kibana is not installed.") configuration_directory = environment_variables.get('KIBANA_PATH_CONF') kb_config = kibana_configs.ConfigManager(configuration_directory) if prompt_user: sys.stderr.write( '\n\033[93m[-] WARNING! Removing Kibana will uninstall all visualizations and saved searches previously ' 'created.\033[0m\n') resp = utilities.prompt_input('\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m ') while resp not in ['', 'no', 'yes']: resp = utilities.prompt_input('\n\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m ') if resp != 'yes': if stdout: sys.stdout.write('\n[+] Exiting\n') exit(0) if kb_profiler.is_running(): kibana_process.ProcessManager().stop() try: shutil.rmtree(kb_config.kibana_path_conf) shutil.rmtree(kb_config.kibana_home) shutil.rmtree(kb_config.kibana_logs) shutil.rmtree(const.INSTALL_CACHE, ignore_errors=True) env_lines = '' with open(env_file) as env_fr: for line in env_fr.readlines(): if 'KIBANA_PATH_CONF' in line: continue elif 'KIBANA_HOME' in line: continue elif 'KIBANA_LOGS' in line: continue elif line.strip() == '': continue env_lines += line.strip() + '\n' with open(env_file, 'w') as env_fw: env_fw.write(env_lines) except Exception as e: logger.error("General error occurred while attempting to uninstall Kibana.".format(e)) logger.debug("General error occurred while attempting to uninstall Kibana; {}".format(e)) raise kibana_exceptions.UninstallKibanaError( "General error occurred while attempting to uninstall kibana; {}".format(e)) try: sysctl = systemctl.SystemCtl() except general_exceptions.CallProcessError: raise kibana_exceptions.UninstallKibanaError("Could not find systemctl.") sysctl.uninstall_and_disable('kibana')
def install_monitor(elasticsearch_password='******', verbose=False): """ Installs Logstash (with ElastiFlow templates modified to work with Zeek), ElasticSearch, and Kibana. :param elasticsearch_password: The password used for authentication across all builtin ES users :param verbose: Include output from system utilities :return: True, if installation succeeded """ es_pre_profiler = elasticsearch.ElasticProfiler() ls_pre_profiler = logstash.LogstashProfiler() kb_pre_profiler = kibana.KibanaProfiler() if ls_pre_profiler.is_installed and es_pre_profiler.is_installed and kb_pre_profiler.is_installed: sys.stderr.write( '[-] Monitor is already installed. If you wish to re-install, first uninstall.\n' ) return False if utilities.get_memory_available_bytes() < 14 * (1000**3): sys.stderr.write( '[-] WARNING Dynamite standalone monitor requires ' 'at-least 14GB to run currently available [{} GB]\n'.format( utilities.get_memory_available_bytes() / (1024**3))) if str(utilities.prompt_input('Continue? [y|N]: ')).lower() != 'y': return False utilities.create_dynamite_user(utilities.generate_random_password(50)) utilities.download_java(stdout=True) utilities.extract_java(stdout=True) utilities.setup_java() if not es_pre_profiler.is_installed: sys.stdout.write('[+] Installing Elasticsearch on localhost.\n') es_installer = elasticsearch.ElasticInstaller( host='0.0.0.0', port=9200, download_elasticsearch_archive=not ls_pre_profiler.is_downloaded, password=elasticsearch_password, stdout=True, verbose=verbose) es_installer.setup_elasticsearch() if not elasticsearch.ElasticProfiler().is_installed: sys.stderr.write( '[-] ElasticSearch failed to install on localhost.\n') return False sys.stdout.write('[+] Starting ElasticSearch on localhost.\n') es_process = elasticsearch.ElasticProcess() es_process.start() if not ls_pre_profiler.is_installed: ls_installer = logstash.LogstashInstaller( host='0.0.0.0', elasticsearch_password=elasticsearch_password, download_logstash_archive=not es_pre_profiler.is_downloaded, stdout=True, verbose=verbose) ls_installer.setup_logstash() if not logstash.LogstashProfiler().is_installed: sys.stderr.write('[-] LogStash failed to install on localhost.\n') return False if not kb_pre_profiler.is_installed and elasticsearch.ElasticProfiler( ).is_installed: sys.stdout.write('[+] Installing Kibana on localhost.\n') kb_installer = kibana.KibanaInstaller( host='0.0.0.0', port=5601, elasticsearch_host='localhost', elasticsearch_port=9200, elasticsearch_password=elasticsearch_password, download_kibana_archive=not kb_pre_profiler.is_downloaded, stdout=True, verbose=verbose) if not kb_pre_profiler.is_downloaded: kb_installer.download_kibana(stdout=True) kb_installer.extract_kibana(stdout=True) kb_installer.setup_kibana() if not kibana.KibanaProfiler().is_installed: sys.stderr.write('[-] Kibana failed to install on localhost.\n') return False sys.stdout.write( '[+] Monitor installation complete. Start the monitor: \'dynamite start monitor\'.\n' ) sys.stdout.flush() return True
def install_logstash(configuration_directory, install_directory, log_directory, host='0.0.0.0', elasticsearch_host='localhost', elasticsearch_port=9200, elasticsearch_password='******', heap_size_gigs=4, install_jdk=True, create_dynamite_user=True, stdout=False, verbose=False): """ Install Logstash with ElastiFlow & Synesis :param configuration_directory: Path to the configuration directory (E.G /etc/dynamite/logstash/) :param install_directory: Path to the install directory (E.G /opt/dynamite/logstash/) :param log_directory: Path to the log directory (E.G /var/log/dynamite/logstash/) :param host: The IP address to bind LogStash listeners too :param elasticsearch_password: The password used for authentication across all builtin ES users :param elasticsearch_host: A hostname/IP of the target elasticsearch instance :param elasticsearch_port: A port number for the target elasticsearch instance :param elasticsearch_password: The password used for authentication across all builtin ES users :param heap_size_gigs: The initial/max java heap space to allocate :param install_jdk: Install the latest OpenJDK that will be used by Logstash/ElasticSearch :param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run Logstash/ElasticSearch :param stdout: Print the output to console :param verbose: Include output from system utilities :return: True, if installation succeeded """ log_level = logging.INFO if verbose: log_level = logging.DEBUG logger = get_logger('LOGSTASH', level=log_level, stdout=stdout) ls_profiler = logstash_profile.ProcessProfiler() if ls_profiler.is_installed(): logger.error('LogStash is already installed.') raise logstash_exceptions.AlreadyInstalledLogstashError() if utilities.get_memory_available_bytes() < 6 * (1000**3): sys.stderr.write( '\n\033[93m[-] WARNING! LogStash should have at-least 6GB to run ' 'currently available [{} GB]\033[0m\n'.format( utilities.get_memory_available_bytes() / (1000**3))) if str(utilities.prompt_input( '\033[93m[?] Continue? [y|N]:\033[0m ')).lower() != 'y': sys.stdout.write('\n[+] Exiting\n') exit(0) ls_installer = InstallManager( configuration_directory, install_directory, log_directory, host=host, elasticsearch_host=elasticsearch_host, elasticsearch_port=elasticsearch_port, elasticsearch_password=elasticsearch_password, heap_size_gigs=heap_size_gigs, download_logstash_archive=not ls_profiler.is_downloaded(), stdout=stdout, verbose=verbose) if install_jdk: try: utilities.download_java(stdout=stdout) utilities.extract_java() utilities.setup_java() except Exception as e: logger.error( 'General error occurred while attempting to setup Java.') logger.debug( "General error occurred while attempting to setup Java; {}". format(e)) raise logstash_exceptions.InstallLogstashError( "General error occurred while attempting to setup Java; {}". format(e)) if create_dynamite_user: utilities.create_dynamite_user(utilities.generate_random_password(50)) ls_installer.setup_logstash()
def uninstall_dynamite_lab(prompt_user=True, stdout=True, verbose=False): """ Uninstall DynamiteLab :param prompt_user: Print a warning before continuing :param stdout: Print the output to console :param verbose: Include detailed debug messages """ log_level = logging.INFO if verbose: log_level = logging.DEBUG logger = get_logger('LAB', level=log_level, stdout=stdout) env_file = os.path.join(const.CONFIG_PATH, 'environment') environment_variables = utilities.get_environment_file_dict() configuration_directory = environment_variables.get('DYNAMITE_LAB_CONFIG') notebook_home = environment_variables.get('NOTEBOOK_HOME') dynamite_lab_profiler = lab_profile.ProcessProfiler() if not (dynamite_lab_profiler.is_installed and dynamite_lab_profiler.is_configured): raise lab_exceptions.UninstallLabError("DynamiteLab is not installed.") dynamite_lab_config = lab_configs.ConfigManager(configuration_directory) if prompt_user: sys.stderr.write( '\n\033[93m[-] WARNING! Removing Dynamite Lab Will Remove All Jupyter Notebooks\033[0m\n' ) resp = utilities.prompt_input( '\033[93m[?] Are you sure you wish to continue? ([no]|yes): \033[0m' ) while resp not in ['', 'no', 'yes']: resp = utilities.prompt_input( '\033[93m[?] Are you sure you wish to continue? ([no]|yes): \033[0m' ) if resp != 'yes': if stdout: sys.stdout.write('\n[+] Exiting\n') return if dynamite_lab_profiler.is_running: lab_process.ProcessManager().stop() shutil.rmtree(configuration_directory) shutil.rmtree(notebook_home) shutil.rmtree(const.INSTALL_CACHE, ignore_errors=True) env_lines = '' try: with open(env_file) as env_fr: for line in env_fr.readlines(): if 'DYNAMITE_LAB_CONFIG' in line: continue elif 'NOTEBOOK_HOME' in line: continue elif line.strip() == '': continue env_lines += line.strip() + '\n' with open(env_file, 'w') as env_fw: env_fw.write(env_lines) except Exception as e: logger.error( 'General error occurred while attempting to uninstall lab.') logger.debug( "General error occurred while attempting to uninstall lab; {}". format(e)) raise lab_exceptions.UninstallLabError( "General error occurred while attempting to uninstall lab; {}". format(e)) InstallManager( configuration_directory, notebook_home, elasticsearch_host=dynamite_lab_config.elasticsearch_url.split( '//')[1].split(':')[0], elasticsearch_password=dynamite_lab_config.elasticsearch_password, elasticsearch_port=dynamite_lab_config.elasticsearch_url.split( '//')[1].split(':')[1].replace('/', ''), download_dynamite_sdk_archive=False, extract_dynamite_sdk_archive=False).uninstall_kibana_lab_icon()
def uninstall_dynamited(prompt_user=True, stdout=True, verbose=False): """ Uninstall dynamited :param prompt_user: Print a warning before continuing :param stdout: Print the output to console :param verbose: Include detailed debug messages """ log_level = logging.INFO if verbose: log_level = logging.DEBUG logger = get_logger('DYNAMITED', level=log_level, stdout=stdout) env_file = os.path.join(const.CONFIG_PATH, 'environment') environment_variables = utilities.get_environment_file_dict() dynamited_profiler = dynamited_profile.ProcessProfiler() if not dynamited_profiler.is_installed(): raise dynamited_exceptions.UninstallDynamiteDaemonError("dynamited is not installed.") if prompt_user: sys.stderr.write( '\n\033[93m[-] WARNING! Removing dynamited will disable various performance metric gathering from ' 'occurring.\033[0m\n') resp = utilities.prompt_input('\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m ') while resp not in ['', 'no', 'yes']: resp = utilities.prompt_input('\n\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m ') if resp != 'yes': if stdout: sys.stdout.write('\n[+] Exiting\n') exit(0) if dynamited_profiler.is_running(): dynamited_process.ProcessManager().stop() try: shutil.rmtree(environment_variables['DYNAMITED_LOGS']) shutil.rmtree(environment_variables['DYNAMITED_INSTALL']) shutil.rmtree(environment_variables['DYNAMITED_CONFIG']) shutil.rmtree(const.INSTALL_CACHE, ignore_errors=True) env_lines = '' with open(env_file) as env_fr: for line in env_fr.readlines(): if 'DYNAMITED_LOGS' in line: continue elif 'DYNAMITED_INSTALL' in line: continue elif 'DYNAMITED_CONFIG' in line: continue elif line.strip() == '': continue env_lines += line.strip() + '\n' with open(env_file, 'w') as env_fw: env_fw.write(env_lines) except Exception as e: logger.error("General error occurred while attempting to uninstall dynamited.".format(e)) logger.debug("General error occurred while attempting to uninstall dynamited; {}".format(e)) raise dynamited_exceptions.UninstallDynamiteDaemonError( "General error occurred while attempting to uninstall dynamited; {}".format(e)) try: sysctl = systemctl.SystemCtl() except general_exceptions.CallProcessError: raise dynamited_exceptions.UninstallDynamiteDaemonError("Could not find systemctl.") sysctl.uninstall_and_disable('dynamited')
def uninstall_logstash(prompt_user=True, stdout=True, verbose=False): """ Install Logstash with ElastiFlow & Synesis :param prompt_user: Print a warning before continuing :param stdout: Print the output to console :param verbose: Include detailed debug messages """ log_level = logging.INFO if verbose: log_level = logging.DEBUG logger = get_logger('LOGSTASH', level=log_level, stdout=stdout) env_file = os.path.join(const.CONFIG_PATH, 'environment') environment_variables = utilities.get_environment_file_dict() configuration_directory = environment_variables.get('LS_PATH_CONF') ls_profiler = logstash_profile.ProcessProfiler() ls_config = logstash_config.ConfigManager( configuration_directory=configuration_directory) if not ls_profiler.is_installed(): logger.error('LogStash is not installed.') raise logstash_exceptions.UninstallLogstashError( "LogStash is not installed.") if prompt_user: sys.stderr.write( '\n\033[93m[-] WARNING! Removing Logstash Will Prevent ElasticSearch From Receiving Events.\033[0m\n' ) resp = utilities.prompt_input( '\n\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m ' ) while resp not in ['', 'no', 'yes']: resp = utilities.prompt_input( '\033[93m[?] Are you sure you wish to continue? ([no]|yes): \033[0m' ) if resp != 'yes': if stdout: sys.stdout.write('\n[+] Exiting\n') exit(0) if ls_profiler.is_running(): logstash_process.ProcessManager().stop() try: shutil.rmtree(ls_config.ls_path_conf) shutil.rmtree(ls_config.ls_home) shutil.rmtree(ls_config.path_logs) shutil.rmtree(const.INSTALL_CACHE, ignore_errors=True) env_lines = '' with open(env_file) as env_fr: for line in env_fr.readlines(): if 'LS_PATH_CONF' in line: continue elif 'LS_HOME' in line: continue elif 'LS_LOGS' in line: continue elif 'ELASTIFLOW_' in line: continue elif 'SYNLITE_' in line: continue elif 'ES_PASSWD' in line: continue elif line.strip() == '': continue env_lines += line.strip() + '\n' with open(env_file, 'w') as env_fw: env_fw.write(env_lines) except Exception as e: logger.error( "General error occurred while attempting to uninstall LogStash.". format(e)) logger.debug( "General error occurred while attempting to uninstall LogStash; {}" .format(e)) raise logstash_exceptions.UninstallLogstashError( "General error occurred while attempting to uninstall LogStash; {}" .format(e)) try: sysctl = systemctl.SystemCtl() except general_exceptions.CallProcessError: raise logstash_exceptions.UninstallLogstashError( "Could not find systemctl.") sysctl.uninstall_and_disable('logstash')
def install_logstash(host='0.0.0.0', elasticsearch_host='localhost', elasticsearch_port=9200, elasticsearch_password='******', install_jdk=True, create_dynamite_user=True, stdout=False, verbose=False): """ Install Logstash/ElastiFlow :param host: The IP address to bind LogStash listeners too :param elasticsearch_password: The password used for authentication across all builtin ES users :param elasticsearch_host: A hostname/IP of the target elasticsearch instance :param elasticsearch_port: A port number for the target elasticsearch instance :param elasticsearch_password: The password used for authentication across all builtin ES users :param install_jdk: Install the latest OpenJDK that will be used by Logstash/ElasticSearch :param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run Logstash/ElasticSearch :param stdout: Print the output to console :param verbose: Include output from system utilities :return: True, if installation succeeded """ ls_profiler = LogstashProfiler() if ls_profiler.is_installed: sys.stderr.write( '[-] LogStash is already installed. If you wish to re-install, first uninstall.\n' ) return False if utilities.get_memory_available_bytes() < 6 * (1000**3): sys.stderr.write( '[-] WARNING! Dynamite Logstash should have at-least 6GB to run ' 'currently available [{} GB]\n'.format( utilities.get_memory_available_bytes() / (1000**3))) if str(utilities.prompt_input('Continue? [y|N]: ')).lower() != 'y': return False try: ls_installer = LogstashInstaller( host=host, elasticsearch_host=elasticsearch_host, elasticsearch_port=elasticsearch_port, elasticsearch_password=elasticsearch_password, download_logstash_archive=not ls_profiler.is_downloaded, stdout=stdout, verbose=verbose) if install_jdk: utilities.download_java(stdout=stdout) utilities.extract_java(stdout=stdout) utilities.setup_java() if create_dynamite_user: utilities.create_dynamite_user( utilities.generate_random_password(50)) ls_installer.setup_logstash() except Exception: sys.stderr.write( '[-] A fatal error occurred while attempting to install LogStash: ' ) traceback.print_exc(file=sys.stderr) return False if stdout: sys.stdout.write( '[+] *** LogStash + ElastiFlow (w/ Zeek Support) installed successfully. ***\n\n' ) sys.stdout.write( '[+] Next, Start your collector: \'dynamite start logstash\'.\n') sys.stdout.flush() return LogstashProfiler(stderr=False).is_installed
def uninstall_dynamite_lab(stdout=False, prompt_user=True): """ Uninstall DynamiteLab :param stdout: Print the output to console :param prompt_user: Print a warning before continuing :return: True, if uninstall succeeded """ environment_variables = utilities.get_environment_file_dict() configuration_directory = environment_variables.get('DYNAMITE_LAB_CONFIG') notebook_home = environment_variables.get('NOTEBOOK_HOME') dynamite_lab_profiler = DynamiteLabProfiler() if not (dynamite_lab_profiler.is_installed and dynamite_lab_profiler.is_configured): sys.stderr.write('[-] DynanmiteLab is not installed.\n') return False dynamite_lab_config = DynamiteLabConfigurator(configuration_directory) if prompt_user: sys.stderr.write( '[-] WARNING! REMOVING DYNAMITE LAB WILL REMOVE ALL JUPYTER NOTEBOOKS.\n' ) resp = utilities.prompt_input( 'Are you sure you wish to continue? ([no]|yes): ') while resp not in ['', 'no', 'yes']: resp = utilities.prompt_input( 'Are you sure you wish to continue? ([no]|yes): ') if resp != 'yes': if stdout: sys.stdout.write('[+] Exiting\n') return False if dynamite_lab_profiler.is_running: JupyterHubProcess().stop(stdout=stdout) try: shutil.rmtree(configuration_directory) shutil.rmtree(notebook_home) shutil.rmtree('/tmp/dynamite/install_cache/', ignore_errors=True) env_lines = '' for line in open('/etc/dynamite/environment').readlines(): if 'DYNAMITE_LAB_CONFIG' in line: continue elif 'NOTEBOOK_HOME' in line: continue elif line.strip() == '': continue env_lines += line.strip() + '\n' open('/etc/dynamite/environment', 'w').write(env_lines) if stdout: sys.stdout.write('[+] Uninstalling DynamiteLab Kibana Icon.\n') icon_remove_result = DynamiteLabInstaller( elasticsearch_host=dynamite_lab_config.elasticsearch_url.split( '//')[1].split(':')[0], elasticsearch_password=dynamite_lab_config.elasticsearch_password, elasticsearch_port=dynamite_lab_config.elasticsearch_url.split( '//')[1].split(':')[1].replace('/', ''), download_dynamite_sdk_archive=False).uninstall_kibana_lab_icon() if not icon_remove_result: sys.stderr.write( '[-] Failed to restore DynamiteLab Kibana icon.\n') # Not fatal...just annoying; if stdout: sys.stdout.write('[+] DynamiteLab uninstalled successfully.\n') except Exception: sys.stderr.write( '[-] A fatal error occurred while attempting to uninstall DynamiteLab: ' ) traceback.print_exc(file=sys.stderr) return False return True
def install_elasticsearch(configuration_directory, install_directory, log_directory, password='******', heap_size_gigs=4, install_jdk=True, create_dynamite_user=True, stdout=True, verbose=False): """ Install ElasticSearch :param configuration_directory: Path to the configuration directory (E.G /etc/dynamite/elasticsearch/) :param install_directory: Path to the install directory (E.G /opt/dynamite/elasticsearch/) :param log_directory: Path to the log directory (E.G /var/log/dynamite/elasticsearch/) :param password: The password used for authentication across all builtin users :param heap_size_gigs: The initial/max java heap space to allocate :param install_jdk: Install the latest OpenJDK that will be used by Logstash/ElasticSearch :param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run Logstash/ElasticSearch :param stdout: Print the output to console :param verbose: Include detailed debug messages """ log_level = logging.INFO if verbose: log_level = logging.DEBUG logger = get_logger('ELASTICSEARCH', level=log_level, stdout=stdout) es_profiler = elastic_profile.ProcessProfiler() if es_profiler.is_installed(): logger.error('ElasticSearch is already installed.') raise elastic_exceptions.AlreadyInstalledElasticsearchError() if utilities.get_memory_available_bytes() < 6 * (1000**3): sys.stderr.write( '\n\033[93m[-] WARNING! ElasticSearch should have at-least 6GB to run ' 'currently available [{} GB]\033[0m\n'.format( utilities.get_memory_available_bytes() / (1000**3))) if str(utilities.prompt_input( '\033[93m[?] Continue? [y|N]:\033[0m ')).lower() != 'y': sys.stdout.write('\n[+] Exiting\n') exit(0) es_installer = InstallManager( configuration_directory=configuration_directory, install_directory=install_directory, log_directory=log_directory, password=password, heap_size_gigs=heap_size_gigs, download_elasticsearch_archive=not es_profiler.is_downloaded(), stdout=stdout, verbose=verbose) if install_jdk: try: utilities.download_java(stdout=stdout) utilities.extract_java() utilities.setup_java() except Exception as e: logger.error( 'General error occurred while attempting to setup Java.') logger.debug( "General error occurred while attempting to setup Java; {}". format(e)) raise elastic_exceptions.InstallElasticsearchError( "General error occurred while attempting to setup Java; {}". format(e)) if create_dynamite_user: utilities.create_dynamite_user(utilities.generate_random_password(50)) es_installer.setup_elasticsearch()