def optimize(self) -> None: """Runs Kibana webpack optimizer among other things. Returns: None """ environ = utilities.get_environment_file_dict() if not os.path.exists(PID_DIRECTORY): utilities.makedirs(PID_DIRECTORY) utilities.set_ownership_of_file(PID_DIRECTORY, user='******', group='dynamite') self.logger.info('Optimizing Kibana Libraries.') # Kibana initially has to be called as root due to a process forking issue when using runuser # builtin subprocess.call('{}/bin/kibana --optimize --allow-root'.format( environ['KIBANA_HOME'], ), shell=True, env=utilities.get_environment_file_dict(), stderr=subprocess.PIPE, stdout=subprocess.PIPE) # Pass permissions back to dynamite user utilities.set_ownership_of_file(environ['KIBANA_LOGS'], user='******', group='dynamite') utilities.set_ownership_of_file(environ['KIBANA_HOME'], user='******', group='dynamite')
def _is_installed(stderr=False): env_dict = utilities.get_environment_file_dict() kibana_home = env_dict.get('KIBANA_HOME') if not kibana_home: if stderr: sys.stderr.write('[-] Kibana installation directory could not be located in ' '/etc/dynamite/environment.\n') return False if not os.path.exists(kibana_home): if stderr: sys.stderr.write('[-] Kibana installation directory could not be located at {}.\n'.format( kibana_home)) return False kibana_home_files_and_dirs = os.listdir(kibana_home) if 'bin' not in kibana_home_files_and_dirs: if stderr: sys.stderr.write('[-] Could not locate Kibana {}/bin directory.\n'.format(kibana_home)) return False if 'webpackShims' not in kibana_home_files_and_dirs: if stderr: sys.stderr.write('[-] Could not locate Kibana {}/webpackShims directory.\n'.format(kibana_home)) return False kibana_binaries = os.listdir(os.path.join(kibana_home, 'bin')) if 'kibana' not in kibana_binaries: if stderr: sys.stderr.write('[-] Could not locate Kibana binary in {}/bin/\n'.format(kibana_home)) return False if not utilities.check_user_exists('dynamite'): sys.stderr.write('[-] dynamite user was not created.\n') return False return True
def _is_installed(stderr=False): env_dict = utilities.get_environment_file_dict() ls_home = env_dict.get('LS_HOME') if not ls_home: if stderr: sys.stderr.write('[-] LogStash installation directory could not be located in ' '/etc/dynamite/environment.\n') return False if not os.path.exists(ls_home): if stderr: sys.stderr.write('[-] LogStash installation directory could not be located at {}.\n'.format(ls_home)) return False ls_home_files_and_dirs = os.listdir(ls_home) if 'bin' not in ls_home_files_and_dirs: if stderr: sys.stderr.write('[-] Could not locate LogStash {}/bin directory.\n'.format(ls_home)) return False if 'lib' not in ls_home_files_and_dirs: if stderr: sys.stderr.write('[-] Could not locate LogStash {}/lib directory.\n'.format(ls_home)) return False ls_binaries = os.listdir(os.path.join(ls_home, 'bin')) if 'logstash' not in ls_binaries: if stderr: sys.stderr.write('[-] Could not locate LogStash binary in {}/bin/\n'.format(ls_home)) return False return True
def change_sdk_elasticsearch_password(password='******', prompt_user=True, stdout=False): """ Change the DynamiteSDK to ElasticSearch password :param password: The password that the SDK will use to connect to ElasticSearch :param prompt_user: Whether or not to warn the user :param stdout: Print output to console :return: True if changed successfully """ environment_variables = utilities.get_environment_file_dict() configuration_directory = environment_variables.get('DYNAMITE_LAB_CONFIG') if prompt_user: resp = utilities.prompt_input( 'Changing the SDK password can cause your notebooks to lose communication with ElasticSearch. ' 'Are you sure you wish to continue? [no]|yes): ') while resp not in ['', 'no', 'yes']: resp = utilities.prompt_input( 'Are you sure you wish to continue? ([no]|yes): ') if resp != 'yes': if stdout: sys.stdout.write('[+] Exiting\n') return False dynamite_lab_config = DynamiteLabConfigurator( configuration_directory=configuration_directory) dynamite_lab_config.elasticsearch_password = password dynamite_lab_config.write_config() return True
def onStart(self): env_vars = get_environment_file_dict() self.zeek_script_config = ZeekScriptConfigurator( env_vars['ZEEK_SCRIPTS']) self.addForm('MAIN', ZeekScriptSettingsForm, name='Zeek Script Configuration')
def __init__(self, stdout: Optional[bool] = True, verbose: Optional[bool] = False, pretty_print_status: Optional[bool] = False): """Manage Kibana Process Args: stdout: Print output to console verbose: Include detailed debug messages pretty_print_status: If enabled, status will be printed in a tabulated style Returns: None """ environ = utilities.get_environment_file_dict() process.BaseProcessManager.__init__( self, 'kibana.service', 'kibana.process', log_path=environ.get('KIBANA_LOGS'), stdout=stdout, verbose=verbose, pretty_print_status=pretty_print_status) if not kibana_profile.ProcessProfiler().is_installed(): self.logger.error( "Kibana is not installed. Install it with 'dynamite kibana install -h'" ) raise CallKibanaProcessError("Kibana is not installed.")
def _is_listening(stderr=False): env_dict = utilities.get_environment_file_dict() kibana_path_conf = env_dict.get('KIBANA_PATH_CONF') if not kibana_path_conf: if stderr: sys.stderr.write( '[-] Kibana configuration directory could not be located in ' '/etc/dynamite/environment.\n') return False if not os.path.exists(os.path.join(kibana_path_conf, 'kibana.yml')): if stderr: sys.stderr.write( '[-] Could not locate kibana.yml in {}\n'.format( kibana_path_conf)) return False try: kibana_config = KibanaConfigurator( configuration_directory=kibana_path_conf) except Exception: if stderr: sys.stderr.write( '[-] Un-parsable elasticsearch.yml or jvm.options \n') return False host = kibana_config.get_server_host() port = kibana_config.get_server_port() if host.strip() == '0.0.0.0': host = 'localhost' return utilities.check_socket(host, port)
def _is_running(): env_dict = utilities.get_environment_file_dict() filebeat_home = env_dict.get('FILEBEAT_HOME') if filebeat_home: return FileBeatProcess( install_directory=filebeat_home).status()['RUNNING'] return False
def install_zeek_package(package_git_url: str, stdout: Optional[bool] = True, verbose: Optional[bool] = False): """ Update Suricata rules specified in the oinkmaster.conf file Args: package_git_url: The path to the git repo containing the Zeek package stdout: Print the output to console verbose: Include detailed debug messages """ log_level = logging.INFO if verbose: log_level = logging.DEBUG logger = get_logger('zeek.zkg.package_install', level=log_level, stdout=stdout) environment_variables = utilities.get_environment_file_dict() zeek_install_dir = environment_variables.get('ZEEK_HOME') logger.info(f'Installing Zeek package: {package_git_url}.') if not zeek_install_dir: logger.error("Could not resolve ZEEK_HOME environment_variable. Is Zeek installed?") raise InstallZeekPackageError('Could not resolve ZEEK_HOME environment_variable. Is ZKG installed? ') zkg_binary_dir = f'{zeek_install_dir}/bin' zkg_install_p = subprocess.Popen(f'./zkg install {package_git_url} --force', cwd=zkg_binary_dir, shell=True, stderr=subprocess.PIPE) err = zkg_install_p.communicate() if zkg_install_p.returncode != 0: logger.error(f'ZKG returned a non-zero exit-code: {zkg_install_p.returncode}.') raise InstallZeekPackageError( f'ZKG returned a non-zero exit-code during install: {zkg_install_p.returncode}; err: {err}.') zkg_load_p = subprocess.Popen(f'./zkg load {package_git_url}', cwd=zkg_binary_dir, shell=True, stderr=subprocess.PIPE) err = zkg_load_p.communicate() if zkg_load_p.returncode != 0: logger.error(f'ZKG returned a non-zero exit-code during load: {zkg_load_p.returncode}.') raise InstallZeekPackageError( f'ZKG returned a non-zero exit-code during load: {zkg_load_p.returncode}; err: {err}.')
def __init__(self, cert_name: Optional[str] = 'admin.pem', key_name: Optional[str] = 'admin-key.pem', subj: Optional[str] = '/C=US/ST=GA/L=Atlanta/O=Dynamite/OU=R&D/CN=dynamite.ai', trusted_ca_cert_name: Optional[str] = 'root-ca.pem', trusted_ca_key_name: Optional[str] = 'root-ca-key.pem'): env = utilities.get_environment_file_dict() self.configuration_directory = env.get('ES_PATH_CONF') self.cert_name = cert_name self.key_name = key_name self.subj = subj self.trusted_ca_cert_name = trusted_ca_cert_name self.trusted_ca_key_name = trusted_ca_key_name self.security_conf_directory = f'{self.configuration_directory}/security' self.cert_directory = f'{self.security_conf_directory}/auth' super(GenerateElasticsearchSSLCertificates, self).__init__( name='generate_elasticsearch_certificates', package_link='N/A', commands=[ ['openssl', 'genrsa', '-out', trusted_ca_key_name, '2048'], ['openssl', 'req', '-new', '-x509', '-sha256', '-key', trusted_ca_key_name, '-out', trusted_ca_cert_name, '-subj', subj], ['openssl', 'genrsa', '-out', 'admin-key-temp.pem', '2048'], ['openssl', 'pkcs8', '-inform', 'PEM', '-outform', 'PEM', '-in', 'admin-key-temp.pem', '-topk8', '-nocrypt', '-v1', 'PBE-SHA1-3DES', '-out', key_name], ['openssl', 'req', '-new', '-key', key_name, '-out', 'admin.csr', '-subj', subj], ['openssl', 'x509', '-req', '-in', 'admin.csr', '-CA', trusted_ca_cert_name, '-CAkey', trusted_ca_key_name, '-CAcreateserial', '-sha256', '-out', cert_name], ])
def onStart(self): env_vars = get_environment_file_dict() self.suricata_rule_config = SuricataConfigurator( env_vars['SURICATA_CONFIG']) self.addForm('MAIN', SuricataRuleSettingsForm, name='Suricata Rule Configuration')
def onStart(self): env_vars = get_environment_file_dict() npyscreen.setTheme(npyscreen.Themes.ColorfulTheme) self.filebeat_config = config.ConfigManager(env_vars.get('FILEBEAT_HOME')) self.addForm('MAIN', FilebeatInstanceSettingsForm, name='FileBeat Configuration') self.addForm('EDITTARGETTYPEFM', EditTargetTypeOutputForm, name='Select between LogStash or Kafka Output.') self.addForm('EDITTARGETFM', EditTargetsForm, name='Edit FileBeat Targets')
def _is_installed(stderr=False): env_dict = utilities.get_environment_file_dict() filebeat_home = env_dict.get('FILEBEAT_HOME') if not filebeat_home: if stderr: sys.stderr.write( '[-] FILEBEAT_HOME installation directory could not be located in /etc/dynamite/environment.\n' ) return False if not os.path.exists(filebeat_home): if stderr: sys.stderr.write( '[-] FILEBEAT_HOME installation directory could not be located on disk at: {}.\n' .format(filebeat_home)) return False filebeat_home_directories_and_files = os.listdir(filebeat_home) if 'filebeat' not in filebeat_home_directories_and_files: if stderr: sys.stderr.write( '[-] Could not locate FILEBEAT {}/filebeat binary.\n'. format(filebeat_home)) return False if 'filebeat.yml' not in filebeat_home_directories_and_files: if stderr: sys.stderr.write( '[-] Could not locate FILEBEAT {}/filebeat.yml config.\n'. format(filebeat_home)) return False return True
def _is_configured(stderr=False): env_dict = utilities.get_environment_file_dict() ls_path_conf = env_dict.get('LS_PATH_CONF') if not ls_path_conf: if stderr: sys.stderr.write( '[-] LogStash configuration directory could not be located in ' '/etc/dynamite/environment.\n') return False if not os.path.exists(os.path.join(ls_path_conf, 'logstash.yml')): if stderr: sys.stderr.write( '[-] Could not locate logstash.yml in {}\n'.format( ls_path_conf)) return False if not os.path.exists(os.path.join(ls_path_conf, 'jvm.options')): if stderr: sys.stderr.write( '[-] Could not locate jvm.options in {}\n'.format( ls_path_conf)) return False try: LogstashConfigurator(configuration_directory=ls_path_conf) except Exception: if stderr: sys.stderr.write( '[-] Un-parsable logstash.yml or jvm.options \n') return False return True
def change_sdk_elasticsearch_password(password='******', prompt_user=True, stdout=False): """ Change the DynamiteSDK to ElasticSearch password :param password: The password that the SDK will use to connect to ElasticSearch :param prompt_user: Whether or not to warn the user :param stdout: Print output to console """ environment_variables = utilities.get_environment_file_dict() configuration_directory = environment_variables.get('DYNAMITE_LAB_CONFIG') if prompt_user: resp = utilities.prompt_input( '\033[93m[-] Changing the SDK password can cause your notebooks to lose communication with ElasticSearch.\n' '[?] Are you sure you wish to continue? [no]|yes):\033[0m ') while resp not in ['', 'no', 'yes']: resp = utilities.prompt_input( '\033[93m[?] Are you sure you wish to continue? ([no]|yes): \033[0m' ) if resp != 'yes': if stdout: sys.stdout.write('\n[+] Exiting\n') return dynamite_lab_config = ConfigManager( configuration_directory=configuration_directory) try: dynamite_lab_config.elasticsearch_password = password dynamite_lab_config.write_config() except lab_exceptions.WriteLabConfigError: raise general_exceptions.ResetPasswordError( "Could not write new password to DynamiteSDK config.cfg.")
def _is_installed(stderr=False): env_dict = utilities.get_environment_file_dict() pf_ring_home = env_dict.get('PF_RING_HOME') if not pf_ring_home: if stderr: sys.stderr.write( '[-] PF_RING installation directory could not be located in ' '/etc/dynamite/environment.\n') return False if not os.path.exists(pf_ring_home): if stderr: sys.stderr.write( '[-] PF_RING installation directory could not be located on disk at: {}.\n' .format(pf_ring_home)) return False pf_ring_home_files_and_dirs = os.listdir(pf_ring_home) if 'bin' not in pf_ring_home_files_and_dirs: if stderr: sys.stderr.write( '[-] Could not locate PF_RING {}/bin directory.\n'.format( pf_ring_home)) return False if 'lib' not in pf_ring_home_files_and_dirs: if stderr: sys.stderr.write( '[-] Could not locate PF_RING {}/lib directory.\n'.format( pf_ring_home)) return False return True
def __init__(self, purge_config: Optional[bool] = True, stdout: Optional[bool] = False, verbose: Optional[bool] = False): """Uninstall Kibana Args: purge_config: If enabled, remove all the configuration files associated with this installation stdout: Print output to console verbose: Include detailed debug messages Returns: None """ from dynamite_nsm.services.kibana.process import ProcessManager env_vars = utilities.get_environment_file_dict() kb_directories = [ env_vars.get('KIBANA_HOME'), env_vars.get('KIBANA_LOGS') ] if purge_config: kb_directories.append(env_vars.get('KIBANA_PATH_CONF')) super().__init__( 'kibana.uninstall', directories=kb_directories, process=ProcessManager(stdout=stdout, verbose=verbose), sysctl_service_name='kibana.service', environ_vars=['KIBANA_HOME', 'KIBANA_LOGS', 'KIBANA_PATH_CONF'], stdout=stdout, verbose=verbose)
def __init__(self, purge_config: Optional[bool] = True, stdout: Optional[bool] = False, verbose: Optional[bool] = False): """Uninstall Elasticsearch Args: purge_config: If enabled, remove all the configuration files associated with this installation stdout: Print output to console verbose: Include detailed debug messages Returns: None """ from dynamite_nsm.services.elasticsearch.process import ProcessManager env_vars = utilities.get_environment_file_dict() es_directories = [env_vars.get('ES_HOME'), env_vars.get('ES_LOGS')] if purge_config: es_directories.append(env_vars.get('ES_PATH_CONF')) install_events_to_hosts.EventsToHostsTask().remove_cronjob() super().__init__('elasticsearch.uninstall', directories=es_directories, environ_vars=['ES_PATH_CONF', 'ES_HOME', 'ES_LOGS'], process=ProcessManager(stdout=stdout, verbose=verbose), sysctl_service_name='elasticsearch.service', stdout=stdout, verbose=verbose)
def _is_configured(stderr=False): env_dict = utilities.get_environment_file_dict() es_path_conf = env_dict.get('ES_PATH_CONF') if not es_path_conf: if stderr: sys.stderr.write( '[-] ElasticSearch configuration directory could not be located in /etc/environment.\n' ) return False if not os.path.exists(os.path.join(es_path_conf, 'elasticsearch.yml')): if stderr: sys.stderr.write( '[-] Could not locate elasticsearch.yml in {}'.format( es_path_conf)) return False if not os.path.exists(os.path.join(es_path_conf, 'jvm.options')): if stderr: sys.stderr.write( '[-] Could not locate jvm.options in {}'.format( es_path_conf)) return False try: ElasticConfigurator(configuration_directory=es_path_conf) except Exception: if stderr: sys.stderr.write( '[-] Un-parsable elasticsearch.yml or jvm.options \n') return False return True
def _is_configured(stderr=False): try: env_dict = utilities.get_environment_file_dict() except IOError: if stderr: sys.stderr.write( '[-] DynamiteLab environment variables haven\'t been created.\n' ) return False dynamite_lab_config = env_dict.get('DYNAMITE_LAB_CONFIG') if not dynamite_lab_config: if stderr: sys.stderr.write( '[-] DynamiteLab configuration directory could not be located in ' '/etc/dynamite/environment.\n') return False if not os.path.exists(dynamite_lab_config): if stderr: sys.stderr.write( '[-] DynamiteLab configuration directory could not be located at {}.\n' .format(dynamite_lab_config)) return False try: DynamiteLabConfigurator( configuration_directory=dynamite_lab_config) except Exception: if stderr: sys.stderr.write('[-] Un-parsable config.cfg \n') return False return True
def _is_listening(stderr=False): env_dict = utilities.get_environment_file_dict() es_path_conf = env_dict.get('ES_PATH_CONF') if not es_path_conf: if stderr: sys.stderr.write( '[-] ElasticSearch configuration directory could not be located in /etc/environment.\n' ) return False if not os.path.exists(os.path.join(es_path_conf, 'elasticsearch.yml')): if stderr: sys.stderr.write( '[-] Could not locate elasticsearch.yml in {}\n'.format( es_path_conf)) return False if not os.path.exists(os.path.join(es_path_conf, 'jvm.options')): if stderr: sys.stderr.write( '[-] Could not locate jvm.options in {}\n'.format( es_path_conf)) return False try: es_config = ElasticConfigurator( configuration_directory=es_path_conf) except Exception: if stderr: sys.stderr.write( '[-] Un-parsable elasticsearch.yml or jvm.options \n') return False host = es_config.get_network_host() port = es_config.get_network_port() if host.strip() == '0.0.0.0': host = 'localhost' return utilities.check_socket(host, port)
def _is_running(): env_dict = utilities.get_environment_file_dict() zeek_home = env_dict.get('ZEEK_HOME') if zeek_home: if 'running' in ZeekProcess(install_directory=zeek_home).status(): return True return False
def uninstall_logstash(stdout=False, prompt_user=True): """ Uninstall Logstash/ElastiFlow :param stdout: Print the output to console :param prompt_user: Print a warning before continuing :return: True, if uninstall succeeded """ environment_variables = utilities.get_environment_file_dict() configuration_directory = environment_variables.get('LS_PATH_CONF') ls_profiler = LogstashProfiler() ls_config = LogstashConfigurator( configuration_directory=configuration_directory) if not ls_profiler.is_installed: sys.stderr.write('[-] LogStash is not installed.\n') return False if prompt_user: sys.stderr.write( '[-] WARNING! REMOVING LOGSTASH WILL PREVENT ELASTICSEARCH FROM RECEIVING EVENTS.\n' ) resp = utilities.prompt_input( 'Are you sure you wish to continue? ([no]|yes): ') while resp not in ['', 'no', 'yes']: resp = utilities.prompt_input( 'Are you sure you wish to continue? ([no]|yes): ') if resp != 'yes': if stdout: sys.stdout.write('[+] Exiting\n') return False if ls_profiler.is_running: LogstashProcess().stop(stdout=stdout) try: shutil.rmtree(ls_config.ls_path_conf) shutil.rmtree(ls_config.ls_home) shutil.rmtree(ls_config.get_log_path()) shutil.rmtree('/tmp/dynamite/install_cache/', ignore_errors=True) env_lines = '' for line in open('/etc/dynamite/environment').readlines(): if 'LS_PATH_CONF' in line: continue elif 'LS_HOME' in line: continue elif 'ELASTIFLOW_' in line: continue elif 'SYNLITE_' in line: continue elif line.strip() == '': continue env_lines += line.strip() + '\n' open('/etc/dynamite/environment', 'w').write(env_lines) if stdout: sys.stdout.write('[+] LogStash uninstalled successfully.\n') except Exception: sys.stderr.write( '[-] A fatal error occurred while attempting to uninstall LogStash: ' ) traceback.print_exc(file=sys.stderr) return False return True
def onStart(self): env_vars = get_environment_file_dict() self.suricata_rule_config = config.ConfigManager( env_vars['SURICATA_CONFIG'], backup_configuration_directory=const.CONFIG_BACKUP_PATH) self.addForm('MAIN', SuricataRuleSettingsForm, name='Suricata Rule Configuration')
def onStart(self): npyscreen.setTheme(npyscreen.Themes.ColorfulTheme) env_vars = get_environment_file_dict() self.zeek_script_config = config.ScriptConfigManager( env_vars['ZEEK_SCRIPTS']) self.addForm('MAIN', ZeekScriptSettingsForm, name='Zeek Script Configuration')
def __init__(self): self.environment_variables = utilities.get_environment_file_dict() self.configuration_directory = self.environment_variables.get('KIBANA_PATH_CONF') self.config = KibanaConfigurator(self.configuration_directory) try: self.pid = int(open('/var/run/dynamite/kibana/kibana.pid').read()) except (IOError, ValueError): self.pid = -1
def start_shell_out(): # We use su instead of runuser here because of nodes' weird dependency on PAM # when calling from within a sub-shell subprocess.call('su -l dynamite -c "{}/bin/kibana -c {} -l {} & > /dev/null &"'.format( self.config.kibana_home, os.path.join(self.config.kibana_path_conf, 'kibana.yml'), os.path.join(self.config.kibana_logs, 'kibana.log') ), shell=True, env=utilities.get_environment_file_dict())
def __init__(self): self.environment_variables = utilities.get_environment_file_dict() self.configuration_directory = self.environment_variables.get( 'DYNAMITE_LAB_CONFIG') try: self.pid = int( open('/var/run/dynamite/jupyterhub/jupyterhub.pid').read()) except (IOError, ValueError): self.pid = -1
def __init__(self, stdout=True, verbose=False, pretty_print_status=False): environ = utilities.get_environment_file_dict() try: process.BaseProcessManager.__init__(self, 'dynamited.service', 'dynamited', log_path=environ.get('DYNAMITED_LOGS'), pid_file=os.path.join(PID_DIRECTORY, 'dynamited.pid'), stdout=stdout, verbose=verbose, pretty_print_status=pretty_print_status) except general_exceptions.CallProcessError: raise dynamited_exceptions.CallDynamiteDaemonProcessError("Could not find systemctl.")
def __init__(self, log_sample_size=10000): self.env_file = os.path.join(const.CONFIG_PATH, 'environment') self.env_dict = utilities.get_environment_file_dict() self.suricata_logs = self.env_dict.get('SURICATA_LOGS') self.log_path = os.path.join(self.suricata_logs, 'eve.json') logs.LogFile.__init__(self, log_path=self.log_path, log_sample_size=log_sample_size)