from logging import DEBUG, INFO from dynamite_nsm.logger import get_logger from dynamite_nsm.services.zeek import process from dynamite_nsm.cmd.zeek.process import interface as zeek_process_interface if __name__ == '__main__': parser = zeek_process_interface.get_parser() args = parser.parse_args() logger = get_logger('ZEEK', level=DEBUG if args.verbose else INFO, stdout=args.stdout) logger.info(f'Calling zeek.process.{args.entry_method_name}.') logger.debug(args.__dict__) result = zeek_process_interface.execute(args) print(result)
from logging import DEBUG, INFO from dynamite_nsm.cmd.elasticsearch.process import interface as elasticsearch_process_interface from dynamite_nsm.logger import get_logger if __name__ == '__main__': parser = elasticsearch_process_interface.get_parser() args = parser.parse_args() logger = get_logger('elasticsearch.process', level=DEBUG if args.verbose else INFO, stdout=args.stdout) logger.info(f'Calling elasticsearch.process.{args.entry_method_name}.') logger.debug(args.__dict__) result = elasticsearch_process_interface.execute(args) print(result)
from logging import DEBUG, INFO from dynamite_nsm.logger import get_logger from dynamite_nsm.cmd.suricata.process import interface as suricata_process_interface if __name__ == '__main__': parser = suricata_process_interface.get_parser() args = parser.parse_args() logger = get_logger('suricata.process', level=DEBUG if args.verbose else INFO, stdout=args.stdout) logger.info(f'Calling suricata.process.{args.entry_method_name}.') logger.debug(args.__dict__) result = suricata_process_interface.execute(args) print(result)
def uninstall_dynamited(prompt_user=True, stdout=True, verbose=False): """ Uninstall dynamited :param prompt_user: Print a warning before continuing :param stdout: Print the output to console :param verbose: Include detailed debug messages """ log_level = logging.INFO if verbose: log_level = logging.DEBUG logger = get_logger('DYNAMITED', level=log_level, stdout=stdout) env_file = os.path.join(const.CONFIG_PATH, 'environment') environment_variables = utilities.get_environment_file_dict() dynamited_profiler = dynamited_profile.ProcessProfiler() if not dynamited_profiler.is_installed(): raise dynamited_exceptions.UninstallDynamiteDaemonError("dynamited is not installed.") if prompt_user: sys.stderr.write( '\n\033[93m[-] WARNING! Removing dynamited will disable various performance metric gathering from ' 'occurring.\033[0m\n') resp = utilities.prompt_input('\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m ') while resp not in ['', 'no', 'yes']: resp = utilities.prompt_input('\n\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m ') if resp != 'yes': if stdout: sys.stdout.write('\n[+] Exiting\n') exit(0) if dynamited_profiler.is_running(): dynamited_process.ProcessManager().stop() try: shutil.rmtree(environment_variables['DYNAMITED_LOGS']) shutil.rmtree(environment_variables['DYNAMITED_INSTALL']) shutil.rmtree(environment_variables['DYNAMITED_CONFIG']) shutil.rmtree(const.INSTALL_CACHE, ignore_errors=True) env_lines = '' with open(env_file) as env_fr: for line in env_fr.readlines(): if 'DYNAMITED_LOGS' in line: continue elif 'DYNAMITED_INSTALL' in line: continue elif 'DYNAMITED_CONFIG' in line: continue elif line.strip() == '': continue env_lines += line.strip() + '\n' with open(env_file, 'w') as env_fw: env_fw.write(env_lines) except Exception as e: logger.error("General error occurred while attempting to uninstall dynamited.".format(e)) logger.debug("General error occurred while attempting to uninstall dynamited; {}".format(e)) raise dynamited_exceptions.UninstallDynamiteDaemonError( "General error occurred while attempting to uninstall dynamited; {}".format(e)) try: sysctl = systemctl.SystemCtl() except general_exceptions.CallProcessError: raise dynamited_exceptions.UninstallDynamiteDaemonError("Could not find systemctl.") sysctl.uninstall_and_disable('dynamited')
def install_logstash(configuration_directory, install_directory, log_directory, host='0.0.0.0', elasticsearch_host='localhost', elasticsearch_port=9200, elasticsearch_password='******', heap_size_gigs=4, install_jdk=True, create_dynamite_user=True, stdout=False, verbose=False): """ Install Logstash with ElastiFlow & Synesis :param configuration_directory: Path to the configuration directory (E.G /etc/dynamite/logstash/) :param install_directory: Path to the install directory (E.G /opt/dynamite/logstash/) :param log_directory: Path to the log directory (E.G /var/log/dynamite/logstash/) :param host: The IP address to bind LogStash listeners too :param elasticsearch_password: The password used for authentication across all builtin ES users :param elasticsearch_host: A hostname/IP of the target elasticsearch instance :param elasticsearch_port: A port number for the target elasticsearch instance :param elasticsearch_password: The password used for authentication across all builtin ES users :param heap_size_gigs: The initial/max java heap space to allocate :param install_jdk: Install the latest OpenJDK that will be used by Logstash/ElasticSearch :param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run Logstash/ElasticSearch :param stdout: Print the output to console :param verbose: Include output from system utilities :return: True, if installation succeeded """ log_level = logging.INFO if verbose: log_level = logging.DEBUG logger = get_logger('LOGSTASH', level=log_level, stdout=stdout) ls_profiler = logstash_profile.ProcessProfiler() if ls_profiler.is_installed(): logger.error('LogStash is already installed.') raise logstash_exceptions.AlreadyInstalledLogstashError() if utilities.get_memory_available_bytes() < 6 * (1000**3): sys.stderr.write( '\n\033[93m[-] WARNING! LogStash should have at-least 6GB to run ' 'currently available [{} GB]\033[0m\n'.format( utilities.get_memory_available_bytes() / (1000**3))) if str(utilities.prompt_input( '\033[93m[?] Continue? [y|N]:\033[0m ')).lower() != 'y': sys.stdout.write('\n[+] Exiting\n') exit(0) ls_installer = InstallManager( configuration_directory, install_directory, log_directory, host=host, elasticsearch_host=elasticsearch_host, elasticsearch_port=elasticsearch_port, elasticsearch_password=elasticsearch_password, heap_size_gigs=heap_size_gigs, download_logstash_archive=not ls_profiler.is_downloaded(), stdout=stdout, verbose=verbose) if install_jdk: try: utilities.download_java(stdout=stdout) utilities.extract_java() utilities.setup_java() except Exception as e: logger.error( 'General error occurred while attempting to setup Java.') logger.debug( "General error occurred while attempting to setup Java; {}". format(e)) raise logstash_exceptions.InstallLogstashError( "General error occurred while attempting to setup Java; {}". format(e)) if create_dynamite_user: utilities.create_dynamite_user(utilities.generate_random_password(50)) ls_installer.setup_logstash()
def uninstall_logstash(prompt_user=True, stdout=True, verbose=False): """ Install Logstash with ElastiFlow & Synesis :param prompt_user: Print a warning before continuing :param stdout: Print the output to console :param verbose: Include detailed debug messages """ log_level = logging.INFO if verbose: log_level = logging.DEBUG logger = get_logger('LOGSTASH', level=log_level, stdout=stdout) env_file = os.path.join(const.CONFIG_PATH, 'environment') environment_variables = utilities.get_environment_file_dict() configuration_directory = environment_variables.get('LS_PATH_CONF') ls_profiler = logstash_profile.ProcessProfiler() ls_config = logstash_config.ConfigManager( configuration_directory=configuration_directory) if not ls_profiler.is_installed(): logger.error('LogStash is not installed.') raise logstash_exceptions.UninstallLogstashError( "LogStash is not installed.") if prompt_user: sys.stderr.write( '\n\033[93m[-] WARNING! Removing Logstash Will Prevent ElasticSearch From Receiving Events.\033[0m\n' ) resp = utilities.prompt_input( '\n\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m ' ) while resp not in ['', 'no', 'yes']: resp = utilities.prompt_input( '\033[93m[?] Are you sure you wish to continue? ([no]|yes): \033[0m' ) if resp != 'yes': if stdout: sys.stdout.write('\n[+] Exiting\n') exit(0) if ls_profiler.is_running(): logstash_process.ProcessManager().stop() try: shutil.rmtree(ls_config.ls_path_conf) shutil.rmtree(ls_config.ls_home) shutil.rmtree(ls_config.path_logs) shutil.rmtree(const.INSTALL_CACHE, ignore_errors=True) env_lines = '' with open(env_file) as env_fr: for line in env_fr.readlines(): if 'LS_PATH_CONF' in line: continue elif 'LS_HOME' in line: continue elif 'LS_LOGS' in line: continue elif 'ELASTIFLOW_' in line: continue elif 'SYNLITE_' in line: continue elif 'ES_PASSWD' in line: continue elif line.strip() == '': continue env_lines += line.strip() + '\n' with open(env_file, 'w') as env_fw: env_fw.write(env_lines) except Exception as e: logger.error( "General error occurred while attempting to uninstall LogStash.". format(e)) logger.debug( "General error occurred while attempting to uninstall LogStash; {}" .format(e)) raise logstash_exceptions.UninstallLogstashError( "General error occurred while attempting to uninstall LogStash; {}" .format(e)) try: sysctl = systemctl.SystemCtl() except general_exceptions.CallProcessError: raise logstash_exceptions.UninstallLogstashError( "Could not find systemctl.") sysctl.uninstall_and_disable('logstash')
def install_dependencies(stdout=False, verbose=False): """ Install the required dependencies required by Zeek :param stdout: Print the output to console :param verbose: Include detailed debug messages """ log_level = logging.INFO if verbose: log_level = logging.DEBUG logger = get_logger('ZEEK', level=log_level, stdout=stdout) logger.info('Installing Dependencies.') pkt_mng = package_manager.OSPackageManager(stdout=stdout, verbose=verbose) packages = None if pkt_mng.package_manager == 'apt-get': packages = [ 'cmake', 'cmake3', 'make', 'gcc', 'g++', 'flex', 'bison', 'libpcap-dev', 'libssl-dev', 'python-dev', 'swig', 'zlib1g-dev', 'linux-headers-$(uname -r)', 'linux-headers-generic', 'tar' ] elif pkt_mng.package_manager == 'yum': packages = [ 'cmake', 'cmake3', 'make', 'gcc', 'gcc-c++', 'flex', 'bison', 'libpcap-devel', 'openssl-devel', 'python-devel', 'python2-devel', 'swig', 'zlib-devel', 'kernel-devel-$(uname -r)', 'kernel-devel', 'tar' ] # Work around for missing dependencies in RHEL/Centos8 try: pkt_mng.install_packages(['dnf-plugins-core']) except general_exceptions.OsPackageManagerInstallError as e: logger.warning( "Failed to install one or more packages: {}".format(e)) enable_powertools_p = subprocess.Popen( ['yum', 'config-manager', '--set-enabled', 'PowerTools'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) enable_powertools_p.communicate() if enable_powertools_p.returncode == 0: logger.info("Installed PowerTools.") logger.info('Refreshing Package Index.') try: pkt_mng.refresh_package_indexes() except general_exceptions.OsPackageManagerRefreshError as e: logger.warning("Failed to refresh packages.") logger.debug("Failed to refresh packages threw: {}".format(e)) raise general_exceptions.OsPackageManagerRefreshError( 'Failed to refresh packages.') logger.info('Installing the following packages: {}.'.format(packages)) try: pkt_mng.install_packages(packages) except general_exceptions.OsPackageManagerInstallError as e: logger.warning( "Failed to install packages one or more packages: {}".format( e))
def install_dependencies(stdout=True, verbose=False): """ Install the required dependencies required by Jupyterhub :param stdout: Print output to console :param verbose: Include detailed debug messages """ log_level = logging.INFO if verbose: log_level = logging.DEBUG logger = get_logger('LAB', level=log_level, stdout=stdout) pkt_mng = package_manager.OSPackageManager(stdout=stdout, verbose=verbose) packages = None logger.info('Updating Package Indexes.') pkt_mng.refresh_package_indexes() logger.info('Installing dependencies.') if pkt_mng.package_manager == 'apt-get': packages = [ 'build-essential', 'python3', 'python3-pip', 'python3-dev', 'nodejs', 'npm' ] elif pkt_mng.package_manager == 'yum': p = subprocess.Popen( 'curl --silent --location https://rpm.nodesource.com/setup_10.x | sudo bash -', stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, close_fds=True) p.communicate() if p.returncode != 0: logger.error('Could not install nodejs source rpm.') raise general_exceptions.OsPackageManagerInstallError( "Could not install nodejs from third-party RPM; https://rpm.nodesource.com/setup_10.x" ) packages = [ 'gcc72-c++', 'gcc', 'gcc-c++', 'nodejs', 'python36', 'python36-devel' ] logger.info('Installing the following packages: {}.'.format(packages)) try: pkt_mng.install_packages(packages) except general_exceptions.OsPackageManagerInstallError as e: logger.warning( "Failed to install one or more packages: {}".format(e)) logger.info( 'Installing configurable-http-proxy. This may take some time.') p = subprocess.Popen('npm install -g configurable-http-proxy', stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) p.communicate() if p.returncode != 0: try: err_msg = p.stderr.read() except ValueError: err_msg = p.returncode logger.error( 'Failed to install configurable-http-proxy, ensure npm is installed and in $PATH: {}' ''.format(err_msg)) raise general_exceptions.OsPackageManagerInstallError( "Could not install configurable-http-proxy via npm; {}".format( err_msg))
def __init__(self, name, verbose=False, stdout=True): log_level = logging.INFO if verbose: log_level = logging.DEBUG self.logger = get_logger(str(name).upper(), level=log_level, stdout=stdout)
def get_zeek_workers(network_capture_interfaces, strategy="aggressive", stdout=True, verbose=False): """ Algorithm for determining the assignment of CPUs for Zeek workers :param network_capture_interfaces: A list of network interface names :param strategy: 'aggressive', results in more CPUs pinned per interface, sometimes overshoots resources 'conservative', results in less CPUs pinned per interface, but never overshoots resources :param stdout: Print the output to console :param verbose: Include detailed debug messages :return: A dictionary containing Zeek worker configuration """ log_level = logging.INFO if verbose: log_level = logging.DEBUG logger = get_logger('ZEEK', level=log_level, stdout=stdout) cpus = [c for c in range(0, utilities.get_cpu_core_count())] logger.info( "Calculating optimal Zeek worker strategy [strategy: {}].".format( strategy)) logger.debug("Detected CPU Cores: {}".format(cpus)) # Reserve 0 for KERNEL/Userland opts available_cpus = cpus[1:] def grouper(n, iterable): args = [iter(iterable)] * n return zip_longest(*args) def create_workers(net_interfaces, avail_cpus): idx = 0 zeek_worker_configs = [] for net_interface in net_interfaces: if idx >= len(avail_cpus): idx = 0 if isinstance(avail_cpus[idx], int): avail_cpus[idx] = [avail_cpus[idx]] zeek_worker_configs.append( dict(name='dynamite-worker-' + net_interface, host='localhost', interface=net_interface, lb_procs=len(avail_cpus[idx]), pinned_cpus=avail_cpus[idx])) idx += 1 return zeek_worker_configs if len(available_cpus) <= len(network_capture_interfaces): # Wrap the number of CPUs around the number of network interfaces; # Since there are more network interfaces than CPUs; CPUs will be assigned more than once # lb_procs will always be 1 zeek_workers = create_workers(network_capture_interfaces, available_cpus) else: # In this scenario we choose from one of two strategies # 1. Aggressive: # - Take the ratio of network_interfaces to available CPUS; ** ROUND UP **. # - Group the available CPUs by this integer # (if the ratio == 2 create as many groupings of 2 CPUs as possible) # - Apply the same wrapping logic used above, but with the CPU groups instead of single CPU instances # 2. Conservative: # - Take the ratio of network_interfaces to available CPUS; ** ROUND DOWN **. # - Group the available CPUs by this integer # (if the ratio == 2 create as many groupings of 2 CPUs as possible) # - Apply the same wrapping logic used above, but with the CPU groups instead of single CPU instances aggressive_ratio = int( math.ceil( len(available_cpus) / float(len(network_capture_interfaces)))) conservative_ratio = int( math.floor( len(available_cpus) / len(network_capture_interfaces))) if strategy == 'aggressive': cpu_groups = grouper(aggressive_ratio, available_cpus) else: cpu_groups = grouper(conservative_ratio, available_cpus) temp_cpu_groups = [] for cpu_group in cpu_groups: cpu_group = [c for c in cpu_group if c] temp_cpu_groups.append(cpu_group) cpu_groups = temp_cpu_groups zeek_workers = create_workers(network_capture_interfaces, cpu_groups) logger.info('Zeek Worker Count: {}'.format(len(zeek_workers))) logger.debug('Zeek Workers: {}'.format(zeek_workers)) return zeek_workers
def install_elasticsearch(configuration_directory, install_directory, log_directory, password='******', heap_size_gigs=4, install_jdk=True, create_dynamite_user=True, stdout=True, verbose=False): """ Install ElasticSearch :param configuration_directory: Path to the configuration directory (E.G /etc/dynamite/elasticsearch/) :param install_directory: Path to the install directory (E.G /opt/dynamite/elasticsearch/) :param log_directory: Path to the log directory (E.G /var/log/dynamite/elasticsearch/) :param password: The password used for authentication across all builtin users :param heap_size_gigs: The initial/max java heap space to allocate :param install_jdk: Install the latest OpenJDK that will be used by Logstash/ElasticSearch :param create_dynamite_user: Automatically create the 'dynamite' user, who has privs to run Logstash/ElasticSearch :param stdout: Print the output to console :param verbose: Include detailed debug messages """ log_level = logging.INFO if verbose: log_level = logging.DEBUG logger = get_logger('ELASTICSEARCH', level=log_level, stdout=stdout) es_profiler = elastic_profile.ProcessProfiler() if es_profiler.is_installed(): logger.error('ElasticSearch is already installed.') raise elastic_exceptions.AlreadyInstalledElasticsearchError() if utilities.get_memory_available_bytes() < 6 * (1000**3): sys.stderr.write( '\n\033[93m[-] WARNING! ElasticSearch should have at-least 6GB to run ' 'currently available [{} GB]\033[0m\n'.format( utilities.get_memory_available_bytes() / (1000**3))) if str(utilities.prompt_input( '\033[93m[?] Continue? [y|N]:\033[0m ')).lower() != 'y': sys.stdout.write('\n[+] Exiting\n') exit(0) es_installer = InstallManager( configuration_directory=configuration_directory, install_directory=install_directory, log_directory=log_directory, password=password, heap_size_gigs=heap_size_gigs, download_elasticsearch_archive=not es_profiler.is_downloaded(), stdout=stdout, verbose=verbose) if install_jdk: try: utilities.download_java(stdout=stdout) utilities.extract_java() utilities.setup_java() except Exception as e: logger.error( 'General error occurred while attempting to setup Java.') logger.debug( "General error occurred while attempting to setup Java; {}". format(e)) raise elastic_exceptions.InstallElasticsearchError( "General error occurred while attempting to setup Java; {}". format(e)) if create_dynamite_user: utilities.create_dynamite_user(utilities.generate_random_password(50)) es_installer.setup_elasticsearch()
def uninstall_dynamite_lab(prompt_user=True, stdout=True, verbose=False): """ Uninstall DynamiteLab :param prompt_user: Print a warning before continuing :param stdout: Print the output to console :param verbose: Include detailed debug messages """ log_level = logging.INFO if verbose: log_level = logging.DEBUG logger = get_logger('LAB', level=log_level, stdout=stdout) env_file = os.path.join(const.CONFIG_PATH, 'environment') environment_variables = utilities.get_environment_file_dict() configuration_directory = environment_variables.get('DYNAMITE_LAB_CONFIG') notebook_home = environment_variables.get('NOTEBOOK_HOME') dynamite_lab_profiler = lab_profile.ProcessProfiler() if not (dynamite_lab_profiler.is_installed and dynamite_lab_profiler.is_configured): raise lab_exceptions.UninstallLabError("DynamiteLab is not installed.") dynamite_lab_config = lab_configs.ConfigManager(configuration_directory) if prompt_user: sys.stderr.write( '\n\033[93m[-] WARNING! Removing Dynamite Lab Will Remove All Jupyter Notebooks\033[0m\n' ) resp = utilities.prompt_input( '\033[93m[?] Are you sure you wish to continue? ([no]|yes): \033[0m' ) while resp not in ['', 'no', 'yes']: resp = utilities.prompt_input( '\033[93m[?] Are you sure you wish to continue? ([no]|yes): \033[0m' ) if resp != 'yes': if stdout: sys.stdout.write('\n[+] Exiting\n') return if dynamite_lab_profiler.is_running: lab_process.ProcessManager().stop() shutil.rmtree(configuration_directory) shutil.rmtree(notebook_home) shutil.rmtree(const.INSTALL_CACHE, ignore_errors=True) env_lines = '' try: with open(env_file) as env_fr: for line in env_fr.readlines(): if 'DYNAMITE_LAB_CONFIG' in line: continue elif 'NOTEBOOK_HOME' in line: continue elif line.strip() == '': continue env_lines += line.strip() + '\n' with open(env_file, 'w') as env_fw: env_fw.write(env_lines) except Exception as e: logger.error( 'General error occurred while attempting to uninstall lab.') logger.debug( "General error occurred while attempting to uninstall lab; {}". format(e)) raise lab_exceptions.UninstallLabError( "General error occurred while attempting to uninstall lab; {}". format(e)) InstallManager( configuration_directory, notebook_home, elasticsearch_host=dynamite_lab_config.elasticsearch_url.split( '//')[1].split(':')[0], elasticsearch_password=dynamite_lab_config.elasticsearch_password, elasticsearch_port=dynamite_lab_config.elasticsearch_url.split( '//')[1].split(':')[1].replace('/', ''), download_dynamite_sdk_archive=False, extract_dynamite_sdk_archive=False).uninstall_kibana_lab_icon()
def __init__(self, configuration_directory, notebook_home, elasticsearch_host=None, elasticsearch_port=None, elasticsearch_password='******', jupyterhub_host=None, jupyterhub_password='******', download_dynamite_sdk_archive=True, extract_dynamite_sdk_archive=True, stdout=True, verbose=False): """ :param configuration_directory: Path to the configuration directory (E.G /etc/dynamite/dynamite_sdk/) :param notebook_home: The path where Jupyter notebooks are stored :param elasticsearch_host: A hostname/IP of the target elasticsearch instance :param elasticsearch_port: A port number for the target elasticsearch instance :param elasticsearch_password: The password used for authentication across all builtin ES users :param jupyterhub_host: The host by which users can access this instance; (Used for creating kibana -> Jupyter hyperlinks) :param jupyterhub_password: The password used for authenticating to jupyterhub (via jupyter user) :param download_dynamite_sdk_archive: If True, download the DynamiteSDK archive from a mirror :param extract_dynamite_sdk_archive: If True, extracts the DynamiteSDK archive :param stdout: Print output to console :param verbose: Include detailed debug messages """ log_level = logging.INFO if verbose: log_level = logging.DEBUG self.logger = get_logger('LAB', level=log_level, stdout=stdout) self.elasticsearch_host = elasticsearch_host self.elasticsearch_port = elasticsearch_port self.elasticsearch_password = elasticsearch_password self.jupyterhub_host = jupyterhub_host self.jupyterhub_password = jupyterhub_password self.configuration_directory = configuration_directory self.notebook_home = notebook_home if download_dynamite_sdk_archive: try: self.download_dynamite_sdk(stdout=stdout) except general_exceptions.DownloadError: self.logger.error('Failed to download DynamiteSDK archive.') raise lab_exceptions.InstallLabError( "Failed to download DynamiteSDK archive.") if extract_dynamite_sdk_archive: try: self.extract_dynamite_sdk() except general_exceptions.ArchiveExtractionError: self.logger.error('Failed to extract DynamiteSDK archive.') raise lab_exceptions.InstallLabError( "Failed to extract DynamiteSDK archive.") try: self.install_dependencies(stdout=stdout, verbose=verbose) self.install_jupyterhub(stdout=stdout) except (general_exceptions.InvalidOsPackageManagerDetectedError, general_exceptions.OsPackageManagerInstallError, general_exceptions.OsPackageManagerRefreshError): self.logger.error('One or more OS dependencies failed to install.') raise lab_exceptions.InstallLabError( "One or more OS dependencies failed to install.") self.logger.info('Creating jupyter user in dynamite group.') utilities.create_jupyter_user(password=self.jupyterhub_password) self.stdout = stdout self.verbose = verbose if not elasticsearch_host: if elastic_profile.ProcessProfiler().is_installed(): self.elasticsearch_host = 'localhost' else: raise lab_exceptions.InstallLabError( "Elasticsearch must either be installed locally, or a remote host must be specified." )
from logging import DEBUG, INFO from dynamite_nsm.logger import get_logger from dynamite_nsm.cmd.agent.process import interface as agent_process_interface if __name__ == '__main__': parser = agent_process_interface.get_parser() args = parser.parse_args() logger = get_logger('agent.process', level=DEBUG if args.verbose else INFO, stdout=args.stdout) logger.info(f'Calling agent.process.{args.entry_method_name}.') logger.debug(args.__dict__) result = agent_process_interface.execute(args) print(result)
def __init__(self, configuration_directory, install_directory, capture_network_interfaces, download_zeek_archive=True, stdout=True, verbose=False): """ Install Zeek :param configuration_directory: Path to the configuration directory (E.G /etc/dynamite/zeek) :param install_directory: Path to the install directory (E.G /opt/dynamite/zeek/) :param capture_network_interfaces: A list of network interfaces to capture on (E.G ["mon0", "mon1"]) :param download_zeek_archive: If True, download the Zeek archive from a mirror :param stdout: Print the output to console :param verbose: Include detailed debug messages """ log_level = logging.INFO if verbose: log_level = logging.DEBUG self.logger = get_logger('ZEEK', level=log_level, stdout=stdout) self.configuration_directory = configuration_directory self.install_directory = install_directory self.capture_network_interfaces = capture_network_interfaces self.stdout = stdout self.verbose = verbose install.BaseInstallManager.__init__(self, 'zeek', verbose=self.verbose, stdout=stdout) utilities.create_dynamite_environment_file() if download_zeek_archive: try: self.logger.info("Attempting to download Zeek archive.") self.download_from_mirror(const.ZEEK_MIRRORS, const.ZEEK_ARCHIVE_NAME, stdout=stdout, verbose=verbose) except general_exceptions.DownloadError as e: self.logger.error("Failed to download Zeek archive.") self.logger.debug( "Failed to download Zeek archive, threw: {}.".format(e)) raise zeek_exceptions.InstallZeekError( "Failed to download Zeek archive.") try: self.logger.info("Attempting to extract Zeek archive ({}).".format( const.ZEEK_ARCHIVE_NAME)) self.extract_archive( os.path.join(const.INSTALL_CACHE, const.ZEEK_ARCHIVE_NAME)) self.logger.info("Extraction completed.") except general_exceptions.ArchiveExtractionError as e: self.logger.error("Failed to extract Zeek archive.") self.logger.debug( "Failed to extract Zeek archive, threw: {}.".format(e)) raise zeek_exceptions.InstallZeekError( "Failed to extract Zeek archive.") try: self.install_dependencies(stdout=stdout, verbose=verbose) except (general_exceptions.InvalidOsPackageManagerDetectedError, general_exceptions.OsPackageManagerRefreshError): raise zeek_exceptions.InstallZeekError( "One or more OS dependencies failed to install.") if not self.validate_capture_network_interfaces( self.capture_network_interfaces): self.logger.error( "One or more defined network interfaces is invalid: {}".format( capture_network_interfaces)) raise zeek_exceptions.InstallZeekError( "One or more defined network interfaces is invalid: {}".format( capture_network_interfaces))
def uninstall_zeek(prompt_user=True, stdout=True, verbose=False): """ Uninstall Zeek :param prompt_user: Print a warning before continuing :param stdout: Print the output to console :param verbose: Include detailed debug messages """ log_level = logging.INFO if verbose: log_level = logging.DEBUG logger = get_logger('ZEEK', level=log_level, stdout=stdout) logger.info("Uninstalling Zeek.") env_file = os.path.join(const.CONFIG_PATH, 'environment') environment_variables = utilities.get_environment_file_dict() zeek_profiler = zeek_profile.ProcessProfiler() if not zeek_profiler.is_installed(): logger.error("Zeek is not installed. Cannot uninstall.") raise zeek_exceptions.UninstallZeekError("Zeek is not installed.") if prompt_user: sys.stderr.write('\n\033[93m[-] WARNING! Removing Zeek Will Remove Critical Agent Functionality.\033[0m\n') resp = utilities.prompt_input('\033[93m[?] Are you sure you wish to continue? ([no]|yes): \033[0m') while resp not in ['', 'no', 'yes']: resp = utilities.prompt_input('\033[93m[?] Are you sure you wish to continue? ([no]|yes): \033[0m') if resp != 'yes': if stdout: sys.stdout.write('\n[+] Exiting\n') exit(0) if zeek_profiler.is_running(): try: zeek_process.ProcessManager().stop() except zeek_exceptions.CallZeekProcessError as e: logger.error("Could not kill Zeek process. Cannot uninstall.") logger.debug("Could not kill Zeek process. Cannot uninstall; {}".format(e)) raise zeek_exceptions.UninstallZeekError("Could not kill Zeek process; {}".format(e)) install_directory = environment_variables.get('ZEEK_HOME') config_directory = environment_variables.get('ZEEK_SCRIPTS') try: with open(env_file) as env_fr: env_lines = '' for line in env_fr.readlines(): if 'ZEEK_HOME' in line: continue elif 'ZEEK_SCRIPTS' in line: continue elif 'PF_RING_HOME' in line: continue elif line.strip() == '': continue env_lines += line.strip() + '\n' with open(env_file, 'w') as env_fw: env_fw.write(env_lines) if zeek_profiler.is_installed(): shutil.rmtree(install_directory, ignore_errors=True) shutil.rmtree(config_directory, ignore_errors=True) except Exception as e: logger.error("General error occurred while attempting to uninstall Zeek.") logger.debug("General error occurred while attempting to uninstall Zeek; {}".format(e)) raise zeek_exceptions.UninstallZeekError( "General error occurred while attempting to uninstall Zeek; {}".format(e)) try: sysctl = systemctl.SystemCtl() except general_exceptions.CallProcessError: raise zeek_exceptions.UninstallZeekError("Could not find systemctl.") sysctl.uninstall_and_disable('zeek')
def uninstall_kibana(prompt_user=True, stdout=True, verbose=False): """ Uninstall Kibana/ElastiFlow/Synesis Dashboards :param prompt_user: Print a warning before continuing :param stdout: Print the output to console :param verbose: Include detailed debug messages """ log_level = logging.INFO if verbose: log_level = logging.DEBUG logger = get_logger('KIBANA', level=log_level, stdout=stdout) env_file = os.path.join(const.CONFIG_PATH, 'environment') environment_variables = utilities.get_environment_file_dict() kb_profiler = kibana_profile.ProcessProfiler() if not kb_profiler.is_installed(): raise kibana_exceptions.UninstallKibanaError("Kibana is not installed.") configuration_directory = environment_variables.get('KIBANA_PATH_CONF') kb_config = kibana_configs.ConfigManager(configuration_directory) if prompt_user: sys.stderr.write( '\n\033[93m[-] WARNING! Removing Kibana will uninstall all visualizations and saved searches previously ' 'created.\033[0m\n') resp = utilities.prompt_input('\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m ') while resp not in ['', 'no', 'yes']: resp = utilities.prompt_input('\n\033[93m[?] Are you sure you wish to continue? ([no]|yes):\033[0m ') if resp != 'yes': if stdout: sys.stdout.write('\n[+] Exiting\n') exit(0) if kb_profiler.is_running(): kibana_process.ProcessManager().stop() try: shutil.rmtree(kb_config.kibana_path_conf) shutil.rmtree(kb_config.kibana_home) shutil.rmtree(kb_config.kibana_logs) shutil.rmtree(const.INSTALL_CACHE, ignore_errors=True) env_lines = '' with open(env_file) as env_fr: for line in env_fr.readlines(): if 'KIBANA_PATH_CONF' in line: continue elif 'KIBANA_HOME' in line: continue elif 'KIBANA_LOGS' in line: continue elif line.strip() == '': continue env_lines += line.strip() + '\n' with open(env_file, 'w') as env_fw: env_fw.write(env_lines) except Exception as e: logger.error("General error occurred while attempting to uninstall Kibana.".format(e)) logger.debug("General error occurred while attempting to uninstall Kibana; {}".format(e)) raise kibana_exceptions.UninstallKibanaError( "General error occurred while attempting to uninstall kibana; {}".format(e)) try: sysctl = systemctl.SystemCtl() except general_exceptions.CallProcessError: raise kibana_exceptions.UninstallKibanaError("Could not find systemctl.") sysctl.uninstall_and_disable('kibana')