def invoke(self, shell: Optional[bool] = False, cwd: Optional[str] = None) -> \ List[Tuple[List, bytes, bytes]]: utilities.makedirs(f'{self.cert_directory}') if not cwd: cwd = self.cert_directory results = super().invoke(shell=shell, cwd=cwd) utilities.safely_remove_file(f'{self.cert_directory}/admin-key-temp.pem') utilities.safely_remove_file(f'{self.cert_directory}/admin.csr') utilities.set_ownership_of_file(path=self.cert_directory, user='******', group='dynamite') utilities.set_permissions_of_file(file_path=self.cert_directory, unix_permissions_integer=700) utilities.set_permissions_of_file(file_path=f'{self.cert_directory}/{self.cert_name}', unix_permissions_integer=600) utilities.set_permissions_of_file(file_path=f'{self.cert_directory}/{self.key_name}', unix_permissions_integer=600) utilities.set_permissions_of_file(file_path=f'{self.cert_directory}/{self.trusted_ca_cert_name}', unix_permissions_integer=600) utilities.set_permissions_of_file(file_path=f'{self.cert_directory}/{self.trusted_ca_key_name}', unix_permissions_integer=600) es_main_config = config.ConfigManager(self.configuration_directory) es_main_config.transport_pem_cert_file = f'security/auth/{self.cert_name}' es_main_config.rest_api_pem_cert_file = es_main_config.transport_pem_cert_file es_main_config.transport_pem_key_file = f'security/auth/{self.key_name}' es_main_config.rest_api_pem_key_file = es_main_config.transport_pem_key_file es_main_config.transport_trusted_cas_file = f'security/auth/{self.trusted_ca_cert_name}' es_main_config.rest_api_trusted_cas_file = es_main_config.transport_trusted_cas_file es_main_config.commit() return results
def commit(self, out_file_path: Optional[str] = None, backup_directory: Optional[str] = None) -> None: """Write out an updated configuration file, and optionally backup the old one. Args: out_file_path: The path to the output file; if none given overwrites existing backup_directory: The path to the backup directory Returns: None """ # Backup old configuration first out_file_name = os.path.basename(out_file_path) backup_file_name = out_file_name + '.backup' if backup_directory: utilities.backup_configuration_file( out_file_path, backup_directory, destination_file_prefix=backup_file_name) try: with open(out_file_path, 'w') as config_raw_f: config_raw_f.write(self.formatted_data) except IOError: raise general_exceptions.WriteConfigError( 'An error occurred while writing the configuration file to disk.' ) utilities.set_permissions_of_file(out_file_path, 644) self.logger.warning( 'Configuration updated. Restart this service to apply.')
def commit(self, out_file_path: Optional[str] = None, backup_directory: Optional[str] = None) -> None: """Write out an updated configuration file, and optionally backup the old one. Args: out_file_path: The path to the output file; if none given overwrites existing backup_directory: The path to the backup directory Returns: None """ # Backup old configuration first out_file_name = os.path.basename(out_file_path) backup_file_name = out_file_name + '.backup' self.formatted_data = f'-Xms{self.initial_memory}\n-Xmx{self.maximum_memory}\n' + \ '\n'.join(self._raw_extra_params) if backup_directory: utilities.backup_configuration_file( out_file_path, backup_directory, destination_file_prefix=backup_file_name) try: with open(out_file_path, 'w') as config_raw_f: config_raw_f.write(self.formatted_data) except IOError: raise general_exceptions.WriteConfigError( 'An error occurred while writing the configuration file to disk.' ) utilities.set_permissions_of_file(out_file_path, 644)
def setup_filebeat(self): """ Creates necessary directory structure, and copies required files, generates a default configuration """ if self.stdout: sys.stdout.write('[+] Creating Filebeat install directory.\n') subprocess.call('mkdir -p {}'.format(self.install_directory), shell=True) if self.stdout: sys.stdout.write('[+] Copying Filebeat to install directory.\n') utilities.copytree( os.path.join(const.INSTALL_CACHE, const.FILE_BEAT_DIRECTORY_NAME), self.install_directory) shutil.copy( os.path.join(const.DEFAULT_CONFIGS, 'filebeat', 'filebeat.yml'), self.install_directory) if self.stdout: sys.stdout.write( '[+] Building configurations and setting up permissions.\n') beats_config = FileBeatConfigurator(self.install_directory) beats_config.set_monitor_target_paths(self.monitor_paths) beats_config.write_config() utilities.set_permissions_of_file(os.path.join(self.install_directory, 'filebeat.yml'), unix_permissions_integer=501) if 'FILEBEAT_HOME' not in open('/etc/dynamite/environment').read(): if self.stdout: sys.stdout.write( '[+] Updating FileBeat default script path [{}]\n'.format( self.install_directory)) subprocess.call( 'echo FILEBEAT_HOME="{}" >> /etc/dynamite/environment'.format( self.install_directory), shell=True)
def invoke(self, shell: Optional[bool] = False, cwd: Optional[str] = None) -> List[ Tuple[List, bytes, bytes]]: utilities.set_permissions_of_file(file_path=self.opendistro_security_admin, unix_permissions_integer='+x') if not cwd: cwd = self.security_conf_directory attempts = 0 es_process_profile = profile.ProcessProfiler() if not es_process_profile.is_listening(): process.ProcessManager().start() while not es_process_profile.is_listening() and attempts < self.max_attempts: attempts += 1 sleep(10) results = super().invoke(shell, cwd) if self.terminate_elasticsearch: process.ProcessManager().stop() return results
def write_config(self): def update_dict_from_path(path, value): """ :param path: A tuple representing each level of a nested path in the yaml document ('vars', 'address-groups', 'HOME_NET') = /vars/address-groups/HOME_NET :param value: The new value :return: None """ partial_config_data = self.config_data for i in range(0, len(path) - 1): try: partial_config_data = partial_config_data[path[i]] except KeyError: pass partial_config_data.update({path[-1]: value}) # Backup old configuration first source_configuration_file_path = os.path.join(self.install_directory, 'filebeat.yml') if self.backup_configuration_directory: destination_configuration_path = os.path.join(self.backup_configuration_directory, 'filebeat.yml.d') try: utilities.backup_configuration_file(source_configuration_file_path, destination_configuration_path, destination_file_prefix='filebeat.yml.backup') except general_exceptions.WriteConfigError: raise filebeat_exceptions.WriteFilebeatConfigError( 'Suricata configuration failed to write [filebeat.yml].') except general_exceptions.ReadConfigError: raise filebeat_exceptions.ReadFilebeatConfigError( 'Suricata configuration failed to read [filebeat.yml].') for k, v in vars(self).items(): if k not in self.tokens: continue token_path = self.tokens[k] update_dict_from_path(token_path, v) try: with open(source_configuration_file_path, 'w') as configyaml: dump(self.config_data, configyaml, default_flow_style=False) utilities.set_permissions_of_file(source_configuration_file_path, 744) except IOError: raise filebeat_exceptions.WriteFilebeatConfigError("Could not locate {}".format(self.install_directory)) except Exception as e: raise filebeat_exceptions.WriteFilebeatConfigError( "General error while attempting to write new filebeat.yml file to {}; {}".format( self.install_directory, e))
def commit(self, out_file_path: Optional[str] = None, backup_directory: Optional[str] = None, top_text: Optional[str] = None) -> None: """Write out an updated configuration file, and optionally backup the old one. Args: out_file_path: The path to the output file; if none given overwrites existing backup_directory: The path to the backup directory top_text: The text to be appended at the top of the config file (typically used for YAML version header) Returns: None """ out_file_name = os.path.basename(out_file_path) backup_file_name = out_file_name + '.backup' def update_dict_from_path(path, value) -> None: """Update the internal YAML dictionary object with the new values from our config Args: path: A tuple representing each level of a nested path in the yaml document ('vars', 'address-groups', 'HOME_NET') = /vars/address-groups/HOME_NET value: The new value Returns: None """ partial_config_data = self.config_data if len(path) > 1: for i in range(0, len(path) - 1): k = path[i] if isinstance(partial_config_data, dict): partial_config_data = partial_config_data[k] elif isinstance(partial_config_data, list): for list_entry in partial_config_data: if isinstance(list_entry, dict): if k in list_entry.keys(): partial_config_data = list_entry[k] else: break if value is None: return partial_config_data.update({path[-1]: value}) # Backup old configuration first if backup_directory: utilities.backup_configuration_file( out_file_path, backup_directory, destination_file_prefix=backup_file_name) for k, v in vars(self).items(): if k not in self.extract_tokens: continue token_path = self.extract_tokens[k] update_dict_from_path(token_path, v) try: with open(out_file_path, 'w') as config_yaml_f: if top_text: config_yaml_f.write(f'{top_text}\n') try: dump(self.config_data, config_yaml_f, default_flow_style=False, Dumper=NoAliasDumper) except RecursionError: dump(self.config_data, config_yaml_f, default_flow_style=False) except IOError: raise general_exceptions.WriteConfigError( 'An error occurred while writing the configuration file to disk.' ) utilities.set_permissions_of_file(out_file_path, 644) self.logger.warning( 'Configuration updated. Restart this service to apply.')
def setup_filebeat(self): """ Creates necessary directory structure, and copies required files, generates a default configuration """ env_file = os.path.join(const.CONFIG_PATH, 'environment') self.logger.info('Creating FileBeat install directory.') utilities.makedirs(self.install_directory, exist_ok=True) self.logger.info('Copying FileBeat to install directory.') try: utilities.copytree( os.path.join(const.INSTALL_CACHE, const.FILE_BEAT_DIRECTORY_NAME), self.install_directory) shutil.copy( os.path.join(const.DEFAULT_CONFIGS, 'filebeat', 'filebeat.yml'), self.install_directory) except Exception as e: self.logger.error( "General error occurred while copying Filebeat configs.") self.logger.debug( "General error occurred while copying Filebeat configs; {}". format(e)) raise filebeat_exceptions.InstallFilebeatError( "General error occurred while copying Filebeat configs; {}". format(e)) self.logger.info("Building configurations and setting up permissions.") try: beats_config = filebeat_configs.ConfigManager( self.install_directory) except filebeat_exceptions.ReadFilebeatConfigError: self.logger.error("Failed to read Filebeat configuration.") raise filebeat_exceptions.InstallFilebeatError( "Failed to read Filebeat configuration.") beats_config.set_monitor_target_paths(self.monitor_paths) beats_config.set_agent_tag(self.agent_tag) if (self.kafka_password or self.kafka_username) and not self.kafka_topic: self.logger.error( "You have specified Kafka config options without specifying a Kafka topic." ) raise filebeat_exceptions.InstallFilebeatError( "You have specified Kafka config options without specifying a Kafka topic." ) if self.kafka_topic: self.logger.warning( "You have enabled the Agent's Kafka output which does integrate natively with Dynamite " "Monitor/LogStash component. You will have to bring your own broker. Happy Hacking!" ) time.sleep(2) beats_config.set_kafka_targets(target_hosts=self.targets, topic=self.kafka_topic, username=self.kafka_username, password=self.kafka_password) # setup example upstream LogStash example, just in case you want to configure later beats_config.set_logstash_targets(target_hosts=['localhost:5601']) beats_config.enable_kafka_output() else: # setup example upstream Kafka example, just in case you want to configure later beats_config.set_kafka_targets(target_hosts=['localhost:9092'], topic='dynamite-nsm-events') beats_config.enable_logstash_output() beats_config.set_logstash_targets(self.targets) try: beats_config.write_config() except filebeat_exceptions.WriteFilebeatConfigError: self.logger.error("Failed to write filebeat configuration.") raise filebeat_exceptions.InstallFilebeatError( "Failed to write filebeat configuration.") try: utilities.set_permissions_of_file(os.path.join( self.install_directory, 'filebeat.yml'), unix_permissions_integer=501) except Exception as e: self.logger.error( "Failed to set permissions of filebeat.yml file.") self.logger.debug( "Failed to set permissions of filebeat.yml file; {}".format(e)) filebeat_exceptions.InstallFilebeatError( "Failed to set permissions of filebeat.yml file; {}".format(e)) try: with open(env_file) as env_f: if 'FILEBEAT_HOME' not in env_f.read(): self.logger.info( 'Updating FileBeat default script path [{}]'.format( self.install_directory)) subprocess.call('echo FILEBEAT_HOME="{}" >> {}'.format( self.install_directory, env_file), shell=True) except Exception as e: self.logger.error( "General error occurred while attempting to install FileBeat.") self.logger.debug( "General error occurred while attempting to install FileBeat; {}" .format(e)) raise filebeat_exceptions.InstallFilebeatError( "General error occurred while attempting to install FileBeat; {}" .format(e)) try: sysctl = systemctl.SystemCtl() except general_exceptions.CallProcessError: raise filebeat_exceptions.InstallFilebeatError( "Could not find systemctl.") self.logger.info("Installing Filebeat systemd service.") if not sysctl.install_and_enable( os.path.join(const.DEFAULT_CONFIGS, 'systemd', 'filebeat.service')): raise filebeat_exceptions.InstallFilebeatError( "Failed to install Filebeat systemd service.") sysctl.install_and_enable( os.path.join(const.DEFAULT_CONFIGS, 'systemd', 'dynamite-agent.target')) zeek_logs = None zeek_home = self.environ.get('ZEEK_HOME') suricata_logs = os.path.join(const.LOG_PATH, 'suricata') if zeek_home: zeek_logs = os.path.join(zeek_home, 'logs', 'current') try: self.logger.info('Patching Zeek/Suricata modules.') beats_config.patch_modules(zeek_log_directory=zeek_logs, suricata_log_directory=suricata_logs) except filebeat_exceptions.WriteFilebeatModuleError: self.logger.error('Could not patch Zeek/Suricata modules.') raise filebeat_exceptions.InstallFilebeatError( "Could not patch Zeek/Suricata modules.")
def setup(self, targets: List[str], target_type: Optional[str] = 'elasticsearch', monitor_log_paths: Optional[List[str]] = None, agent_tag: Optional[str] = None) -> None: """Setup Filebeat Args: targets: A list of Elasticsearch/Kafka/Logstash targets to forward events to (E.G ["192.168.0.9 5044", ...]) target_type: The target type; current supported: elasticsearch (default), logstash, kafka, redis monitor_log_paths: A tuple of log paths to monitor agent_tag: A friendly name for the agent (defaults to the hostname with no spaces and _agt suffix) Returns: None """ from dynamite_nsm.services.zeek import profile as zeek_profile from dynamite_nsm.services.suricata import profile as suricata_profile sysctl = systemctl.SystemCtl() zeek_log_root, suricata_log_root = None, None # Directory setup self.logger.debug(f'Creating directory: {self.install_directory}') utilities.makedirs(self.install_directory) utilities.makedirs(f'{self.install_directory}/logs') self.logger.info('Installing files and directories.') self.copy_filebeat_files_and_directories() self.copy_file_or_directory_to_destination(f'{const.DEFAULT_CONFIGS}/filebeat/filebeat.yml', self.install_directory) # Overwrite with dynamite default configurations self.copy_file_or_directory_to_destination(f'{const.DEFAULT_CONFIGS}/filebeat/module/', self.install_directory) self.copy_file_or_directory_to_destination(f'{const.DEFAULT_CONFIGS}/filebeat/modules.d/', self.install_directory) filebeat_config = config.ConfigManager(self.install_directory, verbose=self.verbose, stdout=self.stdout) if target_type == 'elasticsearch': filebeat_config.switch_to_elasticsearch_target() filebeat_config.elasticsearch_targets.target_strings = targets self.logger.info(f'Enabling Elasticsearch connector: ' f'{filebeat_config.elasticsearch_targets.target_strings}') elif target_type == 'logstash': filebeat_config.switch_to_logstash_target() filebeat_config.logstash_targets.target_strings = targets elif target_type == 'kafka': filebeat_config.switch_to_kafka_target() filebeat_config.kafka_targets.target_strings = targets elif target_type == 'redis': filebeat_config.switch_to_redis_target() filebeat_config.redis_targets.target_strings = targets filebeat_config.input_logs = misc_filebeat_objs.InputLogs( monitor_log_paths=[] ) filebeat_config.field_processors.originating_agent_tag = agent_tag if not monitor_log_paths: environ = utilities.get_environment_file_dict() zeek_log_root = f'{environ.get("ZEEK_HOME", "")}/logs/current/' suricata_log_root = environ.get('SURICATA_LOGS', '') zeek_profiler = zeek_profile.ProcessProfiler() suricata_profiler = suricata_profile.ProcessProfiler() if zeek_profiler.is_installed(): self.logger.info(f'Zeek installation found; monitoring: {zeek_log_root}*.log') filebeat_config.input_logs.monitor_log_paths.append(f'{zeek_log_root}*.log') if suricata_profiler.is_installed(): self.logger.info(f'Suricata installation found; monitoring: {suricata_log_root}/eve.json') filebeat_config.input_logs.monitor_log_paths.append(f'{suricata_log_root}/eve.json') else: filebeat_config.input_logs = misc_filebeat_objs.InputLogs( monitor_log_paths=monitor_log_paths ) self.logger.info(f'Monitoring Paths = {filebeat_config.input_logs.monitor_log_paths}') if not agent_tag: filebeat_config.field_processors.originating_agent_tag = utilities.get_default_agent_tag() self.logger.info(f'Agent Tag = {filebeat_config.field_processors.originating_agent_tag}') self.logger.debug(filebeat_config.elasticsearch_targets.get_raw()) filebeat_config.commit() self.logger.info('Applying configuration.') # Fix Permissions self.logger.info('Installing modules.') filebeat_config.patch_modules(zeek_log_directory=zeek_log_root, suricata_log_directory=suricata_log_root) # Setting up permissions self.logger.info('Setting up file permissions.') config_file = f'{self.install_directory}/filebeat.yml' utilities.set_ownership_of_file(self.install_directory, user='******', group='dynamite') utilities.set_permissions_of_file(f'{self.install_directory}/modules.d/', unix_permissions_integer='go-w') utilities.set_permissions_of_file(f'{self.install_directory}/module/', unix_permissions_integer='go-w') utilities.set_ownership_of_file(config_file, user='******', group='dynamite') utilities.set_permissions_of_file(config_file, unix_permissions_integer=644) filebeat_config.enable_ecs_normalization() # Install and enable service self.logger.info(f'Installing service -> {const.DEFAULT_CONFIGS}/systemd/filebeat.service') sysctl.install_and_enable(f'{const.DEFAULT_CONFIGS}/systemd/filebeat.service') # Update environment file self.create_update_filebeat_environment_variables()