def setup_filebeat(self): """ Creates necessary directory structure, and copies required files, generates a default configuration """ if self.stdout: sys.stdout.write('[+] Creating Filebeat install directory.\n') subprocess.call('mkdir -p {}'.format(self.install_directory), shell=True) if self.stdout: sys.stdout.write('[+] Copying Filebeat to install directory.\n') utilities.copytree( os.path.join(const.INSTALL_CACHE, const.FILE_BEAT_DIRECTORY_NAME), self.install_directory) shutil.copy( os.path.join(const.DEFAULT_CONFIGS, 'filebeat', 'filebeat.yml'), self.install_directory) if self.stdout: sys.stdout.write( '[+] Building configurations and setting up permissions.\n') beats_config = FileBeatConfigurator(self.install_directory) beats_config.set_monitor_target_paths(self.monitor_paths) beats_config.write_config() utilities.set_permissions_of_file(os.path.join(self.install_directory, 'filebeat.yml'), unix_permissions_integer=501) if 'FILEBEAT_HOME' not in open('/etc/dynamite/environment').read(): if self.stdout: sys.stdout.write( '[+] Updating FileBeat default script path [{}]\n'.format( self.install_directory)) subprocess.call( 'echo FILEBEAT_HOME="{}" >> /etc/dynamite/environment'.format( self.install_directory), shell=True)
def _copy_suricata_files_and_directories(self): if self.stdout: sys.stdout.write( '[+] Creating suricata install|configuration|logging directories.\n' ) subprocess.call('mkdir -p {}'.format(self.install_directory), shell=True) subprocess.call('mkdir -p {}'.format(self.configuration_directory), shell=True) subprocess.call('mkdir -p {}'.format(self.log_directory), shell=True) try: os.mkdir(os.path.join(self.configuration_directory, 'rules')) shutil.copy( os.path.join(const.DEFAULT_CONFIGS, 'suricata', 'suricata.yaml'), os.path.join(self.configuration_directory, 'suricata.yaml')) utilities.copytree( os.path.join(const.INSTALL_CACHE, const.SURICATA_DIRECTORY_NAME, 'rules'), os.path.join(self.configuration_directory, 'rules')) except Exception as e: sys.stderr.write( '[-] Unable to re-create Suricata rules directory: {}\n'. format(e)) return False return True
def setup_oinkmaster(self): env_file = os.path.join(const.CONFIG_PATH, 'environment') self.logger.info("Installing Oinkmaster.") try: utilities.makedirs(self.install_directory, exist_ok=True) except Exception as e: self.logger.error("Failed to create required directory structure.") self.logger.debug("Failed to create required directory structure; {}".format(e)) raise oinkmaster_exceptions.InstallOinkmasterError( "Failed to create required directory structure; {}".format(e)) self.logger.info("Copying oinkmaster files.") try: utilities.copytree(os.path.join(const.INSTALL_CACHE, const.OINKMASTER_DIRECTORY_NAME), self.install_directory) except Exception as e: self.logger.error( 'Failed to copy {} -> {}.'.format(os.path.join(const.INSTALL_CACHE, const.OINKMASTER_DIRECTORY_NAME), self.install_directory)) self.logger.debug(sys.stderr.write('Failed to copy {} -> {}: {}'.format( os.path.join(const.INSTALL_CACHE, const.OINKMASTER_DIRECTORY_NAME), self.install_directory, e))) raise oinkmaster_exceptions.InstallOinkmasterError( "General error while copying Oinkmaster from install cache; {}".format(e)) if 'OINKMASTER_HOME' not in open(env_file).read(): self.logger.info('Updating Oinkmaster default home path [{}]'.format(self.install_directory)) subprocess.call('echo OINKMASTER_HOME="{}" >> {}'.format(self.install_directory, env_file), shell=True) self.logger.info('PATCHING oinkmaster.conf with emerging-threats URL.') try: with open(os.path.join(self.install_directory, 'oinkmaster.conf'), 'a') as f: f.write('\nurl = http://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz') except Exception as e: self.logger.error('Failed to update oinkmaster.conf.') self.logger.debug('Failed to update oinkmaster.conf: {}.\n'.format(e)) raise oinkmaster_exceptions.InstallOinkmasterError( "Failed to update oinkmaster configuration file; {}".format(e))
def setup_oinkmaster(self, stdout=False): try: os.mkdir(self.install_directory) except Exception as e: if 'exists' not in str(e).lower(): return False if stdout: sys.stdout.write('[+] Copying oinkmaster files.\n') try: utilities.copytree( os.path.join(const.INSTALL_CACHE, const.OINKMASTER_DIRECTORY_NAME), self.install_directory) except Exception as e: sys.stderr.write('[-] Failed to copy {} -> {}: {}'.format( os.path.join(const.INSTALL_CACHE, const.OINKMASTER_DIRECTORY_NAME), self.install_directory, e)) return False if stdout: sys.stdout.write( '[+] Updating oinkmaster.conf with emerging-threats URL.\n') try: with open(os.path.join(self.install_directory, 'oinkmaster.conf'), 'a') as f: f.write( '\nurl = http://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz' ) except Exception as e: sys.stderr.write( '[-] Failed to update oinkmaster.conf: {}.\n'.format(e)) return False return True
def setup_logstash_elastiflow(self, stdout=False): if stdout: sys.stdout.write( '[+] Creating elastiflow install|configuration directories.\n') subprocess.call('mkdir -p {}'.format(self.install_directory), shell=True) if stdout: sys.stdout.write('[+] Copying elastiflow configurations\n') utilities.copytree( os.path.join(const.DEFAULT_CONFIGS, 'logstash', 'zeek'), self.install_directory) utilities.set_ownership_of_file(self.install_directory, user='******', group='dynamite') if 'ELASTIFLOW_DICT_PATH' not in open( '/etc/dynamite/environment').read(): dict_path = os.path.join(self.install_directory, 'dictionaries') if stdout: sys.stdout.write( '[+] Updating Elastiflow dictionary configuration path [{}]\n' .format(dict_path)) subprocess.call( 'echo ELASTIFLOW_DICT_PATH="{}" >> /etc/dynamite/environment'. format(dict_path), shell=True) if 'ELASTIFLOW_TEMPLATE_PATH' not in open( '/etc/dynamite/environment').read(): template_path = os.path.join(self.install_directory, 'templates') if stdout: sys.stdout.write( '[+] Updating Elastiflow template configuration path [{}]\n' .format(template_path)) subprocess.call( 'echo ELASTIFLOW_TEMPLATE_PATH="{}" >> /etc/dynamite/environment' .format(template_path), shell=True) if 'ELASTIFLOW_GEOIP_DB_PATH' not in open( '/etc/dynamite/environment').read(): geo_path = os.path.join(self.install_directory, 'geoipdbs') if stdout: sys.stdout.write( '[+] Updating Elastiflow geodb configuration path [{}]\n'. format(geo_path)) subprocess.call( 'echo ELASTIFLOW_GEOIP_DB_PATH="{}" >> /etc/dynamite/environment' .format(geo_path), shell=True) if 'ELASTIFLOW_DEFINITION_PATH' not in open( '/etc/dynamite/environment').read(): def_path = os.path.join(self.install_directory, 'definitions') if stdout: sys.stdout.write( '[+] Updating Elastiflow definitions configuration path [{}]\n' .format(def_path)) subprocess.call( 'echo ELASTIFLOW_DEFINITION_PATH="{}" >> /etc/dynamite/environment' .format(def_path), shell=True) ElastiflowConfigurator().write_environment_variables()
def setup(self): utilities.makedirs(self.install_directory) self.create_update_oinkmaster_environment_variables() utilities.copytree(f'{const.INSTALL_CACHE}/{self.local_mirror_root}', self.install_directory) with open(os.path.join(self.install_directory, 'oinkmaster.conf'), 'a') as f: f.write(f'\nurl = {const.EMERGING_THREATS_OPEN}')
def _copy_elasticsearch_files_and_directories(self): config_paths = [ 'config/elasticsearch.yml', 'config/jvm.options', 'config/log4j2.properties' ] install_paths = ['bin/', 'lib/', 'logs/', 'modules/', 'plugins/'] path = None try: for path in config_paths: self.logger.debug('Copying {} -> {}'.format( os.path.join( const.INSTALL_CACHE, '{}/{}'.format(const.ELASTICSEARCH_DIRECTORY_NAME, path)), self.configuration_directory)) try: shutil.copy( os.path.join( const.INSTALL_CACHE, '{}/{}'.format(const.ELASTICSEARCH_DIRECTORY_NAME, path)), self.configuration_directory) except shutil.Error: self.logger.warning( '{} already exists at this path.'.format(path)) except Exception as e: self.logger.error( "General error while attempting to copy {} to {}.".format( path, self.configuration_directory)) self.logger.debug( "General error while attempting to copy {} to {}; {}".format( path, self.configuration_directory, e)) raise elastic_exceptions.InstallElasticsearchError( "General error while attempting to copy {} to {}; {}".format( path, self.configuration_directory, e)) try: for path in install_paths: src_install_path = os.path.join( const.INSTALL_CACHE, const.ELASTICSEARCH_DIRECTORY_NAME, path) dst_install_path = os.path.join(self.install_directory, path) self.logger.debug('Copying {} -> {}'.format( src_install_path, dst_install_path)) try: utilities.makedirs(dst_install_path, exist_ok=True) utilities.copytree(src_install_path, dst_install_path) except shutil.Error: self.logger.warning( '{} already exists at this path.'.format(path)) except Exception as e: self.logger.error( "General error while attempting to copy {} to {}.".format( path, self.install_directory)) self.logger.debug( "General error while attempting to copy {} to {}; {}".format( path, self.install_directory, e)) raise elastic_exceptions.InstallElasticsearchError( "General error while attempting to copy {} to {}; {}".format( path, self.install_directory, e))
def setup_logstash_synesis(self, stdout=False): if stdout: sys.stdout.write( '[+] Creating synesis install|configuration directories.\n') subprocess.call('mkdir -p {}'.format(self.install_directory), shell=True) if stdout: sys.stdout.write('[+] Copying synesis configurations\n') utilities.copytree( os.path.join(const.DEFAULT_CONFIGS, 'logstash', 'suricata'), self.install_directory) utilities.set_ownership_of_file(self.install_directory, user='******', group='dynamite') if 'SYNLITE_SURICATA_DICT_PATH' not in open( '/etc/dynamite/environment').read(): dict_path = os.path.join(self.install_directory, 'dictionaries') if stdout: sys.stdout.write( '[+] Updating Synesis dictionary configuration path [{}]\n' .format(dict_path)) subprocess.call( 'echo SYNLITE_SURICATA_DICT_PATH="{}" >> /etc/dynamite/environment' .format(dict_path), shell=True) if 'SYNLITE_SURICATA_TEMPLATE_PATH' not in open( '/etc/dynamite/environment').read(): template_path = os.path.join(self.install_directory, 'templates') if stdout: sys.stdout.write( '[+] Updating Synesis template configuration path [{}]\n'. format(template_path)) subprocess.call( 'echo SYNLITE_SURICATA_TEMPLATE_PATH="{}" >> /etc/dynamite/environment' .format(template_path), shell=True) if 'SYNLITE_SURICATA_GEOIP_DB_PATH' not in open( '/etc/dynamite/environment').read(): geo_path = os.path.join(self.install_directory, 'geoipdbs') if stdout: sys.stdout.write( '[+] Updating Synesis geodb configuration path [{}]\n'. format(geo_path)) subprocess.call( 'echo SYNLITE_SURICATA_GEOIP_DB_PATH="{}" >> /etc/dynamite/environment' .format(geo_path), shell=True) SynesisConfigurator().write_environment_variables()
def _copy_suricata_files_and_directories(self): self.logger.info('Creating Suricata installation, configuration, and logging directories.') try: utilities.makedirs(self.install_directory, exist_ok=True) utilities.makedirs(self.configuration_directory, exist_ok=True) utilities.makedirs(self.log_directory, exist_ok=True) except Exception as e: self.logger.error('Failed to create required directory structure.') self.logger.debug("Failed to create required directory structure; {}".format(e)) raise suricata_exceptions.InstallSuricataError( "Failed to create required directory structure; {}".format(e)) try: utilities.makedirs(os.path.join(self.configuration_directory, 'rules'), exist_ok=True) except Exception as e: self.logger.error('Unable to re-create Suricata rules directory.') self.logger.debug('Unable to re-create Suricata rules directory: {}'.format(e)) try: shutil.copy(os.path.join(const.DEFAULT_CONFIGS, 'suricata', 'suricata.yaml'), os.path.join(self.configuration_directory, 'suricata.yaml')) except Exception as e: self.logger.error("General error while attempting to copy {} to {}.".format( os.path.join(const.DEFAULT_CONFIGS, 'suricata', 'suricata.yaml'), self.configuration_directory)) self.logger.debug("General error while attempting to copy {} to {}; {}".format( os.path.join(const.DEFAULT_CONFIGS, 'suricata', 'suricata.yaml'), self.configuration_directory, e)) raise suricata_exceptions.InstallSuricataError( "General error while attempting to copy {} to {}; {}".format( os.path.join(const.DEFAULT_CONFIGS, 'suricata', 'suricata.yaml'), self.configuration_directory, e)) try: utilities.copytree(os.path.join(const.INSTALL_CACHE, const.SURICATA_DIRECTORY_NAME, 'rules'), os.path.join(self.configuration_directory, 'rules')) except Exception as e: self.logger.error("General error while attempting to copy {} to {}.".format( os.path.join(const.INSTALL_CACHE, const.SURICATA_DIRECTORY_NAME, 'rules'), os.path.join(self.configuration_directory, 'rules'))) self.logger.debug("General error while attempting to copy {} to {}; {}".format( os.path.join(const.INSTALL_CACHE, const.SURICATA_DIRECTORY_NAME, 'rules'), os.path.join(self.configuration_directory, 'rules'), e)) raise suricata_exceptions.InstallSuricataError( "General error while attempting to copy {} to {}; {}".format( os.path.join(const.INSTALL_CACHE, const.SURICATA_DIRECTORY_NAME, 'rules'), os.path.join(self.configuration_directory, 'rules'), e))
def setup_oinkmaster(self): try: os.mkdir(self.install_directory) except Exception as e: if 'exists' not in str(e).lower(): return False if self.stdout: sys.stdout.write('[+] Copying oinkmaster files.\n') try: utilities.copytree( os.path.join(const.INSTALL_CACHE, const.OINKMASTER_DIRECTORY_NAME), self.install_directory) except Exception as e: sys.stderr.write('[-] Failed to copy {} -> {}: {}'.format( os.path.join(const.INSTALL_CACHE, const.OINKMASTER_DIRECTORY_NAME), self.install_directory, e)) return False if 'OINKMASTER_HOME' not in open('/etc/dynamite/environment').read(): if self.stdout: sys.stdout.write( '[+] Updating Oinkmaster default home path [{}]\n'.format( self.install_directory)) subprocess.call( 'echo OINKMASTER_HOME="{}" >> /etc/dynamite/environment'. format(self.install_directory), shell=True) if self.stdout: sys.stdout.write( '[+] Updating oinkmaster.conf with emerging-threats URL.\n') try: with open(os.path.join(self.install_directory, 'oinkmaster.conf'), 'a') as f: f.write( '\nurl = http://rules.emergingthreats.net/open/suricata/emerging.rules.tar.gz' ) except Exception as e: sys.stderr.write( '[-] Failed to update oinkmaster.conf: {}.\n'.format(e)) return False return True
def copy_file_or_directory_to_destination(self, file_or_dir: str, destination_file_or_dir: str) -> None: """ Copy a file or directory to another file or directory Args: file_or_dir: The file or directory to copy destination_file_or_dir: The file or directory destination Returns: None """ file_or_dir = file_or_dir.rstrip('/') destination_location = f'{destination_file_or_dir}/{os.path.basename(file_or_dir)}' if os.path.isdir(file_or_dir): utilities.makedirs(destination_location, exist_ok=True) self.logger.debug(f'Creating directory: {destination_location}') try: self.logger.debug(f'Copying directory {file_or_dir} -> {destination_location}') utilities.copytree(file_or_dir, destination_location) except shutil.Error as e: if 'exist' in str(e): self.logger.warning(f'{destination_file_or_dir} directory already exists. Skipping.') else: raise e else: try: self.logger.debug(f'Copying file {file_or_dir} -> {destination_file_or_dir}') shutil.copy(file_or_dir, destination_file_or_dir) except FileNotFoundError: parent_directory = os.path.dirname(destination_file_or_dir) self.logger.debug(f'Creating parent directory: {parent_directory}') utilities.makedirs(parent_directory) shutil.copy(file_or_dir, destination_file_or_dir) except shutil.Error as e: if 'exist' in str(e): self.logger.warning(f'{destination_file_or_dir} file already exists. Skipping.') else: raise e
def setup_dynamite_sdk(self): """ Sets up sdk files; and installs globally """ if self.stdout: sys.stdout.write('[+] Copying DynamiteSDK into lab environment.\n') sys.stdout.flush() subprocess.call('mkdir -p {}'.format(self.notebook_home), shell=True) if 'NOTEBOOK_HOME' not in open('/etc/dynamite/environment').read(): if self.stdout: sys.stdout.write( '[+] Updating Notebook home path [{}]\n'.format( self.notebook_home)) subprocess.call( 'echo NOTEBOOK_HOME="{}" >> /etc/dynamite/environment'. format(self.notebook_home), shell=True) subprocess.call('mkdir -p {}'.format(self.configuration_directory), shell=True) if 'DYNAMITE_LAB_CONFIG' not in open( '/etc/dynamite/environment').read(): if self.stdout: sys.stdout.write( '[+] Updating Dynamite Lab Config path [{}]\n'.format( self.configuration_directory)) subprocess.call( 'echo DYNAMITE_LAB_CONFIG="{}" >> /etc/dynamite/environment'. format(self.configuration_directory), shell=True) sdk_install_cache = os.path.join(const.INSTALL_CACHE, const.DYNAMITE_SDK_DIRECTORY_NAME) utilities.copytree(os.path.join(sdk_install_cache, 'notebooks'), self.notebook_home) shutil.copy( os.path.join(sdk_install_cache, 'dynamite_sdk', 'config.cfg.example'), os.path.join(self.configuration_directory, 'config.cfg')) utilities.set_ownership_of_file(self.notebook_home, user='******', group='jupyter') if self.stdout: sys.stdout.write( '[+] Installing dynamite-sdk-lite (https://github.com/DynamiteAI/dynamite-sdk-lite)\n' ) sys.stdout.write( '[+] Depending on your distribution it may take some time to install all requirements.\n' ) sys.stdout.flush() if self.verbose: p = subprocess.Popen(['python3', 'setup.py', 'install'], cwd=sdk_install_cache) else: p = subprocess.Popen(['python3', 'setup.py', 'install'], cwd=sdk_install_cache, stdout=subprocess.PIPE, stderr=subprocess.PIPE) p.communicate() dynamite_sdk_config = DynamiteLabConfigurator( configuration_directory=self.configuration_directory) dynamite_sdk_config.elasticsearch_url = 'http://{}:{}'.format( self.elasticsearch_host, self.elasticsearch_port) dynamite_sdk_config.elasticsearch_user = '******' dynamite_sdk_config.elasticsearch_password = self.elasticsearch_password dynamite_sdk_config.write_config()
def setup_filebeat(self): """ Creates necessary directory structure, and copies required files, generates a default configuration """ env_file = os.path.join(const.CONFIG_PATH, 'environment') self.logger.info('Creating FileBeat install directory.') utilities.makedirs(self.install_directory, exist_ok=True) self.logger.info('Copying FileBeat to install directory.') try: utilities.copytree( os.path.join(const.INSTALL_CACHE, const.FILE_BEAT_DIRECTORY_NAME), self.install_directory) shutil.copy( os.path.join(const.DEFAULT_CONFIGS, 'filebeat', 'filebeat.yml'), self.install_directory) except Exception as e: self.logger.error( "General error occurred while copying Filebeat configs.") self.logger.debug( "General error occurred while copying Filebeat configs; {}". format(e)) raise filebeat_exceptions.InstallFilebeatError( "General error occurred while copying Filebeat configs; {}". format(e)) self.logger.info("Building configurations and setting up permissions.") try: beats_config = filebeat_configs.ConfigManager( self.install_directory) except filebeat_exceptions.ReadFilebeatConfigError: self.logger.error("Failed to read Filebeat configuration.") raise filebeat_exceptions.InstallFilebeatError( "Failed to read Filebeat configuration.") beats_config.set_monitor_target_paths(self.monitor_paths) beats_config.set_agent_tag(self.agent_tag) if (self.kafka_password or self.kafka_username) and not self.kafka_topic: self.logger.error( "You have specified Kafka config options without specifying a Kafka topic." ) raise filebeat_exceptions.InstallFilebeatError( "You have specified Kafka config options without specifying a Kafka topic." ) if self.kafka_topic: self.logger.warning( "You have enabled the Agent's Kafka output which does integrate natively with Dynamite " "Monitor/LogStash component. You will have to bring your own broker. Happy Hacking!" ) time.sleep(2) beats_config.set_kafka_targets(target_hosts=self.targets, topic=self.kafka_topic, username=self.kafka_username, password=self.kafka_password) # setup example upstream LogStash example, just in case you want to configure later beats_config.set_logstash_targets(target_hosts=['localhost:5601']) beats_config.enable_kafka_output() else: # setup example upstream Kafka example, just in case you want to configure later beats_config.set_kafka_targets(target_hosts=['localhost:9092'], topic='dynamite-nsm-events') beats_config.enable_logstash_output() beats_config.set_logstash_targets(self.targets) try: beats_config.write_config() except filebeat_exceptions.WriteFilebeatConfigError: self.logger.error("Failed to write filebeat configuration.") raise filebeat_exceptions.InstallFilebeatError( "Failed to write filebeat configuration.") try: utilities.set_permissions_of_file(os.path.join( self.install_directory, 'filebeat.yml'), unix_permissions_integer=501) except Exception as e: self.logger.error( "Failed to set permissions of filebeat.yml file.") self.logger.debug( "Failed to set permissions of filebeat.yml file; {}".format(e)) filebeat_exceptions.InstallFilebeatError( "Failed to set permissions of filebeat.yml file; {}".format(e)) try: with open(env_file) as env_f: if 'FILEBEAT_HOME' not in env_f.read(): self.logger.info( 'Updating FileBeat default script path [{}]'.format( self.install_directory)) subprocess.call('echo FILEBEAT_HOME="{}" >> {}'.format( self.install_directory, env_file), shell=True) except Exception as e: self.logger.error( "General error occurred while attempting to install FileBeat.") self.logger.debug( "General error occurred while attempting to install FileBeat; {}" .format(e)) raise filebeat_exceptions.InstallFilebeatError( "General error occurred while attempting to install FileBeat; {}" .format(e)) try: sysctl = systemctl.SystemCtl() except general_exceptions.CallProcessError: raise filebeat_exceptions.InstallFilebeatError( "Could not find systemctl.") self.logger.info("Installing Filebeat systemd service.") if not sysctl.install_and_enable( os.path.join(const.DEFAULT_CONFIGS, 'systemd', 'filebeat.service')): raise filebeat_exceptions.InstallFilebeatError( "Failed to install Filebeat systemd service.") sysctl.install_and_enable( os.path.join(const.DEFAULT_CONFIGS, 'systemd', 'dynamite-agent.target')) zeek_logs = None zeek_home = self.environ.get('ZEEK_HOME') suricata_logs = os.path.join(const.LOG_PATH, 'suricata') if zeek_home: zeek_logs = os.path.join(zeek_home, 'logs', 'current') try: self.logger.info('Patching Zeek/Suricata modules.') beats_config.patch_modules(zeek_log_directory=zeek_logs, suricata_log_directory=suricata_logs) except filebeat_exceptions.WriteFilebeatModuleError: self.logger.error('Could not patch Zeek/Suricata modules.') raise filebeat_exceptions.InstallFilebeatError( "Could not patch Zeek/Suricata modules.")
def setup_logstash_elastiflow(self): """ Create required environmental variables; copy configurations to various directories. """ env_file = os.path.join(const.CONFIG_PATH, 'environment') self.logger.info( 'Creating ElastiFlow installation and configuration directories.') utilities.makedirs(self.install_directory, exist_ok=True) self.logger.info('Copying ElastiFlow configurations.') utilities.copytree( os.path.join(const.DEFAULT_CONFIGS, 'logstash', 'zeek'), self.install_directory) utilities.set_ownership_of_file(self.install_directory, user='******', group='dynamite') try: with open(env_file) as env_f: env_str = env_f.read() if 'ELASTIFLOW_DICT_PATH' not in env_str: dict_path = os.path.join(self.install_directory, 'dictionaries') self.logger.info( 'Updating ElastiFlow dictionary configuration path [{}]' .format(dict_path)) subprocess.call( 'echo ELASTIFLOW_DICT_PATH="{}" >> {}'.format( dict_path, env_file), shell=True) if 'ELASTIFLOW_TEMPLATE_PATH' not in env_str: template_path = os.path.join(self.install_directory, 'templates') self.logger.info( 'Updating ElastiFlow template configuration path [{}]'. format(template_path)) subprocess.call( 'echo ELASTIFLOW_TEMPLATE_PATH="{}" >> {}'.format( template_path, env_file), shell=True) if 'ELASTIFLOW_GEOIP_DB_PATH' not in env_str: geo_path = os.path.join(self.install_directory, 'geoipdbs') self.logger.info( 'Updating ElastiFlow GeoDBs configuration path [{}]'. format(geo_path)) subprocess.call( 'echo ELASTIFLOW_GEOIP_DB_PATH="{}" >> {}'.format( geo_path, env_file), shell=True) if 'ELASTIFLOW_DEFINITION_PATH' not in env_str: def_path = os.path.join(self.install_directory, 'definitions') self.logger.info( 'Updating ElastiFlow definitions configuration path [{}]' .format(def_path)) subprocess.call( 'echo ELASTIFLOW_DEFINITION_PATH="{}" >> {}'.format( def_path, env_file), shell=True) except Exception as e: self.logger.error( 'Failed to read ElastiFlow environment variables.') self.logger.debug( "Failed to read ElastiFlow environment variables; {}".format( e)) raise elastiflow_exceptions.InstallElastiflowError( "Failed to read ElastiFlow environment variables; {}".format( e)) try: elastiflow_config.ConfigManager().write_environment_variables() except (elastiflow_exceptions.ReadElastiflowConfigError, elastiflow_exceptions.WriteElastiflowConfigError): self.logger.error( 'Failed to read/write ElastiFlow environment variables.') raise elastiflow_exceptions.InstallElastiflowError( "Could not read/write ElastiFlow environment variables.")
def setup_dynamite_sdk(self): """ Sets up sdk files; and installs globally """ env_file = os.path.join(const.CONFIG_PATH, 'environment') self.logger.info('Copying DynamiteSDK into lab environment.') try: utilities.makedirs(self.notebook_home, exist_ok=True) utilities.makedirs(self.configuration_directory, exist_ok=True) except Exception as e: self.logger.error( 'General error occurred while attempting to create root directories.' ) self.logger.debug( "General error occurred while attempting to create root directories; {}" .format(e)) raise lab_exceptions.InstallLabError( "General error occurred while attempting to create root directories; {}" .format(e)) try: with open(env_file) as env_f: env_str = env_f.read() if 'NOTEBOOK_HOME' not in env_str: self.logger.info('Updating Notebook home path [{}]'.format( self.notebook_home)) subprocess.call('echo NOTEBOOK_HOME="{}" >> {}'.format( self.notebook_home, env_file), shell=True) if 'DYNAMITE_LAB_CONFIG' not in env_str: self.logger.info( 'Updating Dynamite Lab Config path [{}]'.format( self.configuration_directory)) subprocess.call( 'echo DYNAMITE_LAB_CONFIG="{}" >> {}'.format( self.configuration_directory, env_file), shell=True) except IOError: self.logger.error( "Failed to open {} for reading.".format(env_file)) raise lab_exceptions.InstallLabError( "Failed to open {} for reading.".format(env_file)) except Exception as e: self.logger.error( "General error while creating environment variables in {}.") self.logger.debug( "General error while creating environment variables in {}; {}". format(env_file, e)) raise lab_exceptions.InstallLabError( "General error while creating environment variables in {}; {}". format(env_file, e)) sdk_install_cache = os.path.join(const.INSTALL_CACHE, const.DYNAMITE_SDK_DIRECTORY_NAME) utilities.copytree(os.path.join(sdk_install_cache, 'notebooks'), self.notebook_home) shutil.copy( os.path.join(sdk_install_cache, 'dynamite_sdk', 'config.cfg.example'), os.path.join(self.configuration_directory, 'config.cfg')) try: utilities.set_ownership_of_file(self.notebook_home, user='******', group='jupyter') except Exception as e: raise lab_exceptions.InstallLabError( "General error occurred while attempting to set permissions on root directories; {}" .format(e)) self.logger.info( 'Installing dynamite-sdk-lite (https://github.com/DynamiteAI/dynamite-sdk-lite), ' 'Depending on your distribution it may take some time to install all requirements.' ) if self.verbose: p = subprocess.Popen(['python3', 'setup.py', 'install'], cwd=sdk_install_cache) else: p = subprocess.Popen(['python3', 'setup.py', 'install'], cwd=sdk_install_cache, stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: p.communicate() except Exception as e: self.logger.error( "General error occurred while installing DynamiteSDK.") self.logger.debug( "General error occurred while installing DynamiteSDK; {}". format(e)) raise lab_exceptions.InstallLabError( "General error occurred while installing DynamiteSDK; {}". format(e)) if p.returncode != 0: self.logger.error( "DynamiteSDK install process returned non-zero exit-code: {}". format(p.returncode)) raise lab_exceptions.InstallLabError( "DynamiteSDK install process returned non-zero exit-code: {}". format(p.returncode)) try: dynamite_sdk_config = lab_configs.ConfigManager( configuration_directory=self.configuration_directory) except lab_exceptions.ReadLabConfigError: self.logger.error("Failed to read DynamiteSDK config.cfg.") raise lab_exceptions.InstallLabError( "Failed to read DynamiteSDK config.cfg.") dynamite_sdk_config.elasticsearch_url = 'http://{}:{}'.format( self.elasticsearch_host, self.elasticsearch_port) dynamite_sdk_config.elasticsearch_user = '******' dynamite_sdk_config.elasticsearch_password = self.elasticsearch_password try: dynamite_sdk_config.write_config() except lab_exceptions.WriteLabConfigError: self.logger.error("Failed to write DynamiteSDK config.cfg.") raise lab_exceptions.InstallLabError( "Failed to write DynamiteSDK config.cfg.")
def setup_dynamite_zeek_scripts(self): """ Installs and enables extra dynamite Zeek scripts :return: True if zeek scripts were successfully installed """ scripts = '' redefs = '' if self.stdout: sys.stdout.write('[+] Setting up Zeek scripts.\n') install_cache_extra_scripts_path = \ os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'dynamite_extra_scripts') if not os.path.exists(install_cache_extra_scripts_path): sys.stderr.write( '[-] dynamite_extra_scripts not found in install_cache.\n') sys.stderr.flush() return False try: os.mkdir( os.path.join(self.configuration_directory, 'dynamite_extra_scripts')) except OSError: pass if self.stdout: sys.stdout.write('[+] Installing third-party Zeek scripts.\n') extra_scripts_path = os.path.join(self.configuration_directory, 'dynamite_extra_scripts') utilities.copytree(install_cache_extra_scripts_path, extra_scripts_path) with open( os.path.join(self.configuration_directory, 'site', 'local.bro'), 'r') as rf: for line in rf.readlines(): if '@load' in line: scripts += line.strip() + '\n' elif 'redef' in line: redefs += line.strip() + '\n' with open( os.path.join(self.configuration_directory, 'site', 'local.bro'), 'w') as wf: extra_script_install_path = os.path.join( self.configuration_directory, 'dynamite_extra_scripts') wf.write(scripts) for script_dir in os.listdir(extra_script_install_path): wf.write('@load {}\n'.format( os.path.join(extra_script_install_path, script_dir))) wf.write(redefs) if self.stdout: sys.stdout.write('[+] Disabling unneeded Zeek scripts.\n') # Disable Unneeded Zeek scripts script_config = ZeekScriptConfigurator() script_config.disable_script('protocols/ftp/detect') script_config.disable_script('protocols/ftp/software') script_config.disable_script('protocols/ftp/detect-bruteforcing') script_config.disable_script('protocols/dns/detect-external-names') script_config.disable_script('protocols/http/detect-sqli') script_config.disable_script('protocols/http/detect-webapps') script_config.disable_script('protocols/krb/ticket-logging') script_config.disable_script('protocols/rdp/indicate_ssl') script_config.disable_script('protocols/smb/log-cmds') script_config.disable_script('protocols/smtp/blocklists') script_config.disable_script('protocols/smtp/detect-suspicious-orig') script_config.disable_script('protocols/smtp/entities-excerpt') script_config.disable_script('protocols/smtp/blocklists') script_config.disable_script('protocols/smtp/software') script_config.disable_script('protocols/ssh/detect-bruteforcing') script_config.disable_script('protocols/ssh/geo-data') script_config.disable_script('protocols/ssh/interesting-hostnames') script_config.disable_script('protocols/ssh/software') script_config.disable_script('protocols/ssl/expiring-certs') script_config.disable_script('protocols/ssl/extract-certs-pem') script_config.disable_script('protocols/ssl/heartbleed') script_config.disable_script('protocols/ssl/known-certs') script_config.disable_script('protocols/ssl/notary') script_config.disable_script('protocols/ssl/validate-ocsp') script_config.disable_script('protocols/ssl/validate-sct') script_config.disable_script('protocols/ssl/weak-keys') script_config.disable_script('frameworks/dpd/detect-protocols') script_config.disable_script('frameworks/dpd/packet-segment-logging') script_config.disable_script('frameworks/files/detect-MHR') script_config.disable_script('frameworks/files/entropy-test-all-files') script_config.disable_script('frameworks/files/extract-all-files') script_config.disable_script('frameworks/files/hash-all-files') script_config.disable_script( 'policy/frameworks/notice/extend-email/hostnames') script_config.write_config() return True
def setup_dynamite_zeek_scripts(self): """ Installs and enables extra dynamite Zeek scripts """ scripts = '' redefs = '' self.logger.info('Setting up Zeek scripts.') install_cache_extra_scripts_path = \ os.path.join(const.DEFAULT_CONFIGS, 'zeek', 'dynamite_extra_scripts') if not os.path.exists(install_cache_extra_scripts_path): self.logger.error('dynamite_extra_scripts not found in install_cache.') raise zeek_exceptions.InstallZeekError( "Third party scripts could not be installed; could not locate {}".format( install_cache_extra_scripts_path)) try: utilities.makedirs(os.path.join(self.configuration_directory, 'dynamite_extra_scripts'), exist_ok=True) except Exception as e: self.logger.error('General error occurred while creating dynamite_extra_scripts directory.') self.logger.debug("General error occurred while creating dynamite_extra_scripts directory; {}".format(e)) zeek_exceptions.InstallZeekError( "General error occurred while creating dynamite_extra_scripts directory; {}".format(e)) self.logger.info("Installing third-party Zeek scripts.") extra_scripts_path = os.path.join(self.configuration_directory, 'dynamite_extra_scripts') try: utilities.copytree(install_cache_extra_scripts_path, extra_scripts_path) except Exception as e: self.logger.error("General error occurred while copying files to dynamite_extra_scripts directory.") self.logger.debug( "General error occurred while copying files to dynamite_extra_scripts directory; {}".format(e)) zeek_exceptions.InstallZeekError( "General error occurred while copying files to dynamite_extra_scripts directory; {}".format(e)) zeek_site_local_path = os.path.join(self.configuration_directory, 'site', 'local.zeek') try: with open(zeek_site_local_path, 'r') as rf: for line in rf.readlines(): if '@load' in line: scripts += line.strip() + '\n' elif 'redef' in line: redefs += line.strip() + '\n' except Exception as e: self.logger.error("General error occurred while reading {}.".format(e)) self.logger.debug("General error occurred while reading {}; {}".format(zeek_site_local_path, e)) raise zeek_exceptions.InstallZeekError( "General error occurred while reading {}; {}".format(zeek_site_local_path, e)) try: with open(zeek_site_local_path, 'w') as wf: extra_script_install_path = os.path.join(self.configuration_directory, 'dynamite_extra_scripts') wf.write(scripts) for script_dir in os.listdir(extra_script_install_path): wf.write('@load {}\n'.format(os.path.join(extra_script_install_path, script_dir))) wf.write(redefs) except Exception as e: self.logger.error("General error occurred while writing {}.".format(e)) self.logger.debug("General error occurred while writing {}; {}".format(zeek_site_local_path, e)) raise zeek_exceptions.InstallZeekError( "General error occurred while writing {}; {}".format(zeek_site_local_path, e) ) self.setup_zeek_af_packet_plugin() self.setup_zeek_community_id_plugin() self.logger.info('Disabling unneeded Zeek scripts.') # Disable Unneeded Zeek scripts try: script_config = zeek_configs.ScriptConfigManager(self.configuration_directory) except zeek_exceptions.ReadsZeekConfigError: self.logger.error('Could not read Zeek script configuration.') raise zeek_exceptions.InstallZeekError("Could not read Zeek script configuration.") try: self.logger.debug('Disabling Zeek Script: protocols/ftp/detect') script_config.disable_script('protocols/ftp/detect') self.logger.debug('Disabling Zeek Script: protocols/ftp/software') script_config.disable_script('protocols/ftp/software') self.logger.debug('Disabling Zeek Script: protocols/ftp/detect-bruteforcing') script_config.disable_script('protocols/ftp/detect-bruteforcing') self.logger.debug('Disabling Zeek Script: protocols/dns/detect-external-names') script_config.disable_script('protocols/dns/detect-external-names') self.logger.debug('Disabling Zeek Script: protocols/http/detect-sqli') script_config.disable_script('protocols/http/detect-sqli') self.logger.debug('Disabling Zeek Script: protocols/http/detect-webapps') script_config.disable_script('protocols/http/detect-webapps') self.logger.debug('Disabling Zeek Script: protocols/krb/ticket-logging') script_config.disable_script('protocols/krb/ticket-logging') self.logger.debug('Disabling Zeek Script: protocols/rdp/indicate_ssl') script_config.disable_script('protocols/rdp/indicate_ssl') self.logger.debug('Disabling Zeek Script: protocols/smb/log-cmds') script_config.disable_script('protocols/smb/log-cmds') self.logger.debug('Disabling Zeek Script: protocols/smtp/blocklists') script_config.disable_script('protocols/smtp/blocklists') self.logger.debug('Disabling Zeek Script: protocols/smtp/detect-suspicious-orig') script_config.disable_script('protocols/smtp/detect-suspicious-orig') self.logger.debug('Disabling Zeek Script: protocols/smtp/entities-excerpt') script_config.disable_script('protocols/smtp/entities-excerpt') self.logger.debug('Disabling Zeek Script: protocols/smtp/blocklists') script_config.disable_script('protocols/smtp/blocklists') self.logger.debug('Disabling Zeek Script: protocols/smtp/software') script_config.disable_script('protocols/smtp/software') self.logger.debug('Disabling Zeek Script: protocols/ssh/detect-bruteforcing') script_config.disable_script('protocols/ssh/detect-bruteforcing') self.logger.debug('Disabling Zeek Script: protocols/ssh/geo-data') script_config.disable_script('protocols/ssh/geo-data') self.logger.debug('Disabling Zeek Script: protocols/ssh/interesting-hostnames') script_config.disable_script('protocols/ssh/interesting-hostnames') self.logger.debug('Disabling Zeek Script: protocols/ssh/software') script_config.disable_script('protocols/ssh/software') self.logger.debug('Disabling Zeek Script: protocols/ssl/expiring-certs') script_config.disable_script('protocols/ssl/expiring-certs') self.logger.debug('Disabling Zeek Script: protocols/ssl/extract-certs-pem') script_config.disable_script('protocols/ssl/extract-certs-pem') self.logger.debug('Disabling Zeek Script: protocols/ssl/heartbleed') script_config.disable_script('protocols/ssl/heartbleed') self.logger.debug('Disabling Zeek Script: protocols/ssl/known-certs') script_config.disable_script('protocols/ssl/known-certs') self.logger.debug('Disabling Zeek Script: protocols/ssl/notary') script_config.disable_script('protocols/ssl/notary') self.logger.debug('Disabling Zeek Script: protocols/ssl/validate-ocsp') script_config.disable_script('protocols/ssl/validate-ocsp') self.logger.debug('Disabling Zeek Script: protocols/ssl/validate-sct') script_config.disable_script('protocols/ssl/validate-sct') self.logger.debug('Disabling Zeek Script: protocols/ssl/weak-keys') script_config.disable_script('protocols/ssl/weak-keys') self.logger.debug('Disabling Zeek Script: frameworks/dpd/detect-protocols') script_config.disable_script('frameworks/dpd/detect-protocols') self.logger.debug('Disabling Zeek Script: frameworks/dpd/packet-segment-logging') script_config.disable_script('frameworks/dpd/packet-segment-logging') self.logger.debug('Disabling Zeek Script: frameworks/files/detect-MHR') script_config.disable_script('frameworks/files/detect-MHR') self.logger.debug('Disabling Zeek Script: frameworks/files/entropy-test-all-files') script_config.disable_script('frameworks/files/entropy-test-all-files') self.logger.debug('Disabling Zeek Script: frameworks/files/extract-all-files') script_config.disable_script('frameworks/files/extract-all-files') self.logger.debug('Disabling Zeek Script: frameworks/files/hash-all-files') script_config.disable_script('frameworks/files/hash-all-files') self.logger.debug('Disabling Zeek Script: policy/frameworks/notice/extend-email/hostnames') script_config.disable_script('policy/frameworks/notice/extend-email/hostnames') except zeek_exceptions.ZeekScriptNotFoundError: self.logger.error('Could not disable one or more Zeek scripts.') raise zeek_exceptions.InstallZeekError("Could not disable one or more Zeek scripts.") try: script_config.write_config() except zeek_exceptions.WriteZeekConfigError: self.logger.error('Could not write Zeek script configuration.') raise zeek_exceptions.InstallZeekError("Could not write Zeek script configuration.")
def setup_logstash_synesis(self): """ Create required environmental variables; copy configurations to various directories. """ env_file = os.path.join(const.CONFIG_PATH, 'environment') self.logger.info( 'Creating Synesis installation and configuration directories.') utilities.makedirs(self.install_directory, exist_ok=True) self.logger.info('Copying Synesis configurations.') utilities.copytree( os.path.join(const.DEFAULT_CONFIGS, 'logstash', 'suricata'), self.install_directory) utilities.set_ownership_of_file(self.install_directory, user='******', group='dynamite') try: with open(env_file) as env_f: env_str = env_f.read() if 'SYNLITE_SURICATA_DICT_PATH' not in env_str: dict_path = os.path.join(self.install_directory, 'dictionaries') self.logger.info( 'Updating Synesis dictionary configuration path [{}]'. format(dict_path)) subprocess.call( 'echo SYNLITE_SURICATA_DICT_PATH="{}" >> {}'.format( dict_path, env_file), shell=True) if 'SYNLITE_SURICATA_TEMPLATE_PATH' not in env_str: template_path = os.path.join(self.install_directory, 'templates') self.logger.info( 'Updating Synesis template configuration path [{}]'. format(template_path)) subprocess.call( 'echo SYNLITE_SURICATA_TEMPLATE_PATH="{}" >> {}'. format(template_path, env_file), shell=True) if 'SYNLITE_SURICATA_GEOIP_DB_PATH' not in env_str: geo_path = os.path.join(self.install_directory, 'geoipdbs') self.logger.info( 'Updating Synesis GeoDBs configuration path [{}]'. format(geo_path)) subprocess.call( 'echo SYNLITE_SURICATA_GEOIP_DB_PATH="{}" >> {}'. format(geo_path, env_file), shell=True) except Exception as e: self.logger.error('Failed to read Synesis environment variables.') self.logger.debug( "Failed to read Synesis environment variables; {}".format(e)) raise synesis_exceptions.InstallSynesisError( "Failed to read Synesis environment variables; {} ".format(e)) try: synesis_config.ConfigManager().write_environment_variables() except (synesis_exceptions.ReadSynesisConfigError, synesis_exceptions.WriteSynesisConfigError): self.logger.error( 'Failed to read/write Synesis environment variables.') raise synesis_exceptions.InstallSynesisError( "Could not read/write Synesis environment variables.")