def _terraform_init(config): """Initialize infrastructure using Terraform Args: config (CLIConfig): Loaded StreamAlert CLI """ LOGGER_CLI.info('Initializing StreamAlert') # generate init Terraform files if not terraform_generate(config=config, init=True): return LOGGER_CLI.info('Initializing Terraform') if not run_command(['terraform', 'init']): sys.exit(1) # build init infrastructure LOGGER_CLI.info('Building Initial Infrastructure') init_targets = [ 'aws_s3_bucket.lambda_source', 'aws_s3_bucket.logging_bucket', 'aws_s3_bucket.stream_alert_secrets', 'aws_s3_bucket.terraform_remote_state', 'aws_s3_bucket.streamalerts', 'aws_kms_key.stream_alert_secrets', 'aws_kms_alias.stream_alert_secrets' ] if not tf_runner(targets=init_targets): LOGGER_CLI.error('An error occurred while running StreamAlert init') sys.exit(1) # generate the main.tf with remote state enabled LOGGER_CLI.info('Configuring Terraform Remote State') if not terraform_generate(config=config): return if not run_command(['terraform', 'init']): return # Use a named tuple to match the 'processor' attribute in the argparse options deploy_opts = namedtuple('DeployOptions', ['processor', 'clusters']) LOGGER_CLI.info('Deploying Lambda Functions') deploy(deploy_opts(['rule', 'alert', 'alert_merger', 'athena'], []), config) # we need to manually create the streamalerts table since terraform does not support this # See: https://github.com/terraform-providers/terraform-provider-aws/issues/1486 alerts_bucket = '{}.streamalerts'.format( config['global']['account']['prefix']) create_table('alerts', alerts_bucket, config) LOGGER_CLI.info('Building Remainder Infrastructure') tf_runner(refresh=False)
def _resolve_third_party(self, temp_package_path): """Install all third-party packages into the deployment package folder Args: temp_package_path (str): Full path to temp package path Returns: bool: False if the pip command failed to install requirements, True otherwise """ # Install all required core libs that were not precompiled for this package third_party_libs = self.third_party_libs.difference( self.precompiled_libs) # Add any custom libs needed by rules, etc if self.config_key in self.config['lambda']: third_party_libs.update( set(self.config['lambda'][self.config_key].get( 'third_party_libraries', []))) # Return a default of True here if no libraries to install if not third_party_libs: LOGGER_CLI.info('No third-party libraries to install.') return True LOGGER_CLI.info('Installing third-party libraries: %s', ', '.join(third_party_libs)) pip_command = ['pip', 'install'] pip_command.extend(third_party_libs) pip_command.extend(['--upgrade', '--target', temp_package_path]) # Return True if the pip command is successfully run return run_command(pip_command, cwd=temp_package_path, quiet=True)
def terraform_check(): """Verify that Terraform is configured correctly Returns: bool: Success or failure of the command ran""" prereqs_message = ('Terraform not found! Please install and add to ' 'your $PATH:\n' '\t$ export PATH=$PATH:/usr/local/terraform/bin') return run_command(['terraform', 'version'], error_message=prereqs_message, quiet=True)
def terraform_handler(options, config): """Handle all Terraform CLI operations Args: options (namedtuple): Parsed arguments from manage.py config (CLIConfig): Loaded StreamAlert CLI """ # Check for valid credentials if not check_credentials(): return # Verify terraform is installed if not terraform_check(): return # Plan and Apply our streamalert infrastructure if options.subcommand == 'build': _terraform_build(options, config) # generate terraform files elif options.subcommand == 'generate': if not terraform_generate(config=config): return elif options.subcommand == 'init-backend': run_command(['terraform', 'init']) # initialize streamalert infrastructure from a blank state elif options.subcommand == 'init': _terraform_init(config) elif options.subcommand == 'clean': if not continue_prompt( message='Are you sure you want to clean all Terraform files?'): sys.exit(1) _terraform_clean(config) elif options.subcommand == 'destroy': _terraform_destroy(options, config) # get a quick status on our declared infrastructure elif options.subcommand == 'status': terraform_status(config)
def _terraform_init_backend(): """Initialize the infrastructure backend (S3) using Terraform Returns: bool: False if errors occurred, True otherwise """ # Check for valid credentials if not check_credentials(): return False # Verify terraform is installed if not terraform_check(): return False LOGGER.info('Initializing StreamAlert backend') return run_command(['terraform', 'init'])
def terraform_destroy_handler(options, config): """Use Terraform to destroy any existing infrastructure Args: options (argparse.Namespace): Parsed arguments from manage.py config (CLIConfig): Loaded StreamAlert config Returns: bool: False if errors occurred, True otherwise """ # Check for valid credentials if not check_credentials(): return False # Verify terraform is installed if not terraform_check(): return False # Ask for approval here since multiple Terraform commands may be necessary if not continue_prompt(message='Are you sure you want to destroy?'): return False if options.target: target_modules, valid = _get_valid_tf_targets(config, options.target) if not valid: return False return tf_runner(action='destroy', auto_approve=True, targets=target_modules if target_modules else None) # Migrate back to local state so Terraform can successfully # destroy the S3 bucket used by the backend. # Do not check for terraform or aws creds again since these were checked above if not terraform_generate_handler( config=config, init=True, check_tf=False, check_creds=False): return False if not run_command(['terraform', 'init']): return False # Destroy all of the infrastructure if not tf_runner(action='destroy', auto_approve=True): return False # Remove old Terraform files return terraform_clean_handler()
def _terraform_destroy(options, config): """Use Terraform to destroy any existing infrastructure Args: options (namedtuple): Parsed arguments from manage.py config (CLIConfig): Loaded StreamAlert CLI """ # Ask for approval here since multiple Terraform commands may be necessary if not continue_prompt(message='Are you sure you want to destroy?'): sys.exit(1) if options.target: targets = [] # Iterate over any targets to destroy. Global modules, like athena # are prefixed with `stream_alert_` while cluster based modules # are a combination of the target and cluster name for target in options.target: if target == 'athena': targets.append('module.stream_alert_{}'.format(target)) elif target == 'threat_intel_downloader': targets.append('module.threat_intel_downloader') else: targets.extend([ 'module.{}_{}'.format(target, cluster) for cluster in config.clusters() ]) tf_runner(action='destroy', auto_approve=True, targets=targets) return # Migrate back to local state so Terraform can successfully # destroy the S3 bucket used by the backend. if not terraform_generate(config=config, init=True): return if not run_command(['terraform', 'init']): return # Destroy all of the infrastructure if not tf_runner(action='destroy', auto_approve=True): return # Remove old Terraform files _terraform_clean(config)
def _resolve_libraries(self, temp_package_path): """Install all libraries into the deployment package folder Args: temp_package_path (str): Full path to temp package path Returns: bool: False if the pip command failed to install requirements, True otherwise """ # Install all required core libs that were not precompiled for this package package_libs = self._required_libs.difference(self.precompiled_libs) libs_to_install = set() for item in package_libs: if item not in self.PACKAGE_LIBS: LOGGER.error( 'Please ensure a pinned version of package \'%s\' is included in PACKAGE_LIBS', item) return False libs_to_install.add(self.PACKAGE_LIBS[item]) # Add any custom libs needed by rules, etc if self.config_key in self.config['lambda']: libs_to_install.update( set(self.config['lambda'][self.config_key].get( 'third_party_libraries', []))) # Return a default of True here if no libraries to install if not libs_to_install: LOGGER.info('No libraries to install') return True LOGGER.info('Installing libraries: %s', ', '.join(libs_to_install)) pip_command = ['pip', 'install'] pip_command.extend(libs_to_install) pip_command.extend(['--upgrade', '--target', temp_package_path]) # Return True if the pip command is successfully run return run_command(pip_command, cwd=temp_package_path, quiet=True)
def _resolve_third_party(self, temp_package_path): """Install all third-party packages into the deployment package folder Args: temp_package_path (str): Full path to temp package path Returns: bool: False if the pip command failed to install requirements, True otherwise """ third_party_libs = self.config['lambda'][self.config_key]['third_party_libraries'] # Return a default of True here if no libraries to install if not third_party_libs: LOGGER_CLI.info('No third-party libraries to install.') return True LOGGER_CLI.info( 'Installing third-party libraries: %s', ', '.join(third_party_libs)) pip_command = ['pip', 'install'] pip_command.extend(third_party_libs) pip_command.extend(['--upgrade', '--target', temp_package_path]) # Return True if the pip command is successfully run return run_command(pip_command, cwd=temp_package_path, quiet=True)
def terraform_handler(options, config): """Handle all Terraform CLI operations Args: options (namedtuple): Parsed arguments from manage.py """ # Check for valid credentials if not check_credentials(): return # Verify terraform is installed if not terraform_check(): return # Use a named tuple to match the 'processor' attribute in the argparse options deploy_opts = namedtuple('DeployOptions', ['processor', 'clusters']) # Plan and Apply our streamalert infrastructure if options.subcommand == 'build': terraform_build(options, config) # generate terraform files elif options.subcommand == 'generate': if not terraform_generate(config=config): return elif options.subcommand == 'init-backend': run_command(['terraform', 'init']) # initialize streamalert infrastructure from a blank state elif options.subcommand == 'init': LOGGER_CLI.info('Initializing StreamAlert') # generate init Terraform files if not terraform_generate(config=config, init=True): return LOGGER_CLI.info('Initializing Terraform') if not run_command(['terraform', 'init']): sys.exit(1) # build init infrastructure LOGGER_CLI.info('Building Initial Infrastructure') init_targets = [ 'aws_s3_bucket.lambda_source', 'aws_s3_bucket.logging_bucket', 'aws_s3_bucket.stream_alert_secrets', 'aws_s3_bucket.terraform_remote_state', 'aws_s3_bucket.streamalerts', 'aws_kms_key.stream_alert_secrets', 'aws_kms_alias.stream_alert_secrets' ] if not tf_runner(targets=init_targets): LOGGER_CLI.error('An error occured while running StreamAlert init') sys.exit(1) # generate the main.tf with remote state enabled LOGGER_CLI.info('Configuring Terraform Remote State') if not terraform_generate(config=config): return if not run_command(['terraform', 'init']): return LOGGER_CLI.info('Deploying Lambda Functions') # deploy both lambda functions deploy(deploy_opts(['rule', 'alert'], []), config) # create all remainder infrastructure LOGGER_CLI.info('Building Remainder Infrastructure') tf_runner() elif options.subcommand == 'clean': if not continue_prompt( message='Are you sure you want to clean all Terraform files?'): sys.exit(1) terraform_clean(config) elif options.subcommand == 'destroy': if not continue_prompt(message='Are you sure you want to destroy?'): sys.exit(1) if options.target: targets = [] # Iterate over any targets to destroy. Global modules, like athena # are prefixed with `stream_alert_` while cluster based modules # are a combination of the target and cluster name for target in options.target: if target == 'athena': targets.append('module.stream_alert_{}'.format(target)) elif target == 'threat_intel_downloader': targets.append('module.threat_intel_downloader') else: targets.extend([ 'module.{}_{}'.format(target, cluster) for cluster in config.clusters() ]) tf_runner(targets=targets, action='destroy') return # Migrate back to local state so Terraform can successfully # destroy the S3 bucket used by the backend. if not terraform_generate(config=config, init=True): return if not run_command(['terraform', 'init']): return # Destroy all of the infrastructure if not tf_runner(action='destroy'): return # Remove old Terraform files terraform_clean(config) # get a quick status on our declared infrastructure elif options.subcommand == 'status': terraform_status(config)
def terraform_init(options, config): """Initialize infrastructure using Terraform Args: config (CLIConfig): Loaded StreamAlert config Returns: bool: False if errors occurred, True otherwise """ # Stop here if only initializing the backend if options.backend: return _terraform_init_backend() LOGGER.info('Initializing StreamAlert') # generate init Terraform files if not terraform_generate_handler(config=config, init=True): return False LOGGER.info('Initializing Terraform') if not run_command(['terraform', 'init']): return False # build init infrastructure LOGGER.info('Building initial infrastructure') init_targets = [ 'aws_s3_bucket.lambda_source', 'aws_s3_bucket.logging_bucket', 'aws_s3_bucket.stream_alert_secrets', 'aws_s3_bucket.terraform_remote_state', 'aws_s3_bucket.streamalerts', 'aws_kms_key.server_side_encryption', 'aws_kms_alias.server_side_encryption', 'aws_kms_key.stream_alert_secrets', 'aws_kms_alias.stream_alert_secrets' ] if not tf_runner(targets=init_targets): LOGGER.error('An error occurred while running StreamAlert init') return False # generate the main.tf with remote state enabled LOGGER.info('Configuring Terraform Remote State') if not terraform_generate_handler( config=config, check_tf=False, check_creds=False): return False if not run_command(['terraform', 'init']): return False LOGGER.info('Deploying Lambda Functions') functions = ['rule', 'alert', 'alert_merger', 'athena', 'classifier'] deploy(functions, config) # we need to manually create the streamalerts table since terraform does not support this # See: https://github.com/terraform-providers/terraform-provider-aws/issues/1486 alerts_bucket = '{}.streamalerts'.format( config['global']['account']['prefix']) create_table('alerts', alerts_bucket, config) LOGGER.info('Building remainding infrastructure') return tf_runner(refresh=False)
def run_command(args=None, **kwargs): """Alias to CLI Helpers.run_command""" return helpers.run_command(args, **kwargs)