def _terraform_build(options, config): """Run Terraform with an optional set of targets and clusters Args: options (namedtuple): Parsed arguments from manage.py config (CLIConfig): Loaded StreamAlert CLI """ if not terraform_generate(config=config): return # Define the set of custom targets to apply tf_runner_targets = set() # If resource are not clustered, it is most likely required to # fall in the custom mapping below: custom_module_mapping = { 'athena': 'module.stream_alert_athena', 'threat_intel_downloader': 'module.threat_intel_downloader' } clusters = set(options.clusters or config.clusters()) if options.target: tf_runner_targets.update({ 'module.{}_{}'.format(target, cluster) for cluster in clusters for target in options.target }) for name in custom_module_mapping: if name in options.target: tf_runner_targets.add(custom_module_mapping[name]) tf_runner(targets=tf_runner_targets)
def deploy(options, config): """Deploy new versions of all Lambda functions Args: options (namedtuple): ArgParsed command from the CLI config (CLIConfig): Loaded StreamAlert config Steps: Build AWS Lambda deployment package Upload to S3 Update lambda.json with uploaded package checksum and S3 key Publish new version Update each cluster's Lambda configuration with latest published version Run Terraform Apply """ # Terraform apply only to the module which contains our lambda functions deploy_targets = set() packages = [] if 'all' in options.processor: processors = { 'alert', 'alert_merger', 'apps', 'athena', 'rule', 'threat_intel_downloader' } else: processors = options.processor for processor in processors: package, targets = _create_and_upload(processor, config, options.clusters) # Continue if the package isn't enabled if not all([package, targets]): continue packages.append(package) deploy_targets.update(targets) # Regenerate the Terraform configuration with the new S3 keys if not terraform_generate(config=config): return # Run Terraform: Update the Lambda source code in $LATEST if not helpers.tf_runner(targets=deploy_targets): sys.exit(1) # Update the rule table now if the rule processor is being deployed if 'rule' in options.processor: _update_rule_table(options, config) # Publish a new production Lambda version if not _publish_version(packages, config, options.clusters): return # Regenerate the Terraform configuration with the new Lambda versions if not terraform_generate(config=config): return # Apply the changes to the Lambda aliases helpers.tf_runner(targets=deploy_targets, refresh=False, auto_approve=True)
def _terraform_init(config): """Initialize infrastructure using Terraform Args: config (CLIConfig): Loaded StreamAlert CLI """ LOGGER_CLI.info('Initializing StreamAlert') # generate init Terraform files if not terraform_generate(config=config, init=True): return LOGGER_CLI.info('Initializing Terraform') if not run_command(['terraform', 'init']): sys.exit(1) # build init infrastructure LOGGER_CLI.info('Building Initial Infrastructure') init_targets = [ 'aws_s3_bucket.lambda_source', 'aws_s3_bucket.logging_bucket', 'aws_s3_bucket.stream_alert_secrets', 'aws_s3_bucket.terraform_remote_state', 'aws_s3_bucket.streamalerts', 'aws_kms_key.stream_alert_secrets', 'aws_kms_alias.stream_alert_secrets' ] if not tf_runner(targets=init_targets): LOGGER_CLI.error('An error occurred while running StreamAlert init') sys.exit(1) # generate the main.tf with remote state enabled LOGGER_CLI.info('Configuring Terraform Remote State') if not terraform_generate(config=config): return if not run_command(['terraform', 'init']): return # Use a named tuple to match the 'processor' attribute in the argparse options deploy_opts = namedtuple('DeployOptions', ['processor', 'clusters']) LOGGER_CLI.info('Deploying Lambda Functions') deploy(deploy_opts(['rule', 'alert', 'alert_merger', 'athena'], []), config) # we need to manually create the streamalerts table since terraform does not support this # See: https://github.com/terraform-providers/terraform-provider-aws/issues/1486 alerts_bucket = '{}.streamalerts'.format( config['global']['account']['prefix']) create_table('alerts', alerts_bucket, config) LOGGER_CLI.info('Building Remainder Infrastructure') tf_runner(refresh=False)
def kinesis_handler(options, config): """Main handler for the Kinesis parser Args: options (argparse.Namespace): Parsed arguments config (CLIConfig): Loaded StreamAlert config Returns: bool: False if errors occurred, True otherwise """ enable = options.action == 'enable-events' LOGGER.info('%s Kinesis Events', 'Enabling' if enable else 'Disabling') for cluster in options.clusters or config.clusters(): if 'kinesis_events' in config['clusters'][cluster]['modules']: config['clusters'][cluster]['modules']['kinesis_events']['enabled'] = enable config.write() if options.skip_terraform: return True # not an error if not terraform_generate_handler(config): return False return tf_runner( action='apply', targets=[ 'module.{}_{}'.format('kinesis_events', cluster) for cluster in config.clusters() ] )
def deploy(functions, config, clusters=None): """Deploy the functions Args: functions (set): Set of functions being deployed config (CLIConfig): Loaded StreamAlert config clusters (set=None): Optional clusters to target for this deploy Returns: bool: False if errors occurred, True otherwise """ LOGGER.info('Deploying: %s', ' '.join(sorted(functions))) # Terraform apply only to the module which contains our lambda functions deploy_targets = set() packages = [] for function in functions: package, targets = _create(function, config, clusters) # Continue if the package isn't enabled if not all([package, targets]): continue packages.append(package) deploy_targets.update(targets) # Terraform applies the new package and publishes a new version return helpers.tf_runner(targets=deploy_targets)
def terraform_destroy_handler(options, config): """Use Terraform to destroy any existing infrastructure Args: options (argparse.Namespace): Parsed arguments from manage.py config (CLIConfig): Loaded StreamAlert config Returns: bool: False if errors occurred, True otherwise """ # Check for valid credentials if not check_credentials(): return False # Verify terraform is installed if not terraform_check(): return False # Ask for approval here since multiple Terraform commands may be necessary if not continue_prompt(message='Are you sure you want to destroy?'): return False if options.target: target_modules, valid = _get_valid_tf_targets(config, options.target) if not valid: return False return tf_runner(action='destroy', auto_approve=True, targets=target_modules if target_modules else None) # Migrate back to local state so Terraform can successfully # destroy the S3 bucket used by the backend. # Do not check for terraform or aws creds again since these were checked above if not terraform_generate_handler( config=config, init=True, check_tf=False, check_creds=False): return False if not run_command(['terraform', 'init']): return False # Destroy all of the infrastructure if not tf_runner(action='destroy', auto_approve=True): return False # Remove old Terraform files return terraform_clean_handler()
def _terraform_destroy(options, config): """Use Terraform to destroy any existing infrastructure Args: options (namedtuple): Parsed arguments from manage.py config (CLIConfig): Loaded StreamAlert CLI """ # Ask for approval here since multiple Terraform commands may be necessary if not continue_prompt(message='Are you sure you want to destroy?'): sys.exit(1) if options.target: targets = [] # Iterate over any targets to destroy. Global modules, like athena # are prefixed with `stream_alert_` while cluster based modules # are a combination of the target and cluster name for target in options.target: if target == 'athena': targets.append('module.stream_alert_{}'.format(target)) elif target == 'threat_intel_downloader': targets.append('module.threat_intel_downloader') else: targets.extend([ 'module.{}_{}'.format(target, cluster) for cluster in config.clusters() ]) tf_runner(action='destroy', auto_approve=True, targets=targets) return # Migrate back to local state so Terraform can successfully # destroy the S3 bucket used by the backend. if not terraform_generate(config=config, init=True): return if not run_command(['terraform', 'init']): return # Destroy all of the infrastructure if not tf_runner(action='destroy', auto_approve=True): return # Remove old Terraform files _terraform_clean(config)
def deploy(options, config): """Deploy new versions of all Lambda functions Args: options (namedtuple): ArgParsed command from the CLI config (CLIConfig): Loaded StreamAlert config Steps: Build AWS Lambda deployment package Upload to S3 Update lambda.json with uploaded package checksum and S3 key Publish new version Update each cluster's Lambda configuration with latest published version Run Terraform Apply """ # Terraform apply only to the module which contains our lambda functions deploy_targets = set() packages = [] if 'all' in options.processor: processors = { 'alert', 'alert_merger', 'apps', 'athena', 'rule', 'rule_promo', 'threat_intel_downloader' } else: processors = options.processor for processor in processors: package, targets = _create(processor, config, options.clusters) # Continue if the package isn't enabled if not all([package, targets]): continue packages.append(package) deploy_targets.update(targets) # Update the rule table now if the rule processor is being deployed if 'rule' in options.processor: _update_rule_table(options, config) # Terraform applies the new package and publishes a new version helpers.tf_runner(targets=deploy_targets)
def rollback(options): """Rollback the current production AWS Lambda version by 1 Notes: Ignores if the production version is $LATEST Only rollsback if published version is greater than 1 """ clusters = CONFIG.clusters() if 'all' in options.processor: lambda_functions = { 'rule_processor', 'alert_processor', 'athena_partition_refresh' } else: lambda_functions = { '{}_processor'.format(proc) for proc in options.processor if proc != 'athena' } if 'athena' in options.processor: lambda_functions.add('athena_partition_refresh') for cluster in clusters: for lambda_function in lambda_functions: stream_alert_key = CONFIG['clusters'][cluster]['modules'][ 'stream_alert'] current_vers = stream_alert_key[lambda_function]['current_version'] if current_vers != '$LATEST': current_vers = int(current_vers) if current_vers > 1: new_vers = current_vers - 1 CONFIG['clusters'][cluster]['modules']['stream_alert'][ lambda_function]['current_version'] = new_vers CONFIG.write() targets = ['module.stream_alert_{}'.format(x) for x in CONFIG.clusters()] if not terraform_generate(config=CONFIG): return helpers.tf_runner(targets=targets)
def set_kinesis_events(options, config, enable=True): """Enable or disable Kinesis events for given clusters Args: options (namedtuple): Parsed arguments config (CLIConfig): Loaded StreamAlert config enable (bool): Enable/Disable switch """ for cluster in options.clusters or config.clusters(): if 'kinesis_events' in config['clusters'][cluster]['modules']: config['clusters'][cluster]['modules']['kinesis_events'][ 'enabled'] = enable config.write() if not options.skip_terraform: terraform_generate(config) tf_runner(action='apply', targets=[ 'module.{}_{}'.format('kinesis_events', cluster) for cluster in config.clusters() ])
def rollback(options, config): """Rollback the current production AWS Lambda version by 1 Notes: Ignores if the production version is $LATEST Only rollsback if published version is greater than 1 """ clusters = options.clusters or config.clusters() rollback_all = 'all' in options.processor tf_targets = [] if rollback_all or 'alert' in options.processor: tf_targets.extend(_rollback_alert(config) or []) if rollback_all or 'alert_merger' in options.processor: tf_targets.extend(_rollback_alert_merger(config) or []) if rollback_all or 'apps' in options.processor: tf_targets.extend(_rollback_apps(config, clusters) or []) if rollback_all or 'athena' in options.processor: tf_targets.extend(_rollback_athena(config) or []) if rollback_all or 'rule' in options.processor: tf_targets.extend(_rollback_rule(config, clusters) or []) if rollback_all or 'threat_intel_downloader' in options.processor: tf_targets.extend(_rollback_downloader(config) or []) if not tf_targets: # No changes made return config.write() if not terraform_generate(config=config): return helpers.tf_runner(targets=sorted(tf_targets))
def terraform_build_handler(options, config): """Run Terraform with an optional set of targets and clusters Args: options (argparse.Namespace): Parsed arguments from manage.py config (CLIConfig): Loaded StreamAlert config Returns: bool: False if errors occurred, True otherwise """ if not terraform_generate_handler(config=config): return False target_modules, valid = _get_valid_tf_targets(config, options.target) if not valid: return False return tf_runner(targets=target_modules if target_modules else None)
def terraform_build(options, config): """Run Terraform with an optional set of targets Args: options (namedtuple): Parsed arguments from manage.py """ # Generate Terraform files if not terraform_generate(config=config): return # Target is for terraforming a specific streamalert module. # This value is passed as a list if options.target == ['athena']: tf_runner(targets=['module.stream_alert_athena']) elif options.target: targets = [ 'module.{}_{}'.format(target, cluster) for cluster in config.clusters() for target in options.target ] tf_runner(targets=targets) else: tf_runner()
def deploy(options, config): """Deploy new versions of all Lambda functions Steps: - Build AWS Lambda deployment package - Upload to S3 - Update lambda.json with uploaded package checksum and S3 key - Publish new version - Update each cluster's Lambda configuration with latest published version - Run Terraform Apply """ processor = options.processor # Terraform apply only to the module which contains our lambda functions targets = set() packages = [] def _publish_version(packages): """Publish Lambda versions""" for package in packages: if package.package_name in {'athena_partition_refresh', 'threat_intel_downloader'}: published = LambdaVersion( config=config, package=package, clustered_deploy=False).publish_function() else: published = LambdaVersion(config=config, package=package).publish_function() if not published: return False return True def _deploy_rule_processor(): """Create Rule Processor package and publish versions""" rule_package = RuleProcessorPackage(config=config, version=current_version) rule_package.create_and_upload() return rule_package def _deploy_alert_processor(): """Create Alert Processor package and publish versions""" alert_package = AlertProcessorPackage(config=config, version=current_version) alert_package.create_and_upload() return alert_package def _deploy_athena_partition_refresh(): """Create Athena Partition Refresh package and publish""" athena_package = AthenaPackage(config=config, version=current_version) athena_package.create_and_upload() return athena_package def _deploy_apps_function(): """Create app integration package and publish versions""" app_integration_package = AppIntegrationPackage(config=config, version=apps_version) app_integration_package.create_and_upload() return app_integration_package def _deploy_threat_intel_downloader(): """Create Threat Intel downloader package and publish version""" threat_intel_package = ThreatIntelDownloaderPackage( config=config, version=ti_downloader_version ) threat_intel_package.create_and_upload() return threat_intel_package if 'all' in processor: targets.update({'module.stream_alert_{}'.format(x) for x in config.clusters()}) targets.update({ 'module.app_{}_{}'.format(app_name, cluster) for cluster, info in config['clusters'].iteritems() for app_name in info['modules'].get('stream_alert_apps', {}) }) packages.append(_deploy_rule_processor()) packages.append(_deploy_alert_processor()) packages.append(_deploy_apps_function()) # Only include the Athena function if it exists and is enabled athena_config = config['lambda'].get('athena_partition_refresh_config') if athena_config and athena_config.get('enabled', False): targets.add('module.stream_alert_athena') packages.append(_deploy_athena_partition_refresh()) else: if 'rule' in processor: targets.update({'module.stream_alert_{}'.format(x) for x in config.clusters()}) packages.append(_deploy_rule_processor()) if 'alert' in processor: targets.update({'module.stream_alert_{}'.format(x) for x in config.clusters()}) packages.append(_deploy_alert_processor()) if 'apps' in processor: targets.update({ 'module.app_{}_{}'.format(app_name, cluster) for cluster, info in config['clusters'].iteritems() for app_name in info['modules'].get('stream_alert_apps', {}) }) packages.append(_deploy_apps_function()) if 'athena' in processor: targets.add('module.stream_alert_athena') packages.append(_deploy_athena_partition_refresh()) if 'threat_intel_downloader' in processor: targets.add('module.threat_intel_downloader') packages.append(_deploy_threat_intel_downloader()) # Regenerate the Terraform configuration with the new S3 keys if not terraform_generate(config=config): return # Run Terraform: Update the Lambda source code in $LATEST if not helpers.tf_runner(targets=targets): sys.exit(1) # TODO(jack) write integration test to verify newly updated function # Publish a new production Lambda version if not _publish_version(packages): return # Regenerate the Terraform configuration with the new Lambda versions if not terraform_generate(config=config): return # Apply the changes to the Lambda aliases helpers.tf_runner(targets=targets)
def terraform_init(options, config): """Initialize infrastructure using Terraform Args: config (CLIConfig): Loaded StreamAlert config Returns: bool: False if errors occurred, True otherwise """ # Stop here if only initializing the backend if options.backend: return _terraform_init_backend() LOGGER.info('Initializing StreamAlert') # generate init Terraform files if not terraform_generate_handler(config=config, init=True): return False LOGGER.info('Initializing Terraform') if not run_command(['terraform', 'init']): return False # build init infrastructure LOGGER.info('Building initial infrastructure') init_targets = [ 'aws_s3_bucket.lambda_source', 'aws_s3_bucket.logging_bucket', 'aws_s3_bucket.stream_alert_secrets', 'aws_s3_bucket.terraform_remote_state', 'aws_s3_bucket.streamalerts', 'aws_kms_key.server_side_encryption', 'aws_kms_alias.server_side_encryption', 'aws_kms_key.stream_alert_secrets', 'aws_kms_alias.stream_alert_secrets' ] if not tf_runner(targets=init_targets): LOGGER.error('An error occurred while running StreamAlert init') return False # generate the main.tf with remote state enabled LOGGER.info('Configuring Terraform Remote State') if not terraform_generate_handler( config=config, check_tf=False, check_creds=False): return False if not run_command(['terraform', 'init']): return False LOGGER.info('Deploying Lambda Functions') functions = ['rule', 'alert', 'alert_merger', 'athena', 'classifier'] deploy(functions, config) # we need to manually create the streamalerts table since terraform does not support this # See: https://github.com/terraform-providers/terraform-provider-aws/issues/1486 alerts_bucket = '{}.streamalerts'.format( config['global']['account']['prefix']) create_table('alerts', alerts_bucket, config) LOGGER.info('Building remainding infrastructure') return tf_runner(refresh=False)
def deploy(options): """Deploy new versions of all Lambda functions Steps: - Build AWS Lambda deployment package - Upload to S3 - Update lambda.json with uploaded package checksum and S3 key - Publish new version - Update each cluster's Lambda configuration with latest published version - Run Terraform Apply """ processor = options.processor # Terraform apply only to the module which contains our lambda functions targets = [] packages = [] def _publish_version(packages): """Publish Lambda versions""" for package in packages: if package.package_name == 'athena_partition_refresh': published = LambdaVersion( config=CONFIG, package=package, clustered_deploy=False).publish_function() else: published = LambdaVersion(config=CONFIG, package=package).publish_function() if not published: return False return True def _deploy_rule_processor(): """Create Rule Processor package and publish versions""" rule_package = RuleProcessorPackage(config=CONFIG, version=current_version) rule_package.create_and_upload() return rule_package def _deploy_alert_processor(): """Create Alert Processor package and publish versions""" alert_package = AlertProcessorPackage(config=CONFIG, version=current_version) alert_package.create_and_upload() return alert_package def _deploy_athena_partition_refresh(): """Create Athena Partition Refresh package and publish""" athena_package = AthenaPackage(config=CONFIG, version=current_version) athena_package.create_and_upload() return athena_package if 'all' in processor: targets.extend( ['module.stream_alert_{}'.format(x) for x in CONFIG.clusters()]) packages.append(_deploy_rule_processor()) packages.append(_deploy_alert_processor()) # Only include the Athena function if it exists and is enabled athena_config = CONFIG['lambda'].get('athena_partition_refresh_config') if athena_config and athena_config.get('enabled', False): targets.append('module.stream_alert_athena') packages.append(_deploy_athena_partition_refresh()) else: if 'rule' in processor: targets.extend([ 'module.stream_alert_{}'.format(x) for x in CONFIG.clusters() ]) packages.append(_deploy_rule_processor()) if 'alert' in processor: targets.extend([ 'module.stream_alert_{}'.format(x) for x in CONFIG.clusters() ]) packages.append(_deploy_alert_processor()) if 'athena' in processor: targets.append('module.stream_alert_athena') packages.append(_deploy_athena_partition_refresh()) # Regenerate the Terraform configuration with the new S3 keys if not terraform_generate(config=CONFIG): return # Run Terraform: Update the Lambda source code in $LATEST if not helpers.tf_runner(targets=targets): sys.exit(1) # TODO(jack) write integration test to verify newly updated function # Publish a new production Lambda version if not _publish_version(packages): return # Regenerate the Terraform configuration with the new Lambda versions if not terraform_generate(config=CONFIG): return # Apply the changes to the Lambda aliases helpers.tf_runner(targets=targets)
def terraform_handler(options, config): """Handle all Terraform CLI operations Args: options (namedtuple): Parsed arguments from manage.py """ # Check for valid credentials if not check_credentials(): return # Verify terraform is installed if not terraform_check(): return # Use a named tuple to match the 'processor' attribute in the argparse options deploy_opts = namedtuple('DeployOptions', ['processor', 'clusters']) # Plan and Apply our streamalert infrastructure if options.subcommand == 'build': terraform_build(options, config) # generate terraform files elif options.subcommand == 'generate': if not terraform_generate(config=config): return elif options.subcommand == 'init-backend': run_command(['terraform', 'init']) # initialize streamalert infrastructure from a blank state elif options.subcommand == 'init': LOGGER_CLI.info('Initializing StreamAlert') # generate init Terraform files if not terraform_generate(config=config, init=True): return LOGGER_CLI.info('Initializing Terraform') if not run_command(['terraform', 'init']): sys.exit(1) # build init infrastructure LOGGER_CLI.info('Building Initial Infrastructure') init_targets = [ 'aws_s3_bucket.lambda_source', 'aws_s3_bucket.logging_bucket', 'aws_s3_bucket.stream_alert_secrets', 'aws_s3_bucket.terraform_remote_state', 'aws_s3_bucket.streamalerts', 'aws_kms_key.stream_alert_secrets', 'aws_kms_alias.stream_alert_secrets' ] if not tf_runner(targets=init_targets): LOGGER_CLI.error('An error occured while running StreamAlert init') sys.exit(1) # generate the main.tf with remote state enabled LOGGER_CLI.info('Configuring Terraform Remote State') if not terraform_generate(config=config): return if not run_command(['terraform', 'init']): return LOGGER_CLI.info('Deploying Lambda Functions') # deploy both lambda functions deploy(deploy_opts(['rule', 'alert'], []), config) # create all remainder infrastructure LOGGER_CLI.info('Building Remainder Infrastructure') tf_runner() elif options.subcommand == 'clean': if not continue_prompt( message='Are you sure you want to clean all Terraform files?'): sys.exit(1) terraform_clean(config) elif options.subcommand == 'destroy': if not continue_prompt(message='Are you sure you want to destroy?'): sys.exit(1) if options.target: targets = [] # Iterate over any targets to destroy. Global modules, like athena # are prefixed with `stream_alert_` while cluster based modules # are a combination of the target and cluster name for target in options.target: if target == 'athena': targets.append('module.stream_alert_{}'.format(target)) elif target == 'threat_intel_downloader': targets.append('module.threat_intel_downloader') else: targets.extend([ 'module.{}_{}'.format(target, cluster) for cluster in config.clusters() ]) tf_runner(targets=targets, action='destroy') return # Migrate back to local state so Terraform can successfully # destroy the S3 bucket used by the backend. if not terraform_generate(config=config, init=True): return if not run_command(['terraform', 'init']): return # Destroy all of the infrastructure if not tf_runner(action='destroy'): return # Remove old Terraform files terraform_clean(config) # get a quick status on our declared infrastructure elif options.subcommand == 'status': terraform_status(config)