def _terraform_init(config):
"""Initialize infrastructure using Terraform
Args:
config (CLIConfig): Loaded StreamAlert CLI
"""
LOGGER_CLI.info('Initializing StreamAlert')
# generate init Terraform files
if not terraform_generate(config=config, init=True):
return
LOGGER_CLI.info('Initializing Terraform')
if not run_command(['terraform', 'init']):
sys.exit(1)
# build init infrastructure
LOGGER_CLI.info('Building Initial Infrastructure')
init_targets = [
'aws_s3_bucket.lambda_source', 'aws_s3_bucket.logging_bucket',
'aws_s3_bucket.stream_alert_secrets',
'aws_s3_bucket.terraform_remote_state', 'aws_s3_bucket.streamalerts',
'aws_kms_key.stream_alert_secrets',
'aws_kms_alias.stream_alert_secrets'
]
if not tf_runner(targets=init_targets):
LOGGER_CLI.error('An error occurred while running StreamAlert init')
sys.exit(1)
# generate the main.tf with remote state enabled
LOGGER_CLI.info('Configuring Terraform Remote State')
if not terraform_generate(config=config):
return
if not run_command(['terraform', 'init']):
return
# Use a named tuple to match the 'processor' attribute in the argparse options
deploy_opts = namedtuple('DeployOptions', ['processor', 'clusters'])
LOGGER_CLI.info('Deploying Lambda Functions')
deploy(deploy_opts(['rule', 'alert', 'alert_merger', 'athena'], []),
config)
# we need to manually create the streamalerts table since terraform does not support this
# See: https://github.com/terraform-providers/terraform-provider-aws/issues/1486
alerts_bucket = '{}.streamalerts'.format(
config['global']['account']['prefix'])
create_table('alerts', alerts_bucket, config)
LOGGER_CLI.info('Building Remainder Infrastructure')
tf_runner(refresh=False)
def _resolve_third_party(self, temp_package_path):
"""Install all third-party packages into the deployment package folder
Args:
temp_package_path (str): Full path to temp package path
Returns:
bool: False if the pip command failed to install requirements, True otherwise
"""
# Install all required core libs that were not precompiled for this package
third_party_libs = self.third_party_libs.difference(
self.precompiled_libs)
# Add any custom libs needed by rules, etc
if self.config_key in self.config['lambda']:
third_party_libs.update(
set(self.config['lambda'][self.config_key].get(
'third_party_libraries', [])))
# Return a default of True here if no libraries to install
if not third_party_libs:
LOGGER_CLI.info('No third-party libraries to install.')
return True
LOGGER_CLI.info('Installing third-party libraries: %s',
', '.join(third_party_libs))
pip_command = ['pip', 'install']
pip_command.extend(third_party_libs)
pip_command.extend(['--upgrade', '--target', temp_package_path])
# Return True if the pip command is successfully run
return run_command(pip_command, cwd=temp_package_path, quiet=True)
def terraform_check():
"""Verify that Terraform is configured correctly
Returns:
bool: Success or failure of the command ran"""
prereqs_message = ('Terraform not found! Please install and add to '
'your $PATH:\n'
'\t$ export PATH=$PATH:/usr/local/terraform/bin')
return run_command(['terraform', 'version'], error_message=prereqs_message, quiet=True)
def terraform_handler(options, config):
"""Handle all Terraform CLI operations
Args:
options (namedtuple): Parsed arguments from manage.py
config (CLIConfig): Loaded StreamAlert CLI
"""
# Check for valid credentials
if not check_credentials():
return
# Verify terraform is installed
if not terraform_check():
return
# Plan and Apply our streamalert infrastructure
if options.subcommand == 'build':
_terraform_build(options, config)
# generate terraform files
elif options.subcommand == 'generate':
if not terraform_generate(config=config):
return
elif options.subcommand == 'init-backend':
run_command(['terraform', 'init'])
# initialize streamalert infrastructure from a blank state
elif options.subcommand == 'init':
_terraform_init(config)
elif options.subcommand == 'clean':
if not continue_prompt(
message='Are you sure you want to clean all Terraform files?'):
sys.exit(1)
_terraform_clean(config)
elif options.subcommand == 'destroy':
_terraform_destroy(options, config)
# get a quick status on our declared infrastructure
elif options.subcommand == 'status':
terraform_status(config)
def _terraform_init_backend():
"""Initialize the infrastructure backend (S3) using Terraform
Returns:
bool: False if errors occurred, True otherwise
"""
# Check for valid credentials
if not check_credentials():
return False
# Verify terraform is installed
if not terraform_check():
return False
LOGGER.info('Initializing StreamAlert backend')
return run_command(['terraform', 'init'])
def terraform_destroy_handler(options, config):
"""Use Terraform to destroy any existing infrastructure
Args:
options (argparse.Namespace): Parsed arguments from manage.py
config (CLIConfig): Loaded StreamAlert config
Returns:
bool: False if errors occurred, True otherwise
"""
# Check for valid credentials
if not check_credentials():
return False
# Verify terraform is installed
if not terraform_check():
return False
# Ask for approval here since multiple Terraform commands may be necessary
if not continue_prompt(message='Are you sure you want to destroy?'):
return False
if options.target:
target_modules, valid = _get_valid_tf_targets(config, options.target)
if not valid:
return False
return tf_runner(action='destroy',
auto_approve=True,
targets=target_modules if target_modules else None)
# Migrate back to local state so Terraform can successfully
# destroy the S3 bucket used by the backend.
# Do not check for terraform or aws creds again since these were checked above
if not terraform_generate_handler(
config=config, init=True, check_tf=False, check_creds=False):
return False
if not run_command(['terraform', 'init']):
return False
# Destroy all of the infrastructure
if not tf_runner(action='destroy', auto_approve=True):
return False
# Remove old Terraform files
return terraform_clean_handler()
def _terraform_destroy(options, config):
"""Use Terraform to destroy any existing infrastructure
Args:
options (namedtuple): Parsed arguments from manage.py
config (CLIConfig): Loaded StreamAlert CLI
"""
# Ask for approval here since multiple Terraform commands may be necessary
if not continue_prompt(message='Are you sure you want to destroy?'):
sys.exit(1)
if options.target:
targets = []
# Iterate over any targets to destroy. Global modules, like athena
# are prefixed with `stream_alert_` while cluster based modules
# are a combination of the target and cluster name
for target in options.target:
if target == 'athena':
targets.append('module.stream_alert_{}'.format(target))
elif target == 'threat_intel_downloader':
targets.append('module.threat_intel_downloader')
else:
targets.extend([
'module.{}_{}'.format(target, cluster)
for cluster in config.clusters()
])
tf_runner(action='destroy', auto_approve=True, targets=targets)
return
# Migrate back to local state so Terraform can successfully
# destroy the S3 bucket used by the backend.
if not terraform_generate(config=config, init=True):
return
if not run_command(['terraform', 'init']):
return
# Destroy all of the infrastructure
if not tf_runner(action='destroy', auto_approve=True):
return
# Remove old Terraform files
_terraform_clean(config)
def _resolve_libraries(self, temp_package_path):
"""Install all libraries into the deployment package folder
Args:
temp_package_path (str): Full path to temp package path
Returns:
bool: False if the pip command failed to install requirements, True otherwise
"""
# Install all required core libs that were not precompiled for this package
package_libs = self._required_libs.difference(self.precompiled_libs)
libs_to_install = set()
for item in package_libs:
if item not in self.PACKAGE_LIBS:
LOGGER.error(
'Please ensure a pinned version of package \'%s\' is included in PACKAGE_LIBS',
item)
return False
libs_to_install.add(self.PACKAGE_LIBS[item])
# Add any custom libs needed by rules, etc
if self.config_key in self.config['lambda']:
libs_to_install.update(
set(self.config['lambda'][self.config_key].get(
'third_party_libraries', [])))
# Return a default of True here if no libraries to install
if not libs_to_install:
LOGGER.info('No libraries to install')
return True
LOGGER.info('Installing libraries: %s', ', '.join(libs_to_install))
pip_command = ['pip', 'install']
pip_command.extend(libs_to_install)
pip_command.extend(['--upgrade', '--target', temp_package_path])
# Return True if the pip command is successfully run
return run_command(pip_command, cwd=temp_package_path, quiet=True)
def _resolve_third_party(self, temp_package_path):
"""Install all third-party packages into the deployment package folder
Args:
temp_package_path (str): Full path to temp package path
Returns:
bool: False if the pip command failed to install requirements, True otherwise
"""
third_party_libs = self.config['lambda'][self.config_key]['third_party_libraries']
# Return a default of True here if no libraries to install
if not third_party_libs:
LOGGER_CLI.info('No third-party libraries to install.')
return True
LOGGER_CLI.info(
'Installing third-party libraries: %s',
', '.join(third_party_libs))
pip_command = ['pip', 'install']
pip_command.extend(third_party_libs)
pip_command.extend(['--upgrade', '--target', temp_package_path])
# Return True if the pip command is successfully run
return run_command(pip_command, cwd=temp_package_path, quiet=True)
def terraform_handler(options, config):
"""Handle all Terraform CLI operations
Args:
options (namedtuple): Parsed arguments from manage.py
"""
# Check for valid credentials
if not check_credentials():
return
# Verify terraform is installed
if not terraform_check():
return
# Use a named tuple to match the 'processor' attribute in the argparse options
deploy_opts = namedtuple('DeployOptions', ['processor', 'clusters'])
# Plan and Apply our streamalert infrastructure
if options.subcommand == 'build':
terraform_build(options, config)
# generate terraform files
elif options.subcommand == 'generate':
if not terraform_generate(config=config):
return
elif options.subcommand == 'init-backend':
run_command(['terraform', 'init'])
# initialize streamalert infrastructure from a blank state
elif options.subcommand == 'init':
LOGGER_CLI.info('Initializing StreamAlert')
# generate init Terraform files
if not terraform_generate(config=config, init=True):
return
LOGGER_CLI.info('Initializing Terraform')
if not run_command(['terraform', 'init']):
sys.exit(1)
# build init infrastructure
LOGGER_CLI.info('Building Initial Infrastructure')
init_targets = [
'aws_s3_bucket.lambda_source', 'aws_s3_bucket.logging_bucket',
'aws_s3_bucket.stream_alert_secrets',
'aws_s3_bucket.terraform_remote_state',
'aws_s3_bucket.streamalerts', 'aws_kms_key.stream_alert_secrets',
'aws_kms_alias.stream_alert_secrets'
]
if not tf_runner(targets=init_targets):
LOGGER_CLI.error('An error occured while running StreamAlert init')
sys.exit(1)
# generate the main.tf with remote state enabled
LOGGER_CLI.info('Configuring Terraform Remote State')
if not terraform_generate(config=config):
return
if not run_command(['terraform', 'init']):
return
LOGGER_CLI.info('Deploying Lambda Functions')
# deploy both lambda functions
deploy(deploy_opts(['rule', 'alert'], []), config)
# create all remainder infrastructure
LOGGER_CLI.info('Building Remainder Infrastructure')
tf_runner()
elif options.subcommand == 'clean':
if not continue_prompt(
message='Are you sure you want to clean all Terraform files?'):
sys.exit(1)
terraform_clean(config)
elif options.subcommand == 'destroy':
if not continue_prompt(message='Are you sure you want to destroy?'):
sys.exit(1)
if options.target:
targets = []
# Iterate over any targets to destroy. Global modules, like athena
# are prefixed with `stream_alert_` while cluster based modules
# are a combination of the target and cluster name
for target in options.target:
if target == 'athena':
targets.append('module.stream_alert_{}'.format(target))
elif target == 'threat_intel_downloader':
targets.append('module.threat_intel_downloader')
else:
targets.extend([
'module.{}_{}'.format(target, cluster)
for cluster in config.clusters()
])
tf_runner(targets=targets, action='destroy')
return
# Migrate back to local state so Terraform can successfully
# destroy the S3 bucket used by the backend.
if not terraform_generate(config=config, init=True):
return
if not run_command(['terraform', 'init']):
return
# Destroy all of the infrastructure
if not tf_runner(action='destroy'):
return
# Remove old Terraform files
terraform_clean(config)
# get a quick status on our declared infrastructure
elif options.subcommand == 'status':
terraform_status(config)
def terraform_init(options, config):
"""Initialize infrastructure using Terraform
Args:
config (CLIConfig): Loaded StreamAlert config
Returns:
bool: False if errors occurred, True otherwise
"""
# Stop here if only initializing the backend
if options.backend:
return _terraform_init_backend()
LOGGER.info('Initializing StreamAlert')
# generate init Terraform files
if not terraform_generate_handler(config=config, init=True):
return False
LOGGER.info('Initializing Terraform')
if not run_command(['terraform', 'init']):
return False
# build init infrastructure
LOGGER.info('Building initial infrastructure')
init_targets = [
'aws_s3_bucket.lambda_source', 'aws_s3_bucket.logging_bucket',
'aws_s3_bucket.stream_alert_secrets',
'aws_s3_bucket.terraform_remote_state', 'aws_s3_bucket.streamalerts',
'aws_kms_key.server_side_encryption',
'aws_kms_alias.server_side_encryption',
'aws_kms_key.stream_alert_secrets',
'aws_kms_alias.stream_alert_secrets'
]
if not tf_runner(targets=init_targets):
LOGGER.error('An error occurred while running StreamAlert init')
return False
# generate the main.tf with remote state enabled
LOGGER.info('Configuring Terraform Remote State')
if not terraform_generate_handler(
config=config, check_tf=False, check_creds=False):
return False
if not run_command(['terraform', 'init']):
return False
LOGGER.info('Deploying Lambda Functions')
functions = ['rule', 'alert', 'alert_merger', 'athena', 'classifier']
deploy(functions, config)
# we need to manually create the streamalerts table since terraform does not support this
# See: https://github.com/terraform-providers/terraform-provider-aws/issues/1486
alerts_bucket = '{}.streamalerts'.format(
config['global']['account']['prefix'])
create_table('alerts', alerts_bucket, config)
LOGGER.info('Building remainding infrastructure')
return tf_runner(refresh=False)
def run_command(args=None, **kwargs):
"""Alias to CLI Helpers.run_command"""
return helpers.run_command(args, **kwargs)
