def add_app_integration(self, app_info):
"""Add a configuration for a new streamalert app integration function
Args:
app_info (dict): The necessary values needed to begin configuring
a new app integration
"""
exists, prompt_for_auth, overwrite = False, True, False
app = get_app(app_info, False)
# Check to see if there is an existing configuration for this app integration
cluster_config = self.config['clusters'][app_info['cluster']]
if app_info['app_name'] in cluster_config['modules'].get('stream_alert_apps', {}):
prompt = ('An app with the name \'{}\' is already configured for cluster '
'\'{}\'. Would you like to update the existing app\'s configuration'
'?'.format(app_info['app_name'], app_info['cluster']))
exists = True
# Return if the user is not deliberately updating an existing config
if not continue_prompt(message=prompt):
return
prompt = ('Would you also like to update the authentication information for '
'app integration with name \'{}\'?'.format(app_info['app_name']))
# If this is true, we shouldn't prompt again to warn about overwriting
prompt_for_auth = overwrite = continue_prompt(message=prompt)
if prompt_for_auth and not save_app_auth_info(app, app_info, overwrite):
return
apps_config = cluster_config['modules'].get('stream_alert_apps', {})
local_config_keys = {'interval', 'timeout', 'memory'}
if not exists:
# Save a default log level as info to the config
app_info['log_level'] = 'info'
app_info['current_version'] = '$LATEST'
local_config_keys.update({'log_level', 'current_version', 'type'})
apps_config[app_info['app_name']] = {key: app_info[key]
for key in local_config_keys}
else:
apps_config[app_info['app_name']].update({key: app_info[key]
for key in local_config_keys})
cluster_config['modules']['stream_alert_apps'] = apps_config
# Add this service to the sources for this app integration
# The `stream_alert_app` is purposely singular here
app_sources = self.config['sources'].get('stream_alert_app', {})
app_sources[app_info['function_name']] = {'logs': [app.service()]}
self.config['sources']['stream_alert_app'] = app_sources
LOGGER_CLI.info('Successfully added \'%s\' app integration to \'conf/clusters/%s.json\' '
'for service \'%s\'.', app_info['app_name'],
app_info['cluster'], app_info['type'])
self.write()
def _add_global_metric_alarm(self, alarm_info):
"""Add a metric alarm that corresponds to a predefined metrics globally
Args:
alarm_info (dict): All the necessary values needed to add a CloudWatch
metric alarm
"""
function_name = alarm_info['function']
func_config_name = '{}_config'.format(function_name)
# Check if metrics are not enabled, and ask the user if they would like to enable them
if func_config_name not in self.config['lambda']:
self.config['lambda'][func_config_name] = {}
function_config = self.config['lambda'][func_config_name]
if function_name in CLUSTERED_FUNCTIONS:
if not self._clusters_with_metrics_enabled(function_name):
prompt = (
'Metrics are not currently enabled for the \'{}\' function '
'within any cluster. Creating an alarm will have no effect '
'until metrics are enabled for this function in at least one '
'cluster. Would you still like to continue?'.format(
function_name))
if not continue_prompt(message=prompt):
return False
else:
if not function_config.get('enable_custom_metrics'):
prompt = (
'Metrics are not currently enabled for the \'{}\' function. '
'Would you like to enable metrics for this function?'
).format(function_name)
if continue_prompt(message=prompt):
self.toggle_metrics(function_name, enabled=True)
elif not continue_prompt(
message='Would you still like to add this alarm '
'even though metrics are disabled?'):
return False
metric_alarms = function_config.get('custom_metric_alarms', {})
# Format the metric name for the aggregate metric
alarm_settings = alarm_info.copy()
alarm_settings['metric_name'] = '{}-{}'.format(
metrics.FUNC_PREFIXES[function_name],
alarm_settings['metric_name'])
function_config[
'custom_metric_alarms'] = self._add_metric_alarm_config(
alarm_settings, metric_alarms)
LOGGER.info(
'Successfully added \'%s\' metric alarm to '
'\'conf/lambda.json\'.', alarm_settings['alarm_name'])
return True
def _add_metric_alarm_per_cluster(self, alarm_info, function_name):
"""Add a metric alarm for individual clusters. This is for non-aggregate
CloudWatch metric alarms.
Args:
alarm_info (dict): All the necessary values needed to add a CloudWatch
metric alarm.
function_name (str): The name of the lambda function this metric is
related to.
"""
# If no clusters have been specified by the user, we can assume this alarm
# should be created for all available clusters, so fall back to that
clusters = (alarm_info['clusters'] if alarm_info['clusters'] else list(
self.config['clusters']))
# Go over each of the clusters and see if enable_metrics == True and prompt
# the user to toggle metrics on if this is False
for cluster in clusters:
function_config = (self.config['clusters'][cluster]['modules']
['stream_alert'][function_name])
if not function_config.get('enable_metrics'):
prompt = (
'Metrics are not currently enabled for the \'{}\' function '
'within the \'{}\' cluster. Would you like to enable metrics '
'for this cluster?'.format(function_name, cluster))
if continue_prompt(message=prompt):
self.toggle_metrics(True, [cluster], [function_name])
elif not continue_prompt(
message='Would you still like to add this alarm '
'even though metrics are disabled?'):
continue
metric_alarms = function_config.get('metric_alarms', {})
# Format the metric name for the cluster based metric
# Prepend a prefix for this function and append the cluster name
alarm_settings = alarm_info.copy()
alarm_settings['metric_name'] = '{}-{}-{}'.format(
metrics.FUNC_PREFIXES[function_name],
alarm_settings['metric_name'], cluster.upper())
new_alarms = self._add_metric_alarm_config(alarm_settings,
metric_alarms)
if new_alarms != False:
function_config['metric_alarms'] = new_alarms
LOGGER_CLI.info(
'Successfully added \'%s\' metric alarm for the \'%s\' '
'function to \'conf/clusters/%s.json\'.',
alarm_settings['alarm_name'], function_name, cluster)
def _add_cluster_metric_alarm(self, alarm_info):
"""Add a metric alarm that corresponds to a predefined metrics for clusters
Args:
alarm_info (dict): All the necessary values needed to add a CloudWatch
metric alarm.
"""
function_name = alarm_info['function']
# Go over each of the clusters and see if enable_metrics == True and prompt
# the user to toggle metrics on if this is False
config_name = '{}_config'.format(function_name)
for cluster in alarm_info['clusters']:
function_config = (self.config['clusters'][cluster]['modules']
['stream_alert'][config_name])
if not function_config.get('enable_custom_metrics'):
prompt = (
'Metrics are not currently enabled for the \'{}\' function '
'within the \'{}\' cluster. Would you like to enable metrics '
'for this cluster?'.format(function_name, cluster))
if continue_prompt(message=prompt):
self.toggle_metrics(function_name,
enabled=True,
clusters=[cluster])
elif not continue_prompt(
message='Would you still like to add this alarm '
'even though metrics are disabled?'):
continue
metric_alarms = function_config.get('custom_metric_alarms', {})
# Format the metric name for the cluster based metric
# Prepend a prefix for this function and append the cluster name
alarm_settings = alarm_info.copy()
alarm_settings['metric_name'] = '{}-{}-{}'.format(
metrics.FUNC_PREFIXES[function_name],
alarm_settings['metric_name'], cluster.upper())
function_config[
'custom_metric_alarms'] = self._add_metric_alarm_config(
alarm_settings, metric_alarms)
LOGGER.info(
'Successfully added \'%s\' metric alarm for the \'%s\' '
'function to \'conf/clusters/%s.json\'.',
alarm_settings['alarm_name'], function_name, cluster)
return True
def drop_all_tables(athena_client):
"""Drop all 'streamalert' Athena tables
Used when cleaning up an existing deployment
Args:
athena_client (boto3.client): Instantiated CLI AthenaClient
"""
if not continue_prompt(
message='Are you sure you want to drop all Athena tables?'):
return
success, all_tables = athena_client.run_athena_query(
query='SHOW TABLES', database='streamalert')
if not success:
LOGGER_CLI.error('There was an issue getting all tables')
return
unique_tables = athena_helpers.unique_values_from_query(all_tables)
for table in unique_tables:
success, all_tables = athena_client.run_command(
query='DROP TABLE {}'.format(table), database='streamalert')
if not success:
LOGGER_CLI.error('Unable to drop the %s table', table)
else:
LOGGER_CLI.info('Dropped %s', table)
def drop_all_tables(config):
"""Drop all 'streamalert' Athena tables
Used when cleaning up an existing deployment
Args:
config (CLIConfig): Loaded StreamAlert CLI
"""
if not continue_prompt(
message='Are you sure you want to drop all Athena tables?'):
return
athena_client = StreamAlertAthenaClient(
config, results_key_prefix='stream_alert_cli')
success, all_tables = athena_client.run_athena_query(
query='SHOW TABLES', database=athena_client.sa_database)
if not success:
LOGGER_CLI.error('There was an issue getting all tables')
return
unique_tables = athena_helpers.unique_values_from_query(all_tables)
for table in unique_tables:
success, all_tables = athena_client.run_athena_query(
query='DROP TABLE {}'.format(table),
database=athena_client.sa_database)
if not success:
LOGGER_CLI.error('Unable to drop the %s table', table)
else:
LOGGER_CLI.info('Dropped %s', table)
def drop_all_tables(config):
"""Drop all 'streamalert' Athena tables
Used when cleaning up an existing deployment
Args:
config (CLIConfig): Loaded StreamAlert config
Returns:
bool: False if errors occurred, True otherwise
"""
if not continue_prompt(
message='Are you sure you want to drop all Athena tables?'):
return False
athena_client = get_athena_client(config)
if not athena_client.drop_all_tables():
LOGGER.error('Failed to drop one or more tables from database: %s',
athena_client.database)
return False
LOGGER.info('Successfully dropped all tables from database: %s',
athena_client.database)
return True
def terraform_destroy_handler(options, config):
"""Use Terraform to destroy any existing infrastructure
Args:
options (argparse.Namespace): Parsed arguments from manage.py
config (CLIConfig): Loaded StreamAlert config
Returns:
bool: False if errors occurred, True otherwise
"""
# Check for valid credentials
if not check_credentials():
return False
# Verify terraform is installed
if not terraform_check():
return False
# Ask for approval here since multiple Terraform commands may be necessary
if not continue_prompt(message='Are you sure you want to destroy?'):
return False
if options.target:
target_modules, valid = _get_valid_tf_targets(config, options.target)
if not valid:
return False
return tf_runner(action='destroy',
auto_approve=True,
targets=target_modules if target_modules else None)
# Migrate back to local state so Terraform can successfully
# destroy the S3 bucket used by the backend.
# Do not check for terraform or aws creds again since these were checked above
if not terraform_generate_handler(
config=config, init=True, check_tf=False, check_creds=False):
return False
if not run_command(['terraform', 'init']):
return False
# Destroy all of the infrastructure
if not tf_runner(action='destroy', auto_approve=True):
return False
# Remove old Terraform files
return terraform_clean_handler()
def save_parameter(region, name, value, description, force_overwrite=False):
"""Function to save the designated value to parameter store
Args:
name (str): Name of the parameter being saved
value (str): Value to be saved to the parameter store
"""
ssm_client = boto3.client('ssm', region_name=region)
param_value = json.dumps(value)
# The name of the parameter should follow the format of:
# <function_name>_<type> where <type> is one of {'auth', 'config', 'state'}
# and <function_name> follows the the format:
# '<prefix>_<cluster>_<service>_<app_name>_app'
# Example: prefix_prod_duo_auth_production_collector_app_config
def save(overwrite=False):
ssm_client.put_parameter(Name=name,
Description=description,
Value=param_value,
Type='SecureString',
Overwrite=overwrite)
try:
save(overwrite=force_overwrite)
except ClientError as err:
if err.response['Error']['Code'] == 'ExpiredTokenException':
# Log an error if this response was due to no credentials being found
LOGGER_CLI.error(
'Could not save \'%s\' to parameter store because no '
'valid credentials were loaded.', name)
if err.response['Error']['Code'] != 'ParameterAlreadyExists':
raise
prompt = (
'A parameter already exists with name \'{}\'. Would you like '
'to overwrite the existing value?'.format(name))
# Ask to overwrite
if not continue_prompt(message=prompt):
return False
save(overwrite=True)
return True
def _terraform_destroy(options, config):
"""Use Terraform to destroy any existing infrastructure
Args:
options (namedtuple): Parsed arguments from manage.py
config (CLIConfig): Loaded StreamAlert CLI
"""
# Ask for approval here since multiple Terraform commands may be necessary
if not continue_prompt(message='Are you sure you want to destroy?'):
sys.exit(1)
if options.target:
targets = []
# Iterate over any targets to destroy. Global modules, like athena
# are prefixed with `stream_alert_` while cluster based modules
# are a combination of the target and cluster name
for target in options.target:
if target == 'athena':
targets.append('module.stream_alert_{}'.format(target))
elif target == 'threat_intel_downloader':
targets.append('module.threat_intel_downloader')
else:
targets.extend([
'module.{}_{}'.format(target, cluster)
for cluster in config.clusters()
])
tf_runner(action='destroy', auto_approve=True, targets=targets)
return
# Migrate back to local state so Terraform can successfully
# destroy the S3 bucket used by the backend.
if not terraform_generate(config=config, init=True):
return
if not run_command(['terraform', 'init']):
return
# Destroy all of the infrastructure
if not tf_runner(action='destroy', auto_approve=True):
return
# Remove old Terraform files
_terraform_clean(config)
def terraform_handler(options, config):
"""Handle all Terraform CLI operations
Args:
options (namedtuple): Parsed arguments from manage.py
config (CLIConfig): Loaded StreamAlert CLI
"""
# Check for valid credentials
if not check_credentials():
return
# Verify terraform is installed
if not terraform_check():
return
# Plan and Apply our streamalert infrastructure
if options.subcommand == 'build':
_terraform_build(options, config)
# generate terraform files
elif options.subcommand == 'generate':
if not terraform_generate(config=config):
return
elif options.subcommand == 'init-backend':
run_command(['terraform', 'init'])
# initialize streamalert infrastructure from a blank state
elif options.subcommand == 'init':
_terraform_init(config)
elif options.subcommand == 'clean':
if not continue_prompt(
message='Are you sure you want to clean all Terraform files?'):
sys.exit(1)
_terraform_clean(config)
elif options.subcommand == 'destroy':
_terraform_destroy(options, config)
# get a quick status on our declared infrastructure
elif options.subcommand == 'status':
terraform_status(config)
def drop_all_tables(config):
"""Drop all 'streamalert' Athena tables
Used when cleaning up an existing deployment
Args:
config (CLIConfig): Loaded StreamAlert CLI
"""
if not continue_prompt(message='Are you sure you want to drop all Athena tables?'):
return
athena_client = get_athena_client(config)
if not athena_client.drop_all_tables():
LOGGER_CLI.error('Failed to drop one or more tables from database: %s',
athena_client.database)
else:
LOGGER_CLI.info('Successfully dropped all tables from database: %s',
athena_client.database)
def add_metric_alarm(self, alarm_info):
"""Add a metric alarm that corresponds to a predefined metrics
Args:
alarm_info (dict): All the necessary values needed to add a CloudWatch
metric alarm
"""
# Check to see if an alarm with this name already exists
if self._alarm_exists(alarm_info['alarm_name']):
return
# Get the current metrics for each function
current_metrics = metrics.MetricLogger.get_available_metrics()
# Extract the function name this metric is associated with
metric_function = {metric: function for function in current_metrics
for metric in current_metrics[function]}[alarm_info['metric_name']]
# Do not continue if the user is trying to apply a metric alarm for an athena
# metric to a specific cluster (since the athena function operates on all clusters)
if (alarm_info['metric_target'] != 'aggregate' and
metric_function == metrics.ATHENA_PARTITION_REFRESH_NAME):
LOGGER_CLI.error('Metrics for the athena function can only be applied '
'to an aggregate metric target, not on a per-cluster basis.')
return
# If the metric is related to either the rule processor or alert processor, we should
# check to see if any cluster has metrics enabled for that function before continuing
if (metric_function in {metrics.ALERT_PROCESSOR_NAME, metrics.RULE_PROCESSOR_NAME} and
not any(self.config['clusters'][cluster]['modules']['stream_alert']
[metric_function].get('enable_metrics') for cluster in
self.config['clusters'])):
prompt = ('Metrics are not currently enabled for the \'{}\' function '
'within any cluster. Creating an alarm will have no effect '
'until metrics are enabled for this function in at least one '
'cluster. Would you still like to continue?'.format(metric_function))
if not continue_prompt(message=prompt):
return
elif metric_function == metrics.ATHENA_PARTITION_REFRESH_NAME:
# If the user is attempting to add a metric for athena, make sure the athena
# function is initialized first
if 'athena_partition_refresh_config' not in self.config['lambda']:
LOGGER_CLI.error('No configuration found for Athena Partition Refresh. '
'Please run: `$ python manage.py athena init` first.')
return
# If the athena function is initialized, but metrics are not enabled, ask
# the user if they would like to enable them now
if not self.config['lambda']['athena_partition_refresh_config'].get('enable_metrics'):
prompt = ('Metrics are not currently enabled for the \'athena\' function. '
'Would you like to enable metrics for athena?')
if continue_prompt(message=prompt):
self.toggle_metrics(True, None, [metric_function])
elif not continue_prompt(message='Would you still like to add this alarm '
'even though metrics are disabled?'):
return
# Add metric alarms for the aggregate metrics - these are added to the global config
if (alarm_info['metric_target'] == 'aggregate' or
metric_function == metrics.ATHENA_PARTITION_REFRESH_NAME):
global_config = self.config['global']['infrastructure']['monitoring']
metric_alarms = global_config.get('metric_alarms', {})
if not metric_alarms:
global_config['metric_alarms'] = {}
metric_alarms = global_config['metric_alarms'].get(metric_function, {})
if not metric_alarms:
global_config['metric_alarms'][metric_function] = {}
# Format the metric name for the aggregate metric
alarm_settings = alarm_info.copy()
alarm_settings['metric_name'] = '{}-{}'.format(metrics.FUNC_PREFIXES[metric_function],
alarm_info['metric_name'])
new_alarms = self._add_metric_alarm_config(alarm_settings, metric_alarms)
if new_alarms != False:
global_config['metric_alarms'][metric_function] = new_alarms
LOGGER_CLI.info('Successfully added \'%s\' metric alarm to '
'\'conf/global.json\'.', alarm_settings['alarm_name'])
else:
# Add metric alarms on a per-cluster basis - these are added to the cluster config
self._add_metric_alarm_per_cluster(alarm_info, metric_function)
# Save all of the alarm updates to disk
self.write()
def add_app(self, func_name, app_info):
"""Add a configuration for a new streamalert app integration function
Args:
app_info (dict): The necessary values needed to begin configuring
a new app integration
Returns:
bool: False if errors occurred, True otherwise
"""
exists, prompt_for_auth, overwrite = False, True, False
app = StreamAlertApp.get_app(app_info['type'])
cluster_name = app_info['cluster']
app_name = app_info['app_name']
# Check to see if there is an existing configuration for this app integration
cluster_config = self.config['clusters'][cluster_name]
if func_name in cluster_config['modules'].get('stream_alert_apps', {}):
prompt = (
'An app with the name \'{}\' is already configured for cluster '
'\'{}\'. Would you like to update the existing app\'s configuration'
'?'.format(app_name, cluster_name))
exists = True
# Return if the user is not deliberately updating an existing config
if not continue_prompt(message=prompt):
return
prompt = (
'Would you also like to update the authentication information for '
'app integration with name \'{}\'?'.format(app_name))
# If this is true, we shouldn't prompt again to warn about overwriting
prompt_for_auth = overwrite = continue_prompt(message=prompt)
if prompt_for_auth and not save_app_auth_info(app, app_info, func_name,
overwrite):
return False
apps_config = cluster_config['modules'].get('stream_alert_apps', {})
if not exists:
# Save a default app settings to the config for new apps
new_app_config = {
'app_name': app_info['app_name'],
'concurrency_limit': 2,
'log_level': 'info',
'log_retention_days': 14,
'memory': app_info['memory'],
'metric_alarms': {
'errors': {
'enabled': True,
'evaluation_periods': 1,
'period_secs': 120
}
},
'schedule_expression': app_info['schedule_expression'],
'timeout': app_info['timeout'],
'type': app_info['type']
}
apps_config[func_name] = new_app_config
else:
# Allow for updating certain attributes for the app without overwriting
# current parts of the configuration
updated_app_config = {
'memory': app_info['memory'],
'schedule_expression': app_info['schedule_expression'],
'timeout': app_info['timeout']
}
apps_config[func_name].update(updated_app_config)
cluster_config['modules']['stream_alert_apps'] = apps_config
# Add this service to the sources for this app integration
# The `stream_alert_app` is purposely singular here
app_sources = self.config['sources'].get('stream_alert_app', {})
app_sources[func_name] = {'logs': [app.service()]}
self.config['sources']['stream_alert_app'] = app_sources
LOGGER.info(
'Successfully added \'%s\' app integration to \'conf/clusters/%s.json\' '
'for service \'%s\'.', app_info['app_name'], app_info['cluster'],
app_info['type'])
self.write()
return True
def terraform_handler(options, config):
"""Handle all Terraform CLI operations
Args:
options (namedtuple): Parsed arguments from manage.py
"""
# Check for valid credentials
if not check_credentials():
return
# Verify terraform is installed
if not terraform_check():
return
# Use a named tuple to match the 'processor' attribute in the argparse options
deploy_opts = namedtuple('DeployOptions', ['processor', 'clusters'])
# Plan and Apply our streamalert infrastructure
if options.subcommand == 'build':
terraform_build(options, config)
# generate terraform files
elif options.subcommand == 'generate':
if not terraform_generate(config=config):
return
elif options.subcommand == 'init-backend':
run_command(['terraform', 'init'])
# initialize streamalert infrastructure from a blank state
elif options.subcommand == 'init':
LOGGER_CLI.info('Initializing StreamAlert')
# generate init Terraform files
if not terraform_generate(config=config, init=True):
return
LOGGER_CLI.info('Initializing Terraform')
if not run_command(['terraform', 'init']):
sys.exit(1)
# build init infrastructure
LOGGER_CLI.info('Building Initial Infrastructure')
init_targets = [
'aws_s3_bucket.lambda_source', 'aws_s3_bucket.logging_bucket',
'aws_s3_bucket.stream_alert_secrets',
'aws_s3_bucket.terraform_remote_state',
'aws_s3_bucket.streamalerts', 'aws_kms_key.stream_alert_secrets',
'aws_kms_alias.stream_alert_secrets'
]
if not tf_runner(targets=init_targets):
LOGGER_CLI.error('An error occured while running StreamAlert init')
sys.exit(1)
# generate the main.tf with remote state enabled
LOGGER_CLI.info('Configuring Terraform Remote State')
if not terraform_generate(config=config):
return
if not run_command(['terraform', 'init']):
return
LOGGER_CLI.info('Deploying Lambda Functions')
# deploy both lambda functions
deploy(deploy_opts(['rule', 'alert'], []), config)
# create all remainder infrastructure
LOGGER_CLI.info('Building Remainder Infrastructure')
tf_runner()
elif options.subcommand == 'clean':
if not continue_prompt(
message='Are you sure you want to clean all Terraform files?'):
sys.exit(1)
terraform_clean(config)
elif options.subcommand == 'destroy':
if not continue_prompt(message='Are you sure you want to destroy?'):
sys.exit(1)
if options.target:
targets = []
# Iterate over any targets to destroy. Global modules, like athena
# are prefixed with `stream_alert_` while cluster based modules
# are a combination of the target and cluster name
for target in options.target:
if target == 'athena':
targets.append('module.stream_alert_{}'.format(target))
elif target == 'threat_intel_downloader':
targets.append('module.threat_intel_downloader')
else:
targets.extend([
'module.{}_{}'.format(target, cluster)
for cluster in config.clusters()
])
tf_runner(targets=targets, action='destroy')
return
# Migrate back to local state so Terraform can successfully
# destroy the S3 bucket used by the backend.
if not terraform_generate(config=config, init=True):
return
if not run_command(['terraform', 'init']):
return
# Destroy all of the infrastructure
if not tf_runner(action='destroy'):
return
# Remove old Terraform files
terraform_clean(config)
# get a quick status on our declared infrastructure
elif options.subcommand == 'status':
terraform_status(config)
