def generate_alert_merger(config): """Generate Terraform for the Alert Merger Args: config (dict): The loaded config from the 'conf/' directory Returns: dict: Alert Merger Terraform definition to be marshaled to JSON """ prefix = config['global']['account']['prefix'] result = infinitedict() # Set variables for the alert merger's IAM permissions result['module']['alert_merger_iam'] = { 'source': './modules/tf_alert_merger_iam', 'account_id': config['global']['account']['aws_account_id'], 'region': config['global']['account']['region'], 'prefix': config['global']['account']['prefix'], 'role_id': '${module.alert_merger_lambda.role_id}' } # Set variables for the Lambda module result['module']['alert_merger_lambda'] = generate_lambda( '{}_streamalert_{}'.format(config['global']['account']['prefix'], ALERT_MERGER_NAME), AlertMergerPackage.package_name + '.zip', AlertMergerPackage.lambda_handler, config['lambda']['alert_merger_config'], config, environment={ 'ALERTS_TABLE': '{}_streamalert_alerts'.format(prefix), 'ALERT_PROCESSOR': '{}_streamalert_alert_processor'.format(prefix), 'ALERT_PROCESSOR_TIMEOUT_SEC': config['lambda']['alert_processor_config']['timeout'], } ) return result
def generate_alert_processor(config): """Generate Terraform for the Alert Processor Args: config (dict): The loaded config from the 'conf/' directory Returns: dict: Alert Processor dict to be marshaled to JSON """ prefix = config['global']['account']['prefix'] result = infinitedict() # Set variables for the IAM permissions module result['module']['alert_processor_iam'] = { 'source': './modules/tf_alert_processor_iam', 'account_id': config['global']['account']['aws_account_id'], 'region': config['global']['account']['region'], 'prefix': prefix, 'role_id': '${module.alert_processor_lambda.role_id}', 'kms_key_arn': '${aws_kms_key.streamalert_secrets.arn}', 'sse_kms_key_arn': '${aws_kms_key.server_side_encryption.arn}', 'output_lambda_functions': [ # Strip qualifiers: only the function name is needed for the IAM permissions func.split(':')[0] for func in list(config['outputs'].get('aws-lambda', {}).values()) ], 'output_s3_buckets': list(config['outputs'].get('aws-s3', {}).values()), 'output_sns_topics': list(config['outputs'].get('aws-sns', {}).values()), 'output_sqs_queues': list(config['outputs'].get('aws-sqs', {}).values()) } # Set variables for the Lambda module result['module']['alert_processor_lambda'] = generate_lambda( '{}_streamalert_{}'.format(config['global']['account']['prefix'], ALERT_PROCESSOR_NAME), AlertProcessorPackage.package_name + '.zip', AlertProcessorPackage.lambda_handler, config['lambda']['alert_processor_config'], config, environment={ 'ALERTS_TABLE': '{}_streamalert_alerts'.format(prefix), 'AWS_ACCOUNT_ID': config['global']['account']['aws_account_id'], 'STREAMALERT_PREFIX': prefix }) return result
def generate_apps(cluster_name, cluster_dict, config): """Add the app integrations module to the Terraform cluster dict. Args: cluster_name (str): The name of the currently generating cluster cluster_dict (defaultdict): The dict containing all Terraform config for a given cluster. config (dict): The loaded config from the 'conf/' directory Returns: bool: Result of applying the app integration module """ prefix = config['global']['account']['prefix'] for function_name, app_info in config['clusters'][cluster_name][ 'modules'].get('streamalert_apps', {}).items(): tf_module_prefix = 'app_{}_{}'.format(app_info['app_name'], cluster_name) destination_func = '{}_{}_streamalert_classifier'.format( prefix, cluster_name) app_config = { 'app_type': app_info['type'], 'destination_function_name': destination_func, 'schedule_expression': app_info['schedule_expression'] } # Format the iam module with 'app_<app_name_<cluster>_iam' cluster_dict['module']['{}_iam'.format(tf_module_prefix)] = { 'account_id': config['global']['account']['aws_account_id'], 'destination_function_name': destination_func, 'function_name': function_name, 'region': config['global']['account']['region'], 'function_role_id': '${{module.{}_lambda.role_id}}'.format(tf_module_prefix), 'source': './modules/tf_app_iam' } # Format the lambda module with 'app_<app_name_<cluster>_lambda' cluster_dict['module']['{}_lambda'.format( tf_module_prefix)] = generate_lambda( function_name, 'streamalert.apps.main.handler', config['clusters'][cluster_name]['modules']['streamalert_apps'] [function_name], config, input_event=app_config, include_layers=True, )
def generate_rule_promotion(config): """Generate Terraform for the Rule Promotion function Args: config (dict): The loaded config from the 'conf/' directory Returns: dict: Rule Promotion dict to be marshaled to JSON """ # The Rule Promotion Lambda function is dependent on the rule staging feature being # enabled, so do not generate the code for this Lambda function if it not enabled if not config['global']['infrastructure']['rule_staging'].get( 'enabled', False): return False result = infinitedict() alerts_bucket = firehose_alerts_bucket(config) # Set variables for the IAM permissions, etc module result['module']['rule_promotion_iam'] = { 'source': './modules/tf_rule_promotion_iam', 'send_digest_schedule_expression': config['lambda']['rule_promotion_config'] ['send_digest_schedule_expression'], 'digest_sns_topic': StatsPublisher.formatted_sns_topic_arn(config).split(':')[-1], 'role_id': '${module.rule_promotion_lambda.role_id}', 'rules_table_arn': '${module.globals.rules_table_arn}', 'function_alias_arn': '${module.rule_promotion_lambda.function_alias_arn}', 'function_name': '${module.rule_promotion_lambda.function_name}', 'athena_results_bucket_arn': '${module.streamalert_athena.results_bucket_arn}', 'alerts_bucket': alerts_bucket, 's3_kms_key_arn': '${aws_kms_key.server_side_encryption.arn}' } # Set variables for the Lambda module result['module']['rule_promotion_lambda'] = generate_lambda( '{}_streamalert_{}'.format(config['global']['account']['prefix'], RULE_PROMOTION_NAME), RulePromotionPackage.package_name + '.zip', RulePromotionPackage.lambda_handler, config['lambda']['rule_promotion_config'], config) return result
def generate_threat_intel_downloader(config): """Generate Threat Intel Downloader Terrafrom Args: config (dict): The loaded config from the 'conf/' directory Returns: dict: Athena dict to be marshalled to JSON """ # Use the monitoring topic as a dead letter queue dlq_topic, _ = monitoring_topic_name(config) prefix = config['global']['account']['prefix'] # Threat Intel Downloader module tid_config = config['lambda']['threat_intel_downloader_config'] # old format of config used interval, but tf_lambda expects 'schedule_expression' if 'schedule_expression' not in tid_config: tid_config['schedule_expression'] = tid_config.get( 'interval', 'rate(1 day)') result = infinitedict() # Set variables for the threat intel downloader configuration result['module']['threat_intel_downloader_iam'] = { 'source': './modules/tf_threat_intel_downloader', 'account_id': config['global']['account']['aws_account_id'], 'region': config['global']['account']['region'], 'prefix': prefix, 'function_role_id': '${module.threat_intel_downloader.role_id}', 'function_alias_arn': '${module.threat_intel_downloader.function_alias_arn}', 'function_cloudwatch_log_group_name': '${module.threat_intel_downloader.log_group_name}', 'monitoring_sns_topic': dlq_topic, 'table_rcu': tid_config.get('table_rcu', '10'), 'table_wcu': tid_config.get('table_wcu', '10'), 'max_read_capacity': tid_config.get('max_read_capacity', '5'), 'min_read_capacity': tid_config.get('min_read_capacity', '5'), 'target_utilization': tid_config.get('target_utilization', '70') } result['module']['threat_intel_downloader'] = generate_lambda( '{}_streamalert_{}'.format(prefix, THREAT_INTEL_DOWNLOADER_NAME), 'streamalert.threat_intel_downloader.main.handler', tid_config, config, ) return result
def generate_rules_engine(config): """Generate Terraform for the Rules Engine Args: config (dict): The loaded config from the 'conf/' directory Returns: dict: Rules Engine Terraform definition to be marshaled to JSON """ prefix = config['global']['account']['prefix'] result = infinitedict() # Set variables for the rules engine IAM permissions result['module']['rules_engine_iam'] = { 'source': './modules/tf_rules_engine', 'account_id': config['global']['account']['aws_account_id'], 'region': config['global']['account']['region'], 'prefix': prefix, 'function_role_id': '${module.rules_engine_lambda.role_id}', 'function_alias_arn': '${module.rules_engine_lambda.function_alias_arn}', 'function_name': '${module.rules_engine_lambda.function_name}', 'threat_intel_enabled': config.get('threat_intel', {}).get('enabled'), 'dynamodb_table_name': config.get('threat_intel', {}).get('dynamodb_table_name'), 'rules_table_arn': '${module.globals.rules_table_arn}', 'enable_rule_staging': config['global']['infrastructure']['rule_staging'].get( 'enabled', False ), 'classifier_sqs_queue_arn': '${module.globals.classifier_sqs_queue_arn}', 'classifier_sqs_sse_kms_key_arn': '${module.globals.classifier_sqs_sse_kms_key_arn}', 'sqs_record_batch_size': min(config.get('sqs_record_batch_size', 10), 10) } environment = { 'ALERTS_TABLE': '{}_streamalert_alerts'.format(prefix), 'STREAMALERT_PREFIX': prefix, } if config['lambda']['rules_engine_config'].get('log_rule_statistics'): environment['STREAMALERT_TRACK_RULE_STATS'] = '1' # Set variables for the Lambda module result['module']['rules_engine_lambda'] = generate_lambda( '{}_streamalert_{}'.format(prefix, RULES_ENGINE_FUNCTION_NAME), 'streamalert.rules_engine.main.handler', config['lambda']['rules_engine_config'], config, environment=environment, ) return result
def generate_classifier(cluster_name, cluster_dict, config): """Add this cluster's classifier module to the Terraform cluster dict. Args: cluster_name (str): The name of the currently generating cluster cluster_dict (defaultdict): The dict containing all Terraform config for a given cluster. config (dict): The loaded config from the 'conf/' directory JSON Input from the config: { "classifier_config": { "log_level": "info", "log_retention_days": 14, "memory": 128, "metric_alarms": { "errors": { "enabled": true, "evaluation_periods": 1, "period_secs": 120, "threshold": 0 }, "throttles": { "enabled": true, "evaluation_periods": 1, "period_secs": 120, "threshold": 0 } }, "timeout": 60, "vpc_config": { "security_group_ids": [], "subnet_ids": [] } } } """ classifier_config = config['clusters'][cluster_name]['classifier_config'] firehose_config = config['global']['infrastructure'].get('firehose', {}) # The default value here must be consistent with the firehose client default use_firehose_prefix = firehose_config.get('use_prefix', True) tf_module_prefix = 'classifier_{}'.format(cluster_name) iam_module = '{}_iam'.format(tf_module_prefix) # Set variables for the alert merger's IAM permissions cluster_dict['module'][iam_module] = { 'source': './modules/tf_classifier', 'account_id': config['global']['account']['aws_account_id'], 'region': config['global']['account']['region'], 'prefix': config['global']['account']['prefix'], 'firehose_use_prefix': use_firehose_prefix, 'function_role_id': '${{module.{}_lambda.role_id}}'.format(tf_module_prefix), 'function_alias_arn': '${{module.{}_lambda.function_alias_arn}}'.format(tf_module_prefix), 'function_name': '${{module.{}_lambda.function_name}}'.format(tf_module_prefix), 'classifier_sqs_queue_arn': '${module.globals.classifier_sqs_queue_arn}', 'classifier_sqs_sse_kms_key_arn': '${module.globals.classifier_sqs_sse_kms_key_arn}', } # Add Classifier input config from the loaded cluster file input_config = classifier_config.get('inputs') if input_config: input_mapping = { 'input_sns_topics': 'aws-sns' } for tf_key, input_key in input_mapping.items(): if input_key in input_config: cluster_dict['module'][iam_module][tf_key] = input_config[input_key] # Set variables for the Lambda module cluster_dict['module']['{}_lambda'.format(tf_module_prefix)] = generate_lambda( '{}_{}_streamalert_classifier'.format(config['global']['account']['prefix'], cluster_name), 'streamalert.classifier.main.handler', classifier_config, config, environment={ 'CLUSTER': cluster_name, 'SQS_QUEUE_URL': '${module.globals.classifier_sqs_queue_url}', }, tags={ 'Cluster': cluster_name }, )
def generate_athena(config): """Generate Athena Terraform. Args: config (dict): The loaded config from the 'conf/' directory Returns: dict: Athena dict to be marshalled to JSON """ result = infinitedict() prefix = config['global']['account']['prefix'] athena_config = config['lambda']['athena_partitioner_config'] data_buckets = athena_partition_buckets_tf(config) database = athena_config.get('database_name', '{}_streamalert'.format(prefix)) results_bucket_name = athena_query_results_bucket(config) queue_name = athena_config.get( 'queue_name', '{}_streamalert_athena_s3_notifications'.format(prefix)).strip() logging_bucket, _ = s3_access_logging_bucket(config) # Set variables for the athena partitioner's IAM permissions result['module']['athena_partitioner_iam'] = { 'source': './modules/tf_athena', 'account_id': config['global']['account']['aws_account_id'], 'prefix': prefix, 's3_logging_bucket': logging_bucket, 'database_name': database, 'queue_name': queue_name, 'athena_data_buckets': data_buckets, 'results_bucket': results_bucket_name, 'lambda_timeout': athena_config['timeout'], 'kms_key_id': '${aws_kms_key.server_side_encryption.key_id}', 'function_role_id': '${module.athena_partitioner_lambda.role_id}', 'function_name': '${module.athena_partitioner_lambda.function_name}', 'function_alias_arn': '${module.athena_partitioner_lambda.function_alias_arn}', } # Set variables for the Lambda module result['module']['athena_partitioner_lambda'] = generate_lambda( '{}_streamalert_{}'.format(prefix, ATHENA_PARTITIONER_NAME), 'streamalert.athena_partitioner.main.handler', athena_config, config, tags={'Subcomponent': 'AthenaPartitioner'}) return result