def terraform_generate(config, init=False):
"""Generate all Terraform plans for the configured clusters.
Keyword Args:
config (dict): The loaded config from the 'conf/' directory
init (bool): Indicates if main.tf.json is generated for `terraform init`
Returns:
bool: Result of cluster generating
"""
cleanup_old_tf_files(config)
# Setup the main.tf.json file
LOGGER_CLI.debug('Generating cluster file: main.tf.json')
with open('terraform/main.tf.json', 'w') as tf_file:
json.dump(generate_main(init=init, config=config),
tf_file,
indent=2,
sort_keys=True)
# Return early during the init process, clusters are not needed yet
if init:
return True
# Setup cluster files
for cluster in config.clusters():
if cluster in RESTRICTED_CLUSTER_NAMES:
raise InvalidClusterName(
'Rename cluster "main" or "athena" to something else!')
LOGGER_CLI.debug('Generating cluster file: %s.tf.json', cluster)
cluster_dict = generate_cluster(cluster_name=cluster, config=config)
if not cluster_dict:
LOGGER_CLI.error(
'An error was generated while creating the %s cluster',
cluster)
return False
with open('terraform/{}.tf.json'.format(cluster), 'w') as tf_file:
json.dump(cluster_dict, tf_file, indent=2, sort_keys=True)
# Setup Athena if it is enabled
athena_config = config['lambda'].get('athena_partition_refresh_config')
if athena_config:
athena_file = 'terraform/athena.tf.json'
if athena_config['enabled']:
athena_generated_config = generate_athena(config=config)
if athena_generated_config:
with open(athena_file, 'w') as tf_file:
json.dump(athena_generated_config,
tf_file,
indent=2,
sort_keys=True)
# Remove Athena file if it's disabled
else:
if os.path.isfile(athena_file):
LOGGER_CLI.info('Removing old Athena Terraform file')
os.remove(athena_file)
return True
def test_generate_athena():
"""CLI - Terraform Generate Athena"""
CONFIG['lambda']['athena_partition_refresh_config'] = {
'current_version': '$LATEST',
'buckets': {
'unit-testing.streamalerts': 'alerts',
'unit-testing.streamalert.data': 'data'
},
'handler': 'main.handler',
'timeout': '60',
'memory': '128',
'source_bucket': 'unit-testing.streamalert.source',
'source_current_hash': '12345',
'source_object_key': 'lambda/athena/source.zip',
'third_party_libraries': []
}
prefix = CONFIG['global']['account']['prefix']
expected_athena_config = {
'module': {
'stream_alert_athena': {
's3_logging_bucket':
'{}.streamalert.s3-logging'.format(prefix),
'source':
'modules/tf_stream_alert_athena',
'database_name':
'{}_streamalert'.format(prefix),
'queue_name':
'{}_streamalert_athena_s3_notifications'.format(prefix),
'results_bucket':
'{}.streamalert.athena-results'.format(prefix),
'current_version':
'$LATEST',
'enable_metrics':
False,
'lambda_handler':
'main.handler',
'lambda_log_level':
'info',
'lambda_memory':
'128',
'lambda_timeout':
'60',
'lambda_s3_bucket':
'unit-testing.streamalert.source',
'lambda_s3_key':
'lambda/athena/source.zip',
'athena_data_buckets':
['unit-testing.streamalerts', 'unit-testing.streamalert.data'],
'prefix':
'unit-testing',
'refresh_interval':
'rate(10 minutes)'
},
'athena_monitoring': {
'source':
'modules/tf_stream_alert_monitoring',
'sns_topic_arn':
'arn:aws:sns:us-west-1:12345678910:stream_alert_monitoring',
'kinesis_alarms_enabled':
False,
'lambda_functions':
['unit-testing_streamalert_athena_partition_refresh']
},
'athena_metric_filters': []
}
}
athena_config = athena.generate_athena(config=CONFIG)
# List order messes up the comparison between both dictionaries
assert_equal(set(athena_config['module']['stream_alert_athena']['athena_data_buckets']),
set(expected_athena_config['module']['stream_alert_athena']\
['athena_data_buckets']))
# Delete the keys to compare the rest of the generated module
del athena_config['module']['stream_alert_athena']['athena_data_buckets']
del expected_athena_config['module']['stream_alert_athena'][
'athena_data_buckets']
# Compare each generated Athena module from the expected module
assert_equal(athena_config['module']['stream_alert_athena'],
expected_athena_config['module']['stream_alert_athena'])
assert_equal(athena_config['module']['athena_monitoring'],
expected_athena_config['module']['athena_monitoring'])