def setup(self):
"""Setup before each method"""
config_data = basic_streamalert_config()
self.fs_patcher = fake_filesystem_unittest.Patcher()
self.fs_patcher.setUp()
self.fs_patcher.fs.create_file('./conf/global.json',
contents=json.dumps(
config_data['global']))
self.fs_patcher.fs.create_file('./conf/threat_intel.json',
contents=json.dumps(
config_data['threat_intel']))
self.fs_patcher.fs.create_file('./conf/normalized_types.json',
contents=json.dumps(
config_data['normalized_types']))
self.fs_patcher.fs.create_file('./conf/lambda.json',
contents=json.dumps(
config_data['lambda']))
self.fs_patcher.fs.create_file('./conf/clusters/prod.json',
contents=json.dumps(
config_data['clusters']['prod']))
# Create the config instance after creating the fake filesystem so that
# CLIConfig uses our mocked config files instead of the real ones.
self.config = CLIConfig('./conf/')
def test_firehose_enabled_log_json(self):
"""CLI - Terraform Generate Kinesis Firehose, Enabled Log with output in JSON format"""
cluster_dict = common.infinitedict()
# Add an enabled log, with no alarm configuration (aka: alarms disabled)
self.config['global']['infrastructure']['firehose']['enabled_logs'] = {
'json:embedded': {}
}
self.config = CLIConfig(config_path='tests/unit/conf_athena')
firehose.generate_firehose(self._logging_bucket_name, cluster_dict,
self.config)
expected_result = {
'module': {
'kinesis_firehose_setup': self._default_firehose_config(),
'kinesis_firehose_json_embedded': {
'source': './modules/tf_kinesis_firehose_delivery_stream',
'buffer_size': 128,
'buffer_interval': 900,
'file_format': 'json',
'stream_name': 'unit_test_streamalert_json_embedded',
'role_arn':
'${module.kinesis_firehose_setup.firehose_role_arn}',
's3_bucket_name': 'unit-test-streamalert-data',
'kms_key_arn': '${aws_kms_key.server_side_encryption.arn}',
'glue_catalog_db_name': 'unit-test_streamalert',
'glue_catalog_table_name': 'json_embedded',
'schema': self._get_expected_schema()
}
}
}
assert_equal(cluster_dict, expected_result)
def cli_runner(args):
"""Main StreamAlert CLI handler
Args:
options (argparse.Namespace): command line arguments passed from the argparser.
Contains the following keys for terraform commands:
(command, subcommand, target)
Contains the following keys for lambda commands:
(command, subcommand, env, func, source)
Returns:
bool: False if errors occurred, True otherwise
"""
config = CLIConfig(args.config_dir, args.terraform_files,
args.build_directory)
set_logger_levels(args.debug)
LOGGER.info(
'Issues? Report here: https://github.com/airbnb/streamalert/issues')
cmds = StreamAlertCLICommandRepository.command_handlers(config)
result = cmds[args.command](args)
LOGGER.info('Completed')
return result
def setUp(self):
cli_config = CLIConfig(config_path='tests/unit/conf')
with patch('streamalert.rules_engine.rules_engine.load_config',
Mock(return_value=load_config(self.TEST_CONFIG_PATH))):
self.runner = TestRunner(MagicMock(), cli_config)
self.setUpPyfakefs()
def setUp(self):
self.setUpPyfakefs()
self.fs.add_real_directory(self.TEST_CONFIG_PATH)
config = CLIConfig(self.TEST_CONFIG_PATH)
with patch('tempfile.gettempdir') as temp_dir_mock:
temp_dir_mock.return_value = self.MOCK_TEMP_PATH
self.packager = package.LambdaPackage(config)
def test_create_table_with_dots():
"""CLI - Athena create table helper when log name contains dots"""
config = CLIConfig(config_path='tests/unit/conf')
config['global']['infrastructure']['firehose']['enabled_logs'] = {
'test:log.name.with.dots': {}
}
assert_true(
handler.create_table('test:log.name.with.dots', 'bucket', config))
def test_create_table_with_underscores():
"""CLI - Athena create table helper when log name contains underscores"""
config = CLIConfig(config_path='tests/unit/conf')
config['global']['infrastructure']['firehose']['enabled_logs'] = {
'cloudwatch:test_match_types': {}
}
assert_true(
handler.create_table('cloudwatch:test_match_types', 'bucket',
config))
def test_generate_data_table_schema():
"""CLI - Athena generate_data_table_schema helper"""
config = CLIConfig(config_path='tests/unit/conf')
config['global']['infrastructure']['firehose']['enabled_logs'] = {
'test:log.name.with.dots': {}
}
assert_true(
helpers.generate_data_table_schema(config, 'test:log.name.with.dots'))
FirehoseClient._ENABLED_LOGS.clear()
def setup(self):
"""Setup before each method"""
self.cluster_dict = common.infinitedict()
self.config = CLIConfig(config_path='tests/unit/conf')
def setup(self):
"""Setup before each method"""
self._logging_bucket_name = 'logging-bucket-name'
self.config = CLIConfig(config_path='tests/unit/conf')
def __init__(self):
"""Init config for the test cases"""
self.config = CLIConfig(config_path='tests/unit/conf')
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from mock import patch
from nose.tools import assert_equal, assert_false, assert_true
from streamalert_cli.config import CLIConfig
from streamalert_cli.terraform import common, monitoring
CONFIG = CLIConfig(config_path='tests/unit/conf')
def test_generate_cloudwatch_monitoring():
"""CLI - Terraform Generate Cloudwatch Monitoring"""
cluster_dict = common.infinitedict()
result = monitoring.generate_monitoring('test', cluster_dict, CONFIG)
# Test the default SNS topic option
expected_cloudwatch_tf = {
'source': './modules/tf_monitoring',
'sns_topic_arn':
'arn:aws:sns:us-west-1:12345678910:unit-test_streamalert_monitoring',
'lambda_functions': ['unit-test_test_streamalert_classifier'],
'kinesis_stream': '${module.kinesis_test.stream_name}',
'lambda_alarms_enabled': True,
class TestCLIConfig:
"""Test class for CLIConfig"""
def __init__(self):
self.config = None
self.fs_patcher = None
@patch('streamalert_cli.config.CLIConfig._copy_terraform_files', Mock())
def setup(self):
"""Setup before each method"""
config_data = basic_streamalert_config()
self.fs_patcher = fake_filesystem_unittest.Patcher()
self.fs_patcher.setUp()
self.fs_patcher.fs.create_file('./conf/global.json',
contents=json.dumps(
config_data['global']))
self.fs_patcher.fs.create_file('./conf/threat_intel.json',
contents=json.dumps(
config_data['threat_intel']))
self.fs_patcher.fs.create_file('./conf/normalized_types.json',
contents=json.dumps(
config_data['normalized_types']))
self.fs_patcher.fs.create_file('./conf/lambda.json',
contents=json.dumps(
config_data['lambda']))
self.fs_patcher.fs.create_file('./conf/clusters/prod.json',
contents=json.dumps(
config_data['clusters']['prod']))
# Create the config instance after creating the fake filesystem so that
# CLIConfig uses our mocked config files instead of the real ones.
self.config = CLIConfig('./conf/')
def teardown(self):
"""Teardown after each method"""
self.fs_patcher.tearDown()
def test_load_config(self):
"""CLI - Load config"""
assert_equal(self.config['global']['account']['prefix'], 'unit-test')
def test_terraform_files(self):
"""CLI - Terraform Files"""
assert_equal(self.config.terraform_files, {'/test/terraform/file.tf'})
def test_toggle_metric(self):
"""CLI - Metric toggling"""
self.config.toggle_metrics('athena_partitioner', enabled=True)
assert_equal(
self.config['lambda']['athena_partitioner_config']
['enable_custom_metrics'], True)
self.config.toggle_metrics('alert_processor', enabled=False)
assert_equal(
self.config['lambda']['alert_processor_config']
['enable_custom_metrics'], False)
def test_aggregate_alarm_exists(self):
"""CLI - Aggregate alarm check"""
result = self.config._alarm_exists(
'Aggregate Unit Testing Failed Parses Alarm')
assert_true(result)
def test_cluster_alarm_exists(self):
"""CLI - Cluster alarm check"""
result = self.config._alarm_exists(
'Prod Unit Testing Failed Parses Alarm')
assert_true(result)
def test_cluster_alarm_creation(self):
"""CLI - Adding CloudWatch metric alarm, cluster"""
alarm_info = {
'function': 'classifier',
'metric_name': 'TotalRecords',
'evaluation_periods': 1,
'alarm_description': '',
'alarm_name': 'Prod Unit Testing Total Records Alarm',
'period': 300,
'threshold': 100.0,
'statistic': 'Sum',
'clusters': {'prod'},
'comparison_operator': 'LessThanThreshold'
}
expected_result = {
'Prod Unit Testing Total Records Alarm': {
'metric_name': 'Classifier-TotalRecords-PROD',
'evaluation_periods': 1,
'alarm_description': '',
'period': 300,
'threshold': 100.0,
'statistic': 'Sum',
'comparison_operator': 'LessThanThreshold'
},
'Prod Unit Testing Failed Parses Alarm': {
'alarm_description': '',
'comparison_operator': 'GreaterThanOrEqualToThreshold',
'evaluation_periods': 1,
'metric_name': 'Classifier-FailedParses-PROD',
'period': 300,
'statistic': 'Sum',
'threshold': 1.0
}
}
self.config.add_metric_alarm(alarm_info)
result = self.config['clusters']['prod']['classifier_config'][
'custom_metric_alarms']
assert_equal(result, expected_result)
def test_aggregate_alarm_creation(self):
"""CLI - Adding CloudWatch metric alarm, aggregate"""
alarm_info = {
'function': 'classifier',
'metric_name': 'TotalRecords',
'evaluation_periods': 1,
'alarm_description': '',
'alarm_name': 'Aggregate Unit Testing Total Records Alarm',
'period': 300,
'threshold': 100.0,
'statistic': 'Sum',
'comparison_operator': 'LessThanThreshold'
}
expected_result = {
'Aggregate Unit Testing Total Records Alarm': {
'metric_name': 'Classifier-TotalRecords',
'evaluation_periods': 1,
'alarm_description': '',
'period': 300,
'threshold': 100.0,
'statistic': 'Sum',
'comparison_operator': 'LessThanThreshold'
}
}
self.config.add_metric_alarm(alarm_info)
result = self.config['lambda']['classifier_config'][
'custom_metric_alarms']
assert_equal(result, expected_result)
def test_add_threat_intel_with_table_name(self):
"""CLI - Add Threat Intel config with default dynamodb table name"""
threat_intel_info = {
'command': 'threat-intel',
'debug': 'False',
'dynamodb_table_name': 'my_ioc_table',
'enable': True
}
self.config.add_threat_intel(threat_intel_info)
expected_config = {
'enabled': True,
'dynamodb_table_name': 'my_ioc_table',
'excluded_iocs': {},
'normalized_ioc_types': {
'ip': ['sourceAddress', 'destinationAddress']
}
}
assert_equal(self.config['threat_intel'], expected_config)
def test_add_threat_intel_without_table_name(self):
"""CLI - Add Threat Intel config without dynamodb table name from cli"""
threat_intel_info = {
'command': 'threat-intel',
'debug': 'False',
'dynamodb_table_name': None,
'enable': True
}
del self.config['threat_intel']['dynamodb_table_name']
self.config.add_threat_intel(threat_intel_info)
expected_config = {
'enabled': True,
'dynamodb_table_name':
'unit-test_streamalert_threat_intel_downloader',
'excluded_iocs': {},
'normalized_ioc_types': {
'ip': ['sourceAddress', 'destinationAddress']
}
}
assert_equal(self.config['threat_intel'], expected_config)
@patch('logging.Logger.info')
@patch('streamalert_cli.config.CLIConfig.write')
def test_add_threat_intel_downloader(self, write_mock, log_mock):
"""CLI - Add Threat Intel Downloader config"""
del self.config['lambda']['threat_intel_downloader_config']
ti_downloader_info = {
'autoscale': True,
'command': 'threat_intel_downloader',
'debug': False,
'interval': 'rate(1 day)',
'memory': '128',
'subcommand': 'enable',
'timeout': '240',
'table_wcu': 25,
'max_read_capacity': 100,
'min_read_capacity': 5,
'target_utilization': 70
}
result = self.config.add_threat_intel_downloader(ti_downloader_info)
assert_true(result)
expected_config = {
'autoscale': True,
'enabled': True,
'interval': 'rate(1 day)',
'ioc_filters': [],
'ioc_keys': [],
'ioc_types': [],
'excluded_sub_types': [],
'log_level': 'info',
'memory': '128',
'table_rcu': 10,
'table_wcu': 25,
'timeout': '240',
'max_read_capacity': 100,
'min_read_capacity': 5,
'target_utilization': 70
}
assert_equal(self.config['lambda']['threat_intel_downloader_config'],
expected_config)
write_mock.assert_called()
log_mock.assert_not_called()
# no config changed if threat intel downloader already been enabled via CLI
result = self.config.add_threat_intel_downloader(ti_downloader_info)
assert_false(result)
write_mock.assert_called_once()
log_mock.assert_called_with(
'Threat Intel Downloader has been enabled. '
'Please edit config/lambda.json if you want to '
'change lambda function settings.')
def setup(self):
"""Create the CLIConfig and the expected template for these tests."""
# pylint: disable=attribute-defined-outside-init
self.config = dict(CLIConfig(config_path='tests/unit/conf'))
self.rule_promo_config = self.config['lambda']['rule_promotion_config']
def setUp(self):
"""Create the CLIConfig and the expected template for these tests."""
self.config = dict(CLIConfig(config_path='tests/unit/conf'))
self.alert_proc_config = self.config['lambda'][
'alert_processor_config']
