def test_rollback_all(self, mock_helper): """CLI - Lambda rollback all""" mock_helper.return_value = True funcs = [ 'alert', 'alert_merger', 'apps', 'athena', 'classifier', 'rule', 'rule_promo', 'scheduled_queries', 'threat_intel_downloader' ] assert_equal( rollback.RollbackCommand.handler( MockOptions(None, funcs), MockCLIConfig(config=basic_streamalert_config())), True) mock_helper.assert_has_calls([ mock.call(mock.ANY, 'unit-test_streamalert_alert_processor'), mock.call(mock.ANY, 'unit-test_streamalert_alert_merger'), mock.call(mock.ANY, 'unit-test_corp_box_admin_events_box_collector_app'), mock.call(mock.ANY, 'unit-test_corp_duo_admin_duo_admin_collector_app'), mock.call(mock.ANY, 'unit-test_streamalert_athena_partitioner'), mock.call(mock.ANY, 'unit-test_corp_streamalert_classifier'), mock.call(mock.ANY, 'unit-test_prod_streamalert_classifier'), mock.call(mock.ANY, 'unit-test_streamalert_rules_engine'), mock.call(mock.ANY, 'unit-test_streamalert_rule_promotion'), mock.call(mock.ANY, 'unit-test_streamalert_scheduled_queries_runner'), mock.call(mock.ANY, 'unit-test_streamalert_threat_intel_downloader') ])
def setup(self): """Setup before each method""" config_data = basic_streamalert_config() self.fs_patcher = fake_filesystem_unittest.Patcher() self.fs_patcher.setUp() self.fs_patcher.fs.create_file('./conf/global.json', contents=json.dumps( config_data['global'])) self.fs_patcher.fs.create_file('./conf/threat_intel.json', contents=json.dumps( config_data['threat_intel'])) self.fs_patcher.fs.create_file('./conf/normalized_types.json', contents=json.dumps( config_data['normalized_types'])) self.fs_patcher.fs.create_file('./conf/lambda.json', contents=json.dumps( config_data['lambda'])) self.fs_patcher.fs.create_file('./conf/clusters/prod.json', contents=json.dumps( config_data['clusters']['prod'])) # Create the config instance after creating the fake filesystem so that # CLIConfig uses our mocked config files instead of the real ones. self.config = CLIConfig('./conf/')
def test_config_no_logs_key(self): """Shared - Config Validator - No Logs Key in Source""" # Load a valid config config = basic_streamalert_config() # Remove everything from the sources entry config['clusters']['prod']['data_sources']['kinesis']['stream_1'] = {} assert_raises(ConfigError, _validate_config, config)
def test_config_invalid_datasources(self): """Shared - Config Validator - Invalid Datasources""" # Load a valid config config = basic_streamalert_config() # Set the sources value to contain an invalid data source ('sqs') config['clusters']['prod']['data_sources'] = {'sqs': {'queue_1': {}}} assert_raises(ConfigError, _validate_config, config)
def test_config_empty_logs_list(self): """Shared - Config Validator - Empty Logs List in Source""" # Load a valid config config = basic_streamalert_config() # Set the logs key to an empty list config['clusters']['prod']['data_sources']['kinesis']['stream_1'] = [] assert_raises(ConfigError, _validate_config, config)
def test_config_no_parsers(self): """Shared - Config Validator - No Parser in Log""" # Load a valid config config = basic_streamalert_config() # Remove the 'parser' keys from the config config['logs']['json_log'].pop('parser') config['logs']['csv_log'].pop('parser') assert_raises(ConfigError, _validate_config, config)
def test_config_ioc_types_no_normalized_types(self): """Shared - Config Validator - IOC Types, Without Normalized Types""" # Load a valid config config = basic_streamalert_config() # Set the sources value to contain an invalid data source ('sqs') config['threat_intel'] = {'normalized_ioc_types': {'ip': ['foobar']}} if 'normalized_types' in config: del config['normalized_types'] assert_raises(ConfigError, _validate_config, config)
def test_config_invalid_ioc_types(self): """Shared - Config Validator - IOC Types, Invalid""" # Load a valid config config = basic_streamalert_config() # Set the sources value to contain an invalid data source ('sqs') config['threat_intel'] = { 'normalized_ioc_types': {'ip': ['foobar']} } config['normalized_types'] = {'log_type': {'sourceAddress': ['ip_address']}} assert_raises(ConfigError, _validate_config, config)
def test_lambda_terraform_targets(self): """CLI - Deploy, Lambda Terraform Targets""" config = basic_streamalert_config() functions = ['rule', 'classifier'] clusters = ['prod'] result = deploy._lambda_terraform_targets(config, functions, clusters) expected_result = { 'module.rules_engine_iam', 'module.rules_engine_lambda', 'module.classifier_prod_iam', 'module.classifier_prod_lambda', } assert_equal(result, expected_result)
def test_rollback_subset(self, mock_helper): """CLI - Lambda rollback apps and rule""" mock_helper.return_value = True assert_equal( rollback.rollback_handler( MockOptions(None, ['apps', 'rule']), MockCLIConfig(config=basic_streamalert_config())), True) mock_helper.assert_has_calls([ mock.call(mock.ANY, 'unit-testing_corp_box_admin_events_box_collector_app'), mock.call(mock.ANY, 'unit-testing_corp_duo_admin_duo_admin_collector_app'), mock.call(mock.ANY, 'unit-testing_streamalert_rules_engine') ])
def test_lambda_terraform_targets_invalid_target(self, log_mock): """CLI - Deploy, Lambda Terraform Targets, Invalid Target""" config = basic_streamalert_config() # The scheduled_queries function is not enabled functions = ['scheduled_queries'] clusters = [] result = deploy._lambda_terraform_targets(config, functions, clusters) assert_equal(result, set()) log_mock.assert_called_with( 'Function is not enabled and will be ignored: %s', 'scheduled_queries' )
def setUp(self): self.setUpPyfakefs() config_data = basic_streamalert_config() mock_cluster_contents = '{"data_sources": {}, "classifier_config": {"foo": "bar"}}' # Add config files which should be loaded self.fs.create_file('conf/clusters/prod.json', contents=mock_cluster_contents) self.fs.create_file('conf/clusters/dev.json', contents=mock_cluster_contents) self.fs.create_file('conf/global.json', contents='{}') self.fs.create_file('conf/lambda.json', contents='{}') self.fs.create_file('conf/logs.json', contents='{}') self.fs.create_file('conf/outputs.json', contents='{}') self.fs.create_file('conf/threat_intel.json', contents=json.dumps(config_data['threat_intel'])) self.fs.create_file('conf/normalized_types.json', contents=json.dumps( config_data['normalized_types'])) self.fs.create_file( 'conf/schemas/csv.json', contents= '{"csv_log2": {"schema": {"data": "string","uid": "integer"},"parser": "csv"}}' ) # Create similar structure but with schemas folder instead of logs.json and 2 clusters. self.fs.create_file('conf_schemas/clusters/prod.json', contents=mock_cluster_contents) self.fs.create_file('conf_schemas/clusters/dev.json', contents=mock_cluster_contents) self.fs.create_file('conf_schemas/global.json', contents='{}') self.fs.create_file('conf_schemas/lambda.json', contents='{}') self.fs.create_file('conf_schemas/outputs.json', contents='{}') self.fs.create_file( 'conf_schemas/schemas/csv.json', contents= '{"csv_log": {"schema": {"data": "string","uid": "integer"},"parser": "csv"}}' ) self.fs.create_file( 'conf_schemas/schemas/json.json', contents= '{"json_log": {"schema": {"name": "string"},"parser": "json"}}') self.fs.create_file( 'conf_schemas/schemas/json_log_with_dots.json', contents= '{"json:log.with.dots": {"schema": {"name": "string"},"parser": "json"}}' )
def setUp(self): self.setUpPyfakefs() config_data = basic_streamalert_config() # Add config files which should be loaded self.fs.create_file('conf/clusters/prod.json', contents='{}') self.fs.create_file('conf/clusters/dev.json', contents='{}') self.fs.create_file('conf/global.json', contents='{}') self.fs.create_file('conf/lambda.json', contents='{}') self.fs.create_file('conf/logs.json', contents='{}') self.fs.create_file('conf/outputs.json', contents='{}') self.fs.create_file('conf/sources.json', contents='{}') self.fs.create_file( 'conf/threat_intel.json', contents=json.dumps(config_data['threat_intel']) ) self.fs.create_file( 'conf/normalized_types.json', contents=json.dumps(config_data['normalized_types']) )
def test_rollback_all(self, mock_helper): """CLI - Lambda rollback all""" mock_helper.return_value = True assert_equal( rollback.rollback_handler( MockOptions(None, ['all']), MockCLIConfig(config=basic_streamalert_config())), True) mock_helper.assert_has_calls([ mock.call(mock.ANY, 'unit-testing_streamalert_alert_processor'), mock.call(mock.ANY, 'unit-testing_streamalert_alert_merger'), mock.call(mock.ANY, 'unit-testing_corp_box_admin_events_box_collector_app'), mock.call(mock.ANY, 'unit-testing_corp_duo_admin_duo_admin_collector_app'), mock.call(mock.ANY, 'unit-testing_streamalert_athena_partition_refresh'), mock.call(mock.ANY, 'unit-testing_corp_streamalert_classifier'), mock.call(mock.ANY, 'unit-testing_prod_streamalert_classifier'), mock.call(mock.ANY, 'unit-testing_streamalert_rules_engine'), mock.call(mock.ANY, 'unit-testing_streamalert_threat_intel_downloader') ])
def test_config_duplicate_sources(self): """Shared - Config Validator - Duplicate Data Sources in Cluster Configs""" config = basic_streamalert_config() config['clusters']['dev'] = config['clusters']['prod'] assert_raises(ConfigError, _validate_config, config)
def test_missing_streamalert_module(self): """Shared - Config Validator, Missing streamalert Module""" config = basic_streamalert_config() del config['clusters']['prod']['classifier_config'] assert_raises(ConfigError, _validate_config, config)
def test_load_schemas(): """Shared - Config Loading - Schemas""" # Load from separate dir where logs.json doesn't exist config = load_config(conf_dir='conf_schemas') basic_config = basic_streamalert_config() assert_equal(config['logs'], basic_config['logs'])