def test_kv_rule(self): """Rule Engine - KV Rule""" @rule(logs=['test_log_type_kv_auditd'], outputs=['pagerduty']) def auditd_bin_cat(rec): return (rec['type'] == 'SYSCALL' and rec['exe'] == '"/bin/cat"') @rule(logs=['test_log_type_kv_auditd'], outputs=['pagerduty']) def gid_500(rec): return (rec['gid'] == 500 and rec['euid'] == 500) auditd_test_data = ( 'type=SYSCALL msg=audit(1364481363.243:24287): ' 'arch=c000003e syscall=2 success=no exit=-13 a0=7fffd19c5592 a1=0 ' 'a2=7fffd19c4b50 a3=a items=1 ppid=2686 pid=3538 auid=500 uid=500 ' 'gid=500 euid=500 suid=500 fsuid=500 egid=500 sgid=500 fsgid=500 tty=pts0 ' 'ses=1 comm="cat" exe="/bin/cat" ' 'subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 ' 'key="sshd_config" type=CWD msg=audit(1364481363.243:24287): ' 'cwd="/home/shadowman" type=PATH ' 'msg=audit(1364481363.243:24287): item=0 name="/etc/ssh/sshd_config" ' 'inode=409248 dev=fd:00 mode=0100600 ouid=0 ogid=0 ' 'rdev=00:00 obj=system_u:object_r:etc_t:s0') # prepare the payloads payload = self.make_kinesis_payload( kinesis_stream='test_kinesis_stream', kinesis_data=auditd_test_data) # process payloads alerts = StreamRules.process(payload) # alert tests assert_equal(len(alerts), 2) rule_name_alerts = set([x['rule_name'] for x in alerts]) assert_equal(rule_name_alerts, set(['gid_500', 'auditd_bin_cat']))
def test_syslog_rule(self): @rule('syslog_sudo', logs=['test_log_type_syslog'], outputs=['s3']) def syslog_sudo(rec): return ( rec['application'] == 'sudo' and 'root' in rec['message'] ) kinesis_data = ( 'Jan 26 19:35:33 vagrant-ubuntu-trusty-64 ' 'sudo: pam_unix(sudo:session): ' 'session opened for user root by (uid=0)' ) # prepare the payloads payload = self.make_kinesis_payload(kinesis_stream='test_stream_2', kinesis_data=kinesis_data) # process payloads alerts = StreamRules.process(payload) # alert tests assert_equal(len(alerts), 1) assert_equal(alerts[0]['rule_name'], 'syslog_sudo') assert_equal(alerts[0]['payload'].record['host'], 'vagrant-ubuntu-trusty-64') assert_equal(alerts[0]['payload'].type, 'syslog')
def process_alerts(payload, alerts_to_send): if payload.valid: alerts = StreamRules.process(payload) if alerts: alerts_to_send.extend(alerts) else: logger.debug('Invalid data: %s', payload)
def process_alerts(self, payload): """Process records for alerts""" if payload.valid: alerts = StreamRules.process(payload) if alerts: self.alerts.extend(alerts) else: logger.debug('Invalid data: %s', payload)
def test_process_req_subkeys(self): @rule('data_location', logs=['test_log_type_json_nested'], outputs=['s3'], req_subkeys={'data': ['location']}) def data_location(rec): return rec['data']['location'].startswith('us') @rule('web_server', logs=['test_log_type_json_nested'], outputs=['s3'], req_subkeys={'data': ['category']}) def web_server(rec): return rec['data']['category'] == 'web-server' kinesis_data = [ { 'date': 'Dec 01 2016', 'unixtime': '1483139547', 'host': 'host1.web.prod.net', 'data': { 'category': 'web-server', 'type': '1', 'source': 'eu' } }, { 'date': 'Dec 01 2016', 'unixtime': '1483139547', 'host': 'host1.web.prod.net', 'data': { 'location': 'us-west-2' } } ] # prepare payloads payloads = [] for data in kinesis_data: kinesis_data_json = json.dumps(data) payload = self.make_kinesis_payload(kinesis_stream='test_kinesis_stream', kinesis_data=kinesis_data_json) payloads.append(payload) alerts = [] for payload in payloads: # process payloads alerts.extend(StreamRules.process(payload)) # check alert output assert_equal(len(alerts), 2) # alert tests assert_equal(alerts[0]['rule_name'], 'web_server') assert_equal(alerts[1]['rule_name'], 'data_location')
def test_basic_rule_matcher_process(self): @matcher('prod') def prod(rec): return rec['environment'] == 'prod' @rule('incomplete_rule') def incomplete_rule(rec): return True @rule('minimal_rule', logs=['test_log_type_json_nested_with_data'], outputs=['s3']) def minimal_rule(rec): return rec['unixtime'] == 1483139547 @rule('chef_logs', matchers=['foobar', 'prod'], logs=['test_log_type_json_nested_with_data'], outputs=['pagerduty']) def chef_logs(rec): return rec['application'] == 'chef' kinesis_data = { 'date': 'Dec 01 2016', 'unixtime': '1483139547', 'host': 'host1.web.prod.net', 'application': 'chef', 'environment': 'prod', 'data': { 'category': 'web-server', 'type': '1', 'source': 'eu' } } # prepare the payloads kinesis_data_json = json.dumps(kinesis_data) payload = self.make_kinesis_payload( kinesis_stream='test_kinesis_stream', kinesis_data=kinesis_data_json) # process payloads alerts = StreamRules.process(payload) # check alert output assert_equal(len(alerts), 2) # alert 1 tests assert_equal(alerts[1]['rule_name'], 'chef_logs') assert_equal(alerts[1]['outputs'], ['pagerduty']) # alert 0 tests assert_equal(alerts[0]['rule_name'], 'minimal_rule') assert_equal(alerts[0]['outputs'], ['s3'])
def test_csv_rule(self): """Rule Engine - CSV Rule""" @rule(logs=['test_log_type_csv_nested'], outputs=['pagerduty']) def nested_csv(rec): return (rec['message']['application'] == 'chef' and rec['message']['cluster_size'] == 100) kinesis_data = ('"Jan 10, 2017","1485739910","host1.prod.test","Corp",' '"chef,web-server,1,100,fail"') # prepare the payloads payload = self.make_kinesis_payload( kinesis_stream='test_kinesis_stream', kinesis_data=kinesis_data) # process payloads alerts = StreamRules.process(payload) # alert tests assert_equal(len(alerts), 1) assert_equal(alerts[0]['rule_name'], 'nested_csv')
def test_alert_format(self): @rule('alert_format_test', logs=['test_log_type_json_nested_with_data'], outputs=['s3']) def alert_format_test(rec): return rec['application'] == 'web-app' kinesis_data = { 'date': 'Dec 01 2016', 'unixtime': '1483139547', 'host': 'host1.web.prod.net', 'application': 'web-app', 'environment': 'prod', 'data': { 'category': 'web-server', 'type': '1', 'source': 'eu' } } # prepare the payloads kinesis_data_json = json.dumps(kinesis_data) payload = self.make_kinesis_payload( kinesis_stream='test_kinesis_stream', kinesis_data=kinesis_data_json) # process payloads alerts = StreamRules.process(payload) alert_keys = {'rule_name', 'metadata', 'record'} metadata_keys = {'log', 'outputs', 'type', 'source'} assert_equal(set(alerts[0].keys()), alert_keys) assert_equal(set(alerts[0]['metadata'].keys()), metadata_keys) # test alert fields assert_equal(type(alerts[0]['rule_name']), str) assert_equal(type(alerts[0]['record']), dict) assert_equal(type(alerts[0]['metadata']['outputs']), list) assert_equal(type(alerts[0]['metadata']['type']), str) assert_equal(type(alerts[0]['metadata']['source']), dict) assert_equal(type(alerts[0]['metadata']['log']), str)