def test_nested_data(self): """Test that highly structured is also searchable.""" event_1 = { 'top': [{ 'middle': { 'abc': 0 } }, { 'middle2': ['def', 'ghi'] }] } event_2 = { 'top': [{ 'middle': { 'abc': 123 } }, { 'middle2': ['tuv', 'wxyz'] }] } events = [ Event(EVENT_TYPE_GENERIC, 1, event_1), Event(EVENT_TYPE_GENERIC, 2, event_2) ] query = parse_query('generic where top[0].middle.abc == 123') results = self.get_output(queries=[query], events=events, config={'flatten': True}) self.assertEqual(len(results), 1, "Missing or extra results") self.assertEqual(results[0].data, event_2, "Failed to match on correct event")
def test_event_load(self): """Test that events can be loaded from valid data buffers or full json.""" event_type = 'process' event_time = 131509374020000000 event_data = { "command_line": "\\??\\C:\\Windows\\system32\\conhost.exe", "event_type_full": "process_event", "parent_process_name": "csrss.exe", "parent_process_path": "C:\\Windows\\System32\\csrss.exe", "pid": 3080, "ppid": 372, "process_name": "conhost.exe", "process_path": "C:\\Windows\\System32\\conhost.exe", "serial_event_id": 49, "timestamp": 131509374020000000, "user_domain": "vagrant", "user_name": "vagrant", } expected_event = Event(event_type, event_time, event_data) from_full_event = Event.from_data({ 'event_timestamp': event_time, 'event_type': 4, 'data_buffer': event_data }) from_buffer = Event.from_data(event_data) self.assertEqual(from_full_event, expected_event, "Full event didn't load properly") self.assertEqual(from_buffer, expected_event, "Event buffer didn't load properly")
def count_tuple_callback(events): # type: (list[Event]) -> None if events is PIPE_EOF: converter = get_type_converter(results) converted_results = { converter(k): v for k, v in results.items() } total = sum(result['count'] for result in converted_results.values()) for key, result in sorted(converted_results.items(), key=lambda kr: (kr[1]['count'], kr[0])): hosts = result.pop('hosts') # type: set if len(hosts) > 0: result['hosts'] = list(sorted(hosts)) result['total_hosts'] = len(hosts) result['key'] = key result['percent'] = float(result['count']) / total next_pipe([Event(EVENT_TYPE_GENERIC, 0, result)]) next_pipe(PIPE_EOF) else: piece = events[0].data key = events[0].data['key'] key = tuple(key) if len(node.arguments) > 1 else key results[key]['count'] += piece['count'] if host_key in piece: results[key]['hosts'].add(piece[host_key]) elif 'hosts' in piece: results[key]['hosts'].update(piece['hosts'])
def count_tuple_callback(events): # type: (list[Event]) -> None if events is PIPE_EOF: # This may seem a little tricky, but we need to effectively learn the type(s) to perform comparison # Python 3 doesn't allow you to use a key function that returns various types converter = get_type_converter(count_table) converted_count_table = { converter(k): v for k, v in count_table.items() } total = sum(tbl['count'] for tbl in count_table.values()) for key, details in sorted(converted_count_table.items(), key=lambda kv: (kv[1]['count'], kv[0])): hosts = details.pop('hosts') if len(hosts): details['hosts'] = list(sorted(hosts)) details['total_hosts'] = len(hosts) details['key'] = key details['percent'] = float(details['count']) / total next_pipe([Event(EVENT_TYPE_GENERIC, 0, details)]) next_pipe(PIPE_EOF) else: key = get_key(events) count_table[key]['count'] += 1 if host_key in events[0].data: count_table[key]['hosts'].add(events[0].data[host_key])
def count_total_callback(events): if events is PIPE_EOF: if len(hosts): summary['total_hosts'] = len(hosts) summary['hosts'] = list(sorted(hosts)) next_pipe([Event(EVENT_TYPE_GENERIC, 0, summary)]) next_pipe(PIPE_EOF) else: summary['count'] += 1 if host_key in events[0].data: hosts.add(events[0].data[host_key])
def count_total_aggregates(events): # type: (list[Event]) -> None if events is PIPE_EOF: hosts = result.pop('hosts') # type: set if len(hosts) > 0: result['hosts'] = list(sorted(hosts)) result['total_hosts'] = len(hosts) next_pipe([Event(EVENT_TYPE_GENERIC, 0, result)]) next_pipe(PIPE_EOF) else: piece = events[0].data result['count'] += piece['count'] if host_key in piece: result['hosts'].add(piece[host_key]) elif 'hosts' in piece: results['hosts'].update(piece['hosts'])