def test_nested_data(self):
"""Test that highly structured is also searchable."""
event_1 = {
'top': [{
'middle': {
'abc': 0
}
}, {
'middle2': ['def', 'ghi']
}]
}
event_2 = {
'top': [{
'middle': {
'abc': 123
}
}, {
'middle2': ['tuv', 'wxyz']
}]
}
events = [
Event(EVENT_TYPE_GENERIC, 1, event_1),
Event(EVENT_TYPE_GENERIC, 2, event_2)
]
query = parse_query('generic where top[0].middle.abc == 123')
results = self.get_output(queries=[query],
events=events,
config={'flatten': True})
self.assertEqual(len(results), 1, "Missing or extra results")
self.assertEqual(results[0].data, event_2,
"Failed to match on correct event")
def test_event_load(self):
"""Test that events can be loaded from valid data buffers or full json."""
event_type = 'process'
event_time = 131509374020000000
event_data = {
"command_line": "\\??\\C:\\Windows\\system32\\conhost.exe",
"event_type_full": "process_event",
"parent_process_name": "csrss.exe",
"parent_process_path": "C:\\Windows\\System32\\csrss.exe",
"pid": 3080,
"ppid": 372,
"process_name": "conhost.exe",
"process_path": "C:\\Windows\\System32\\conhost.exe",
"serial_event_id": 49,
"timestamp": 131509374020000000,
"user_domain": "vagrant",
"user_name": "vagrant",
}
expected_event = Event(event_type, event_time, event_data)
from_full_event = Event.from_data({
'event_timestamp': event_time,
'event_type': 4,
'data_buffer': event_data
})
from_buffer = Event.from_data(event_data)
self.assertEqual(from_full_event, expected_event,
"Full event didn't load properly")
self.assertEqual(from_buffer, expected_event,
"Event buffer didn't load properly")
def count_tuple_callback(events): # type: (list[Event]) -> None
if events is PIPE_EOF:
converter = get_type_converter(results)
converted_results = {
converter(k): v
for k, v in results.items()
}
total = sum(result['count']
for result in converted_results.values())
for key, result in sorted(converted_results.items(),
key=lambda kr:
(kr[1]['count'], kr[0])):
hosts = result.pop('hosts') # type: set
if len(hosts) > 0:
result['hosts'] = list(sorted(hosts))
result['total_hosts'] = len(hosts)
result['key'] = key
result['percent'] = float(result['count']) / total
next_pipe([Event(EVENT_TYPE_GENERIC, 0, result)])
next_pipe(PIPE_EOF)
else:
piece = events[0].data
key = events[0].data['key']
key = tuple(key) if len(node.arguments) > 1 else key
results[key]['count'] += piece['count']
if host_key in piece:
results[key]['hosts'].add(piece[host_key])
elif 'hosts' in piece:
results[key]['hosts'].update(piece['hosts'])
def count_tuple_callback(events): # type: (list[Event]) -> None
if events is PIPE_EOF:
# This may seem a little tricky, but we need to effectively learn the type(s) to perform comparison
# Python 3 doesn't allow you to use a key function that returns various types
converter = get_type_converter(count_table)
converted_count_table = {
converter(k): v
for k, v in count_table.items()
}
total = sum(tbl['count'] for tbl in count_table.values())
for key, details in sorted(converted_count_table.items(),
key=lambda kv:
(kv[1]['count'], kv[0])):
hosts = details.pop('hosts')
if len(hosts):
details['hosts'] = list(sorted(hosts))
details['total_hosts'] = len(hosts)
details['key'] = key
details['percent'] = float(details['count']) / total
next_pipe([Event(EVENT_TYPE_GENERIC, 0, details)])
next_pipe(PIPE_EOF)
else:
key = get_key(events)
count_table[key]['count'] += 1
if host_key in events[0].data:
count_table[key]['hosts'].add(events[0].data[host_key])
def count_total_callback(events):
if events is PIPE_EOF:
if len(hosts):
summary['total_hosts'] = len(hosts)
summary['hosts'] = list(sorted(hosts))
next_pipe([Event(EVENT_TYPE_GENERIC, 0, summary)])
next_pipe(PIPE_EOF)
else:
summary['count'] += 1
if host_key in events[0].data:
hosts.add(events[0].data[host_key])
def count_total_aggregates(events): # type: (list[Event]) -> None
if events is PIPE_EOF:
hosts = result.pop('hosts') # type: set
if len(hosts) > 0:
result['hosts'] = list(sorted(hosts))
result['total_hosts'] = len(hosts)
next_pipe([Event(EVENT_TYPE_GENERIC, 0, result)])
next_pipe(PIPE_EOF)
else:
piece = events[0].data
result['count'] += piece['count']
if host_key in piece:
result['hosts'].add(piece[host_key])
elif 'hosts' in piece:
results['hosts'].update(piece['hosts'])