def test_next_cswitch(self, thread_registry_mock, kevent_mock):
context_switch_registry = ContextSwitchRegistry(thread_registry_mock, kevent_mock)
ts = datetime.strptime("12:05:45.233", '%H:%M:%S.%f')
kcs1 = ddict({'old_thread_wait_ideal_processor': 2, 'previous_c_state': 1, 'old_thread_state': 2,
'old_thread_priority': 0, 'reserved': 777748717, 'spare_byte': 0,
'old_thread_wait_reason': 0, 'new_thread_wait_time': '0x0', 'old_thread_wait_mode': 0,
'new_thread_priority': 15, 'new_thread_id': '0x1054', 'old_thread_id': '0x0'})
new_thread_id = int(kcs1.new_thread_id, 16)
context_switch_registry.next_cswitch(1, ts, kcs1)
thread_registry_mock.get_thread.assert_has_calls([call(new_thread_id),
call(int(kcs1.old_thread_id, 16))])
assert (1, new_thread_id,) in context_switch_registry.context_switches()
cs = context_switch_registry.context_switches()[(1, new_thread_id,)]
assert cs
assert cs.timestamp is ts
assert cs.next_proc_name == "explorer.exe"
assert cs.next_thread_wait_time == 0
assert cs.next_thread_prio == 15
assert cs.prev_thread_prio == 0
assert cs.prev_thread_state is ThreadState.RUNNING
assert cs.count == 1
assert cs.prev_thread_wait_mode is WaitMode.KERNEL
assert cs.prev_thread_wait_reason is WaitReason.EXECUTIVE
def test_next_cswitch_in_registry(self, thread_registry_empty_mock,
kevent_mock):
context_switch_registry = ContextSwitchRegistry(
thread_registry_empty_mock, kevent_mock)
kcs1 = ddict({
'old_thread_wait_ideal_processor': 3,
'previous_c_state': 0,
'old_thread_state': 5,
'old_thread_priority': 8,
'reserved': 4294967294,
'spare_byte': 0,
'old_thread_wait_reason': 8,
'new_thread_wait_time': '0x0',
'old_thread_wait_mode': 1,
'new_thread_priority': 8,
'new_thread_id': '0x1fc8',
'old_thread_id': '0x2348'
})
kcs2 = ddict({
'old_thread_wait_ideal_processor': 3,
'previous_c_state': 0,
'old_thread_state': 3,
'old_thread_priority': 8,
'reserved': 4294967295,
'spare_byte': 0,
'old_thread_wait_reason': 8,
'new_thread_wait_time': '0x0',
'old_thread_wait_mode': 1,
'new_thread_priority': 5,
'new_thread_id': '0x1fc8',
'old_thread_id': '0x2348'
})
context_switch_registry.next_cswitch(
1, datetime.strptime("12:05:45.233", '%H:%M:%S.%f'), kcs1)
context_switch_registry.next_cswitch(
1, datetime.strptime("12:05:45.234", '%H:%M:%S.%f'), kcs2)
k = (1, int(kcs1.new_thread_id, 16))
cs = context_switch_registry.context_switches()[k]
assert cs
assert cs.count == 2
assert cs.next_thread_prio == 5
assert cs.prev_thread_state is ThreadState.STANDBY
assert cs.timestamp == datetime.strptime("12:05:45.234", '%H:%M:%S.%f')
assert cs.prev_thread_wait_reason is WaitReason.FREE_PAGE
def test_next_cswitch_in_registry(self, thread_registry_empty_mock, kevent_mock):
context_switch_registry = ContextSwitchRegistry(thread_registry_empty_mock, kevent_mock)
kcs1 = ddict({'old_thread_wait_ideal_processor': 3, 'previous_c_state': 0, 'old_thread_state': 5,
'old_thread_priority': 8, 'reserved': 4294967294, 'spare_byte': 0,
'old_thread_wait_reason': 8, 'new_thread_wait_time': '0x0', 'old_thread_wait_mode': 1,
'new_thread_priority': 8, 'new_thread_id': '0x1fc8', 'old_thread_id': '0x2348'})
kcs2 = ddict({'old_thread_wait_ideal_processor': 3, 'previous_c_state': 0, 'old_thread_state': 3,
'old_thread_priority': 8, 'reserved': 4294967295, 'spare_byte': 0,
'old_thread_wait_reason': 8, 'new_thread_wait_time': '0x0', 'old_thread_wait_mode': 1,
'new_thread_priority': 5, 'new_thread_id': '0x1fc8', 'old_thread_id': '0x2348'})
context_switch_registry.next_cswitch(1, datetime.strptime("12:05:45.233", '%H:%M:%S.%f'), kcs1)
context_switch_registry.next_cswitch(1, datetime.strptime("12:05:45.234", '%H:%M:%S.%f'), kcs2)
k = (1, int(kcs1.new_thread_id, 16))
cs = context_switch_registry.context_switches()[k]
assert cs
assert cs.count == 2
assert cs.next_thread_prio == 5
assert cs.prev_thread_state is ThreadState.STANDBY
assert cs.timestamp == datetime.strptime("12:05:45.234", '%H:%M:%S.%f')
assert cs.prev_thread_wait_reason is WaitReason.FREE_PAGE
def test_next_cswitch(self, thread_registry_mock, kevent_mock):
context_switch_registry = ContextSwitchRegistry(
thread_registry_mock, kevent_mock)
ts = datetime.strptime("12:05:45.233", '%H:%M:%S.%f')
kcs1 = ddict({
'old_thread_wait_ideal_processor': 2,
'previous_c_state': 1,
'old_thread_state': 2,
'old_thread_priority': 0,
'reserved': 777748717,
'spare_byte': 0,
'old_thread_wait_reason': 0,
'new_thread_wait_time': '0x0',
'old_thread_wait_mode': 0,
'new_thread_priority': 15,
'new_thread_id': '0x1054',
'old_thread_id': '0x0'
})
new_thread_id = int(kcs1.new_thread_id, 16)
context_switch_registry.next_cswitch(1, ts, kcs1)
thread_registry_mock.get_thread.assert_has_calls(
[call(new_thread_id),
call(int(kcs1.old_thread_id, 16))])
assert (
1,
new_thread_id,
) in context_switch_registry.context_switches()
cs = context_switch_registry.context_switches()[(
1,
new_thread_id,
)]
assert cs
assert cs.timestamp is ts
assert cs.next_proc_name == "explorer.exe"
assert cs.next_thread_wait_time == 0
assert cs.next_thread_prio == 15
assert cs.prev_thread_prio == 0
assert cs.prev_thread_state is ThreadState.RUNNING
assert cs.count == 1
assert cs.prev_thread_wait_mode is WaitMode.KERNEL
assert cs.prev_thread_wait_reason is WaitReason.EXECUTIVE
def run(self):
"""Filament main routine.
Setups the interval repeating function and polls for
the kernel events from the queue.
"""
on_interval = self._find_filament_func('on_interval')
if on_interval:
self.scheduler.add_executor(ThreadPoolExecutor(max_workers=4))
self.scheduler.start()
self.scheduler.add_job(self._filament_module.on_interval,
'interval',
seconds=self._interval,
max_instances=4,
misfire_grace_time=60)
while self._poll():
try:
kevent = self._keventq.get()
self._filament_module.on_next_kevent(ddict(kevent))
except Exception:
self._logger.error('Unexpected filament error',
exc_info=sys.exc_info())
def add_image_meta(self, path):
"""Registers image meta information.
This method parses the PE (Portable Executable) binary format
of the the image passed in the `path` parameter.
It then extracts some basic headers present in the PE, as well
as sections which form the binary image.
Parameters
----------
path: str
the absolute path of the image file
"""
if not self.enabled:
return None
try:
if (path.endswith('exe') or
path.endswith('dll') or
path.endswith('sys')) and \
path not in self.image_metas:
pe = pefile.PE(path, fast_load=True)
file_header = ddict(underscore_dict_keys(pe.FILE_HEADER.dump_dict()))
# create image meta instance
image_meta = ImageMeta(file_header.machine.value,
file_header.time_date_stamp.value,
file_header.number_of_sections.value)
image_meta.sections = [dict(name=__decode__(ddict(se.dump_dict()).Name.Value),
entropy=se.get_entropy(),
md5=se.get_hash_md5(),
sha1=se.get_hash_sha1(),
sha256=se.get_hash_sha256(),
sha512=se.get_hash_sha512())
for se in pe.sections]
# parse directory entry imports
if self.imports:
pe.full_load()
self.full_loaded = True
for module in self.__directory_entry_import__(pe):
dll = __decode__(module.dll)
imports = [__decode__(i.name)
for i in module.imports
if not i.import_by_ordinal]
image_meta.imports[dll] = imports
# parse the string table to extract
# the copyright, company, description
# and other attributes
if self.file_info:
if not self.full_loaded:
pe.full_load()
if self.__pe_has_version_info__(pe):
file_info = pe.FileInfo
if file_info and len(file_info) > 0:
file_info = file_info[0]
if self.__fi_has_string_table__(file_info):
string_table = sorted(list(file_info.StringTable[0].entries.items()))
# get file info entries from table index
image_meta.org = __from_idx__(string_table, 0)
image_meta.description = __from_idx__(string_table, 1)
image_meta.version = __from_idx__(string_table, 2)
image_meta.internal_name = __from_idx__(string_table, 3)
image_meta.copyright = __from_idx__(string_table, 4)
self.image_metas[path] = image_meta
return image_meta
except Exception:
# ignore the exception for now
# but consider logging it to file
# in case it can provide hints for
# troubleshooting purposes
pass
def _on_next_kevent(self, ktype, cpuid, ts, kparams):
"""Callback which fires when new kernel event arrives.
This callback is invoked for every new kernel event
forwarded from the kernel stream collector.
Parameters
----------
ktype: tuple
Kernel event type.
cpuid: int
Indentifies the CPU core where the event
has been captured.
ts: str
Temporal reference of the kernel event.
kparams: dict
Kernel event's parameters.
"""
# initialize kernel event properties
self.kevent.ts = ts
self.kevent.cpuid = cpuid
self.kevent.name = ktuple_to_name(ktype)
kparams = ddict(kparams)
# thread / process kernel events
if ktype in [CREATE_PROCESS, CREATE_THREAD, ENUM_PROCESS, ENUM_THREAD]:
self.thread_registry.add_thread(ktype, kparams)
if ktype in [CREATE_PROCESS, CREATE_THREAD]:
self.thread_registry.init_thread_kevent(
self.kevent, ktype, kparams)
self._render(ktype)
elif ktype in [TERMINATE_PROCESS, TERMINATE_THREAD]:
self.thread_registry.init_thread_kevent(self.kevent, ktype,
kparams)
self._render(ktype)
self.thread_registry.remove_thread(ktype, kparams)
# file system/disk kernel events
elif ktype in [
CREATE_FILE, DELETE_FILE, CLOSE_FILE, READ_FILE, WRITE_FILE
]:
self.fsio.parse_fsio(ktype, kparams)
self._render(ktype)
# dll kernel events
elif ktype in [LOAD_IMAGE, ENUM_IMAGE]:
self.dll_repository.register_dll(kparams)
if ktype == LOAD_IMAGE:
self._render(ktype)
elif ktype == UNLOAD_IMAGE:
self.dll_repository.unregister_dll(kparams)
self._render(ktype)
# registry kernel events
elif ktype == REG_CREATE_KCB:
self.hive_parser.add_kcb(kparams)
elif ktype == REG_DELETE_KCB:
self.hive_parser.remove_kcb(kparams.key_handle)
elif ktype in [
REG_CREATE_KEY, REG_DELETE_KEY, REG_OPEN_KEY, REG_QUERY_KEY,
REG_SET_VALUE, REG_DELETE_VALUE, REG_QUERY_VALUE
]:
self.hive_parser.parse_hive(ktype, kparams)
self._render(ktype)
# network kernel events
elif ktype in [
SEND_SOCKET_TCPV4, SEND_SOCKET_UDPV4, RECV_SOCKET_TCPV4,
RECV_SOCKET_UDPV4, ACCEPT_SOCKET_TCPV4, CONNECT_SOCKET_TCPV4,
DISCONNECT_SOCKET_TCPV4, RECONNECT_SOCKET_TCPV4
]:
self.tcpip_parser.parse_tcpip(ktype, kparams)
self._render(ktype)
if self._filament:
# call filament method
# to process the next
# kernel event from the stream
if ktype not in [ENUM_PROCESS, ENUM_THREAD, ENUM_IMAGE]:
if self.kevent.name:
self._filament.process(self.kevent)
def add_image_meta(self, path):
"""Registers image meta information.
This method parses the PE (Portable Executable) binary format
of the the image passed in the `path` parameter.
It then extracts some basic headers present in the PE, as well
as sections which form the binary image.
Parameters
----------
path: str
the absolute path of the image file
"""
if not self.enabled:
return None
try:
if (path.endswith('exe') or
path.endswith('dll') or
path.endswith('sys')) and \
path not in self.image_metas:
pe = pefile.PE(path, fast_load=True)
file_header = ddict(
underscore_dict_keys(pe.FILE_HEADER.dump_dict()))
# create image meta instance
image_meta = ImageMeta(file_header.machine.value,
file_header.time_date_stamp.value,
file_header.number_of_sections.value)
image_meta.sections = [
dict(name=__decode__(ddict(se.dump_dict()).Name.Value),
entropy=se.get_entropy(),
md5=se.get_hash_md5(),
sha1=se.get_hash_sha1(),
sha256=se.get_hash_sha256(),
sha512=se.get_hash_sha512()) for se in pe.sections
]
# parse directory entry imports
if self.imports:
pe.full_load()
self.full_loaded = True
for module in self.__directory_entry_import__(pe):
dll = __decode__(module.dll)
imports = [
__decode__(i.name) for i in module.imports
if not i.import_by_ordinal
]
image_meta.imports[dll] = imports
# parse the string table to extract
# the copyright, company, description
# and other attributes
if self.file_info:
if not self.full_loaded:
pe.full_load()
if self.__pe_has_version_info__(pe):
file_info = pe.FileInfo
if file_info and len(file_info) > 0:
file_info = file_info[0]
if self.__fi_has_string_table__(file_info):
string_table = sorted(
list(file_info.StringTable[0].entries.
items()))
# get file info entries from table index
image_meta.org = __from_idx__(string_table, 0)
image_meta.description = __from_idx__(
string_table, 1)
image_meta.version = __from_idx__(
string_table, 2)
image_meta.internal_name = __from_idx__(
string_table, 3)
image_meta.copyright = __from_idx__(
string_table, 4)
self.image_metas[path] = image_meta
return image_meta
except Exception:
# ignore the exception for now
# but consider logging it to file
# in case it can provide hints for
# troubleshooting purposes
pass
def _on_next_kevent(self, ktype, cpuid, ts, kparams):
"""Callback which fires when new kernel event arrives.
This callback is invoked for every new kernel event
forwarded from the kernel stream collector.
Parameters
----------
ktype: tuple
Kernel event type.
cpuid: int
Indentifies the CPU core where the event
has been captured.
ts: str
Temporal reference of the kernel event.
kparams: dict
Kernel event's parameters.
"""
# initialize kernel event properties
self.kevent.ts = ts
self.kevent.cpuid = cpuid
self.kevent.name = ktuple_to_name(ktype)
kparams = ddict(kparams)
# thread / process kernel events
if ktype in [CREATE_PROCESS,
CREATE_THREAD,
ENUM_PROCESS,
ENUM_THREAD]:
self.thread_registry.add_thread(ktype, kparams)
if ktype in [CREATE_PROCESS, CREATE_THREAD]:
self.thread_registry.init_thread_kevent(self.kevent,
ktype,
kparams)
self._render(ktype)
elif ktype in [TERMINATE_PROCESS, TERMINATE_THREAD]:
self.thread_registry.init_thread_kevent(self.kevent,
ktype,
kparams)
self._render(ktype)
self.thread_registry.remove_thread(ktype, kparams)
# file system/disk kernel events
elif ktype in [CREATE_FILE,
DELETE_FILE,
CLOSE_FILE,
READ_FILE,
WRITE_FILE]:
self.fsio.parse_fsio(ktype, kparams)
self._render(ktype)
# dll kernel events
elif ktype in [LOAD_IMAGE, ENUM_IMAGE]:
self.dll_repository.register_dll(kparams)
if ktype == LOAD_IMAGE:
self._render(ktype)
elif ktype == UNLOAD_IMAGE:
self.dll_repository.unregister_dll(kparams)
self._render(ktype)
# registry kernel events
elif ktype == REG_CREATE_KCB:
self.hive_parser.add_kcb(kparams)
elif ktype == REG_DELETE_KCB:
self.hive_parser.remove_kcb(kparams.key_handle)
elif ktype in [REG_CREATE_KEY,
REG_DELETE_KEY,
REG_OPEN_KEY,
REG_QUERY_KEY,
REG_SET_VALUE,
REG_DELETE_VALUE,
REG_QUERY_VALUE]:
self.hive_parser.parse_hive(ktype, kparams)
self._render(ktype)
# network kernel events
elif ktype in [SEND_SOCKET_TCPV4,
SEND_SOCKET_UDPV4,
RECV_SOCKET_TCPV4,
RECV_SOCKET_UDPV4,
ACCEPT_SOCKET_TCPV4,
CONNECT_SOCKET_TCPV4,
DISCONNECT_SOCKET_TCPV4,
RECONNECT_SOCKET_TCPV4]:
self.tcpip_parser.parse_tcpip(ktype, kparams)
self._render(ktype)
if self._filament:
# call filament method
# to process the next
# kernel event from the stream
if ktype not in [ENUM_PROCESS,
ENUM_THREAD, ENUM_IMAGE]:
if self.kevent.name:
self._filament.process(self.kevent)
def add_thread(self, ketype, kti):
"""Adds a new process or thread to thread registry.
Parameters
----------
ketype: tuple
kernel event type
kti: dict
event payload as coming from the
kernel event stream collector
"""
if ketype == CREATE_PROCESS or ketype == ENUM_PROCESS:
parent_pid = int(kti.parent_id, 16)
process_id = int(kti.process_id, 16)
# we assume the process id is
# equal to thread id (in a single
# threaded process)
thread_id = process_id
name = kti.image_file_name
comm = kti.command_line
thread = ThreadInfo(process_id, thread_id,
parent_pid,
name,
comm)
if ketype == ENUM_PROCESS:
thread.handles = [handle for handle in self._handles if handle.pid == process_id]
if ketype == CREATE_PROCESS:
image_meta = self.image_meta_registry.get_image_meta(thread.exe)
if not image_meta:
image_meta = self.image_meta_registry.add_image_meta(thread.exe)
thread.image_meta = image_meta
self._threads[process_id] = thread
elif ketype == CREATE_THREAD or ketype == ENUM_THREAD:
# new thread created in the
# context of the existing process
# `procces_id` is the parent
# of this thread
process_id = int(kti.process_id, 16)
parent_pid = process_id
thread_id = int(kti.t_thread_id, 16)
if parent_pid in self._threads:
# copy info from the process
# which created this thread
pthread = self._threads[parent_pid]
# increment the number of threads
# for this process
pthread.increment_child_count()
name = pthread.name
comm = pthread.comm
thread = ThreadInfo(process_id, thread_id,
parent_pid,
name,
comm)
thread.ustack_base = hex(kti.user_stack_base)
thread.kstack_base = hex(kti.stack_base)
thread.base_priority = kti.base_priority
thread.io_priority = kti.io_priority
self._threads[thread_id] = thread
else:
# the parent process has not been found
# query the os for process information
handle = open_process(PROCESS_QUERY_INFORMATION | PROCESS_QUERY_LIMITED_INFORMATION | PROCESS_VM_READ,
False,
parent_pid)
info = {}
if handle:
info = self._query_process_info(handle)
close_handle(handle)
else:
if get_last_error() == ERROR_ACCESS_DENIED:
if parent_pid == 0:
info = ddict(name='idle',
comm='idle',
parent_id=0)
else:
# the access to protected / system process
# can't be done with PROCESS_VM_READ or PROCESS_QUERY_INFORMATION
# flags. Open the process again but with
# restricted access rights, so we can get the process image file name
handle = open_process(PROCESS_QUERY_LIMITED_INFORMATION,
False,
parent_pid)
if handle:
info = self._query_process_info(handle, False)
close_handle(handle)
# add a new thread and the parent process
# we just found to avoid continuous lookup
name = info.name if len(info) > 0 and info.name else NA
comm = info.comm if len(info) > 0 and info.comm else NA
ppid = info.parent_pid if len(info) > 0 and info.parent_pid else NA
thread = ThreadInfo(process_id, thread_id,
process_id,
name,
comm)
thread.ustack_base = hex(kti.user_stack_base)
thread.kstack_base = hex(kti.stack_base)
thread.base_priority = kti.base_priority
thread.io_priority = kti.io_priority
parent = ThreadInfo(process_id, process_id,
ppid,
name,
comm)
# enumerate parent handles
parent.handles = self.handle_repository.query_handles(process_id)
self._threads[thread_id] = thread
self._threads[parent_pid] = parent
if self.on_thread_added_callback and callable(self.on_thread_added_callback):
self.on_thread_added_callback(thread)
def on_next_kevent(self, kevent):
try:
self._filament_module.on_next_kevent(ddict(kevent))
except Exception as e:
self._logger.error('Unexpected filament error %s' % e)
def _enum_handles(self, process_id=None):
"""Enumerates handle information.
Enumerates handle info on
the start of the kernel capture.
Returns a dictionary of handle's
information including the handle id,
access mask, and the process which owns
the handle.
"""
buff_size = MAX_BUFFER_SIZE
size = c_ulong()
# allocate the initial buffer
buff = malloc(buff_size)
handles = {}
while True:
status = zw_query_system_information(SYSTEM_HANDLE_INFORMATION_CLASS,
buff,
buff_size,
byref(size))
if status == STATUS_INFO_LENGTH_MISMATCH:
# the buffer is too small
# increment the buffer size and try again
buff_size += MAX_BUFFER_SIZE
elif status == STATUS_SUCCESS:
# cast the buffer to `SYSTEM_HANDLE_INFORMATION` struct
# which contains an array of `SYSTEM_HANDLE` structures
sys_handle_info = cast(buff, POINTER(SYSTEM_HANDLE_INFORMATION))
sys_handle_info = sys_handle_info.contents
handle_count = sys_handle_info.number_of_handles
# resize the array size to the
# actual number of file handles
sys_handles = (SYSTEM_HANDLE * buff_size).from_address(addressof(sys_handle_info.handles))
for i in range(handle_count):
sys_handle = sys_handles[i]
pid = sys_handle.process_id
handle = sys_handle.handle
obj = sys_handle.object
obj_type_index = sys_handle.object_type_number
access_mask = sys_handle.access_mask
if process_id and process_id == pid:
handles[obj] = ddict(pid=process_id,
handle=handle,
obj=obj,
access_mask=access_mask,
obj_type_index=obj_type_index)
elif process_id is None:
handles[obj] = ddict(pid=pid,
handle=handle,
obj=obj,
access_mask=access_mask,
obj_type_index=obj_type_index)
break
else:
raise HandleEnumError(status)
# reallocate the buffer
buff = realloc(buff, buff_size)
# free the buffer memory
free(buff)
return handles
def _on_next_kevent(self, ktype, cpuid, ts, kparams):
"""Callback which fires when new kernel event arrives.
This callback is invoked for every new kernel event
forwarded from the kernel stream collector.
Parameters
----------
ktype: tuple
Kernel event type.
cpuid: int
Indentifies the CPU core where the event
has been captured.
ts: str
Temporal reference of the kernel event.
kparams: dict
Kernel event's parameters.
"""
# initialize kernel event properties
self.kevent.ts = ts
self.kevent.cpuid = cpuid
self.kevent.name = ktuple_to_name(ktype)
kparams = ddict(kparams)
# thread / process kernel events
if ktype in [CREATE_PROCESS, CREATE_THREAD, ENUM_PROCESS, ENUM_THREAD]:
self.thread_registry.add_thread(ktype, kparams)
if ktype in [CREATE_PROCESS, CREATE_THREAD]:
self.thread_registry.init_thread_kevent(
self.kevent, ktype, kparams)
self._aggregate(ktype)
elif ktype in [TERMINATE_PROCESS, TERMINATE_THREAD]:
self.thread_registry.init_thread_kevent(self.kevent, ktype,
kparams)
self._aggregate(ktype)
self.thread_registry.remove_thread(ktype, kparams)
# file system/disk kernel events
elif ktype in [
CREATE_FILE, DELETE_FILE, CLOSE_FILE, READ_FILE, WRITE_FILE
]:
self.fsio.parse_fsio(ktype, kparams)
self._aggregate(ktype)
# dll kernel events
elif ktype in [LOAD_IMAGE, ENUM_IMAGE]:
self.dll_repository.register_dll(kparams)
if ktype == LOAD_IMAGE:
self._aggregate(ktype)
elif ktype == UNLOAD_IMAGE:
self.dll_repository.unregister_dll(kparams)
self._aggregate(ktype)
#
# # registry kernel events
elif ktype == REG_CREATE_KCB:
self.hive_parser.add_kcb(kparams)
elif ktype == REG_DELETE_KCB:
self.hive_parser.remove_kcb(kparams.key_handle)
elif ktype in [
REG_CREATE_KEY, REG_DELETE_KEY, REG_OPEN_KEY, REG_QUERY_KEY,
REG_SET_VALUE, REG_DELETE_VALUE, REG_QUERY_VALUE
]:
self.hive_parser.parse_hive(ktype, kparams)
self._aggregate(ktype)
# network kernel events
elif ktype in [
SEND_SOCKET_TCPV4, SEND_SOCKET_UDPV4, RECV_SOCKET_TCPV4,
RECV_SOCKET_UDPV4, ACCEPT_SOCKET_TCPV4, CONNECT_SOCKET_TCPV4,
DISCONNECT_SOCKET_TCPV4, RECONNECT_SOCKET_TCPV4
]:
self.tcpip_parser.parse_tcpip(ktype, kparams)
self._aggregate(ktype)
# context switch events
elif ktype == CONTEXT_SWITCH:
self.context_switch_registry.next_cswitch(cpuid, ts, kparams)
self._aggregate(ktype)
if self._filament:
# put the event on the queue
# from where the filaments process
# will poll for kernel events
if ktype not in [
ENUM_PROCESS, ENUM_THREAD, ENUM_IMAGE, REG_CREATE_KCB,
REG_DELETE_KCB
]:
ok = self.output_kevents[ktype] if ktype in self.output_kevents \
else False
if self.kevent.name and ok:
thread = self.kevent.thread
# push the kernel event dict
# to processing queue
kevt = dict(params=self.kevent.params,
name=self.kevent.name,
pid=self.kevent.pid,
tid=self.kevent.tid,
timestamp=self.kevent.ts,
cpuid=self.kevent.cpuid,
category=self.kevent.category,
thread=dict(name=thread.name,
exe=thread.exe,
comm=thread.comm,
pid=thread.pid,
ppid=thread.ppid))
self.keventq.put(kevt)
def skips(self):
return ddict(self._yaml.pop('skips', {}))
def image_meta(self):
return ddict(self._yaml.pop('image_meta', {}))
def params(self, params):
self._params = ddict(params)
def params(self, params):
self._params = ddict(params)
def image_meta(self):
return ddict(self._yaml.pop('image_meta', {}))
def skips(self):
return ddict(self._yaml.pop('skips', {}))
def _query_process_info(self, handle, read_peb=True):
"""Gets an extended proc info.
Parameters
-----------
handle: HANDLE
handle to process for which the info
should be acquired
read_peb: boolean
true in case the process PEB should be read
"""
pbi_buff = malloc(sizeof(PROCESS_BASIC_INFORMATION))
status = zw_query_information_process(
handle, PROCESS_BASIC_INFO, pbi_buff,
sizeof(PROCESS_BASIC_INFORMATION), byref(ULONG()))
info = {}
if status == STATUS_SUCCESS:
pbi = cast(pbi_buff, POINTER(PROCESS_BASIC_INFORMATION))
ppid = pbi.contents.inherited_from_unique_process_id
if read_peb:
# read the PEB to get the process parameters.
# Because the PEB structure resides
# in the address space of another process
# we must read the memory block in order
# to access the structure's fields
peb_addr = pbi.contents.peb_base_address
peb_buff = read_process_memory(handle, peb_addr, sizeof(PEB))
if peb_buff:
peb = cast(peb_buff, POINTER(PEB))
# read the RTL_USER_PROCESS_PARAMETERS struct
# which contains the command line and the image
# name of the process
pp = peb.contents.process_parameters
pp_buff = read_process_memory(
handle, pp, sizeof(RTL_USER_PROCESS_PARAMETERS))
if pp_buff:
pp = cast(pp_buff,
POINTER(RTL_USER_PROCESS_PARAMETERS))
comm = pp.contents.command_line.buffer
comm_len = pp.contents.command_line.length
exe = pp.contents.image_path_name.buffer
exe_len = pp.contents.image_path_name.length
# these memory reads are required
# to copy the command line and image name buffers
cb = read_process_memory(handle, comm, comm_len)
eb = read_process_memory(handle, exe, exe_len)
if cb and eb:
# cast the buffers to
# UNICODE strings
comm = cast(cb, c_wchar_p).value
exe = cast(eb, c_wchar_p).value
# the image name contains the full path
# split the string to get the exec name
name = exe[exe.rfind('\\') + 1:]
info = ddict(name=name, comm=comm, parent_pid=ppid)
free(cb)
free(eb)
free(pp_buff)
free(peb_buff)
else:
# query only the process image file name
exe = ctypes.create_unicode_buffer(MAX_PATH)
size = DWORD(MAX_PATH)
name = None
status = query_full_process_image_name(handle, 0, exe,
byref(size))
if status:
exe = exe.value
name = exe[exe.rfind('\\') + 1:]
info = ddict(name=name if name else NA,
comm=exe,
parent_pid=ppid)
if pbi_buff:
free(pbi_buff)
return info
def _query_process_info(self, handle, read_peb=True):
"""Gets an extended proc info.
Parameters
-----------
handle: HANDLE
handle to process for which the info
should be acquired
read_peb: boolean
true in case the process PEB should be read
"""
pbi_buff = malloc(sizeof(PROCESS_BASIC_INFORMATION))
status = zw_query_information_process(handle,
PROCESS_BASIC_INFO,
pbi_buff,
sizeof(PROCESS_BASIC_INFORMATION),
byref(ULONG()))
info = {}
if status == STATUS_SUCCESS:
pbi = cast(pbi_buff, POINTER(PROCESS_BASIC_INFORMATION))
ppid = pbi.contents.inherited_from_unique_process_id
if read_peb:
# read the PEB to get the process parameters.
# Because the PEB structure resides
# in the address space of another process
# we must read the memory block in order
# to access the structure's fields
peb_addr = pbi.contents.peb_base_address
peb_buff = read_process_memory(handle, peb_addr, sizeof(PEB))
if peb_buff:
peb = cast(peb_buff, POINTER(PEB))
# read the RTL_USER_PROCESS_PARAMETERS struct
# which contains the command line and the image
# name of the process
pp = peb.contents.process_parameters
pp_buff = read_process_memory(handle,
pp,
sizeof(RTL_USER_PROCESS_PARAMETERS))
if pp_buff:
pp = cast(pp_buff, POINTER(RTL_USER_PROCESS_PARAMETERS))
comm = pp.contents.command_line.buffer
comm_len = pp.contents.command_line.length
exe = pp.contents.image_path_name.buffer
exe_len = pp.contents.image_path_name.length
# these memory reads are required
# to copy the command line and image name buffers
cb = read_process_memory(handle, comm, comm_len)
eb = read_process_memory(handle, exe, exe_len)
if cb and eb:
# cast the buffers to
# UNICODE strings
comm = cast(cb, c_wchar_p).value
exe = cast(eb, c_wchar_p).value
# the image name contains the full path
# split the string to get the exec name
name = exe[exe.rfind('\\') + 1:]
info = ddict(name=name,
comm=comm,
parent_pid=ppid)
free(cb)
free(eb)
free(pp_buff)
free(peb_buff)
else:
# query only the process image file name
exe = ctypes.create_unicode_buffer(MAX_PATH)
size = DWORD(MAX_PATH)
name = None
status = query_full_process_image_name(handle,
0,
exe,
byref(size))
if status:
exe = exe.value
name = exe[exe.rfind('\\') + 1:]
info = ddict(name=name if name else NA,
comm=exe if type(exe) is str else None,
parent_pid=ppid)
if pbi_buff:
free(pbi_buff)
return info
def add_thread(self, ketype, kti):
"""Adds a new process or thread to thread registry.
Parameters
----------
ketype: tuple
kernel event type
kti: dict
event payload as coming from the
kernel event stream collector
"""
if ketype == CREATE_PROCESS or ketype == ENUM_PROCESS:
parent_pid = int(kti.parent_id, 16)
process_id = int(kti.process_id, 16)
# we assume the process id is
# equal to thread id (in a single
# threaded process)
thread_id = process_id
name = kti.image_file_name
comm = kti.command_line
suid = kti.user_sid
thread = ThreadInfo(process_id, thread_id, parent_pid, name, comm,
suid)
if ketype == ENUM_PROCESS:
thread.handles = [
handle for handle in self._handles
if handle.pid == process_id
]
else:
thread.handles = self.handle_repository.query_handles(
process_id)
self._threads[process_id] = thread
elif ketype == CREATE_THREAD or ketype == ENUM_THREAD:
# new thread created in the
# context of the existing process
# `procces_id` is the parent
# of this thread
process_id = int(kti.process_id, 16)
parent_pid = process_id
thread_id = int(kti.t_thread_id, 16)
if parent_pid in self._threads:
# copy info from the process
# which created this thread
pthread = self._threads[parent_pid]
# increment the number of threads
# for this process
pthread.increment_child_count()
name = pthread.name
comm = pthread.comm
suid = pthread.suid
thread = ThreadInfo(process_id, thread_id, parent_pid, name,
comm, suid)
thread.ustack_base = hex(kti.user_stack_base)
thread.kstack_base = hex(kti.stack_base)
thread.base_priority = kti.base_priority
thread.io_priority = kti.io_priority
self._threads[thread_id] = thread
else:
# the parent process has not been found
# query the os for process information
handle = open_process(
PROCESS_QUERY_INFORMATION
| PROCESS_QUERY_LIMITED_INFORMATION | PROCESS_VM_READ,
False, parent_pid)
info = {}
if handle:
info = self._query_process_info(handle)
close_handle(handle)
else:
if get_last_error() == ERROR_ACCESS_DENIED:
if parent_pid == 0:
info = ddict(name='idle', comm='idle', parent_id=0)
else:
# the access to protected / system process
# can't be done with PROCESS_VM_READ or PROCESS_QUERY_INFORMATION
# flags. Open the process again but with
# restricted access rights, so we can get the process image file name
handle = open_process(
PROCESS_QUERY_LIMITED_INFORMATION, False,
parent_pid)
if handle:
info = self._query_process_info(handle, False)
close_handle(handle)
# add a new thread and the parent process
# we just found to avoid continuous lookup
name = info.name if len(info) > 0 and info.name else NA
comm = info.comm if len(info) > 0 and info.comm else NA
ppid = info.parent_pid if len(
info) > 0 and info.parent_pid else NA
thread = ThreadInfo(process_id, thread_id, process_id, name,
comm, None)
thread.ustack_base = hex(kti.user_stack_base)
thread.kstack_base = hex(kti.stack_base)
thread.base_priority = kti.base_priority
thread.io_priority = kti.io_priority
parent = ThreadInfo(process_id, process_id, ppid, name, comm,
None)
# enumerate parent handles
parent.handles = self.handle_repository.query_handles(
process_id)
self._threads[thread_id] = thread
self._threads[parent_pid] = parent
if self.on_thread_added_callback and callable(
self.on_thread_added_callback):
self.on_thread_added_callback(thread)
def _on_next_kevent(self, ktype, cpuid, ts, kparams):
"""Callback which fires when new kernel event arrives.
This callback is invoked for every new kernel event
forwarded from the kernel stream collector.
Parameters
----------
ktype: tuple
Kernel event type.
cpuid: int
Indentifies the CPU core where the event
has been captured.
ts: str
Temporal reference of the kernel event.
kparams: dict
Kernel event's parameters.
"""
# initialize kernel event properties
self.kevent.ts = ts
self.kevent.cpuid = cpuid
self.kevent.name = ktuple_to_name(ktype)
kparams = ddict(kparams)
# thread / process kernel events
if ktype in [CREATE_PROCESS, CREATE_THREAD, ENUM_PROCESS, ENUM_THREAD]:
self.thread_registry.add_thread(ktype, kparams)
if ktype in [CREATE_PROCESS, CREATE_THREAD]:
self.thread_registry.init_thread_kevent(
self.kevent, ktype, kparams)
# apply yara binding by matching against the process's image path
if ktype == CREATE_PROCESS:
yara_binding = self.__find_binding('yara')
pid = int(kparams.process_id, 16)
thread = self.thread_registry.get_thread(pid)
if thread and yara_binding:
yara_binding.run(thread_info=thread,
kevent=self.kevent)
self._aggregate(ktype)
elif ktype in [TERMINATE_PROCESS, TERMINATE_THREAD]:
self.thread_registry.init_thread_kevent(self.kevent, ktype,
kparams)
self._aggregate(ktype)
self.thread_registry.remove_thread(ktype, kparams)
# file system/disk kernel events
elif ktype in [
CREATE_FILE, DELETE_FILE, CLOSE_FILE, READ_FILE, WRITE_FILE,
RENAME_FILE, SET_FILE_INFORMATION
]:
self.fsio.parse_fsio(ktype, kparams)
self._aggregate(ktype)
# dll kernel events
elif ktype in [LOAD_IMAGE, ENUM_IMAGE]:
self.dll_repository.register_dll(kparams)
if ktype == LOAD_IMAGE:
self._aggregate(ktype)
elif ktype == UNLOAD_IMAGE:
self.dll_repository.unregister_dll(kparams)
self._aggregate(ktype)
#
# # registry kernel events
elif ktype == REG_CREATE_KCB:
self.hive_parser.add_kcb(kparams)
elif ktype == REG_DELETE_KCB:
self.hive_parser.remove_kcb(kparams.key_handle)
elif ktype in [
REG_CREATE_KEY, REG_DELETE_KEY, REG_OPEN_KEY, REG_QUERY_KEY,
REG_SET_VALUE, REG_DELETE_VALUE, REG_QUERY_VALUE
]:
self.hive_parser.parse_hive(ktype, kparams)
self._aggregate(ktype)
# network kernel events
elif ktype in [
SEND_SOCKET_TCPV4, SEND_SOCKET_UDPV4, RECV_SOCKET_TCPV4,
RECV_SOCKET_UDPV4, ACCEPT_SOCKET_TCPV4, CONNECT_SOCKET_TCPV4,
DISCONNECT_SOCKET_TCPV4, RECONNECT_SOCKET_TCPV4
]:
self.tcpip_parser.parse_tcpip(ktype, kparams)
self._aggregate(ktype)
# context switch events
elif ktype == CONTEXT_SWITCH:
self.context_switch_registry.next_cswitch(cpuid, ts, kparams)
self._aggregate(ktype)
if self._filament:
if ktype not in [
ENUM_PROCESS, ENUM_THREAD, ENUM_IMAGE, REG_CREATE_KCB,
REG_DELETE_KCB
]:
ok = self.output_kevents[ktype] if ktype in self.output_kevents \
else False
if self.kevent.name and ok:
thread = self.kevent.thread
kevent = {
'params': self.kevent.params,
'name': self.kevent.name,
'pid': self.kevent.pid,
'tid': self.kevent.tid,
'timestamp': self.kevent.ts,
'cpuid': self.kevent.cpuid,
'category': self.kevent.category
}
if thread:
kevent.update({
'thread': {
'name': thread.name,
'exe': thread.exe,
'comm': thread.comm,
'pid': thread.pid,
'ppid': thread.ppid
}
})
self._filament.on_next_kevent(kevent)
def _enum_handles(self, process_id=None):
"""Enumerates handle information.
Enumerates handle info on
the start of the kernel capture.
Returns a dictionary of handle's
information including the handle id,
access mask, and the process which owns
the handle.
"""
buff_size = MAX_BUFFER_SIZE
size = c_ulong()
# allocate the initial buffer
buff = malloc(buff_size)
handles = {}
while True:
status = zw_query_system_information(
SYSTEM_HANDLE_INFORMATION_CLASS, buff, buff_size, byref(size))
if status == STATUS_INFO_LENGTH_MISMATCH:
# the buffer is too small
# increment the buffer size and try again
buff_size += MAX_BUFFER_SIZE
elif status == STATUS_SUCCESS:
# cast the buffer to `SYSTEM_HANDLE_INFORMATION` struct
# which contains an array of `SYSTEM_HANDLE` structures
sys_handle_info = cast(buff,
POINTER(SYSTEM_HANDLE_INFORMATION))
sys_handle_info = sys_handle_info.contents
handle_count = sys_handle_info.number_of_handles
# resize the array size to the
# actual number of file handles
sys_handles = (SYSTEM_HANDLE * buff_size).from_address(
addressof(sys_handle_info.handles))
for i in range(handle_count):
sys_handle = sys_handles[i]
pid = sys_handle.process_id
handle = sys_handle.handle
obj = sys_handle.object
obj_type_index = sys_handle.object_type_number
access_mask = sys_handle.access_mask
if process_id and process_id == pid:
handles[obj] = ddict(pid=process_id,
handle=handle,
obj=obj,
access_mask=access_mask,
obj_type_index=obj_type_index)
elif process_id is None:
handles[obj] = ddict(pid=pid,
handle=handle,
obj=obj,
access_mask=access_mask,
obj_type_index=obj_type_index)
break
else:
raise HandleEnumError(status)
# reallocate the buffer
buff = realloc(buff, buff_size)
# free the buffer memory
free(buff)
return handles
def on_next_kevent(self, kevent):
try:
self._filament_module.on_next_kevent(ddict(kevent))
except Exception as e:
self._logger.error('Unexpected filament error %s' % e)
def _on_next_kevent(self, ktype, cpuid, ts, kparams):
"""Callback which fires when new kernel event arrives.
This callback is invoked for every new kernel event
forwarded from the kernel stream collector.
Parameters
----------
ktype: tuple
Kernel event type.
cpuid: int
Indentifies the CPU core where the event
has been captured.
ts: str
Temporal reference of the kernel event.
kparams: dict
Kernel event's parameters.
"""
# initialize kernel event properties
self.kevent.ts = ts
self.kevent.cpuid = cpuid
self.kevent.name = ktuple_to_name(ktype)
kparams = ddict(kparams)
# thread / process kernel events
if ktype in [CREATE_PROCESS,
CREATE_THREAD,
ENUM_PROCESS,
ENUM_THREAD]:
self.thread_registry.add_thread(ktype, kparams)
if ktype in [CREATE_PROCESS, CREATE_THREAD]:
self.thread_registry.init_thread_kevent(self.kevent,
ktype,
kparams)
# apply yara binding by matching against the process's image path
if ktype == CREATE_PROCESS:
yara_binding = self.__find_binding('yara')
pid = int(kparams.process_id, 16)
thread = self.thread_registry.get_thread(pid)
if thread and yara_binding:
yara_binding.run(thread_info=thread,
kevent=self.kevent)
self._aggregate(ktype)
elif ktype in [TERMINATE_PROCESS, TERMINATE_THREAD]:
self.thread_registry.init_thread_kevent(self.kevent,
ktype,
kparams)
self._aggregate(ktype)
self.thread_registry.remove_thread(ktype, kparams)
# file system/disk kernel events
elif ktype in [CREATE_FILE,
DELETE_FILE,
CLOSE_FILE,
READ_FILE,
WRITE_FILE,
RENAME_FILE,
SET_FILE_INFORMATION]:
self.fsio.parse_fsio(ktype, kparams)
self._aggregate(ktype)
# dll kernel events
elif ktype in [LOAD_IMAGE, ENUM_IMAGE]:
self.dll_repository.register_dll(kparams)
if ktype == LOAD_IMAGE:
self._aggregate(ktype)
elif ktype == UNLOAD_IMAGE:
self.dll_repository.unregister_dll(kparams)
self._aggregate(ktype)
#
# # registry kernel events
elif ktype == REG_CREATE_KCB:
self.hive_parser.add_kcb(kparams)
elif ktype == REG_DELETE_KCB:
self.hive_parser.remove_kcb(kparams.key_handle)
elif ktype in [REG_CREATE_KEY,
REG_DELETE_KEY,
REG_OPEN_KEY,
REG_QUERY_KEY,
REG_SET_VALUE,
REG_DELETE_VALUE,
REG_QUERY_VALUE]:
self.hive_parser.parse_hive(ktype, kparams)
self._aggregate(ktype)
# network kernel events
elif ktype in [SEND_SOCKET_TCPV4,
SEND_SOCKET_UDPV4,
RECV_SOCKET_TCPV4,
RECV_SOCKET_UDPV4,
ACCEPT_SOCKET_TCPV4,
CONNECT_SOCKET_TCPV4,
DISCONNECT_SOCKET_TCPV4,
RECONNECT_SOCKET_TCPV4]:
self.tcpip_parser.parse_tcpip(ktype, kparams)
self._aggregate(ktype)
# context switch events
elif ktype == CONTEXT_SWITCH:
self.context_switch_registry.next_cswitch(cpuid, ts, kparams)
self._aggregate(ktype)
if self._filament:
if ktype not in [ENUM_PROCESS,
ENUM_THREAD,
ENUM_IMAGE,
REG_CREATE_KCB,
REG_DELETE_KCB]:
ok = self.output_kevents[ktype] if ktype in self.output_kevents \
else False
if self.kevent.name and ok:
thread = self.kevent.thread
kevent = {
'params': self.kevent.params,
'name': self.kevent.name,
'pid': self.kevent.pid,
'tid': self.kevent.tid,
'timestamp': self.kevent.ts,
'cpuid': self.kevent.cpuid,
'category': self.kevent.category
}
if thread:
kevent.update({
'thread': {
'name': thread.name,
'exe': thread.exe,
'comm': thread.comm,
'pid': thread.pid,
'ppid': thread.ppid
}
})
self._filament.on_next_kevent(kevent)
