def _getMapInfo(record, info, event_property):
"""
When parsing a field in the event property structure, there may be a mapping between a given
name and the structure it represents. If it exists, we retrieve that mapping here.
Because this may legitimately return a NULL value we return a tuple containing the success or
failure status as well as either None (NULL) or an EVENT_MAP_INFO pointer.
:param record: The EventRecord structure for the event we are parsing
:param info: The TraceEventInfo structure for the event we are parsing
:param event_property: The EVENT_PROPERTY_INFO structure for the TopLevelProperty of the event we are parsing
:return: A tuple of the map_info structure and boolean indicating whether we succeeded or not
"""
map_name = rel_ptr_to_str(
info, event_property.epi_u1.nonStructType.MapNameOffset)
map_size = wt.DWORD()
map_info = ct.POINTER(tdh.EVENT_MAP_INFO)()
status = tdh.TdhGetEventMapInformation(record, map_name, None,
ct.byref(map_size))
if tdh.ERROR_INSUFFICIENT_BUFFER == status:
map_info = ct.cast((ct.c_char * map_size.value)(),
ct.POINTER(tdh.EVENT_MAP_INFO))
status = tdh.TdhGetEventMapInformation(record, map_name, map_info,
ct.byref(map_size))
if tdh.ERROR_SUCCESS == status:
return map_info, True
# ERROR_NOT_FOUND is actually a perfectly acceptable status
if tdh.ERROR_NOT_FOUND == status:
return None, True
# We actually failed.
raise ct.WinError()
def get_keywords_bitmask(guid, keywords):
"""
Queries available keywords of the provider and returns a bitmask of the associated values
:param guid: The GUID of the ETW provider.
:param keywords: List of keywords to resolve.
:return Bitmask of the keyword flags ORed together
"""
bitmask = 0
if keywords is None or len(keywords) == 0:
return bitmask
# enumerate the keywords for the provider as well as the bitmask values
provider_info = None
providers_size = wt.ULONG(0)
status = tdh.TdhEnumerateProviderFieldInformation(
ct.byref(guid),
tdh.EventKeywordInformation,
provider_info,
ct.byref(providers_size))
if status == tdh.ERROR_INSUFFICIENT_BUFFER:
provider_info = ct.cast((ct.c_char * providers_size.value)(), ct.POINTER(tdh.PROVIDER_FIELD_INFOARRAY))
status = tdh.TdhEnumerateProviderFieldInformation(
ct.byref(guid),
tdh.EventKeywordInformation,
provider_info,
ct.byref(providers_size))
if tdh.ERROR_SUCCESS != status and tdh.ERROR_NOT_FOUND != status:
raise ct.WinError(status)
if provider_info:
field_info_array = ct.cast(provider_info.contents.FieldInfoArray, ct.POINTER(tdh.PROVIDER_FIELD_INFO))
provider_keywords = {}
for i in range(provider_info.contents.NumberOfElements):
provider_keyword = rel_ptr_to_str(provider_info, field_info_array[i].NameOffset)
provider_keywords[provider_keyword] = field_info_array[i].Value
for keyword in keywords:
if keyword in provider_keywords:
bitmask |= provider_keywords[keyword]
return bitmask
def _processEvent(self, record):
"""
This is a callback function that fires whenever an event needs handling. It iterates through the structure to
parse the properties of each event. If a user defined callback is specified it then passes the parsed data to
it.
:param record: The EventRecord structure for the event we are parsing
:return: Nothing
"""
info = self._getEventInformation(record)
if info is None:
return
# Some events do not have an associated task_name value. In this case, we should use the provider name instead.
if info.contents.TaskNameOffset == 0:
task_name = rel_ptr_to_str(info, info.contents.ProviderNameOffset)
else:
task_name = rel_ptr_to_str(info, info.contents.TaskNameOffset)
task_name = task_name.strip().upper()
# Add a description for the event
description = rel_ptr_to_str(info, info.contents.EventMessageOffset)
# Add the EventID
event_id = info.contents.EventDescriptor.Id
# Windows 7 does not support predicate filters. Instead, we use a whitelist to filter things on the consumer.
if self.task_name_filters and task_name not in self.task_name_filters:
return
# add all header fields from EVENT_HEADER structure
# https://msdn.microsoft.com/en-us/library/windows/desktop/aa363759(v=vs.85).aspx
out = {
'EventHeader': {
'Size': record.contents.EventHeader.Size,
'HeaderType': record.contents.EventHeader.HeaderType,
'Flags': record.contents.EventHeader.Flags,
'EventProperty': record.contents.EventHeader.EventProperty,
'ThreadId': record.contents.EventHeader.ThreadId,
'ProcessId': record.contents.EventHeader.ProcessId,
'TimeStamp': record.contents.EventHeader.TimeStamp,
'ProviderId': str(record.contents.EventHeader.ProviderId),
'EventDescriptor': {
'Id': record.contents.EventHeader.EventDescriptor.Id,
'Version':
record.contents.EventHeader.EventDescriptor.Version,
'Channel':
record.contents.EventHeader.EventDescriptor.Channel,
'Level': record.contents.EventHeader.EventDescriptor.Level,
'Opcode':
record.contents.EventHeader.EventDescriptor.Opcode,
'Task': record.contents.EventHeader.EventDescriptor.Task,
'Keyword':
record.contents.EventHeader.EventDescriptor.Keyword
},
'KernelTime': record.contents.EventHeader.KernelTime,
'UserTime': record.contents.EventHeader.UserTime,
'ActivityId': str(record.contents.EventHeader.ActivityId)
}
}
user_data = record.contents.UserData
if user_data is None:
user_data = 0
end_of_user_data = user_data + record.contents.UserDataLength
self.index = 0
self.vfield_length = None
property_array = ct.cast(info.contents.EventPropertyInfoArray,
ct.POINTER(tdh.EVENT_PROPERTY_INFO))
for i in range(info.contents.TopLevelPropertyCount):
# If the user_data is the same value as the end_of_user_data, we are ending with a 0-length
# field. Though not documented, this is completely valid.
if user_data == end_of_user_data:
break
# Determine whether we are processing a simple type or a complex type and act accordingly
if property_array[i].Flags & tdh.PropertyStruct:
out.update(
self._unpackComplexType(record, info, property_array[i]))
continue
out.update(self._unpackSimpleType(record, info, property_array[i]))
# Add the description field in
out['Description'] = description
out['Task Name'] = task_name
# Call the user's specified callback function
if self.event_callback:
self.event_callback((event_id, out))
return
def _unpackSimpleType(self, record, info, event_property):
"""
This method handles dumping all simple types of data (i.e., non-struct types).
:param record: The EventRecord structure for the event we are parsing
:param info: The TraceEventInfo structure for the event we are parsing
:param event_property: The EVENT_PROPERTY_INFO structure for the TopLevelProperty of the event we are parsing
:return: Returns a key-value pair as a dictionary. If we fail, the dictionary is {}
"""
# Get the EVENT_MAP_INFO, if it is present.
map_info, success = self._getMapInfo(record, info, event_property)
if not success:
return {}
# Get the length of the value of the property we are dealing with.
property_length = self._getPropertyLength(record, info, event_property)
if property_length is None:
return {}
# The version of the Python interpreter may be different than the system architecture.
if record.contents.EventHeader.Flags & ec.EVENT_HEADER_FLAG_32_BIT_HEADER:
ptr_size = 4
else:
ptr_size = 8
name_field = rel_ptr_to_str(info, event_property.NameOffset)
if property_length == 0 and self.vfield_length is not None:
if self.vfield_length == 0:
self.vfield_length = None
return {name_field: None}
# If vfield_length isn't 0, we should be able to parse the property.
property_length = self.vfield_length
# After calling the TdhFormatProperty function, use the UserDataConsumed parameter value to set the new values
# of the UserData and UserDataLength parameters (Subtract UserDataConsumed from UserDataLength and use
# UserDataLength to increment the UserData pointer).
# All of the variables needed to actually use TdhFormatProperty retrieve the value
user_data = record.contents.UserData + self.index
user_data_remaining = record.contents.UserDataLength - self.index
# if there is no data remaining then return
if user_data_remaining <= 0:
return {}
in_type = event_property.epi_u1.nonStructType.InType
out_type = event_property.epi_u1.nonStructType.OutType
formatted_data_size = wt.DWORD()
formatted_data = wt.LPWSTR()
user_data_consumed = ct.c_ushort()
status = tdh.TdhFormatProperty(
info, map_info, ptr_size, in_type, out_type,
ct.c_ushort(property_length), user_data_remaining,
ct.cast(user_data, ct.POINTER(ct.c_byte)),
ct.byref(formatted_data_size), None, ct.byref(user_data_consumed))
if status == tdh.ERROR_INSUFFICIENT_BUFFER:
formatted_data = ct.cast((ct.c_char * formatted_data_size.value)(),
wt.LPWSTR)
status = tdh.TdhFormatProperty(
info, map_info, ptr_size, in_type, out_type,
ct.c_ushort(property_length), user_data_remaining,
ct.cast(user_data, ct.POINTER(ct.c_byte)),
ct.byref(formatted_data_size), formatted_data,
ct.byref(user_data_consumed))
if status != tdh.ERROR_SUCCESS:
if status != tdh.ERROR_EVT_INVALID_EVENT_DATA:
raise ct.WinError(status)
# We can handle this error and still capture the data.
user_data_consumed, formatted_data = self._handleEvtInvalidEvtData(
user_data, user_data_remaining)
# Increment where we are in the user data segment that we are parsing.
self.index += user_data_consumed.value
if name_field.lower().endswith('length'):
try:
self.vfield_length = int(formatted_data.value, 10)
except ValueError:
logger.warning('Setting vfield_length to None')
self.vfield_length = None
data = formatted_data.value
# Convert the formatted data if necessary
if out_type in tdh.TDH_CONVERTER_LOOKUP:
data = tdh.TDH_CONVERTER_LOOKUP[out_type](data)
return {name_field: data}
def _processEvent(self, record):
"""
This is a callback function that fires whenever an event needs handling. It iterates through the structure to
parse the properties of each event. If a user defined callback is specified it then passes the parsed data to
it.
:param record: The EventRecord structure for the event we are parsing
:return: Nothing
"""
parsed_data = {}
field_parse_error = False
if self.callback_data_flag == RETURN_RAW_UNFORMATTED_DATA:
event_id = 0
out = record
else:
event_id = record.contents.EventHeader.EventDescriptor.Id
if self.event_id_filters and event_id not in self.event_id_filters:
return
# set task name to provider guid for the time being
task_name = str(record.contents.EventHeader.ProviderId)
# add all header fields from EVENT_HEADER structure
# https://msdn.microsoft.com/en-us/library/windows/desktop/aa363759(v=vs.85).aspx
out = {'EventHeader': {
'Size': record.contents.EventHeader.Size,
'HeaderType': record.contents.EventHeader.HeaderType,
'Flags': record.contents.EventHeader.Flags,
'EventProperty': record.contents.EventHeader.EventProperty,
'ThreadId': record.contents.EventHeader.ThreadId,
'ProcessId': record.contents.EventHeader.ProcessId,
'TimeStamp': record.contents.EventHeader.TimeStamp,
'ProviderId': task_name,
'EventDescriptor': {'Id': event_id,
'Version': record.contents.EventHeader.EventDescriptor.Version,
'Channel': record.contents.EventHeader.EventDescriptor.Channel,
'Level': record.contents.EventHeader.EventDescriptor.Level,
'Opcode': record.contents.EventHeader.EventDescriptor.Opcode,
'Task': record.contents.EventHeader.EventDescriptor.Task,
'Keyword':
record.contents.EventHeader.EventDescriptor.Keyword},
'KernelTime': record.contents.EventHeader.KernelTime,
'UserTime': record.contents.EventHeader.UserTime,
'ActivityId': str(record.contents.EventHeader.ActivityId)},
'Task Name': task_name}
if self.callback_data_flag != RETURN_RAW_DATA_ONLY:
try:
info = self._getEventInformation(record)
# Some events do not have an associated task_name value. In this case, we should use the provider
# name instead.
if info.contents.TaskNameOffset == 0:
task_name = rel_ptr_to_str(info, info.contents.ProviderNameOffset)
else:
task_name = rel_ptr_to_str(info, info.contents.TaskNameOffset)
task_name = task_name.strip().upper()
# Add a description for the event, if present
if info.contents.EventMessageOffset:
description = rel_ptr_to_str(info, info.contents.EventMessageOffset)
else:
description = ''
# Windows 7 does not support predicate filters. Instead, we use a whitelist to filter things on the
# consumer.
if self.task_name_filters and task_name not in self.task_name_filters:
return
user_data = record.contents.UserData
if user_data is None:
user_data = 0
end_of_user_data = user_data + record.contents.UserDataLength
self.index = 0
self.vfield_length = None
property_array = ct.cast(info.contents.EventPropertyInfoArray, ct.POINTER(tdh.EVENT_PROPERTY_INFO))
for i in range(info.contents.TopLevelPropertyCount):
# If the user_data is the same value as the end_of_user_data, we are ending with a 0-length
# field. Though not documented, this is completely valid.
if user_data == end_of_user_data:
break
# Determine whether we are processing a simple type or a complex type and act accordingly
if property_array[i].Flags & tdh.PropertyStruct:
field = self._unpackComplexType(record, info, property_array[i])
else:
field = self._unpackSimpleType(record, info, property_array[i])
if field == {} or None in field.values():
field_parse_error = True
parsed_data.update(field)
# Add the description field in
parsed_data['Description'] = description
parsed_data['Task Name'] = task_name
except Exception as e:
logger.warning('Unable to parse event: {}'.format(e))
try:
if self.callback_data_flag == RETURN_RAW_DATA_ONLY or \
((self.callback_data_flag == RETURN_RAW_DATA_ON_ERROR or
self.callback_data_flag == RETURN_ONLY_RAW_DATA_ON_ERROR) and field_parse_error is True):
out['UserData'] = b''.join([ct.cast(record.contents.UserData + i, wt.PBYTE).contents
for i in range(record.contents.UserDataLength)])
if (self.callback_data_flag == RETURN_ONLY_RAW_DATA_ON_ERROR and field_parse_error is False) or \
self.callback_data_flag == RETURN_RAW_DATA_ON_ERROR or self.callback_data_flag == 0:
out.update(parsed_data)
# Call the user's specified callback function
if self.event_callback:
self.event_callback((event_id, out))
except Exception as e:
logger.error('Exception during callback: {}'.format(e))
logger.error(traceback.format_exc())
return