def _AddEventData(self, event_data, serialized_data=None):
"""Adds event data.
Args:
event_data (EventData): event data.
serialized_data (bytes): serialized form of the event data.
"""
row_identifier = getattr(event_data,
'_event_data_stream_row_identifier', None)
if row_identifier is not None:
event_data_stream_identifier = identifiers.SQLTableIdentifier(
self._CONTAINER_TYPE_EVENT_DATA_STREAM, row_identifier)
lookup_key = event_data_stream_identifier.CopyToString()
event_data_stream_identifier = (
self._event_data_stream_identifier_mappings.get(
lookup_key, None))
if event_data_stream_identifier:
event_data.SetEventDataStreamIdentifier(
event_data_stream_identifier)
elif lookup_key in self._deserialization_errors:
event_data_identifier = event_data.GetIdentifier()
event_data_identifier = event_data_identifier.CopyToString()
# TODO: store this as an extraction warning so this is preserved
# in the storage file.
logger.error((
'Unable to merge event data attribute container: {0:s} since '
'corresponding event data stream: {1:s} could not be '
'deserialized.').format(event_data_identifier, lookup_key))
return
identifier = event_data.GetIdentifier()
lookup_key = identifier.CopyToString()
self._storage_writer.AddEventData(event_data,
serialized_data=serialized_data)
last_write_identifier = event_data.GetIdentifier()
self._event_data_identifier_mappings[
lookup_key] = last_write_identifier
def _AddEvent(self, event, serialized_data=None):
"""Adds an event.
Args:
event (EventObject): event.
serialized_data (Optional[bytes]): serialized form of the event.
"""
row_identifier = getattr(event, '_event_data_row_identifier', None)
# TODO: error if row_identifier is None
if row_identifier is not None:
event_data_identifier = identifiers.SQLTableIdentifier(
self._CONTAINER_TYPE_EVENT_DATA, row_identifier)
lookup_key = event_data_identifier.CopyToString()
event_data_identifier = self._event_data_identifier_mappings.get(
lookup_key, None)
if not event_data_identifier:
event_identifier = event.GetIdentifier()
event_identifier = event_identifier.CopyToString()
if lookup_key in self._deserialization_errors:
reason = 'deserialized'
else:
reason = 'found'
# TODO: store this as an extraction warning so this is preserved
# in the storage file.
logger.error(
('Unable to merge event attribute container: {0:s} since '
'corresponding event data: {1:s} could not be {2:s}.'
).format(event_identifier, lookup_key, reason))
return
event.SetEventDataIdentifier(event_data_identifier)
# TODO: add event identifier mappings for event tags.
self._storage_writer.AddEvent(event)
def AddAttributeContainer(self, container):
"""Adds an attribute container.
Args:
container (AttributeContainer): attribute container.
"""
if container.CONTAINER_TYPE == self._CONTAINER_TYPE_EVENT:
event_data_identifier = container.GetEventDataIdentifier()
event_data_lookup_key = event_data_identifier.CopyToString()
event_data_identifier = self._event_data_identifier_mappings.get(
event_data_lookup_key, None)
if event_data_identifier:
container.SetEventDataIdentifier(event_data_identifier)
else:
identifier = container.GetIdentifier()
identifier_string = identifier.CopyToString()
# TODO: store this as a merge warning so this is preserved
# in the storage file.
logger.error(
('Unable to merge event attribute container: {0:s} since '
'corresponding event data: {1:s} could not be found.'
).format(identifier_string, event_data_lookup_key))
return
elif container.CONTAINER_TYPE == self._CONTAINER_TYPE_EVENT_DATA:
event_data_stream_identifier = container.GetEventDataStreamIdentifier(
)
event_data_stream_lookup_key = None
if event_data_stream_identifier:
event_data_stream_lookup_key = (
event_data_stream_identifier.CopyToString())
event_data_stream_identifier = (
self._event_data_stream_identifier_mappings.get(
event_data_stream_lookup_key, None))
if event_data_stream_identifier:
container.SetEventDataStreamIdentifier(
event_data_stream_identifier)
elif event_data_stream_lookup_key:
identifier = container.GetIdentifier()
identifier_string = identifier.CopyToString()
# TODO: store this as a merge warning so this is preserved
# in the storage file.
logger.error((
'Unable to merge event data attribute container: {0:s} since '
'corresponding event data stream: {1:s} could not be '
'found.').format(identifier_string,
event_data_stream_lookup_key))
return
if container.CONTAINER_TYPE in (
self._CONTAINER_TYPE_EVENT_DATA,
self._CONTAINER_TYPE_EVENT_DATA_STREAM):
# Preserve the lookup key before adding it to the attribute container
# store.
identifier = container.GetIdentifier()
lookup_key = identifier.CopyToString()
if container.CONTAINER_TYPE == self._CONTAINER_TYPE_EVENT_TAG:
self._storage_writer.AddOrUpdateEventTag(container)
else:
self._storage_writer.AddAttributeContainer(container)
if container.CONTAINER_TYPE == self._CONTAINER_TYPE_EVENT:
parser_name = self._event_data_parser_mappings.get(
event_data_lookup_key, 'N/A')
self._session.parsers_counter[parser_name] += 1
self._session.parsers_counter['total'] += 1
elif container.CONTAINER_TYPE == self._CONTAINER_TYPE_EVENT_DATA:
identifier = container.GetIdentifier()
self._event_data_identifier_mappings[lookup_key] = identifier
parser_name = container.parser.split('/')[-1]
self._event_data_parser_mappings[lookup_key] = parser_name
elif container.CONTAINER_TYPE == self._CONTAINER_TYPE_EVENT_DATA_STREAM:
identifier = container.GetIdentifier()
self._event_data_stream_identifier_mappings[
lookup_key] = identifier
def MergeAttributeContainers(self,
callback=None,
maximum_number_of_containers=0):
"""Reads attribute containers from a task storage file into the writer.
Args:
callback (function[StorageWriter, AttributeContainer]): function to call
after each attribute container is deserialized.
maximum_number_of_containers (Optional[int]): maximum number of
containers to merge, where 0 represent no limit.
Returns:
bool: True if the entire task storage file has been merged.
Raises:
RuntimeError: if the add method for the active attribute container
type is missing.
OSError: if the task storage file cannot be deleted.
ValueError: if the maximum number of containers is a negative value.
"""
if maximum_number_of_containers < 0:
raise ValueError('Invalid maximum number of containers')
if not self._cursor:
self._Open()
self._ReadStorageMetadata()
self._container_types = self._GetContainerTypes()
self._deserialization_errors = []
number_of_containers = 0
while self._active_cursor or self._container_types:
if not self._active_cursor:
self._PrepareForNextContainerType()
if maximum_number_of_containers == 0:
rows = self._active_cursor.fetchall()
else:
number_of_rows = maximum_number_of_containers - number_of_containers
rows = self._active_cursor.fetchmany(size=number_of_rows)
if not rows:
self._active_cursor = None
continue
for row in rows:
identifier = identifiers.SQLTableIdentifier(
self._active_container_type, row[0])
if self._compression_format == definitions.COMPRESSION_FORMAT_ZLIB:
serialized_data = zlib.decompress(row[1])
else:
serialized_data = row[1]
try:
attribute_container = self._DeserializeAttributeContainer(
self._active_container_type, serialized_data)
except IOError as exception:
# TODO: store this as an extraction warning so this is preserved
# in the storage file.
logger.error((
'Unable to deserialize attribute container with error: '
'{0!s}').format(exception))
identifier = identifier.CopyToString()
self._deserialization_errors.append(identifier)
continue
attribute_container.SetIdentifier(identifier)
if self._active_container_type == self._CONTAINER_TYPE_EVENT_TAG:
event_identifier = identifiers.SQLTableIdentifier(
self._CONTAINER_TYPE_EVENT,
attribute_container.event_row_identifier)
attribute_container.SetEventIdentifier(event_identifier)
del attribute_container.event_row_identifier
if callback:
callback(self._storage_writer, attribute_container)
self._add_active_container_method(
attribute_container, serialized_data=serialized_data)
number_of_containers += 1
if (maximum_number_of_containers != 0
and number_of_containers >= maximum_number_of_containers):
return False
self._Close()
os.remove(self._path)
return True
def AddAttributeContainer(self, container):
"""Adds an attribute container.
Args:
container (AttributeContainer): attribute container.
"""
if container.CONTAINER_TYPE == self._CONTAINER_TYPE_EVENT:
event_data_identifier = container.GetEventDataIdentifier()
event_data_lookup_key = event_data_identifier.CopyToString()
event_data_identifier = self._event_data_identifier_mappings.get(
event_data_lookup_key, None)
if event_data_identifier:
container.SetEventDataIdentifier(event_data_identifier)
else:
identifier = container.GetIdentifier()
identifier_string = identifier.CopyToString()
# TODO: store this as a merge warning so this is preserved
# in the storage file.
logger.error(
('Unable to merge event attribute container: {0:s} since '
'corresponding event data: {1:s} could not be found.'
).format(identifier_string, event_data_lookup_key))
return
elif container.CONTAINER_TYPE == self._CONTAINER_TYPE_EVENT_DATA:
event_data_stream_identifier = container.GetEventDataStreamIdentifier(
)
event_data_stream_lookup_key = None
if event_data_stream_identifier:
event_data_stream_lookup_key = (
event_data_stream_identifier.CopyToString())
event_data_stream_identifier = (
self._event_data_stream_identifier_mappings.get(
event_data_stream_lookup_key, None))
if event_data_stream_identifier:
container.SetEventDataStreamIdentifier(
event_data_stream_identifier)
elif event_data_stream_lookup_key:
identifier = container.GetIdentifier()
identifier_string = identifier.CopyToString()
# TODO: store this as a merge warning so this is preserved
# in the storage file.
logger.error((
'Unable to merge event data attribute container: {0:s} since '
'corresponding event data stream: {1:s} could not be '
'found.').format(identifier_string,
event_data_stream_lookup_key))
return
elif container.CONTAINER_TYPE == 'windows_eventlog_message_string':
message_file_identifier = container.GetMessageFileIdentifier()
message_file_lookup_key = message_file_identifier.CopyToString()
message_file_identifier = self._message_file_identifier_mappings.get(
message_file_lookup_key, None)
if message_file_identifier:
container.SetMessageFileIdentifier(message_file_identifier)
else:
identifier = container.GetIdentifier()
identifier_string = identifier.CopyToString()
# TODO: store this as a merge warning so this is preserved
# in the storage file.
logger.error((
'Unable to merge Windows EventLog message string attribute '
'container: {0:s} since corresponding Windows EventLog message '
'file: {1:s} could not be found.').format(
identifier_string, message_file_lookup_key))
return
if container.CONTAINER_TYPE in (self._CONTAINER_TYPE_EVENT_DATA,
self._CONTAINER_TYPE_EVENT_DATA_STREAM,
'windows_eventlog_message_file'):
# Preserve the lookup key before adding it to the attribute container
# store.
identifier = container.GetIdentifier()
lookup_key = identifier.CopyToString()
if container.CONTAINER_TYPE == self._CONTAINER_TYPE_EVENT_TAG:
self._storage_writer.AddOrUpdateEventTag(container)
else:
self._storage_writer.AddAttributeContainer(container)
if container.CONTAINER_TYPE == self._CONTAINER_TYPE_ANALYSIS_REPORT:
self.analysis_reports_counter[container.plugin_name] += 1
self.analysis_reports_counter['total'] += 1
elif container.CONTAINER_TYPE == self._CONTAINER_TYPE_EVENT:
parser_name = self._event_data_parser_mappings.get(
event_data_lookup_key, 'N/A')
self.parsers_counter[parser_name] += 1
self.parsers_counter['total'] += 1
elif container.CONTAINER_TYPE == self._CONTAINER_TYPE_EVENT_DATA:
identifier = container.GetIdentifier()
self._event_data_identifier_mappings[lookup_key] = identifier
parser_name = container.parser.split('/')[-1]
self._event_data_parser_mappings[lookup_key] = parser_name
elif container.CONTAINER_TYPE == self._CONTAINER_TYPE_EVENT_DATA_STREAM:
identifier = container.GetIdentifier()
self._event_data_stream_identifier_mappings[
lookup_key] = identifier
elif container.CONTAINER_TYPE == self._CONTAINER_TYPE_EVENT_TAG:
for label in container.labels:
self.event_labels_counter[label] += 1
self.event_labels_counter['total'] += 1
elif container.CONTAINER_TYPE == 'windows_eventlog_message_file':
identifier = container.GetIdentifier()
self._message_file_identifier_mappings[lookup_key] = identifier