def _UpdateAttributeContainerAfterDeserialize(self, container):
"""Updates an attribute container after deserialization.
Args:
container (AttributeContainer): attribute container.
Raises:
ValueError: if an attribute container identifier is missing.
"""
if container.CONTAINER_TYPE == self._CONTAINER_TYPE_EVENT:
row_identifier = getattr(container, '_event_data_row_identifier',
None)
if row_identifier:
event_data_identifier = identifiers.RedisKeyIdentifier(
self._CONTAINER_TYPE_EVENT_DATA, row_identifier)
container.SetEventDataIdentifier(event_data_identifier)
delattr(container, '_event_data_row_identifier')
elif container.CONTAINER_TYPE == self._CONTAINER_TYPE_EVENT_DATA:
row_identifier = getattr(container,
'_event_data_stream_row_identifier', None)
if row_identifier:
event_data_stream_identifier = identifiers.RedisKeyIdentifier(
self._CONTAINER_TYPE_EVENT_DATA_STREAM, row_identifier)
container.SetEventDataStreamIdentifier(
event_data_stream_identifier)
delattr(container, '_event_data_stream_row_identifier')
def GetAttributeContainers(self, container_type, filter_expression=None):
"""Retrieves attribute containers
Args:
container_type (str): container type attribute of the container being
added.
filter_expression (Optional[str]): expression to filter the resulting
attribute containers by.
Yields:
AttributeContainer: attribute container.
"""
redis_hash_name = self._GetRedisHashName(container_type)
for redis_key, serialized_data in self._redis_client.hscan_iter(
redis_hash_name):
redis_key = redis_key.decode('utf-8')
attribute_container = self._DeserializeAttributeContainer(
container_type, serialized_data)
_, sequence_number = redis_key.split('.')
sequence_number = int(sequence_number, 10)
identifier = identifiers.RedisKeyIdentifier(
container_type, sequence_number)
attribute_container.SetIdentifier(identifier)
self._UpdateAttributeContainerAfterDeserialize(attribute_container)
# TODO: map filter expression to Redis native filter.
if attribute_container.MatchesExpression(filter_expression):
yield attribute_container
def GetSortedEvents(self, time_range=None):
"""Retrieves the events in increasing chronological order.
Args:
time_range (Optional[TimeRange]): This argument is not supported by the
Redis store.
Yields:
EventObject: event.
Raises:
RuntimeError: if a time_range argument is specified.
"""
event_index_name = self._GetRedisHashName(self._EVENT_INDEX_NAME)
if time_range:
raise RuntimeError('Not supported')
for redis_key, _ in self._redis_client.zscan_iter(event_index_name):
redis_key = redis_key.decode('utf-8')
container_type, sequence_number = redis_key.split('.')
sequence_number = int(sequence_number, 10)
identifier = identifiers.RedisKeyIdentifier(
container_type, sequence_number)
yield self.GetAttributeContainerByIdentifier(
self._CONTAINER_TYPE_EVENT, identifier)
def _WriteNewAttributeContainer(self, container):
"""Writes a new attribute container to the store.
Args:
container (AttributeContainer): attribute container.
"""
next_sequence_number = self._GetAttributeContainerNextSequenceNumber(
container.CONTAINER_TYPE)
identifier = identifiers.RedisKeyIdentifier(container.CONTAINER_TYPE,
next_sequence_number)
container.SetIdentifier(identifier)
redis_hash_name = self._GetRedisHashName(container.CONTAINER_TYPE)
redis_key = identifier.CopyToString()
self._UpdateAttributeContainerBeforeSerialize(container)
serialized_data = self._SerializeAttributeContainer(container)
self._redis_client.hsetnx(redis_hash_name, redis_key, serialized_data)
if container.CONTAINER_TYPE == self._CONTAINER_TYPE_EVENT:
index_name = self._GetRedisHashName(self._EVENT_INDEX_NAME)
self._redis_client.zincrby(index_name, container.timestamp,
redis_key)
def GetAttributeContainerByIndex(self, container_type, index):
"""Retrieves a specific attribute container.
Args:
container_type (str): attribute container type.
index (int): attribute container index.
Returns:
AttributeContainer: attribute container or None if not available.
"""
sequence_number = index + 1
redis_hash_name = self._GetRedisHashName(container_type)
redis_key = '{0:s}.{1:d}'.format(container_type, sequence_number)
serialized_data = self._redis_client.hget(redis_hash_name, redis_key)
if not serialized_data:
return None
attribute_container = self._DeserializeAttributeContainer(
container_type, serialized_data)
identifier = identifiers.RedisKeyIdentifier(container_type,
sequence_number)
attribute_container.SetIdentifier(identifier)
self._UpdateAttributeContainerAfterDeserialize(attribute_container)
return attribute_container
def GetAttributeContainers(self, container_type):
"""Retrieves attribute containers
Args:
container_type (str): container type attribute of the container being
added.
Yields:
AttributeContainer: attribute container.
"""
redis_hash_name = self._GetRedisHashName(container_type)
for redis_key, serialized_data in self._redis_client.hscan_iter(
redis_hash_name):
redis_key = redis_key.decode('utf-8')
attribute_container = self._DeserializeAttributeContainer(
container_type, serialized_data)
_, sequence_number = redis_key.split('.')
sequence_number = int(sequence_number, 10)
identifier = identifiers.RedisKeyIdentifier(
container_type, sequence_number)
attribute_container.SetIdentifier(identifier)
yield attribute_container
def GetAttributeContainerByIndex(self, container_type, index):
"""Retrieves a specific attribute container.
Args:
container_type (str): attribute container type.
index (int): attribute container index.
Returns:
AttributeContainer: attribute container or None if not available.
Raises:
IOError: if the attribute container type is not supported.
OSError: if the attribute container type is not supported.
"""
if container_type not in self._CONTAINER_TYPES:
raise IOError('Unsupported attribute container type: {0:s}'.format(
container_type))
sequence_number = index + 1
redis_hash_name = self._GetRedisHashName(container_type)
redis_key = '{0:s}.{1:d}'.format(container_type, sequence_number)
serialized_data = self._redis_client.hget(redis_hash_name, redis_key)
if not serialized_data:
return None
attribute_container = self._DeserializeAttributeContainer(
container_type, serialized_data)
identifier = identifiers.RedisKeyIdentifier(container_type,
sequence_number)
attribute_container.SetIdentifier(identifier)
return attribute_container
def GetSortedEvents(self, time_range=None):
"""Retrieves the events in increasing chronological order.
Args:
time_range (Optional[TimeRange]): This argument is not supported by the
Redis store.
Yields:
EventObject: event.
Raises:
RuntimeError: if a time_range argument is specified.
"""
event_index_name = self._GenerateRedisKey(self._EVENT_INDEX_NAME)
if time_range:
raise RuntimeError('Not supported')
sorted_event_identifiers = self._redis_client.zscan_iter(
event_index_name)
for event_identifier, _ in sorted_event_identifiers:
identifier_string = event_identifier.decode('utf-8')
event_identifier = identifiers.RedisKeyIdentifier(
identifier_string)
yield self._GetAttributeContainerByIdentifier(
self._CONTAINER_TYPE_EVENT, event_identifier)
def _GetAttributeContainers(self,
container_type,
callback=None,
cursor=0,
maximum_number_of_items=0):
"""Retrieves attribute containers of the specified type.
Args:
container_type (str): attribute container type.
callback (function[StorageWriter, AttributeContainer]): function to call
after each attribute container is deserialized.
cursor (int): Redis cursor for scanning items.
maximum_number_of_items (Optional[int]): maximum number of
containers to retrieve, where 0 represent no limit.
Returns:
list(AttributeContainer): attribute containers from Redis.
"""
if not cursor:
cursor = 0
cursor, items = self._store.GetSerializedAttributeContainers(
container_type, cursor, maximum_number_of_items)
containers = []
identifiers_to_delete = []
for identifier_bytes, serialized_container in items.items():
identifier_string = codecs.decode(identifier_bytes, 'utf-8')
identifier = identifiers.RedisKeyIdentifier(identifier_string)
identifiers_to_delete.append(identifier)
container = self._DeserializeAttributeContainer(
self._active_container_type, serialized_container)
container.SetIdentifier(identifier)
if callback:
callback(self._storage_writer, container)
containers.append(container)
self._store.RemoveAttributeContainers(container_type,
identifiers_to_delete)
self._active_cursor = cursor
containers = self._active_extra_containers + containers
if maximum_number_of_items:
self._active_extra_containers = containers[
maximum_number_of_items:]
return containers[:maximum_number_of_items]
def _GetAttributeContainers(self, container_type):
"""Retrieves attribute containers
Args:
container_type (str): container type attribute of the container being
added.
Yields:
AttributeContainer: attribute container.
"""
container_key = self._GenerateRedisKey(container_type)
for identifier, serialized_data in self._redis_client.hscan_iter(
container_key):
attribute_container = self._DeserializeAttributeContainer(
container_type, serialized_data)
identifier_string = identifier.decode('utf-8')
redis_identifier = identifiers.RedisKeyIdentifier(identifier_string)
attribute_container.SetIdentifier(redis_identifier)
yield attribute_container
def _AddAttributeContainer(
self, container_type, container, serialized_data=None):
"""Adds an attribute container to the store.
Args:
container_type (str): container type attribute of the container being
added.
container (AttributeContainer): unserialized attribute container.
serialized_data (Optional[bytes]): serialized form of the container.
"""
self._RaiseIfNotWritable()
identifier = identifiers.RedisKeyIdentifier()
container.SetIdentifier(identifier)
if not serialized_data:
serialized_data = self._SerializeAttributeContainer(container)
container_key = self._GenerateRedisKey(container_type)
string_identifier = identifier.CopyToString()
self._redis_client.hset(container_key, string_identifier, serialized_data)