def _UpdateAttributeContainerAfterDeserialize(self, container): """Updates an attribute container after deserialization. Args: container (AttributeContainer): attribute container. Raises: ValueError: if an attribute container identifier is missing. """ if container.CONTAINER_TYPE == self._CONTAINER_TYPE_EVENT: row_identifier = getattr(container, '_event_data_row_identifier', None) if row_identifier: event_data_identifier = identifiers.RedisKeyIdentifier( self._CONTAINER_TYPE_EVENT_DATA, row_identifier) container.SetEventDataIdentifier(event_data_identifier) delattr(container, '_event_data_row_identifier') elif container.CONTAINER_TYPE == self._CONTAINER_TYPE_EVENT_DATA: row_identifier = getattr(container, '_event_data_stream_row_identifier', None) if row_identifier: event_data_stream_identifier = identifiers.RedisKeyIdentifier( self._CONTAINER_TYPE_EVENT_DATA_STREAM, row_identifier) container.SetEventDataStreamIdentifier( event_data_stream_identifier) delattr(container, '_event_data_stream_row_identifier')
def GetAttributeContainers(self, container_type, filter_expression=None): """Retrieves attribute containers Args: container_type (str): container type attribute of the container being added. filter_expression (Optional[str]): expression to filter the resulting attribute containers by. Yields: AttributeContainer: attribute container. """ redis_hash_name = self._GetRedisHashName(container_type) for redis_key, serialized_data in self._redis_client.hscan_iter( redis_hash_name): redis_key = redis_key.decode('utf-8') attribute_container = self._DeserializeAttributeContainer( container_type, serialized_data) _, sequence_number = redis_key.split('.') sequence_number = int(sequence_number, 10) identifier = identifiers.RedisKeyIdentifier( container_type, sequence_number) attribute_container.SetIdentifier(identifier) self._UpdateAttributeContainerAfterDeserialize(attribute_container) # TODO: map filter expression to Redis native filter. if attribute_container.MatchesExpression(filter_expression): yield attribute_container
def GetSortedEvents(self, time_range=None): """Retrieves the events in increasing chronological order. Args: time_range (Optional[TimeRange]): This argument is not supported by the Redis store. Yields: EventObject: event. Raises: RuntimeError: if a time_range argument is specified. """ event_index_name = self._GetRedisHashName(self._EVENT_INDEX_NAME) if time_range: raise RuntimeError('Not supported') for redis_key, _ in self._redis_client.zscan_iter(event_index_name): redis_key = redis_key.decode('utf-8') container_type, sequence_number = redis_key.split('.') sequence_number = int(sequence_number, 10) identifier = identifiers.RedisKeyIdentifier( container_type, sequence_number) yield self.GetAttributeContainerByIdentifier( self._CONTAINER_TYPE_EVENT, identifier)
def _WriteNewAttributeContainer(self, container): """Writes a new attribute container to the store. Args: container (AttributeContainer): attribute container. """ next_sequence_number = self._GetAttributeContainerNextSequenceNumber( container.CONTAINER_TYPE) identifier = identifiers.RedisKeyIdentifier(container.CONTAINER_TYPE, next_sequence_number) container.SetIdentifier(identifier) redis_hash_name = self._GetRedisHashName(container.CONTAINER_TYPE) redis_key = identifier.CopyToString() self._UpdateAttributeContainerBeforeSerialize(container) serialized_data = self._SerializeAttributeContainer(container) self._redis_client.hsetnx(redis_hash_name, redis_key, serialized_data) if container.CONTAINER_TYPE == self._CONTAINER_TYPE_EVENT: index_name = self._GetRedisHashName(self._EVENT_INDEX_NAME) self._redis_client.zincrby(index_name, container.timestamp, redis_key)
def GetAttributeContainerByIndex(self, container_type, index): """Retrieves a specific attribute container. Args: container_type (str): attribute container type. index (int): attribute container index. Returns: AttributeContainer: attribute container or None if not available. """ sequence_number = index + 1 redis_hash_name = self._GetRedisHashName(container_type) redis_key = '{0:s}.{1:d}'.format(container_type, sequence_number) serialized_data = self._redis_client.hget(redis_hash_name, redis_key) if not serialized_data: return None attribute_container = self._DeserializeAttributeContainer( container_type, serialized_data) identifier = identifiers.RedisKeyIdentifier(container_type, sequence_number) attribute_container.SetIdentifier(identifier) self._UpdateAttributeContainerAfterDeserialize(attribute_container) return attribute_container
def GetAttributeContainers(self, container_type): """Retrieves attribute containers Args: container_type (str): container type attribute of the container being added. Yields: AttributeContainer: attribute container. """ redis_hash_name = self._GetRedisHashName(container_type) for redis_key, serialized_data in self._redis_client.hscan_iter( redis_hash_name): redis_key = redis_key.decode('utf-8') attribute_container = self._DeserializeAttributeContainer( container_type, serialized_data) _, sequence_number = redis_key.split('.') sequence_number = int(sequence_number, 10) identifier = identifiers.RedisKeyIdentifier( container_type, sequence_number) attribute_container.SetIdentifier(identifier) yield attribute_container
def GetAttributeContainerByIndex(self, container_type, index): """Retrieves a specific attribute container. Args: container_type (str): attribute container type. index (int): attribute container index. Returns: AttributeContainer: attribute container or None if not available. Raises: IOError: if the attribute container type is not supported. OSError: if the attribute container type is not supported. """ if container_type not in self._CONTAINER_TYPES: raise IOError('Unsupported attribute container type: {0:s}'.format( container_type)) sequence_number = index + 1 redis_hash_name = self._GetRedisHashName(container_type) redis_key = '{0:s}.{1:d}'.format(container_type, sequence_number) serialized_data = self._redis_client.hget(redis_hash_name, redis_key) if not serialized_data: return None attribute_container = self._DeserializeAttributeContainer( container_type, serialized_data) identifier = identifiers.RedisKeyIdentifier(container_type, sequence_number) attribute_container.SetIdentifier(identifier) return attribute_container
def GetSortedEvents(self, time_range=None): """Retrieves the events in increasing chronological order. Args: time_range (Optional[TimeRange]): This argument is not supported by the Redis store. Yields: EventObject: event. Raises: RuntimeError: if a time_range argument is specified. """ event_index_name = self._GenerateRedisKey(self._EVENT_INDEX_NAME) if time_range: raise RuntimeError('Not supported') sorted_event_identifiers = self._redis_client.zscan_iter( event_index_name) for event_identifier, _ in sorted_event_identifiers: identifier_string = event_identifier.decode('utf-8') event_identifier = identifiers.RedisKeyIdentifier( identifier_string) yield self._GetAttributeContainerByIdentifier( self._CONTAINER_TYPE_EVENT, event_identifier)
def _GetAttributeContainers(self, container_type, callback=None, cursor=0, maximum_number_of_items=0): """Retrieves attribute containers of the specified type. Args: container_type (str): attribute container type. callback (function[StorageWriter, AttributeContainer]): function to call after each attribute container is deserialized. cursor (int): Redis cursor for scanning items. maximum_number_of_items (Optional[int]): maximum number of containers to retrieve, where 0 represent no limit. Returns: list(AttributeContainer): attribute containers from Redis. """ if not cursor: cursor = 0 cursor, items = self._store.GetSerializedAttributeContainers( container_type, cursor, maximum_number_of_items) containers = [] identifiers_to_delete = [] for identifier_bytes, serialized_container in items.items(): identifier_string = codecs.decode(identifier_bytes, 'utf-8') identifier = identifiers.RedisKeyIdentifier(identifier_string) identifiers_to_delete.append(identifier) container = self._DeserializeAttributeContainer( self._active_container_type, serialized_container) container.SetIdentifier(identifier) if callback: callback(self._storage_writer, container) containers.append(container) self._store.RemoveAttributeContainers(container_type, identifiers_to_delete) self._active_cursor = cursor containers = self._active_extra_containers + containers if maximum_number_of_items: self._active_extra_containers = containers[ maximum_number_of_items:] return containers[:maximum_number_of_items]
def _GetAttributeContainers(self, container_type): """Retrieves attribute containers Args: container_type (str): container type attribute of the container being added. Yields: AttributeContainer: attribute container. """ container_key = self._GenerateRedisKey(container_type) for identifier, serialized_data in self._redis_client.hscan_iter( container_key): attribute_container = self._DeserializeAttributeContainer( container_type, serialized_data) identifier_string = identifier.decode('utf-8') redis_identifier = identifiers.RedisKeyIdentifier(identifier_string) attribute_container.SetIdentifier(redis_identifier) yield attribute_container
def _AddAttributeContainer( self, container_type, container, serialized_data=None): """Adds an attribute container to the store. Args: container_type (str): container type attribute of the container being added. container (AttributeContainer): unserialized attribute container. serialized_data (Optional[bytes]): serialized form of the container. """ self._RaiseIfNotWritable() identifier = identifiers.RedisKeyIdentifier() container.SetIdentifier(identifier) if not serialized_data: serialized_data = self._SerializeAttributeContainer(container) container_key = self._GenerateRedisKey(container_type) string_identifier = identifier.CopyToString() self._redis_client.hset(container_key, string_identifier, serialized_data)