def Open(self, path=None, read_only=True, **unused_kwargs): """Opens the store. Args: path (Optional[str]): path to the storage file. read_only (Optional[bool]): True if the file should be opened in read-only mode. Raises: IOError: if the storage file is already opened or if the database cannot be connected. OSError: if the storage file is already opened or if the database cannot be connected. ValueError: if path is missing. """ if self._is_open: raise IOError('Storage file already opened.') if not path: raise ValueError('Missing path.') path = os.path.abspath(path) try: path_uri = pathlib.Path(path).as_uri() if read_only: path_uri = '{0:s}?mode=ro'.format(path_uri) except ValueError: path_uri = None detect_types = sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES if path_uri: connection = sqlite3.connect( path_uri, detect_types=detect_types, uri=True) else: connection = sqlite3.connect(path, detect_types=detect_types) cursor = connection.cursor() if not cursor: return self._connection = connection self._cursor = cursor self._is_open = True self._read_only = read_only if read_only: self._ReadAndCheckStorageMetadata(check_readable_only=True) else: # self._cursor.execute('PRAGMA journal_mode=MEMORY') try: # Turn off insert transaction integrity since we want to do bulk insert. self._cursor.execute('PRAGMA synchronous=OFF') except sqlite3.OperationalError as exception: raise IOError('Unable to query storage file with error: {0!s}'.format( exception)) if not self._HasTable('metadata'): self._WriteMetadata() else: self._ReadAndCheckStorageMetadata() # Update the storage metadata format version in case we are adding # new format features that are not backwards compatible. self._UpdateStorageMetadataFormatVersion() container_types = set(self._CONTAINER_TYPES) if self._use_schema: container_types.update(self._CONTAINER_SCHEMAS.keys()) for container_type in container_types: if (self.storage_type == definitions.STORAGE_TYPE_SESSION and container_type in self._TASK_STORE_ONLY_CONTAINER_TYPES): continue if (self.storage_type == definitions.STORAGE_TYPE_TASK and container_type in self._SESSION_STORE_ONLY_CONTAINER_TYPES): continue if not self._HasTable(container_type): self._CreateAttributeContainerTable(container_type) self._connection.commit() last_session_start = self.GetNumberOfAttributeContainers( self._CONTAINER_TYPE_SESSION_START) last_session_completion = self.GetNumberOfAttributeContainers( self._CONTAINER_TYPE_SESSION_COMPLETION) # Initialize next_sequence_number based on the file contents so that # SQLTableIdentifier points to the correct attribute container. for container_type in self._REFERENCED_CONTAINER_TYPES: next_sequence_number = self.GetNumberOfAttributeContainers( container_type) self._SetAttributeContainerNextSequenceNumber( container_type, next_sequence_number) # TODO: handle open sessions. if last_session_start != last_session_completion: logger.warning('Detected unclosed session.') self._last_session = last_session_completion
def Open(self, path=None, read_only=True, **unused_kwargs): """Opens the storage. Args: path (Optional[str]): path to the storage file. read_only (Optional[bool]): True if the file should be opened in read-only mode. Raises: IOError: if the storage file is already opened or if the database cannot be connected. ValueError: if path is missing. """ if self._is_open: raise IOError('Storage file already opened.') if not path: raise ValueError('Missing path.') path = os.path.abspath(path) connection = sqlite3.connect( path, detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES) cursor = connection.cursor() if not cursor: return self._connection = connection self._cursor = cursor self._is_open = True self._read_only = read_only if read_only: self._ReadStorageMetadata() else: # self._cursor.execute('PRAGMA journal_mode=MEMORY') # Turn off insert transaction integrity since we want to do bulk insert. self._cursor.execute('PRAGMA synchronous=OFF') if not self._HasTable('metadata'): self._WriteStorageMetadata() else: self._ReadStorageMetadata() if self.compression_format == definitions.COMPRESSION_FORMAT_ZLIB: data_column_type = 'BLOB' else: data_column_type = 'TEXT' for container_type in self._CONTAINER_TYPES: if not self._HasTable(container_type): if container_type == self._CONTAINER_TYPE_EVENT: query = self._CREATE_EVENT_TABLE_QUERY.format( container_type, data_column_type) else: query = self._CREATE_TABLE_QUERY.format( container_type, data_column_type) self._cursor.execute(query) self._connection.commit() last_session_start = self._CountStoredAttributeContainers( self._CONTAINER_TYPE_SESSION_START) last_session_completion = self._CountStoredAttributeContainers( self._CONTAINER_TYPE_SESSION_COMPLETION) # Initialize next_sequence_number based on the file contents so that # SQLTableIdentifier points to the correct attribute container. for container_type in self._REFERENCED_CONTAINER_TYPES: container_list = self._GetSerializedAttributeContainerList(container_type) container_list.next_sequence_number = ( self._CountStoredAttributeContainers(container_type)) # TODO: handle open sessions. if last_session_start != last_session_completion: logger.warning('Detected unclosed session.') self._last_session = last_session_completion
def Open(self, path=None, read_only=True, **unused_kwargs): """Opens the storage. Args: path (Optional[str]): path to the storage file. read_only (Optional[bool]): True if the file should be opened in read-only mode. Raises: IOError: if the storage file is already opened or if the database cannot be connected. OSError: if the storage file is already opened or if the database cannot be connected. ValueError: if path is missing. """ if self._is_open: raise IOError('Storage file already opened.') if not path: raise ValueError('Missing path.') path = os.path.abspath(path) connection = sqlite3.connect(path, detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES) cursor = connection.cursor() if not cursor: return self._connection = connection self._cursor = cursor self._is_open = True self._read_only = read_only if read_only: self._ReadAndCheckStorageMetadata(check_readable_only=True) else: # self._cursor.execute('PRAGMA journal_mode=MEMORY') # Turn off insert transaction integrity since we want to do bulk insert. self._cursor.execute('PRAGMA synchronous=OFF') if not self._HasTable('metadata'): self._WriteStorageMetadata() else: self._ReadAndCheckStorageMetadata() if self.compression_format == definitions.COMPRESSION_FORMAT_ZLIB: data_column_type = 'BLOB' else: data_column_type = 'TEXT' for container_type in self._CONTAINER_TYPES: if not self._HasTable(container_type): if container_type == self._CONTAINER_TYPE_EVENT: query = self._CREATE_EVENT_TABLE_QUERY.format( container_type, data_column_type) else: query = self._CREATE_TABLE_QUERY.format( container_type, data_column_type) self._cursor.execute(query) self._connection.commit() last_session_start = self._GetNumberOfAttributeContainers( self._CONTAINER_TYPE_SESSION_START) last_session_completion = self._GetNumberOfAttributeContainers( self._CONTAINER_TYPE_SESSION_COMPLETION) # Initialize next_sequence_number based on the file contents so that # SQLTableIdentifier points to the correct attribute container. for container_type in self._REFERENCED_CONTAINER_TYPES: container_list = self._GetSerializedAttributeContainerList( container_type) container_list.next_sequence_number = ( self._GetNumberOfAttributeContainers(container_type)) # TODO: handle open sessions. if last_session_start != last_session_completion: logger.warning('Detected unclosed session.') self._last_session = last_session_completion
def GetSessions(self): """Retrieves the sessions. Yields: Session: session attribute container. Raises: IOError: if there is a mismatch in session identifiers between the session start and completion attribute containers. OSError: if there is a mismatch in session identifiers between the session start and completion attribute containers. """ last_session_start = self._store.GetNumberOfAttributeContainers( self._CONTAINER_TYPE_SESSION_START) last_session_completion = self._store.GetNumberOfAttributeContainers( self._CONTAINER_TYPE_SESSION_COMPLETION) # TODO: handle open sessions. if last_session_start != last_session_completion: logger.warning('Detected unclosed session.') session_start_generator = self._store.GetAttributeContainers( self._CONTAINER_TYPE_SESSION_START) session_completion_generator = self._store.GetAttributeContainers( self._CONTAINER_TYPE_SESSION_COMPLETION) if self.HasAttributeContainers( self._CONTAINER_TYPE_SESSION_CONFIGURATION): session_configuration_generator = self._store.GetAttributeContainers( self._CONTAINER_TYPE_SESSION_CONFIGURATION) else: session_configuration_generator = None for session_index in range(1, last_session_completion + 1): try: session_start = next(session_start_generator) except StopIteration: raise IOError( 'Missing session start: {0:d}'.format(session_index)) try: session_completion = next(session_completion_generator) except StopIteration: pass session_configuration = None if session_configuration_generator: try: session_configuration = next( session_configuration_generator) except StopIteration: raise IOError( 'Missing session configuration: {0:d}'.format( session_index)) session = sessions.Session() session.CopyAttributesFromSessionStart(session_start) if session_configuration: try: session.CopyAttributesFromSessionConfiguration( session_configuration) except ValueError: raise IOError(( 'Session identifier mismatch for session configuration: ' '{0:d}').format(session_index)) if session_completion: try: session.CopyAttributesFromSessionCompletion( session_completion) except ValueError: raise IOError( ('Session identifier mismatch for session completion: ' '{0:d}').format(session_index)) yield session