def Open(self, path=None, read_only=True, **unused_kwargs):
"""Opens the store.
Args:
path (Optional[str]): path to the storage file.
read_only (Optional[bool]): True if the file should be opened in
read-only mode.
Raises:
IOError: if the storage file is already opened or if the database
cannot be connected.
OSError: if the storage file is already opened or if the database
cannot be connected.
ValueError: if path is missing.
"""
if self._is_open:
raise IOError('Storage file already opened.')
if not path:
raise ValueError('Missing path.')
path = os.path.abspath(path)
try:
path_uri = pathlib.Path(path).as_uri()
if read_only:
path_uri = '{0:s}?mode=ro'.format(path_uri)
except ValueError:
path_uri = None
detect_types = sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES
if path_uri:
connection = sqlite3.connect(
path_uri, detect_types=detect_types, uri=True)
else:
connection = sqlite3.connect(path, detect_types=detect_types)
cursor = connection.cursor()
if not cursor:
return
self._connection = connection
self._cursor = cursor
self._is_open = True
self._read_only = read_only
if read_only:
self._ReadAndCheckStorageMetadata(check_readable_only=True)
else:
# self._cursor.execute('PRAGMA journal_mode=MEMORY')
try:
# Turn off insert transaction integrity since we want to do bulk insert.
self._cursor.execute('PRAGMA synchronous=OFF')
except sqlite3.OperationalError as exception:
raise IOError('Unable to query storage file with error: {0!s}'.format(
exception))
if not self._HasTable('metadata'):
self._WriteMetadata()
else:
self._ReadAndCheckStorageMetadata()
# Update the storage metadata format version in case we are adding
# new format features that are not backwards compatible.
self._UpdateStorageMetadataFormatVersion()
container_types = set(self._CONTAINER_TYPES)
if self._use_schema:
container_types.update(self._CONTAINER_SCHEMAS.keys())
for container_type in container_types:
if (self.storage_type == definitions.STORAGE_TYPE_SESSION and
container_type in self._TASK_STORE_ONLY_CONTAINER_TYPES):
continue
if (self.storage_type == definitions.STORAGE_TYPE_TASK and
container_type in self._SESSION_STORE_ONLY_CONTAINER_TYPES):
continue
if not self._HasTable(container_type):
self._CreateAttributeContainerTable(container_type)
self._connection.commit()
last_session_start = self.GetNumberOfAttributeContainers(
self._CONTAINER_TYPE_SESSION_START)
last_session_completion = self.GetNumberOfAttributeContainers(
self._CONTAINER_TYPE_SESSION_COMPLETION)
# Initialize next_sequence_number based on the file contents so that
# SQLTableIdentifier points to the correct attribute container.
for container_type in self._REFERENCED_CONTAINER_TYPES:
next_sequence_number = self.GetNumberOfAttributeContainers(
container_type)
self._SetAttributeContainerNextSequenceNumber(
container_type, next_sequence_number)
# TODO: handle open sessions.
if last_session_start != last_session_completion:
logger.warning('Detected unclosed session.')
self._last_session = last_session_completion
def Open(self, path=None, read_only=True, **unused_kwargs):
"""Opens the storage.
Args:
path (Optional[str]): path to the storage file.
read_only (Optional[bool]): True if the file should be opened in
read-only mode.
Raises:
IOError: if the storage file is already opened or if the database
cannot be connected.
ValueError: if path is missing.
"""
if self._is_open:
raise IOError('Storage file already opened.')
if not path:
raise ValueError('Missing path.')
path = os.path.abspath(path)
connection = sqlite3.connect(
path, detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
cursor = connection.cursor()
if not cursor:
return
self._connection = connection
self._cursor = cursor
self._is_open = True
self._read_only = read_only
if read_only:
self._ReadStorageMetadata()
else:
# self._cursor.execute('PRAGMA journal_mode=MEMORY')
# Turn off insert transaction integrity since we want to do bulk insert.
self._cursor.execute('PRAGMA synchronous=OFF')
if not self._HasTable('metadata'):
self._WriteStorageMetadata()
else:
self._ReadStorageMetadata()
if self.compression_format == definitions.COMPRESSION_FORMAT_ZLIB:
data_column_type = 'BLOB'
else:
data_column_type = 'TEXT'
for container_type in self._CONTAINER_TYPES:
if not self._HasTable(container_type):
if container_type == self._CONTAINER_TYPE_EVENT:
query = self._CREATE_EVENT_TABLE_QUERY.format(
container_type, data_column_type)
else:
query = self._CREATE_TABLE_QUERY.format(
container_type, data_column_type)
self._cursor.execute(query)
self._connection.commit()
last_session_start = self._CountStoredAttributeContainers(
self._CONTAINER_TYPE_SESSION_START)
last_session_completion = self._CountStoredAttributeContainers(
self._CONTAINER_TYPE_SESSION_COMPLETION)
# Initialize next_sequence_number based on the file contents so that
# SQLTableIdentifier points to the correct attribute container.
for container_type in self._REFERENCED_CONTAINER_TYPES:
container_list = self._GetSerializedAttributeContainerList(container_type)
container_list.next_sequence_number = (
self._CountStoredAttributeContainers(container_type))
# TODO: handle open sessions.
if last_session_start != last_session_completion:
logger.warning('Detected unclosed session.')
self._last_session = last_session_completion
def Open(self, path=None, read_only=True, **unused_kwargs):
"""Opens the storage.
Args:
path (Optional[str]): path to the storage file.
read_only (Optional[bool]): True if the file should be opened in
read-only mode.
Raises:
IOError: if the storage file is already opened or if the database
cannot be connected.
OSError: if the storage file is already opened or if the database
cannot be connected.
ValueError: if path is missing.
"""
if self._is_open:
raise IOError('Storage file already opened.')
if not path:
raise ValueError('Missing path.')
path = os.path.abspath(path)
connection = sqlite3.connect(path,
detect_types=sqlite3.PARSE_DECLTYPES
| sqlite3.PARSE_COLNAMES)
cursor = connection.cursor()
if not cursor:
return
self._connection = connection
self._cursor = cursor
self._is_open = True
self._read_only = read_only
if read_only:
self._ReadAndCheckStorageMetadata(check_readable_only=True)
else:
# self._cursor.execute('PRAGMA journal_mode=MEMORY')
# Turn off insert transaction integrity since we want to do bulk insert.
self._cursor.execute('PRAGMA synchronous=OFF')
if not self._HasTable('metadata'):
self._WriteStorageMetadata()
else:
self._ReadAndCheckStorageMetadata()
if self.compression_format == definitions.COMPRESSION_FORMAT_ZLIB:
data_column_type = 'BLOB'
else:
data_column_type = 'TEXT'
for container_type in self._CONTAINER_TYPES:
if not self._HasTable(container_type):
if container_type == self._CONTAINER_TYPE_EVENT:
query = self._CREATE_EVENT_TABLE_QUERY.format(
container_type, data_column_type)
else:
query = self._CREATE_TABLE_QUERY.format(
container_type, data_column_type)
self._cursor.execute(query)
self._connection.commit()
last_session_start = self._GetNumberOfAttributeContainers(
self._CONTAINER_TYPE_SESSION_START)
last_session_completion = self._GetNumberOfAttributeContainers(
self._CONTAINER_TYPE_SESSION_COMPLETION)
# Initialize next_sequence_number based on the file contents so that
# SQLTableIdentifier points to the correct attribute container.
for container_type in self._REFERENCED_CONTAINER_TYPES:
container_list = self._GetSerializedAttributeContainerList(
container_type)
container_list.next_sequence_number = (
self._GetNumberOfAttributeContainers(container_type))
# TODO: handle open sessions.
if last_session_start != last_session_completion:
logger.warning('Detected unclosed session.')
self._last_session = last_session_completion
def GetSessions(self):
"""Retrieves the sessions.
Yields:
Session: session attribute container.
Raises:
IOError: if there is a mismatch in session identifiers between the
session start and completion attribute containers.
OSError: if there is a mismatch in session identifiers between the
session start and completion attribute containers.
"""
last_session_start = self._store.GetNumberOfAttributeContainers(
self._CONTAINER_TYPE_SESSION_START)
last_session_completion = self._store.GetNumberOfAttributeContainers(
self._CONTAINER_TYPE_SESSION_COMPLETION)
# TODO: handle open sessions.
if last_session_start != last_session_completion:
logger.warning('Detected unclosed session.')
session_start_generator = self._store.GetAttributeContainers(
self._CONTAINER_TYPE_SESSION_START)
session_completion_generator = self._store.GetAttributeContainers(
self._CONTAINER_TYPE_SESSION_COMPLETION)
if self.HasAttributeContainers(
self._CONTAINER_TYPE_SESSION_CONFIGURATION):
session_configuration_generator = self._store.GetAttributeContainers(
self._CONTAINER_TYPE_SESSION_CONFIGURATION)
else:
session_configuration_generator = None
for session_index in range(1, last_session_completion + 1):
try:
session_start = next(session_start_generator)
except StopIteration:
raise IOError(
'Missing session start: {0:d}'.format(session_index))
try:
session_completion = next(session_completion_generator)
except StopIteration:
pass
session_configuration = None
if session_configuration_generator:
try:
session_configuration = next(
session_configuration_generator)
except StopIteration:
raise IOError(
'Missing session configuration: {0:d}'.format(
session_index))
session = sessions.Session()
session.CopyAttributesFromSessionStart(session_start)
if session_configuration:
try:
session.CopyAttributesFromSessionConfiguration(
session_configuration)
except ValueError:
raise IOError((
'Session identifier mismatch for session configuration: '
'{0:d}').format(session_index))
if session_completion:
try:
session.CopyAttributesFromSessionCompletion(
session_completion)
except ValueError:
raise IOError(
('Session identifier mismatch for session completion: '
'{0:d}').format(session_index))
yield session