class Table(attrib.Container):
"""Table Class."""
#
# Public Attributes
#
fields: property = attrib.Virtual(fget='_get_fields')
colnames: property = attrib.Virtual(fget='_get_colnames')
#
# Protected Attributes
#
_store: property = attrib.Content(classinfo=list, default=[])
_diff: property = attrib.Temporary(classinfo=list, default=[])
_index: property = attrib.Temporary(classinfo=list, default=[])
_iter_index: property = attrib.Temporary()
_Record: property = attrib.Temporary(classinfo=type)
#
# Events
#
def __init__(self, columns: OptFieldLike = None) -> None:
""" """
super().__init__()
if columns:
self._create_header(columns)
def __iter__(self) -> Iterator:
self._iter_index = iter(self._index)
return self
def __next__(self) -> Record:
row = self.get_row(next(self._iter_index))
while not row:
row = self.get_row(next(self._iter_index))
return row
def __len__(self) -> int:
return len(self._index)
#
# Public Methods
#
def commit(self) -> None:
"""Apply changes to table."""
# Delete / Update rows in storage table
for rowid in list(range(len(self._store))):
row = self.get_row(rowid)
if not row:
continue
state = row.state
if state & ROW_STATE_DELETE:
self._store[rowid] = None
try:
self._index.remove(rowid)
except ValueError:
pass
elif state & (ROW_STATE_CREATE | ROW_STATE_UPDATE):
self._store[rowid] = self._diff[rowid]
self._store[rowid].state = 0
# Flush diff table
self._diff = [None] * len(self._store)
def rollback(self) -> None:
"""Revoke changes from table."""
# Remove newly created rows from index and reset states of already
# existing rows
for rowid in list(range(len(self._store))):
row = self.get_row(rowid)
if not row:
continue
state = row.state
if state & ROW_STATE_CREATE:
try:
self._index.remove(rowid)
except ValueError:
pass
else:
self._store[rowid].state = 0
# Flush diff table
self._diff = [None] * len(self._store)
def get_cursor(self,
predicate: OptCallable = None,
mapper: OptCallable = None,
mode: OptStr = None) -> Cursor:
""" """
return Cursor(getter=self.get_row,
predicate=predicate,
mapper=mapper,
mode=mode,
parent=self)
def get_row(self, rowid: int) -> OptRow:
""" """
return self._diff[rowid] or self._store[rowid]
def get_rows(self,
predicate: OptCallable = None,
mode: OptStr = None) -> Cursor:
""" """
return self.get_cursor(predicate=predicate, mode=mode)
def append_row(self, *args: Any, **kwds: Any) -> None:
""" """
row = self._create_row(*args, **kwds)
self._store.append(None)
self._diff.append(row)
self._append_row_id(row.id)
def delete_row(self, rowid: int) -> None:
""" """
row = self.get_row(rowid)
if not row:
raise RowLookupError(rowid)
row.delete()
def delete_rows(self, predicate: OptCallable = None) -> None:
""" """
for row in self.get_rows(predicate):
row.delete()
def update_row(self, rowid: int, **kwds: Any) -> None:
""" """
row = self.get_row(rowid)
if not row:
raise RowLookupError(rowid)
row.update(**kwds)
def update_rows(self, predicate: OptCallable = None, **kwds: Any) -> None:
""" """
for row in self.get_rows(predicate):
row.update(**kwds)
def select(self,
columns: OptStrTuple = None,
predicate: OptCallable = None,
fmt: type = tuple,
mode: OptStr = None) -> RowLikeList:
""" """
if not columns:
mapper = self._get_mapper(self.colnames, fmt=fmt)
else:
check.is_subset("'columns'", set(columns), "table column names",
set(self.colnames))
mapper = self._get_mapper(columns, fmt=fmt)
return self.get_cursor( # type: ignore
predicate=predicate, mapper=mapper, mode=mode)
def pack(self) -> None:
"""Remove empty records from storage table and rebuild table index."""
# Commit pending changes
self.commit()
# Remove empty records
self._store = list(filter(None.__ne__, self._store))
# Rebuild table index
self._index = list(range(len(self._store)))
for rowid in self._index:
self._store[rowid].id = rowid
# Rebuild diff table
self._diff = [None] * len(self._store)
#
# Protected Methods
#
def _get_mapper(self, columns: StrTuple, fmt: type = tuple) -> Callable:
def mapper_tuple(row: Record) -> tuple:
return tuple(getattr(row, col) for col in columns)
def mapper_dict(row: Record) -> dict:
return {col: getattr(row, col) for col in columns}
if fmt == tuple:
return mapper_tuple
if fmt == dict:
return mapper_dict
raise TableError(f"'fmt' requires to be tuple or dict")
def _get_fields(self) -> FieldTuple:
return dataclasses.fields(self._Record)
def _get_colnames(self) -> StrTuple:
return tuple(field.name for field in self.fields)
def _create_row_id(self) -> int:
return len(self._store)
def _append_row_id(self, rowid: int) -> None:
self._index.append(rowid)
def _remove_row_id(self, rowid: int) -> None:
self._index.remove(rowid)
def _update_row_diff(self, rowid: int, **kwds: Any) -> None:
row = self.get_row(rowid)
if not row:
raise RowLookupError(rowid)
upd = dataclasses.replace(row, **kwds)
upd.id = rowid
upd.state = row.state
self._diff[rowid] = upd
def _remove_row_diff(self, rowid: int) -> None:
self._diff[rowid] = None
def _create_row(self, *args: Any, **kwds: Any) -> Record:
return self._Record(*args, **kwds) # pylint: disable=E0110
def _create_header(self, columns: FieldLike) -> None:
# Check types of fieldlike column descriptors and convert them to field
# descriptors, that are accepted by dataclasses.make_dataclass()
fields: list = []
for each in columns:
if isinstance(each, str):
fields.append(each)
continue
check.has_type(f"field {each}", each, tuple)
check.has_size(f"field {each}", each, min_size=2, max_size=3)
check.has_type("first arg", each[0], str)
check.has_type("second arg", each[1], type)
if len(each) == 2:
fields.append(each)
continue
check.has_type("third arg", each[2], (Field, dict))
if isinstance(each[2], Field):
fields.append(each)
continue
field = dataclasses.field(**each[2])
fields.append(each[:2] + (field, ))
# Create record namespace with table hooks
namespace = {
'_create_row_id': self._create_row_id,
'_delete_hook': self._remove_row_id,
'_restore_hook': self._append_row_id,
'_update_hook': self._update_row_diff,
'_revoke_hook': self._remove_row_diff
}
# Create Record dataclass and constructor
self._Record = dataclasses.make_dataclass('Row',
fields,
bases=(Record, ),
namespace=namespace)
# Create slots
self._Record.__slots__ = ['id', 'state'] + [
field.name for field in dataclasses.fields(self._Record)
]
# Reset store, diff and index
self._store = []
self._diff = []
self._index = []
class Cursor(attrib.Container):
"""Cursor Class.
Args:
index: List of row IDs, that are traversed by the cursor. By default the
attribute '_index' of the parent object is used.
mode: Named string identifier for the cursor :py:attr:`.mode`. The
default cursor mode is 'forward-only indexed'. Note: After
initializing the curser, it's mode can not be changed anymore.
"""
#
# Protected Class Variables
#
_default_mode: ClassVar[int] = CUR_MODE_INDEXED
#
# Public Attributes
#
mode: property = attrib.Virtual(fget='_get_mode')
mode.__doc__ = """
The read-only string attribute *cursor mode* specifies the space separated
*scrolling type* and the *operation mode* of the cursor. Supported scrolling
types are:
:forward-only: The default scrolling type of cursors is called a
forward-only cursor and can move only forward through the result set. A
forward-only cursor does not support scrolling but only fetching rows
from the start to the end of the result set.
:scrollable: A scrollable cursor is commonly used in screen-based
interactive applications, like spreadsheets, in which users are allowed
to scroll back and forth through the result set. However, applications
should use scrollable cursors only when forward-only cursors will not do
the job, as scrollable cursors are generally more expensive, than
forward-only cursors.
:random: Random cursors move randomly through the result set. In difference
to a randomly sorted cursor, the rows are not unique and the number of
fetched rows is not limited to the size of the result set. If the method
:meth:`.fetch` is called with a zero value for size, a
CursorModeError is raised.
Supported operation modes are:
:dynamic: A **dynamic cursor** is built on-the-fly and therefore comprises
any changes made to the rows in the result set during it's traversal,
including new appended rows and the order of it's traversal. This
behaviour is regardless of whether the changes occur from inside the
cursor or by other users from outside the cursor. Dynamic cursors are
threadsafe but do not support counting filtered rows or sorting rows.
:indexed: Indexed cursors (aka Keyset-driven cursors) are built on-the-fly
with respect to an initial copy of the table index and therefore
comprise changes made to the rows in the result set during it's
traversal, but not new appended rows nor changes within their order.
Keyset driven cursors are threadsafe but do not support sorting rows or
counting filtered rows.
:static: Static cursors are buffered and built during it's creation time and
therfore always display the result set as it was when the cursor was
first opened. Static cursors are not threadsafe but support counting the
rows with respect to a given filter and sorting the rows.
"""
batchsize: property = attrib.MetaData(classinfo=int, default=1)
"""
The read-writable integer attribute *batchsize* specifies the default number
of rows which is to be fetched by the method :meth:`.fetch`. It defaults
to 1, meaning to fetch a single row at a time. Whether and which batchsize
to use depends on the application and should be considered with care. The
batchsize can also be adapted during the lifetime of the cursor, which
allows dynamic performance optimization.
"""
rowcount: property = attrib.Virtual(fget='_get_rowcount')
"""
The read-only integer attribute *rowcount* specifies the current number of
rows within the cursor.
"""
#
# Protected Attributes
#
_mode: property = attrib.MetaData(classinfo=int, default=_default_mode)
_index: property = attrib.MetaData(classinfo=list, inherit=True)
_getter: property = attrib.Temporary(classinfo=CallableClasses)
_filter: property = attrib.Temporary(classinfo=CallableClasses)
_mapper: property = attrib.Temporary(classinfo=CallableClasses)
_buffer: property = attrib.Temporary(classinfo=list, default=[])
#
# Events
#
def __init__(self,
index: OptIntList = None,
getter: OptCallable = None,
predicate: OptCallable = None,
mapper: OptCallable = None,
batchsize: OptInt = None,
mode: OptStr = None,
parent: Optional[attrib.Container] = None) -> None:
"""Initialize Cursor."""
super().__init__(parent=parent) # Parent is set by container
if index is not None:
self._index = index
self._getter = getter
self._filter = predicate
self._mapper = mapper
if mode:
self._set_mode(mode)
if batchsize:
self.batchsize = batchsize
if self._mode & CUR_MODE_INDEXED:
self._create_index()
if self._mode & CUR_MODE_BUFFERED:
self._create_buffer()
self.reset() # Initialize iterator
def __iter__(self) -> Iterator:
self.reset()
return self
def __next__(self) -> RowLike:
return self.next()
def __len__(self) -> int:
return self.rowcount
#
# Public Methods
#
def reset(self) -> None:
"""Reset cursor position before the first record."""
mode = self._mode
if mode & CUR_MODE_BUFFERED: # Iterate over fixed result set
self._iter_buffer = iter(self._buffer)
elif mode & CUR_MODE_INDEXED: # Iterate over fixed index
self._iter_index = iter(self._index)
else: # TODO: handle case for dynamic cursors by self._iter_table
self._iter_index = iter(self._index)
def next(self) -> RowLike:
"""Return next row that matches the given filter."""
mode = self._mode
if mode & CUR_MODE_BUFFERED:
return self._get_next_from_buffer()
if mode & CUR_MODE_INDEXED:
return self._get_next_from_fixed_index()
# TODO: For dynamic cursors implement _get_next_from_dynamic_index()
return self._get_next_from_fixed_index()
def fetch(self, size: OptInt = None) -> RowLikeList:
"""Fetch rows from the result set.
Args:
size: Integer value, which represents the number of rows, which is
fetched from the result set. For the given size 0 all remaining
rows from the result set are fetched. By default the number of
rows is given by the cursors batchsize.
"""
if size is None:
size = self.batchsize
if self._mode & CUR_MODE_RANDOM and size <= 0:
raise CursorModeError(self.mode, 'fetching all rows')
finished = False
results: RowLikeList = []
while not finished:
try:
results.append(self.next())
except StopIteration:
finished = True
else:
finished = 0 < size <= len(results)
return results
#
# Protected Methods
#
def _get_next_from_fixed_index(self) -> RowLike:
is_random = self._mode & CUR_MODE_RANDOM
matches = False
while not matches:
if is_random:
row_id = random.randrange(len(self._index))
else:
row_id = next(self._iter_index)
row = self._getter(row_id)
if self._filter:
matches = self._filter(row)
else:
matches = True
if self._mapper:
return self._mapper(row)
return row
def _get_next_from_buffer(self) -> RowLike:
if self._mode & CUR_MODE_RANDOM:
row_id = random.randrange(len(self._buffer))
return self._buffer[row_id]
return next(self._iter_buffer)
def _get_mode(self) -> str:
mode = self._mode
tokens = []
# Add name of traversal mode
if mode & CUR_MODE_RANDOM:
tokens.append('random')
elif mode & CUR_MODE_SCROLLABLE:
tokens.append('scrollable')
# Add name of operation mode
if mode & CUR_MODE_BUFFERED:
tokens.append('static')
elif mode & CUR_MODE_INDEXED:
tokens.append('indexed')
else:
tokens.append('dynamic')
return ' '.join(tokens)
def _set_mode(self, name: str) -> None:
mode = 0
name = name.strip(' ').lower()
# Set traversal mode flags
if 'random' in name:
mode |= CUR_MODE_RANDOM
elif 'scrollable' in name:
mode |= CUR_MODE_SCROLLABLE
# Set operation mode flags
if 'static' in name:
mode |= CUR_MODE_BUFFERED | CUR_MODE_INDEXED
elif 'indexed' in name:
mode |= CUR_MODE_INDEXED
self._mode = mode
def _get_rowcount(self) -> int:
mode = self._mode
if mode & CUR_MODE_RANDOM:
raise CursorModeError(self.mode, 'counting rows')
if mode & CUR_MODE_BUFFERED:
return len(self._buffer)
if self._filter:
raise CursorModeError(self.mode, 'counting filtered rows')
return len(self._index)
def _create_index(self) -> None:
self._index = self._index.copy()
def _create_buffer(self) -> None:
cur = self.__class__( # Create new dynamic cursor
index=self._index,
getter=self._getter,
predicate=self._filter,
mapper=self._mapper)
self._buffer = cur.fetch(0) # Fetch all from result set
class Session(attrib.Container):
"""Session."""
#
# Private Class Variables
#
_config_file_path: ClassVar[str] = '%user_config_dir%/nemoa.ini'
_config_file_struct: ClassVar[SecDict] = {
'session': {
'path': Path,
'restore_on_startup': bool,
'autosave_on_exit': bool
}
}
_default_config: ClassVar[StrDict] = {
'path': None,
'restore_on_startup': False,
'autosave_on_exit': False
}
_default_paths: StrList = [
'%user_data_dir%', '%site_data_dir%', '%package_data_dir%'
]
#
# Public Attributes and Attribute Groups
#
dc: attrib.Group = attrib.create_group(attrib.DCGroup, remote=True)
config: property = attrib.MetaData(classinfo=dict)
config.__doc__ = """Session configuration."""
paths: property = attrib.MetaData(classinfo=list)
paths.__doc__ = """Search paths for workspaces."""
files: property = attrib.Virtual(fget='_get_files')
files.__doc__ = """Files within the current workspace."""
folders: property = attrib.Virtual(fget='_get_folders')
folders.__doc__ = """Folders within the current workspace."""
path: property = attrib.Virtual(fget='_get_path')
path.__doc__ = """Filepath of the current workspace."""
logger: property = attrib.Temporary(classinfo=log.Logger)
logger.__doc__ = """Logger instance."""
#
# Protected Attributes
#
_ws: property = attrib.Content(classinfo=wsfile.WsFile)
#
# Events
#
def __init__(self,
workspace: OptPathLike = None,
basedir: OptPathLike = None,
pwd: OptBytes = None) -> None:
"""Initialize instance variables and load workspace from file."""
super().__init__()
# Initialize instance variables with default values
self.config = self._default_config.copy()
self._ws = wsfile.WsFile()
self.paths = [env.expand(path) for path in self._default_paths]
self.logger = log.get_instance()
# Bind session to workspace
self.parent = self._ws
# Load session configuration from file
if env.is_file(self._config_file_path):
self._load_config()
# Load workspace from file
filepath: OptPath = None
if workspace and isinstance(workspace, (Path, str)):
filepath = Path(workspace)
elif self.config.get('restore_on_startup'):
cfg_path = self.config.get('path')
if isinstance(cfg_path, (Path, str)):
filepath = Path(cfg_path)
if isinstance(filepath, Path):
self.load(workspace=filepath, basedir=basedir, pwd=pwd)
def __enter__(self) -> 'Session':
"""Enter with statement."""
return self
def __exit__(self, cls: ExcType, obj: Exc, tb: Traceback) -> None:
"""Exit with statement."""
self.close() # Close Workspace
self._save_config() # Save config
def __del__(self) -> None:
"""Run destructor for instance."""
#
# Public Methods
#
def load(self,
workspace: OptPathLike = None,
basedir: OptPathLike = None,
pwd: OptBytes = None) -> None:
"""Load Workspace from file.
Args:
workspace:
basedir:
pwd: Bytes representing password of workspace file.
"""
path = self._locate_path(workspace=workspace, basedir=basedir)
self._ws = wsfile.WsFile(filepath=path, pwd=pwd)
self.parent = self._ws
def save(self) -> None:
"""Save Workspace to current file."""
self._ws.save()
def saveas(self, filepath: PathLike) -> None:
"""Save the workspace to a file.
Args:
filepath: String or :term:`path-like object`, that represents the
name of a workspace file.
"""
self._ws.saveas(filepath)
def close(self) -> None:
"""Close current session."""
if self.config.get('autosave_on_exit') and self._ws.changed:
self.save()
if hasattr(self._ws, 'close'):
self._ws.close()
def get_file_accessor(self, path: PathLike) -> FileAccessorBase:
"""Get file accessor to workspace member.
Args:
path: String or :term:`path-like object`, that represents a
workspace member. In reading mode the path has to point to a
valid workspace file, or a FileNotFoundError is raised. In
writing mode the path by default is treated as a file path. New
directories can be written by setting the argument is_dir to
True.
Returns:
:class:`File accessor <nemoa.types.FileAccessorBase>` to workspace
member.
"""
return self._ws.get_file_accessor(path)
def open(self,
filepath: PathLike,
workspace: OptPathLike = None,
basedir: OptPathLike = None,
pwd: OptBytes = None,
mode: str = '',
encoding: OptStr = None,
is_dir: bool = False) -> FileLike:
"""Open file within current or given workspace.
Args:
filepath: String or :term:`path-like object`, that represents a
workspace member. In reading mode the path has to point to a
valid workspace file, or a FileNotFoundError is raised. In
writing mode the path by default is treated as a file path. New
directories can be written by setting the argument is_dir to
True.
workspace:
basedir:
mode: String, which characters specify the mode in which the file is
to be opened. The default mode is reading in text mode. Suported
characters are:
'r': Reading mode (default)
'w': Writing mode
'b': Binary mode
't': Text mode (default)
encoding: In binary mode encoding has not effect. In text mode
encoding specifies the name of the encoding, which in reading
and writing mode respectively is used to decode the stream’s
bytes into strings, and to encode strings into bytes. By default
the preferred encoding of the operating system is used.
is_dir: Boolean value which determines, if the path is to be treated
as a directory or not. This information is required for writing
directories to the workspace. The default behaviour is not to
treat paths as directories.
Returns:
Context manager for :term:`file object` in reading or writing mode.
"""
if workspace:
path = self._locate_path(workspace=workspace, basedir=basedir)
ws = wsfile.WsFile(filepath=path, pwd=pwd)
return ws.open(filepath,
mode=mode,
encoding=encoding,
is_dir=is_dir)
return self._ws.open(filepath,
mode=mode,
encoding=encoding,
is_dir=is_dir)
def append(self, source: PathLike, target: OptPathLike = None) -> bool:
"""Append file to the current workspace.
Args:
source: String or :term:`path-like object`, that points to a valid
file in the directory structure if the system. If the file does
not exist, a FileNotFoundError is raised. If the filepath points
to a directory, a IsADirectoryError is raised.
target: String or :term:`path-like object`, that points to a valid
directory in the directory structure of the workspace. By
default the root directory is used. If the directory does not
exist, a FileNotFoundError is raised. If the target directory
already contains a file, which name equals the filename of the
source, a FileExistsError is raised.
Returns:
Boolean value which is True if the file has been appended.
"""
return self._ws.append(source, target=target)
def unlink(self, filepath: PathLike, ignore_missing: bool = True) -> bool:
"""Remove file from the current workspace.
Args:
filepath: String or :term:`path-like object`, that points to a file
in the directory structure of the workspace. If the filapath
points to a directory, an IsADirectoryError is raised. For the
case, that the file does not exist, the argument ignore_missing
determines, if a FileNotFoundError is raised.
ignore_missing: Boolean value which determines, if FileNotFoundError
is raised, if the target file does not exist. The default
behaviour, is to ignore missing files.
Returns:
Boolean value, which is True if the given file was removed.
"""
return self._ws.unlink(filepath, ignore_missing=ignore_missing)
def mkdir(self, dirpath: PathLike, ignore_exists: bool = False) -> bool:
"""Create a new directory in current workspace.
Args:
dirpath: String or :term:`path-like object`, that represents a valid
directory name in the directory structure of the workspace. If
the directory already exists, the argument ignore_exists
determines, if a FileExistsError is raised.
ignore_exists: Boolean value which determines, if FileExistsError is
raised, if the target directory already exists. The default
behaviour is to raise an error, if the file already exists.
Returns:
Boolean value, which is True if the given directory was created.
"""
return self._ws.mkdir(dirpath, ignore_exists=ignore_exists)
def rmdir(self,
dirpath: PathLike,
recursive: bool = False,
ignore_missing: bool = False) -> bool:
"""Remove directory from current workspace.
Args:
dirpath: String or :term:`path-like object`, that points to a
directory in the directory structure of the workspace. If the
directory does not exist, the argument ignore_missing
determines, if a FileNotFoundError is raised.
ignore_missing: Boolean value which determines, if FileNotFoundError
is raised, if the target directory does not exist. The default
behaviour, is to raise an error if the directory is missing.
recursive: Boolean value which determines, if directories are
removed recursively. If recursive is False, then only empty
directories can be removed. If recursive, however, is True, then
all files and subdirectories are alse removed. By default
recursive is False.
Returns:
Boolean value, which is True if the given directory was removed.
"""
return self._ws.rmdir(dirpath,
recursive=recursive,
ignore_missing=ignore_missing)
def search(self, pattern: OptStr = None) -> StrList:
"""Search for files in the current workspace.
Args:
pattern: Search pattern that contains Unix shell-style wildcards:
'*': Matches arbitrary strings
'?': Matches single characters
[seq]: Matches any character in seq
[!seq]: Matches any character not in seq
By default a list of all files and directories is returned.
Returns:
List of files and directories in the directory structure of the
workspace, that match the search pattern.
"""
return self._ws.search(pattern)
def copy(self, source: PathLike, target: PathLike) -> bool:
"""Copy file within current workspace.
Args:
source: String or :term:`path-like object`, that points to a file in
the directory structure of the workspace. If the file does not
exist, a FileNotFoundError is raised. If the filepath points to
a directory, an IsADirectoryError is raised.
target: String or :term:`path-like object`, that points to a new
filename or an existing directory in the directory structure of
the workspace. If the target is a directory the target file
consists of the directory and the basename of the source file.
If the target file already exists a FileExistsError is raised.
Returns:
Boolean value which is True if the file was copied.
"""
return self._ws.copy(source, target)
def move(self, source: PathLike, target: PathLike) -> bool:
"""Move file within current workspace.
Args:
source: String or :term:`path-like object`, that points to a file in
the directory structure of the workspace. If the file does not
exist, a FileNotFoundError is raised. If the filepath points to
a directory, an IsADirectoryError is raised.
target: String or :term:`path-like object`, that points to a new
filename or an existing directory in the directory structure of
the workspace. If the target is a directory the target file
consists of the directory and the basename of the source file.
If the target file already exists a FileExistsError is raised.
Returns:
Boolean value which is True if the file has been moved.
"""
return self._ws.move(source, target)
def read_text(self, filepath: PathLike, encoding: OptStr = None) -> str:
"""Read text from file in current workspace.
Args:
filepath: String or :term:`path-like object`, that points to a valid
file in the directory structure of the workspace. If the file
does not exist a FileNotFoundError is raised.
encoding: Specifies the name of the encoding, which is used to
decode the stream’s bytes into strings. By default the preferred
encoding of the operating system is used.
Returns:
Contents of the given filepath encoded as string.
"""
return self._ws.read_text(filepath, encoding=encoding)
def read_bytes(self, filepath: PathLike) -> bytes:
"""Read bytes from file in current workspace.
Args:
filepath: String or :term:`path-like object`, that points to a valid
file in the dirctory structure of the workspace. If the file
does not exist a FileNotFoundError is raised.
Returns:
Contents of the given filepath as bytes.
"""
return self._ws.read_bytes(filepath)
def write_text(self,
text: str,
filepath: PathLike,
encoding: OptStr = None) -> int:
"""Write text to file.
Args:
text: String, which has to be written to the given file.
filepath: String or :term:`path-like object`, that represents a
valid filename in the dirctory structure of the workspace.
encoding: Specifies the name of the encoding, which is used to
encode strings into bytes. By default the preferred encoding of
the operating system is used.
Returns:
Number of characters, that are written to the file.
"""
return self._ws.write_text(text, filepath, encoding=encoding)
def write_bytes(self, data: BytesLike, filepath: PathLike) -> int:
"""Write bytes to file.
Args:
data: Bytes, which are to be written to the given file.
filepath: String or :term:`path-like object`, that represents a
valid filename in the dirctory structure of the workspace.
Returns:
Number of bytes, that are written to the file.
"""
return self._ws.write_bytes(data, filepath)
def log(self, level: StrOrInt, msg: str, *args: Any, **kwds: Any) -> None:
"""Log event.
Args:
level: Integer value or string, which describes the severity of the
event. Ordered by ascending severity, the allowed level names
are: 'DEBUG', 'INFO', 'WARNING', 'ERROR' and 'CRITICAL'. The
respectively corresponding level numbers are 10, 20, 30, 40 and
50.
msg: Message ``format string``_, which may can contain literal text
or replacement fields delimited by braces. Each replacement
field contains either the numeric index of a positional
argument, given by *args, or the name of a keyword argument,
given by the keyword *extra*.
*args: Arguments, which can be used by the message format string.
**kwds: Additional Keywords, used by the function `Logger.log()`_.
"""
self.logger.log(level, msg, *args, **kwds)
#
# Private Methods
#
def _load_config(self) -> None:
config = inifile.load(self._config_file_path, self._config_file_struct)
if 'session' in config and isinstance(config['session'], dict):
for key, val in config['session'].items():
self.config[key] = val
def _save_config(self) -> None:
config = {'session': self.config}
inifile.save(config, self._config_file_path)
def _get_path(self) -> OptPath:
return self._ws.path
def _get_files(self) -> StrList:
return self._ws.search()
def _get_folders(self) -> StrList:
return self._ws.folders
def _locate_path(self,
workspace: OptPathLike = None,
basedir: OptPathLike = None) -> OptPath:
if not workspace:
return None
if not basedir:
# If workspace is a fully qualified file path in the directory
# structure of the system, ignore the 'paths' list
if env.is_file(workspace):
return env.expand(workspace)
# Use the 'paths' list to find a workspace
for path in self.paths:
candidate = Path(path, workspace)
if candidate.is_file():
return candidate
raise FileNotFoundError(f"file {workspace} does not exist")
return Path(basedir, workspace)
class Logger(attrib.Container):
"""Logger class.
Args:
name: String identifier of Logger, given as a period-separated
hierarchical value like 'foo.bar.baz'. The name of a Logger also
identifies respective parents and children by the name hierachy,
which equals the Python package hierarchy.
file: String or :term:`path-like object` that identifies a valid
filename in the directory structure of the operating system. If they
do not exist, the parent directories of the file are created. If no
file is given, a default logfile within the applications
*user-log-dir* is created. If the logfile can not be created a
temporary logfile in the systems *temp* folder is created as a
fallback.
level: Integer value or string, which describes the minimum required
severity of events, to be logged. Ordered by ascending severity, the
allowed level names are: 'DEBUG', 'INFO', 'WARNING', 'ERROR' and
'CRITICAL'. The respectively corresponding level numbers are 10, 20,
30, 40 and 50. The default level is 'INFO'.
"""
#
# Protected Class Variables
#
_level_names: ClassVar[StrList] = [
'NOTSET', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
_default_name: ClassVar[str] = env.get_var('name') or __name__
_default_file: ClassVar[Path] = Path(
env.get_dir('user_log_dir'), _default_name + '.log')
_default_level: ClassVar[StrOrInt] = logging.INFO
#
# Public Attributes
#
logger: property = attrib.Virtual(
fget='_get_logger', fset='_set_logger', classinfo=logging.Logger)
name: property = attrib.Virtual(
fget='_get_name', fset='_set_name', classinfo=str)
name.__doc__ = """
String identifier of Logger, given as a period-separated hierarchical value
like 'foo.bar.baz'. The name of a Logger also identifies respective parents
and children by the name hierachy, which equals the Python package
hierarchy.
"""
file: property = attrib.Virtual(
fget='_get_file', fset='_set_file', classinfo=(str, Path))
file.__doc__ = """
String or :term:`path-like object` that identifies a valid filename in the
directory structure of the operating system. If they do not exist, the
parent directories of the file are created. If no file is given, a default
logfile within the applications *user-log-dir* is created. If the logfile
can not be created a temporary logfile in the systems *temp* folder is
created as a fallback.
"""
level: property = attrib.Virtual(
fget='_get_level', fset='_set_level', classinfo=(str, int))
level.__doc__ = """
Integer value or string, which describes the minimum required severity of
events, to be logged. Ordered by ascending severity, the allowed level names
are: 'DEBUG', 'INFO', 'WARNING', 'ERROR' and 'CRITICAL'. The respectively
corresponding level numbers are 10, 20, 30, 40 and 50. The default level is
'INFO'.
"""
#
# Protected Attributes
#
_logger: property = attrib.Temporary(classinfo=logging.Logger)
#
# Events
#
def __init__(self,
name: str = _default_name,
file: PathLike = _default_file,
level: StrOrInt = _default_level) -> None:
"""Initialize instance."""
# Initialize Attribute Container
super().__init__()
# Start logging
self._start_logging(name=name, file=file, level=level)
def __del__(self) -> None:
"""Run destructor for instance."""
self._stop_logging()
def __str__(self) -> str:
"""Represent instance as string."""
return str(self.logger)
#
# Public Methods
#
def log(self, level: StrOrInt, msg: str, *args: Any, **kwds: Any) -> None:
"""Log event.
Args:
level: Integer value or string, which describes the severity of the
event. In the order of ascending severity, the accepted level
names are: 'DEBUG', 'INFO', 'WARNING', 'ERROR' and 'CRITICAL'.
The respectively corresponding level numbers are 10, 20, 30, 40
and 50.
msg: Message :ref:`format string <formatstrings>`, containing
literal text or braces delimited replacement fields. Each
replacement field contains either the numeric index of a
positional argument, given by *args, or the name of a keyword
argument, given by the keyword *extra*.
*args: Arguments, which can be used by the message format string.
**kwds: Additional Keywords, used by :meth:`logging.Logger.log`.
"""
if isinstance(level, str):
level = self._get_level_number(level)
self.logger.log(level, msg, *args, **kwds)
#
# Protected Methods
#
def _start_logging(
self, name: str = _default_name, file: PathLike = _default_file,
level: StrOrInt = _default_level) -> bool:
logger = logging.getLogger(name) # Create new logger instance
self._set_logger(logger) # Bind new logger instance to global variable
self._set_level(level) # Set log level
self._set_file(file) # Add file handler for logfile
if not self.file.is_file(): # If an error occured stop logging
self._stop_logging()
return False
return True
def _stop_logging(self) -> None:
for handler in self.logger.handlers: # Close file handlers
with contextlib.suppress(AttributeError):
handler.close()
self._logger = None
def _get_logger(self, auto_start: bool = True) -> logging.Logger:
if not self._logger:
if auto_start:
self._start_logging()
else:
raise NotExistsError("logging has not been started")
return self._logger
def _set_logger(
self, logger: logging.Logger, auto_stop: bool = True) -> None:
if self._logger:
if auto_stop:
self._stop_logging()
else:
raise ExistsError("logging has already been started")
self._logger = logger
def _get_name(self) -> str:
return self.logger.name
def _set_name(self, name: str) -> None:
self.logger.name = name
def _get_file(self) -> OptPath:
for handler in self.logger.handlers:
with contextlib.suppress(AttributeError):
return Path(handler.baseFilename)
return None
def _set_file(self, filepath: PathLike = _default_file) -> None:
# Locate valid logfile
logfile = self._locate_logfile(filepath)
if not isinstance(logfile, Path):
warnings.warn("could not set logfile")
return None
# Close and remove all previous file handlers
if self.logger.hasHandlers():
remove = [h for h in self.logger.handlers if hasattr(h, 'close')]
for handler in remove:
handler.close()
self.logger.removeHandler(handler)
# Add file handler for logfile
handers = importlib.import_module('logging.handlers')
handler = getattr(handers, 'TimedRotatingFileHandler')(
str(logfile), when="d", interval=1, backupCount=5)
formatter = logging.Formatter(
fmt="%(asctime)s %(levelname)s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S")
handler.setFormatter(formatter)
self.logger.addHandler(handler)
return None
def _get_level(self, as_name: bool = True) -> StrOrInt:
level = getattr(self.logger, 'getEffectiveLevel')()
if not as_name:
return level
return self._get_level_name(level)
def _get_level_name(self, level: int) -> str:
names = self._level_names
return names[int(max(min(level, 50), 0) / 10)]
def _get_level_number(self, name: str) -> int:
name = name.upper()
names = self._level_names
if not name in names:
allowed = ', '.join(names[1:])
raise ValueError(
f"{name} is not a valid level name, "
f"allowed values are: {allowed}")
return names.index(name) * 10
def _set_level(self, level: StrOrInt) -> None:
if isinstance(level, str):
level = level.upper()
getattr(self.logger, 'setLevel')(level)
def _locate_logfile(
self, filepath: PathLike = _default_file) -> OptPath:
# Get valid logfile from filepath
if isinstance(filepath, (str, Path)):
logfile = env.expand(filepath)
if env.touch(logfile):
return logfile
# Get temporary logfile
logfile = Path(tempfile.NamedTemporaryFile().name + '.log')
if env.touch(logfile):
warnings.warn(
f"logfile '{filepath}' is not valid: "
f"using temporary logfile '{logfile}'")
return logfile
return None
class WsFile(attrib.Container):
"""Workspace File.
Workspace files are Zip-Archives, that contain a INI-formatted
configuration file 'workspace.ini' in the archives root, and arbitrary
resource files within subfolders.
Args:
filepath: String or :term:`path-like object`, that points to a valid
workspace file or None. If the filepath points to a valid workspace
file, then the class instance is initialized with a memory copy of
the file. If the given file, however, does not exist, isn't a valid
ZipFile, or does not contain a workspace configuration, respectively
one of the errors FileNotFoundError, BadZipFile or BadWsFile is
raised. The default behaviour, if the filepath is None, is to create
an empty workspace in the memory, that uses the default folders
layout. In this case the attribute maintainer is initialized with
the current username.
pwd: Bytes representing password of workspace file.
"""
#
# Protected Class Variables
#
_config_file: ClassVar[Path] = Path('workspace.ini')
_default_config: ClassVar[ConfigDict] = {
'dc': {
'creator': env.get_username(),
'date': datetime.datetime.now()}}
_default_dir_layout: ClassVar[StrList] = [
'dataset', 'network', 'system', 'model', 'script']
_default_encoding = env.get_encoding()
#
# Public Attributes and Attribute Groups
#
dc: attrib.Group = attrib.create_group(attrib.DCGroup)
startup: property = attrib.MetaData(classinfo=Path, category='hooks')
startup.__doc__ = """
The startup script is a path, that points to a python script inside the
workspace, which is executed after loading the workspace.
"""
path: property = attrib.Virtual(fget='_get_path')
path.__doc__ = """Filepath of the workspace."""
name: property = attrib.Virtual(fget='_get_name')
name.__doc__ = """Filename of the workspace without file extension."""
files: property = attrib.Virtual(fget='search')
files.__doc__ = """List of all files within the workspace."""
folders: property = attrib.Virtual(fget='_get_folders')
folders.__doc__ = """List of all folders within the workspace."""
changed: property = attrib.Virtual(fget='_get_changed')
changed.__doc__ = """Tells whether the workspace file has been changed."""
#
# Protected Attributes
#
_file: property = attrib.Content(classinfo=ZipFile)
_buffer: property = attrib.Content(classinfo=BytesIOBaseClass)
_path: property = attrib.Temporary(classinfo=Path)
_pwd: property = attrib.Temporary(classinfo=bytes)
_changed: property = attrib.Temporary(classinfo=bool, default=False)
#
# Events
#
def __init__(
self, filepath: OptPathLike = None, pwd: OptBytes = None,
parent: Optional[attrib.Container] = None) -> None:
"""Load Workspace from file."""
super().__init__()
if filepath:
self.load(filepath, pwd=pwd)
else:
self._create_new()
def __enter__(self) -> 'WsFile':
"""Enter with statement."""
return self
def __exit__(self, cls: ExcType, obj: Exc, tb: Traceback) -> None:
"""Close workspace file and buffer."""
self.close()
#
# Public Methods
#
def load(self, filepath: PathLike, pwd: OptBytes = None) -> None:
"""Load Workspace from file.
Args:
filepath: String or :term:`path-like object`, that points to a valid
workspace file. If the filepath points to a valid workspace
file, then the class instance is initialized with a memory copy
of the file. If the given file, however, does not exist, isn't a
valid ZipFile, or does not contain a workspace configuration,
respectively one of the errors FileNotFoundError, BadZipFile or
BadWsFile is raised.
pwd: Bytes representing password of workspace file.
"""
# Initialize instance Variables, Buffer and buffered ZipFile
self._changed = False
self._path = env.expand(filepath)
self._pwd = pwd
self._buffer = BytesIO()
self._file = ZipFile(self._buffer, mode='w')
# Copy contents from ZipFile to buffered ZipFile
with warnings.catch_warnings():
warnings.simplefilter("ignore", UserWarning)
try:
with ZipFile(self.path, mode='r') as fh:
for zinfo in fh.infolist():
data = fh.read(zinfo, pwd=pwd)
# TODO ([email protected]): The zipfile standard
# module currently does not support encryption in write
# mode of new ZipFiles. See:
# https://docs.python.org/3/library/zipfile.html
# When support is provided, the below line for writing
# files shall be replaced by:
# self._file.writestr(zinfo, data, pwd=pwd)
self._file.writestr(zinfo, data)
except FileNotFoundError as err:
raise FileNotFoundError(
f"file '{self.path}' does not exist") from err
except BadZipFile as err:
raise BadZipFile(
f"file '{self.path}' is not a valid ZIP file") from err
# Try to open and load workspace configuration from buffer
structure = {
'dc': self._get_attr_types(group='dc'),
'hooks': self._get_attr_types(category='hooks')}
try:
with self.open(self._config_file) as file:
cfg = inifile.load(file, structure=structure)
except KeyError as err:
raise BadWsFile(
f"workspace '{self.path}' is not valid: "
f"file '{self._config_file}' could not be loaded") from err
# Link configuration
self._set_attr_values(cfg.get('dc', {}), group='dc') # type: ignore
def save(self) -> None:
"""Save the workspace to it's filepath."""
if isinstance(self.path, Path):
self.saveas(self.path)
else:
raise FileNotGivenError(
"use saveas() to save the workspace to a file")
def saveas(self, filepath: PathLike) -> None:
"""Save the workspace to a file.
Args:
filepath: String or :term:`path-like object`, that represents the
name of a workspace file.
"""
path = env.expand(filepath)
# Update datetime
self.date = datetime.datetime.now()
# Update 'workspace.ini'
with self.open(self._config_file, mode='w') as file:
inifile.save({
'dc': self._get_attr_values(group='dc'),
'hooks': self._get_attr_values(category='hooks')}, file)
# Remove duplicates from workspace
self._remove_duplicates()
# Mark plattform, which created the files as Windows
# to avoid inference of wrong Unix permissions
for zinfo in self._file.infolist():
zinfo.create_system = 0
# Close ZipArchive (to allow to read the buffer)
self._file.close()
# Read buffer and write workspace file
if not isinstance(self._buffer, BytesIO):
raise TypeError("buffer has not been initialized")
with open(path, 'wb') as file:
file.write(self._buffer.getvalue())
# Close buffer
self._buffer.close()
# Reload saved workpace from file
self.load(path, pwd=self._pwd)
def get_file_accessor(self, path: PathLike) -> FileAccessorBase:
"""Get file accessor to workspace member.
Args:
path: String or :term:`path-like object`, that represents a
workspace member. In reading mode the path has to point to a
valid workspace file, or a FileNotFoundError is raised. In
writing mode the path by default is treated as a file path. New
directories can be written by setting the argument is_dir to
True.
Returns:
:class:`File accessor <nemoa.types.FileAccessorBase>` to workspace
member.
"""
def wrap_open(path: PathLike) -> AnyFunc:
def wrapped_open(
obj: FileAccessorBase, *args: Any, **kwds: Any) -> FileLike:
return self.open(path, *args, **kwds)
return wrapped_open
return type( # pylint: disable=E0110
'FileAccessor', (FileAccessorBase,), {
'name': str(path),
'open': wrap_open(path)})()
def open(
self, path: PathLike, mode: str = 'r', encoding: OptStr = None,
is_dir: bool = False) -> FileLike:
"""Open file within the workspace.
Args:
path: String or :term:`path-like object`, that represents a
workspace member. In reading mode the path has to point to a
valid workspace file, or a FileNotFoundError is raised. In
writing mode the path by default is treated as a file path. New
directories can be written by setting the argument is_dir to
True.
mode: String, which characters specify the mode in which the file is
to be opened. The default mode is reading in text mode. Suported
characters are:
'r': Reading mode (default)
'w': Writing mode
'b': Binary mode
't': Text mode (default)
encoding: In binary mode encoding has not effect. In text mode
encoding specifies the name of the encoding, which in reading
and writing mode respectively is used to decode the stream’s
bytes into strings, and to encode strings into bytes. By default
the preferred encoding of the operating system is used.
is_dir: Boolean value which determines, if the path is to be treated
as a directory or not. This information is required for writing
directories to the workspace. The default behaviour is not to
treat paths as directories.
Returns:
:term:`File object` in reading or writing mode.
Examples:
>>> with self.open('workspace.ini') as file:
>>> print(file.read())
"""
# Open file handler to workspace member
if 'w' in mode:
if 'r' in mode:
raise ValueError(
"'mode' is not allowed to contain the "
"characters 'r' AND 'w'")
file = self._open_write(path, is_dir=is_dir)
else:
file = self._open_read(path)
# Wrap binary files to text files if required
if 'b' in mode:
if 't' in mode:
raise ValueError(
"'mode' is not allowed to contain the "
"characters 'b' AND 't'")
return file
return TextIOWrapper(
file, encoding=encoding or self._default_encoding,
write_through=True)
def close(self) -> None:
"""Close current workspace and buffer."""
if hasattr(self._file, 'close'):
self._file.close()
if hasattr(self._buffer, 'close'):
self._buffer.close()
def copy(self, source: PathLike, target: PathLike) -> bool:
"""Copy file within workspace.
Args:
source: String or :term:`path-like object`, that points to a file in
the directory structure of the workspace. If the file does not
exist, a FileNotFoundError is raised. If the filepath points to
a directory, an IsADirectoryError is raised.
target: String or :term:`path-like object`, that points to a new
filename or an existing directory in the directory structure of
the workspace. If the target is a directory the target file
consists of the directory and the basename of the source file.
If the target file already exists a FileExistsError is raised.
Returns:
Boolean value which is True if the file was copied.
"""
# Check if source file exists and is not a directory
src_file = PurePath(source).as_posix()
src_infos = self._locate(source)
if not src_infos:
raise FileNotFoundError(
f"workspace file '{src_file}' does not exist")
src_info = src_infos[-1]
if getattr(src_info, 'is_dir')():
raise IsADirectoryError(
f"'{src_file}/' is a directory not a file")
# If target is a directory get name of target file from
# source filename
tgt_file = PurePath(target).as_posix()
if tgt_file == '.':
tgt_file = Path(src_file).name
else:
tgt_infos = self._locate(target)
if tgt_infos:
if getattr(tgt_infos[-1], 'is_dir')():
tgt_path = PurePath(tgt_file, Path(src_file).name)
tgt_file = tgt_path.as_posix()
# Check if target file already exists
if self._locate(tgt_file):
raise FileExistsError(
f"workspace file '{tgt_file}' already exist.")
# Read binary data from source file
data = self._file.read(src_info, pwd=self._pwd)
# Create ZipInfo for target file from source file info
tgt_time = getattr(src_info, 'date_time')
tgt_info = ZipInfo(filename=tgt_file, date_time=tgt_time) # type: ignore
# Write binary data to target file
# TODO ([email protected]): The zipfile standard module currently
# does not support encryption in write mode. See:
# https://docs.python.org/3/library/zipfile.html
# When support is provided, the below line shall be replaced by:
# self._file.writestr(tgt_info, data, pwd=self._pwd)
self._file.writestr(tgt_info, data)
self._changed = True
# Check if new file exists
return bool(self._locate(tgt_file))
def move(self, source: PathLike, target: PathLike) -> bool:
"""Move file within workspace.
Args:
source: String or :term:`path-like object`, that points to a file in
the directory structure of the workspace. If the file does not
exist, a FileNotFoundError is raised. If the filepath points to
a directory, an IsADirectoryError is raised.
target: String or :term:`path-like object`, that points to a new
filename or an existing directory in the directory structure of
the workspace. If the target is a directory the target file
consists of the directory and the basename of the source file.
If the target file already exists a FileExistsError is raised.
Returns:
Boolean value which is True if the file has been moved.
"""
# Copy source file to target file or directory
# and on success remove source file
return self.copy(source, target) and self.unlink(source)
def append(self, source: PathLike, target: OptPathLike = None) -> bool:
"""Append file to the workspace.
Args:
source: String or :term:`path-like object`, that points to a valid
file in the directory structure if the system. If the file does
not exist, a FileNotFoundError is raised. If the filepath points
to a directory, a IsADirectoryError is raised.
target: String or :term:`path-like object`, that points to a valid
directory in the directory structure of the workspace. By
default the root directory is used. If the directory does not
exist, a FileNotFoundError is raised. If the target directory
already contains a file, which name equals the filename of the
source, a FileExistsError is raised.
Returns:
Boolean value which is True if the file has been appended.
"""
# Check source file
src_file = env.expand(source)
if not src_file.exists():
raise FileNotFoundError(f"file '{src_file}' does not exist")
if src_file.is_dir():
raise IsADirectoryError(f"'{src_file}' is a directory not a file")
# Check target directory
if target:
tgt_dir = PurePath(target).as_posix() + '/'
if not self._locate(tgt_dir):
raise FileNotFoundError(
f"workspace directory '{tgt_dir}' does not exist")
else:
tgt_dir = '.'
tgt_file = Path(tgt_dir, src_file.name)
if self._locate(tgt_file):
raise FileExistsError(
f"workspace directory '{tgt_dir}' already contains a file "
f"with name '{src_file.name}'")
# Create ZipInfo entry from source file
filename = PurePath(tgt_file).as_posix()
date_time = time.localtime(src_file.stat().st_mtime)[:6]
zinfo = ZipInfo(filename=filename, date_time=date_time) # type: ignore
# Copy file to archive
with src_file.open('rb') as src:
data = src.read()
# TODO ([email protected]): The zipfile standard module currently
# does not support encryption in write mode. See:
# https://docs.python.org/3/library/zipfile.html
# When support is provided, the below line shall be replaced by:
# self._file.writestr(zinfo, data, pwd=pwd)
self._file.writestr(zinfo, data)
return True
def read_text(self, filepath: PathLike, encoding: OptStr = None) -> str:
"""Read text from file.
Args:
filepath: String or :term:`path-like object`, that points to a valid
file in the directory structure of the workspace. If the file
does not exist a FileNotFoundError is raised.
encoding: Specifies the name of the encoding, which is used to
decode the stream’s bytes into strings. By default the preferred
encoding of the operating system is used.
Returns:
Contents of the given filepath encoded as string.
"""
with self.open(filepath, mode='r', encoding=encoding) as file:
text = file.read()
if not isinstance(text, str):
return ''
return text
def read_bytes(self, filepath: PathLike) -> bytes:
"""Read bytes from file.
Args:
filepath: String or :term:`path-like object`, that points to a valid
file in the dirctory structure of the workspace. If the file
does not exist a FileNotFoundError is raised.
Returns:
Contents of the given filepath as bytes.
"""
with self.open(filepath, mode='rb') as file:
blob = file.read()
if not isinstance(blob, bytes):
return b''
return blob
def write_text(
self, text: str, filepath: PathLike,
encoding: OptStr = None) -> int:
"""Write text to file.
Args:
text: String, which has to be written to the given file.
filepath: String or :term:`path-like object`, that represents a
valid filename in the dirctory structure of the workspace.
encoding: Specifies the name of the encoding, which is used to
encode strings into bytes. By default the preferred encoding of
the operating system is used.
Returns:
Number of characters, that are written to the file.
"""
with self.open(filepath, mode='w', encoding=encoding) as file:
if isinstance(file, TextIOBaseClass):
return file.write(text)
return 0
def write_bytes(self, blob: BytesLike, filepath: PathLike) -> int:
"""Write bytes to file.
Args:
blob: Bytes, which are to be written to the given file.
filepath: String or :term:`path-like object`, that represents a
valid filename in the dirctory structure of the workspace.
Returns:
Number of bytes, that are written to the file.
"""
with self.open(filepath, mode='wb') as file:
if isinstance(file, BytesIOBaseClass):
return file.write(blob)
return 0
def unlink(self, filepath: PathLike, ignore_missing: bool = True) -> bool:
"""Remove file from workspace.
Args:
filepath: String or :term:`path-like object`, that points to a file
in the directory structure of the workspace. If the filepath
points to a directory, an IsADirectoryError is raised. For the
case, that the file does not exist, the argument ignore_missing
determines, if a FileNotFoundError is raised.
ignore_missing: Boolean value which determines, if FileNotFoundError
is raised, if the target file does not exist. The default
behaviour, is to ignore missing files.
Returns:
Boolean value, which is True if the given file was removed.
"""
matches = self._locate(filepath)
if not matches:
if ignore_missing:
return True
filename = PurePath(filepath).as_posix()
raise FileNotFoundError(f"file '{filename}' does not exist")
if getattr(matches[-1], 'is_dir')():
dirname = PurePath(filepath).as_posix() + '/'
raise IsADirectoryError(f"'{dirname}' is a directory not a file")
return self._remove_members(matches)
def mkdir(self, dirpath: PathLike, ignore_exists: bool = False) -> bool:
"""Create a new directory at the given path.
Args:
dirpath: String or :term:`path-like object`, that represents a valid
directory name in the directory structure of the workspace. If
the directory already exists, the argument ignore_exists
determines, if a FileExistsError is raised.
ignore_exists: Boolean value which determines, if FileExistsError is
raised, if the target directory already exists. The default
behaviour is to raise an error, if the file already exists.
Returns:
Boolean value, which is True if the given directory was created.
"""
matches = self._locate(dirpath)
if not matches:
with self.open(dirpath, mode='w', is_dir=True):
pass
elif not ignore_exists:
dirname = PurePath(dirpath).as_posix() + '/'
raise FileExistsError(f"directory '{dirname}' already exists")
return True
def rmdir(
self, dirpath: PathLike, recursive: bool = False,
ignore_missing: bool = False) -> bool:
"""Remove directory from workspace.
Args:
dirpath: String or :term:`path-like object`, that points to a
directory in the directory structure of the workspace. If the
directory does not exist, the argument ignore_missing
determines, if a FileNotFoundError is raised.
ignore_missing: Boolean value which determines, if FileNotFoundError
is raised, if the target directory does not exist. The default
behaviour, is to raise an error if the directory is missing.
recursive: Boolean value which determines, if directories are
removed recursively. If recursive is False, then only empty
directories can be removed. If recursive, however, is True, then
all files and subdirectories are alse removed. By default
recursive is False.
Returns:
Boolean value, which is True if the given directory was removed.
"""
matches = self._locate(dirpath)
dirname = PurePath(dirpath).as_posix() + '/'
if not matches:
if ignore_missing:
return True
raise FileNotFoundError(f"directory '{dirname}' does not exist")
files = self.search(dirname + '*')
if not files:
return self._remove_members(matches)
if not recursive:
raise DirNotEmptyError(f"directory '{dirname}' is not empty")
allmatches = matches
for file in files:
allmatches += self._locate(file)
return self._remove_members(allmatches)
def search(self, pattern: OptStr = None) -> StrList:
"""Search for files in the workspace.
Args:
pattern: Search pattern that contains Unix shell-style wildcards:
'*': Matches arbitrary strings
'?': Matches single characters
[seq]: Matches any character in seq
[!seq]: Matches any character not in seq
By default a list of all files and directories is returned.
Returns:
List of files and directories in the directory structure of the
workspace, that match the search pattern.
"""
# Get list of normalized unique paths of workspace members
paths: PathLikeList = []
for zinfo in self._file.infolist():
path = PurePath(zinfo.filename).as_posix()
if getattr(zinfo, 'is_dir')():
path += '/'
if path not in paths:
paths.append(path)
# Match path list with given pattern
if pattern:
paths = env.match_paths(paths, pattern)
# Sort paths
return sorted([str(path) for path in paths])
#
# Protected Methods
#
def _create_new(self) -> None:
# Initialize instance Variables, Buffer and buffered ZipFile
self._set_attr_values(self._default_config['dc'], group='dc')
self._path = None
self._changed = False
self._pwd = None
self._buffer = BytesIO()
self._file = ZipFile(self._buffer, mode='w')
# Create folders
for folder in self._default_dir_layout:
self.mkdir(folder)
def _open_read(self, path: PathLike) -> BytesIOLike:
# Locate workspace member by it's path
# and open file handler for reading the file
matches = self._locate(path)
if not matches:
fname = PurePath(path).as_posix()
raise FileNotFoundError(
f"workspace member with filename '{fname}' does not exist")
# Select latest version of file
zinfo = matches[-1]
return self._file.open(zinfo, pwd=self._pwd, mode='r')
def _open_write(self, path: PathLike, is_dir: bool = False) -> BytesIOLike:
# Determine workspace member name from path
# and get ZipInfo with local time as date_time
filename = PurePath(path).as_posix()
if is_dir:
filename += '/'
zinfo = ZipInfo( # type: ignore
filename=filename,
date_time=time.localtime()[:6])
# Catch Warning for duplicate files
with warnings.catch_warnings():
warnings.simplefilter("ignore", UserWarning)
# TODO ([email protected]): The zipfile standard
# module currently does not support encryption in write
# mode of new ZipFiles. See:
# https://docs.python.org/3/library/zipfile.html
# When support is provided, the below line for writing
# files shall be replaced by:
# file = self._file.open(zinfo, mode='w', pwd=self._pwd)
file = self._file.open(zinfo, mode='w')
self._changed = True
return file
def _locate(self, path: PathLike, sort: bool = True) -> ZipInfoList:
# Get list of member zipinfos
zinfos = self._file.infolist()
# Match members by path-like filenames
matches = [i for i in zinfos if Path(i.filename) == Path(path)]
if sort:
# Sort matches by datetime
matches = sorted(matches, key=lambda i: i.date_time)
# Return sorted matches
return matches
def _get_name(self) -> OptStr:
return getattr(self._path, 'stem', None)
def _get_path(self) -> OptPath:
return self._path
def _get_changed(self) -> bool:
return self._changed
def _get_folders(self) -> StrList:
names: StrList = []
for zinfo in self._file.infolist():
if getattr(zinfo, 'is_dir')():
name = PurePath(zinfo.filename).as_posix() + '/'
names.append(name)
return sorted(names)
def _remove_members(self, zinfos: ZipInfoList) -> bool:
# Return True if list of members is empty
if not zinfos:
return True
# Remove entries in the list of members from workspace
new_zinfos = []
zids = [(zinfo.filename, zinfo.date_time) for zinfo in zinfos]
for zinfo in self._file.infolist():
zid = (zinfo.filename, zinfo.date_time)
if zid in zids:
zids.remove(zid)
else:
new_zinfos.append(zinfo)
# If any entry on the list could not be found raise an error
if zids:
names = [zid[0] for zid in zids]
raise FileNotFoundError(
f"could not locate workspace members: {names}")
# Create new ZipArchive in Memory
new_buffer = BytesIO()
new_file = ZipFile(new_buffer, mode='w')
# Copy all workspace members on the new list from current
# to new workspace
for zinfo in new_zinfos:
data = self._file.read(zinfo, pwd=self._pwd)
new_file.writestr(zinfo, data)
# Close current workspace and buffer and link new workspace and buffer
self._file.close()
self._buffer.close()
self._buffer = new_buffer
self._file = new_file
self._changed = True
return True
def _remove_duplicates(self) -> bool:
# Get list of duplicates
zinfos: ZipInfoList = []
for filename in self.files:
zinfos += self._locate(filename, sort=True)[:-1]
# Remove duplicates
return self._remove_members(zinfos)