def test_handle_os_exceptions():
"""Tests airfs._core.exceptions.handle_os_exceptions"""
from airfs._core.exceptions import (
handle_os_exceptions,
ObjectNotFoundError,
ObjectPermissionError,
ObjectNotADirectoryError,
ObjectExistsError,
)
with pytest.raises(FileNotFoundError):
with handle_os_exceptions():
raise ObjectNotFoundError
with pytest.raises(PermissionError):
with handle_os_exceptions():
raise ObjectPermissionError
with pytest.raises(FileExistsError):
with handle_os_exceptions():
raise ObjectExistsError
with pytest.raises(NotADirectoryError):
with handle_os_exceptions():
raise ObjectNotADirectoryError
with pytest.raises(FileExistsError):
with handle_os_exceptions():
raise FileExistsError()
def __init__(self, name, mode="r", storage_parameters=None, **kwargs):
RawIOBase.__init__(self)
ObjectIOBase.__init__(self, name, mode=mode)
if storage_parameters is not None:
storage_parameters = storage_parameters.copy()
try:
self._cache["_head"] = storage_parameters.pop("airfs.raw_io._head")
except (AttributeError, KeyError):
pass
try:
self._system = storage_parameters.pop("airfs.system_cached")
except (AttributeError, KeyError):
self._system = None
if not self._system:
self._system = self._SYSTEM_CLASS(
storage_parameters=storage_parameters, **kwargs
)
self._client_kwargs = self._system.get_client_kwargs(name)
self._is_raw_of_buffered = False
if self._writable:
self._write_buffer = bytearray()
if "a" in mode:
if self._exists() == 1:
with handle_os_exceptions():
self._init_append()
elif self._exists() == 0:
with handle_os_exceptions():
self._create()
else:
raise PermissionError(
"Insufficient permission to check if file already exists."
)
elif "x" in mode and self._exists() == 1:
raise FileExistsError
elif "x" in mode and self._exists() == -1:
raise PermissionError(
"Insufficient permission to check if file already exists."
)
else:
with handle_os_exceptions():
self._create()
else:
with handle_os_exceptions():
self._head()
def readinto(self, b):
"""
Read bytes into a pre-allocated, writable bytes-like object b, and return the
number of bytes read.
Args:
b (bytes-like object): buffer.
Returns:
int: number of bytes read
"""
if not self._readable:
raise UnsupportedOperation("read")
size = len(b)
with self._seek_lock:
start = self._seek
end = start + size
self._seek = end
with handle_os_exceptions():
read_data = self._read_range(start, end)
read_size = len(read_data)
if read_size:
memoryview(b)[:read_size] = read_data
if read_size != size:
with self._seek_lock:
self._seek = start + read_size
return read_size
def is_dir(self, *, follow_symlinks=True):
"""
Return True if this entry is a directory or a symbolic link pointing to a
directory; return False if the entry is or points to any other kind of file, or
if it doesn’t exist anymore.
The result is cached on the os.DirEntry object.
Args:
follow_symlinks (bool): Follow symlinks.
Returns:
bool: True if directory exists.
"""
with handle_os_exceptions():
try:
return self._system.isdir(
path=self._path,
client_kwargs=self._client_kwargs,
virtual_dir=False,
follow_symlinks=follow_symlinks,
) or bool(
# Some directories only exists virtually in object path and don't
# have headers.
S_ISDIR(self.stat().st_mode))
except ObjectPermissionError:
return True
def flush(self):
"""
Flush the write buffers of the stream if applicable and save the object on the
storage.
"""
if self._writable:
with handle_os_exceptions():
self._flush(self._get_buffer())
def close(self):
"""
Flush the write buffers of the stream if applicable and close the object.
"""
if self._writable and not self._closed:
self._closed = True
with self._seek_lock:
self._flush_raw_or_buffered()
if self._seek:
with handle_os_exceptions():
self._close_writable()
def _peek(self, size=-1):
"""
Return bytes from the stream without advancing the position.
Args:
size (int): Number of bytes to read. -1 to read the full stream.
Returns:
bytes: bytes read
"""
with self._seek_lock:
seek = self._seek
with handle_os_exceptions():
return self._read_range(seek, seek + size)
def flush(self):
"""
Flush the write buffers of the stream if applicable and save the object on the
storage.
"""
if self._writable:
with self._seek_lock:
buffer = self._get_buffer()
end = self._seek
start = end - len(buffer)
self._write_buffer = bytearray()
with handle_os_exceptions():
self._flush(buffer, start, end)
def read(self, size=-1):
"""
Read and return up to size bytes, with at most one call to the underlying raw
stream.
Use at most one call to the underlying raw stream’s read method.
Args:
size (int): Number of bytes to read. -1 to read the stream until end.
Returns:
bytes: Object content
"""
if not self._readable:
raise UnsupportedOperation("read")
elif not self._seekable:
return self._raw.read(size)
if self._seek == self._size:
return b""
if size == self._buffer_size:
queue_index = self._seek
if queue_index == 0:
self._preload_range()
with handle_os_exceptions():
buffer = self._read_queue.pop(queue_index).result()
buffer_size = self._buffer_size
index = queue_index + buffer_size * self._max_buffers
if index < self._size:
self._read_queue[index] = self._workers.submit(
self._read_range, index, index + buffer_size)
self._seek += len(buffer)
return buffer
if size != -1:
buffer = bytearray(size)
else:
buffer = bytearray()
read_size = self.readinto(buffer)
return memoryview(buffer)[:read_size].tobytes()
def decorated(path, *args, **kwargs):
"""
Decorated function.
Args:
path (path-like object or int): Path, URL or file descriptor.
"""
if not isinstance(path, int):
path_str = fsdecode(path).replace("\\", "/")
if is_storage(path_str):
with handle_os_exceptions():
result = cos_function(path_str, *args, **kwargs)
if keep_path_type and isinstance(path, bytes):
result = fsencode(result)
return result
return std_function(path, *args, **kwargs)
def _flush_raw_or_buffered(self):
"""
Flush using raw of buffered methods.
"""
# Flush only if bytes written
# This avoid no required process/thread creation and network call.
# This step is performed by raw stream.
if self._buffer_seek and self._seek:
self._seek += 1
with handle_os_exceptions():
self._flush()
# If data lower than buffer size flush data with raw stream to reduce IO calls
elif self._buffer_seek:
self._raw._write_buffer = self._get_buffer()
self._raw._seek = self._buffer_seek
self._raw.flush()
def symlink(src, dst, target_is_directory=False, *, dir_fd=None):
"""
Create a symbolic link pointing to src named dst.
Equivalent to "os.symlink".
.. versionadded:: 1.5.0
Args:
src (path-like object): Path or URL to the symbolic link.
dst (path-like object): Path or URL to the target.
target_is_directory (bool): On Windows, define if symlink represents either a
file or a directory.
Not supported on storage objects and non-Windows platforms.
dir_fd (int): directory descriptors;
see the os.symlink() description for how it is interpreted.
Not supported on storage objects.
"""
src, src_is_storage = format_and_is_storage(src)
dst, dst_is_storage = format_and_is_storage(dst)
if not src_is_storage and not dst_is_storage:
return os.symlink(src,
dst,
target_is_directory=target_is_directory,
dir_fd=dir_fd)
with handle_os_exceptions():
if not src_is_storage or not dst_is_storage:
ObjectNotImplementedError(
"Cross storage symlinks are not supported")
raises_on_dir_fd(dir_fd)
system_src = get_instance(src)
system_dst = get_instance(dst)
if system_src is not system_dst:
ObjectNotImplementedError(
"Cross storage symlinks are not supported")
elif system_src.relpath(src) == system_dst.relpath(dst):
raise ObjectSameFileError(path1=src, path2=dst)
return get_instance(src).symlink(src, dst)
def shareable_url(path, expires_in=3600):
"""
Get a shareable public URL for the specified path of an existing object.
Only available for some storage and not for local paths.
.. versionadded:: 1.5.0
Args:
path (str): Path or URL.
expires_in (int): Expiration in seconds. Default to 1 hour.
"""
with handle_os_exceptions():
path_str = fsdecode(path).replace("\\", "/")
if not is_storage(path_str):
raise ObjectUnsupportedOperation("shareable_url")
return get_instance(path).shareable_url(path_str, expires_in)
def stat(self, *, follow_symlinks=True):
"""
Return a stat_result object for this entry.
The result is cached on the os.DirEntry object.
Args:
follow_symlinks (bool): Follow symlinks.
Returns:
os.stat_result: Stat result object
"""
with handle_os_exceptions():
return self._system.stat(
path=self._path,
client_kwargs=self._client_kwargs,
header=self._header,
follow_symlinks=follow_symlinks,
)
def is_file(self, *, follow_symlinks=True):
"""
Return True if this entry is a file or a symbolic link pointing to a file;
return False if the entry is or points to a directory or other non-file entry,
or if it doesn’t exist anymore.
The result is cached on the os.DirEntry object.
Args:
follow_symlinks (bool): Follow symlinks.
Returns:
bool: True if directory exists.
"""
with handle_os_exceptions():
return self._system.isfile(
path=self._path,
client_kwargs=self._client_kwargs,
follow_symlinks=follow_symlinks,
)
def readall(self):
"""
Read and return all the bytes from the stream until EOF.
Returns:
bytes: Object content
"""
if not self._readable:
raise UnsupportedOperation("read")
with self._seek_lock:
with handle_os_exceptions():
if self._seek and self._seekable:
data = self._read_range(self._seek)
else:
data = self._readall()
self._seek += len(data)
return data
def _scandir_generator(is_bytes, scandir_path, system):
"""
scandir generator
Args:
is_bytes (bool): True if DirEntry must handle path as bytes.
scandir_path (str): Path.
system (airfs._core.io_system.SystemBase subclass): Storage system.
Yields:
DirEntry: Directory entries
"""
with handle_os_exceptions():
for name, header in system.list_objects(scandir_path,
first_level=True):
yield DirEntry(
scandir_path=scandir_path,
system=system,
name=name,
header=header,
bytes_path=is_bytes,
)
def _copy(src, dst, src_is_storage, dst_is_storage, follow_symlinks):
"""
Copies file from source to destination
Args:
src (str or file-like object): Source file.
dst (str or file-like object): Destination file.
src_is_storage (bool): Source is storage.
dst_is_storage (bool): Destination is storage.
follow_symlinks (bool): If True, follow symlinks.
"""
with handle_os_exceptions():
if src_is_storage and dst_is_storage:
system_src = get_instance(src)
system_dst = get_instance(dst)
if system_src is system_dst:
if system_src.relpath(src) == system_dst.relpath(dst):
raise ObjectSameFileError(path1=src, path2=dst)
try:
return system_dst.copy(src, dst)
except AirfsInternalException:
pass
for caller, called, method in (
(system_dst, system_src, "copy_from_%s"),
(system_src, system_dst, "copy_to_%s"),
):
if hasattr(caller, method % called.storage):
try:
return getattr(caller, method % called.storage)(
src, dst, called, follow_symlinks)
except AirfsInternalException:
continue
_copy_stream(dst, src)
def readinto(self, b):
"""
Read bytes into a pre-allocated, writable bytes-like object b, and return the
number of bytes read.
Args:
b (bytes-like object): buffer.
Returns:
int: number of bytes read
"""
if not self._readable:
raise UnsupportedOperation("read")
elif not self._seekable:
return self._raw.readinto(b)
with self._seek_lock:
seek = self._seek
queue = self._read_queue
if seek == 0:
self._preload_range()
size = len(b)
if size:
# Preallocated buffer: Use memory view to avoid copies
b_view = memoryview(b)
size_left = size
else:
# Dynamic buffer: Can't avoid copy, read until EOF
b_view = b
size_left = -1
b_end = 0
buffer_size = self._buffer_size
while size_left > 0 or size_left == -1:
start = seek % buffer_size
queue_index = seek - start
try:
buffer = queue[queue_index]
except KeyError:
# EOF
break
with handle_os_exceptions():
try:
queue[queue_index] = buffer = buffer.result()
except AttributeError:
# Already evaluated
pass
buffer_view = memoryview(buffer)
data_size = len(buffer)
if not data_size:
break
if size_left != -1:
end = start + size_left
else:
end = data_size - start
if end >= data_size:
end = data_size
del queue[queue_index]
index = queue_index + buffer_size * self._max_buffers
if index < self._size:
queue[index] = self._workers.submit(
self._read_range, index, index + buffer_size)
read_size = end - start
if size_left != -1:
size_left -= read_size
seek += read_size
b_start = b_end
b_end = b_start + read_size
b_view[b_start:b_end] = buffer_view[start:end]
self._seek = seek
self._raw.seek(seek)
return b_end
def write(self, b):
"""
Write the given bytes-like object, b, to the underlying raw stream, and return
the number of bytes written.
Args:
b (bytes-like object): Bytes to write.
Returns:
int: The number of bytes written.
"""
if not self._writable:
raise UnsupportedOperation("write")
size = len(b)
b_view = memoryview(b)
size_left = size
buffer_size = self._buffer_size
max_buffers = self._max_buffers
with self._seek_lock:
end = self._buffer_seek
buffer_view = memoryview(self._write_buffer)
while size_left > 0:
start = end
end = start + size_left
if end > buffer_size:
end = buffer_size
flush = True
else:
flush = False
buffer_range = end - start
b_start = size - size_left
size_left -= buffer_range
buffer_view[start:end] = b_view[b_start:b_start + buffer_range]
if flush:
self._buffer_seek = end
self._seek += 1
if max_buffers:
futures = self._write_futures
flush_wait = self._FLUSH_WAIT
while (sum(1 for future in futures
if not future.done()) >= max_buffers):
sleep(flush_wait)
with handle_os_exceptions():
self._flush()
self._write_buffer = bytearray(buffer_size)
buffer_view = memoryview(self._write_buffer)
end = 0
self._buffer_seek = end
return size
