def test_handle_os_exceptions(): """Tests airfs._core.exceptions.handle_os_exceptions""" from airfs._core.exceptions import ( handle_os_exceptions, ObjectNotFoundError, ObjectPermissionError, ObjectNotADirectoryError, ObjectExistsError, ) with pytest.raises(FileNotFoundError): with handle_os_exceptions(): raise ObjectNotFoundError with pytest.raises(PermissionError): with handle_os_exceptions(): raise ObjectPermissionError with pytest.raises(FileExistsError): with handle_os_exceptions(): raise ObjectExistsError with pytest.raises(NotADirectoryError): with handle_os_exceptions(): raise ObjectNotADirectoryError with pytest.raises(FileExistsError): with handle_os_exceptions(): raise FileExistsError()
def __init__(self, name, mode="r", storage_parameters=None, **kwargs): RawIOBase.__init__(self) ObjectIOBase.__init__(self, name, mode=mode) if storage_parameters is not None: storage_parameters = storage_parameters.copy() try: self._cache["_head"] = storage_parameters.pop("airfs.raw_io._head") except (AttributeError, KeyError): pass try: self._system = storage_parameters.pop("airfs.system_cached") except (AttributeError, KeyError): self._system = None if not self._system: self._system = self._SYSTEM_CLASS( storage_parameters=storage_parameters, **kwargs ) self._client_kwargs = self._system.get_client_kwargs(name) self._is_raw_of_buffered = False if self._writable: self._write_buffer = bytearray() if "a" in mode: if self._exists() == 1: with handle_os_exceptions(): self._init_append() elif self._exists() == 0: with handle_os_exceptions(): self._create() else: raise PermissionError( "Insufficient permission to check if file already exists." ) elif "x" in mode and self._exists() == 1: raise FileExistsError elif "x" in mode and self._exists() == -1: raise PermissionError( "Insufficient permission to check if file already exists." ) else: with handle_os_exceptions(): self._create() else: with handle_os_exceptions(): self._head()
def readinto(self, b): """ Read bytes into a pre-allocated, writable bytes-like object b, and return the number of bytes read. Args: b (bytes-like object): buffer. Returns: int: number of bytes read """ if not self._readable: raise UnsupportedOperation("read") size = len(b) with self._seek_lock: start = self._seek end = start + size self._seek = end with handle_os_exceptions(): read_data = self._read_range(start, end) read_size = len(read_data) if read_size: memoryview(b)[:read_size] = read_data if read_size != size: with self._seek_lock: self._seek = start + read_size return read_size
def is_dir(self, *, follow_symlinks=True): """ Return True if this entry is a directory or a symbolic link pointing to a directory; return False if the entry is or points to any other kind of file, or if it doesn’t exist anymore. The result is cached on the os.DirEntry object. Args: follow_symlinks (bool): Follow symlinks. Returns: bool: True if directory exists. """ with handle_os_exceptions(): try: return self._system.isdir( path=self._path, client_kwargs=self._client_kwargs, virtual_dir=False, follow_symlinks=follow_symlinks, ) or bool( # Some directories only exists virtually in object path and don't # have headers. S_ISDIR(self.stat().st_mode)) except ObjectPermissionError: return True
def flush(self): """ Flush the write buffers of the stream if applicable and save the object on the storage. """ if self._writable: with handle_os_exceptions(): self._flush(self._get_buffer())
def close(self): """ Flush the write buffers of the stream if applicable and close the object. """ if self._writable and not self._closed: self._closed = True with self._seek_lock: self._flush_raw_or_buffered() if self._seek: with handle_os_exceptions(): self._close_writable()
def _peek(self, size=-1): """ Return bytes from the stream without advancing the position. Args: size (int): Number of bytes to read. -1 to read the full stream. Returns: bytes: bytes read """ with self._seek_lock: seek = self._seek with handle_os_exceptions(): return self._read_range(seek, seek + size)
def flush(self): """ Flush the write buffers of the stream if applicable and save the object on the storage. """ if self._writable: with self._seek_lock: buffer = self._get_buffer() end = self._seek start = end - len(buffer) self._write_buffer = bytearray() with handle_os_exceptions(): self._flush(buffer, start, end)
def read(self, size=-1): """ Read and return up to size bytes, with at most one call to the underlying raw stream. Use at most one call to the underlying raw stream’s read method. Args: size (int): Number of bytes to read. -1 to read the stream until end. Returns: bytes: Object content """ if not self._readable: raise UnsupportedOperation("read") elif not self._seekable: return self._raw.read(size) if self._seek == self._size: return b"" if size == self._buffer_size: queue_index = self._seek if queue_index == 0: self._preload_range() with handle_os_exceptions(): buffer = self._read_queue.pop(queue_index).result() buffer_size = self._buffer_size index = queue_index + buffer_size * self._max_buffers if index < self._size: self._read_queue[index] = self._workers.submit( self._read_range, index, index + buffer_size) self._seek += len(buffer) return buffer if size != -1: buffer = bytearray(size) else: buffer = bytearray() read_size = self.readinto(buffer) return memoryview(buffer)[:read_size].tobytes()
def decorated(path, *args, **kwargs): """ Decorated function. Args: path (path-like object or int): Path, URL or file descriptor. """ if not isinstance(path, int): path_str = fsdecode(path).replace("\\", "/") if is_storage(path_str): with handle_os_exceptions(): result = cos_function(path_str, *args, **kwargs) if keep_path_type and isinstance(path, bytes): result = fsencode(result) return result return std_function(path, *args, **kwargs)
def _flush_raw_or_buffered(self): """ Flush using raw of buffered methods. """ # Flush only if bytes written # This avoid no required process/thread creation and network call. # This step is performed by raw stream. if self._buffer_seek and self._seek: self._seek += 1 with handle_os_exceptions(): self._flush() # If data lower than buffer size flush data with raw stream to reduce IO calls elif self._buffer_seek: self._raw._write_buffer = self._get_buffer() self._raw._seek = self._buffer_seek self._raw.flush()
def symlink(src, dst, target_is_directory=False, *, dir_fd=None): """ Create a symbolic link pointing to src named dst. Equivalent to "os.symlink". .. versionadded:: 1.5.0 Args: src (path-like object): Path or URL to the symbolic link. dst (path-like object): Path or URL to the target. target_is_directory (bool): On Windows, define if symlink represents either a file or a directory. Not supported on storage objects and non-Windows platforms. dir_fd (int): directory descriptors; see the os.symlink() description for how it is interpreted. Not supported on storage objects. """ src, src_is_storage = format_and_is_storage(src) dst, dst_is_storage = format_and_is_storage(dst) if not src_is_storage and not dst_is_storage: return os.symlink(src, dst, target_is_directory=target_is_directory, dir_fd=dir_fd) with handle_os_exceptions(): if not src_is_storage or not dst_is_storage: ObjectNotImplementedError( "Cross storage symlinks are not supported") raises_on_dir_fd(dir_fd) system_src = get_instance(src) system_dst = get_instance(dst) if system_src is not system_dst: ObjectNotImplementedError( "Cross storage symlinks are not supported") elif system_src.relpath(src) == system_dst.relpath(dst): raise ObjectSameFileError(path1=src, path2=dst) return get_instance(src).symlink(src, dst)
def shareable_url(path, expires_in=3600): """ Get a shareable public URL for the specified path of an existing object. Only available for some storage and not for local paths. .. versionadded:: 1.5.0 Args: path (str): Path or URL. expires_in (int): Expiration in seconds. Default to 1 hour. """ with handle_os_exceptions(): path_str = fsdecode(path).replace("\\", "/") if not is_storage(path_str): raise ObjectUnsupportedOperation("shareable_url") return get_instance(path).shareable_url(path_str, expires_in)
def stat(self, *, follow_symlinks=True): """ Return a stat_result object for this entry. The result is cached on the os.DirEntry object. Args: follow_symlinks (bool): Follow symlinks. Returns: os.stat_result: Stat result object """ with handle_os_exceptions(): return self._system.stat( path=self._path, client_kwargs=self._client_kwargs, header=self._header, follow_symlinks=follow_symlinks, )
def is_file(self, *, follow_symlinks=True): """ Return True if this entry is a file or a symbolic link pointing to a file; return False if the entry is or points to a directory or other non-file entry, or if it doesn’t exist anymore. The result is cached on the os.DirEntry object. Args: follow_symlinks (bool): Follow symlinks. Returns: bool: True if directory exists. """ with handle_os_exceptions(): return self._system.isfile( path=self._path, client_kwargs=self._client_kwargs, follow_symlinks=follow_symlinks, )
def readall(self): """ Read and return all the bytes from the stream until EOF. Returns: bytes: Object content """ if not self._readable: raise UnsupportedOperation("read") with self._seek_lock: with handle_os_exceptions(): if self._seek and self._seekable: data = self._read_range(self._seek) else: data = self._readall() self._seek += len(data) return data
def _scandir_generator(is_bytes, scandir_path, system): """ scandir generator Args: is_bytes (bool): True if DirEntry must handle path as bytes. scandir_path (str): Path. system (airfs._core.io_system.SystemBase subclass): Storage system. Yields: DirEntry: Directory entries """ with handle_os_exceptions(): for name, header in system.list_objects(scandir_path, first_level=True): yield DirEntry( scandir_path=scandir_path, system=system, name=name, header=header, bytes_path=is_bytes, )
def _copy(src, dst, src_is_storage, dst_is_storage, follow_symlinks): """ Copies file from source to destination Args: src (str or file-like object): Source file. dst (str or file-like object): Destination file. src_is_storage (bool): Source is storage. dst_is_storage (bool): Destination is storage. follow_symlinks (bool): If True, follow symlinks. """ with handle_os_exceptions(): if src_is_storage and dst_is_storage: system_src = get_instance(src) system_dst = get_instance(dst) if system_src is system_dst: if system_src.relpath(src) == system_dst.relpath(dst): raise ObjectSameFileError(path1=src, path2=dst) try: return system_dst.copy(src, dst) except AirfsInternalException: pass for caller, called, method in ( (system_dst, system_src, "copy_from_%s"), (system_src, system_dst, "copy_to_%s"), ): if hasattr(caller, method % called.storage): try: return getattr(caller, method % called.storage)( src, dst, called, follow_symlinks) except AirfsInternalException: continue _copy_stream(dst, src)
def readinto(self, b): """ Read bytes into a pre-allocated, writable bytes-like object b, and return the number of bytes read. Args: b (bytes-like object): buffer. Returns: int: number of bytes read """ if not self._readable: raise UnsupportedOperation("read") elif not self._seekable: return self._raw.readinto(b) with self._seek_lock: seek = self._seek queue = self._read_queue if seek == 0: self._preload_range() size = len(b) if size: # Preallocated buffer: Use memory view to avoid copies b_view = memoryview(b) size_left = size else: # Dynamic buffer: Can't avoid copy, read until EOF b_view = b size_left = -1 b_end = 0 buffer_size = self._buffer_size while size_left > 0 or size_left == -1: start = seek % buffer_size queue_index = seek - start try: buffer = queue[queue_index] except KeyError: # EOF break with handle_os_exceptions(): try: queue[queue_index] = buffer = buffer.result() except AttributeError: # Already evaluated pass buffer_view = memoryview(buffer) data_size = len(buffer) if not data_size: break if size_left != -1: end = start + size_left else: end = data_size - start if end >= data_size: end = data_size del queue[queue_index] index = queue_index + buffer_size * self._max_buffers if index < self._size: queue[index] = self._workers.submit( self._read_range, index, index + buffer_size) read_size = end - start if size_left != -1: size_left -= read_size seek += read_size b_start = b_end b_end = b_start + read_size b_view[b_start:b_end] = buffer_view[start:end] self._seek = seek self._raw.seek(seek) return b_end
def write(self, b): """ Write the given bytes-like object, b, to the underlying raw stream, and return the number of bytes written. Args: b (bytes-like object): Bytes to write. Returns: int: The number of bytes written. """ if not self._writable: raise UnsupportedOperation("write") size = len(b) b_view = memoryview(b) size_left = size buffer_size = self._buffer_size max_buffers = self._max_buffers with self._seek_lock: end = self._buffer_seek buffer_view = memoryview(self._write_buffer) while size_left > 0: start = end end = start + size_left if end > buffer_size: end = buffer_size flush = True else: flush = False buffer_range = end - start b_start = size - size_left size_left -= buffer_range buffer_view[start:end] = b_view[b_start:b_start + buffer_range] if flush: self._buffer_seek = end self._seek += 1 if max_buffers: futures = self._write_futures flush_wait = self._FLUSH_WAIT while (sum(1 for future in futures if not future.done()) >= max_buffers): sleep(flush_wait) with handle_os_exceptions(): self._flush() self._write_buffer = bytearray(buffer_size) buffer_view = memoryview(self._write_buffer) end = 0 self._buffer_seek = end return size