def test_handle_os_exceptions(): """Tests pycosio._core.exceptions.handle_os_exceptions""" from pycosio._core.exceptions import (handle_os_exceptions, ObjectNotFoundError, ObjectPermissionError) with pytest.raises(FileNotFoundError): with handle_os_exceptions(): raise ObjectNotFoundError('error') with pytest.raises(PermissionError): with handle_os_exceptions(): raise ObjectPermissionError('error')
def _copy(src, dst, src_is_storage, dst_is_storage): """ Copies file from source to destination Args: src (str or file-like object): Source file. dst (str or file-like object): Destination file. src_is_storage (bool): Source is storage. dst_is_storage (bool): Destination is storage. """ with handle_os_exceptions(): # If both storage: Tries to perform same storage direct copy if src_is_storage and dst_is_storage: system_src = get_instance(src) system_dst = get_instance(dst) # Same storage copy if system_src is system_dst: # Checks if same file if system_src.relpath(src) == system_dst.relpath(dst): raise SameFileError("'%s' and '%s' are the same file" % (src, dst)) # Tries to copy try: return system_dst.copy(src, dst) except (UnsupportedOperation, ObjectException): pass # Copy from compatible storage using "copy_from_<src_storage>" or # "copy_to_<src_storage>" method if any for caller, called, method in ((system_dst, system_src, 'copy_from_%s'), (system_src, system_dst, 'copy_to_%s')): if hasattr(caller, method % called.storage): try: return getattr(caller, method % called.storage)(src, dst, called) except (UnsupportedOperation, ObjectException): continue # At least one storage object: copies streams with cos_open(src, 'rb') as fsrc: with cos_open(dst, 'wb') as fdst: # Get stream buffer size for stream in (fsrc, fdst): try: buffer_size = getattr(stream, '_buffer_size') break except AttributeError: continue else: buffer_size = COPY_BUFSIZE # Read and write copyfileobj(fsrc, fdst, buffer_size)
def flush(self): """ Flush the write buffers of the stream if applicable and save the object on the cloud. """ if self._writable: with handle_os_exceptions(): self._flush(self._get_buffer())
def read(self, size=-1): """ Read and return up to size bytes, with at most one call to the underlying raw stream’s. Use at most one call to the underlying raw stream’s read method. Args: size (int): Number of bytes to read. -1 to read the stream until end. Returns: bytes: Object content """ if not self._readable: raise UnsupportedOperation('read') # Checks if EOF if self._seek == self._size: return b'' # Returns existing buffer with no copy if size == self._buffer_size: queue_index = self._seek # Starts initial preloading on first call if queue_index == 0: self._preload_range() # Get buffer from future with handle_os_exceptions(): buffer = self._read_queue.pop(queue_index).result() # Append another buffer preload at end of queue buffer_size = self._buffer_size index = queue_index + buffer_size * self._max_buffers if index < self._size: self._read_queue[index] = self._workers.submit( self._read_range, index, index + buffer_size) # Update seek and return buffer self._seek += len(buffer) return buffer # Uses a prealocated buffer if size != -1: buffer = bytearray(size) # Uses a mutable buffer else: buffer = bytearray() read_size = self.readinto(buffer) return memoryview(buffer)[:read_size].tobytes()
def close(self): """ Flush the write buffers of the stream if applicable and close the object. """ if self._writable and not self._closed: self._closed = True with self._seek_lock: self._flush_raw_or_buffered() if self._seek: with handle_os_exceptions(): self._close_writable()
def decorated(path, *args, **kwargs): """Decorated function""" # Handles path-like objects path = fsdecode(path).replace('\\', '/') # Storage object: Handle with Cloud object storage # function if is_storage(path): with handle_os_exceptions(): return cos_function(path, *args, **kwargs) # Local file: Redirect to standard function return std_function(path, *args, **kwargs)
def _peek(self, size=-1): """ Return bytes from the stream without advancing the position. Args: size (int): Number of bytes to read. -1 to read the full stream. Returns: bytes: bytes read """ with self._seek_lock: seek = self._seek with handle_os_exceptions(): return self._read_range(seek, seek + size)
def _scandir_generator(is_bytes, scandir_path, system): """ scandir generator Args: is_bytes (bool): True if DirEntry must handle path as bytes. scandir_path (str): Path. system (pycosio._core.io_system.SystemBase subclass): Storage system. Yields: DirEntry: Directory entries """ with handle_os_exceptions(): for name, header in system.list_objects(scandir_path, first_level=True): yield DirEntry( scandir_path=scandir_path, system=system, name=name, header=header, bytes_path=is_bytes)
def flush(self): """ Flush the write buffers of the stream if applicable and save the object on the cloud. """ if self._writable: with self._seek_lock: buffer = self._get_buffer() # Flush that part of the file end = self._seek start = end - len(buffer) # Clear buffer self._write_buffer = bytearray() # Flush content with handle_os_exceptions(): self._flush(buffer, start, end)
def _flush_raw_or_buffered(self): """ Flush using raw of buffered methods. """ # Flush only if bytes written # This avoid no required process/thread # creation and network call. # This step is performed by raw stream. if self._buffer_seek and self._seek: self._seek += 1 with handle_os_exceptions(): self._flush() # If data lower than buffer size # flush data with raw stream to reduce IO calls elif self._buffer_seek: self._raw._write_buffer = self._get_buffer() self._raw._seek = self._buffer_seek self._raw.flush()
def readall(self): """ Read and return all the bytes from the stream until EOF. Returns: bytes: Object content """ if not self._readable: raise UnsupportedOperation('read') with self._seek_lock: # Get data starting from seek with handle_os_exceptions(): if self._seek and self._seekable: data = self._read_range(self._seek) # Get all data else: data = self._readall() # Update seek self._seek += len(data) return data
def readinto(self, b): """ Read bytes into a pre-allocated, writable bytes-like object b, and return the number of bytes read. Args: b (bytes-like object): buffer. Returns: int: number of bytes read """ if not self._readable: raise UnsupportedOperation('read') # Get and update stream positions size = len(b) with self._seek_lock: start = self._seek end = start + size self._seek = end # Read data range with handle_os_exceptions(): read_data = self._read_range(start, end) # Copy to bytes-like object read_size = len(read_data) if read_size: memoryview(b)[:read_size] = read_data # Update stream position if end of file if read_size != size: with self._seek_lock: self._seek = start + read_size # Return read size return read_size
def __init__(self, name, mode='r', storage_parameters=None, **kwargs): RawIOBase.__init__(self) ObjectIOBase.__init__(self, name, mode=mode) if storage_parameters is not None: storage_parameters = storage_parameters.copy() # Try to get cached head for this file try: self._cache['_head'] = storage_parameters.pop( 'pycosio.raw_io._head') except (AttributeError, KeyError): pass # Initializes system try: # Try to get cached system self._system = storage_parameters.pop('pycosio.system_cached') except (AttributeError, KeyError): self._system = None if not self._system: # If none cached, create a new system self._system = self._SYSTEM_CLASS( storage_parameters=storage_parameters, **kwargs) # Gets storage local path from URL self._path = self._system.relpath(name) self._client_kwargs = self._system.get_client_kwargs(name) # Mark as standalone RAW to avoid flush conflicts on close self._is_raw_of_buffered = False # Configures write mode if self._writable: self._write_buffer = bytearray() # Initializes starting data if 'a' in mode: # Initialize with existing file content if self._exists() == 1: with handle_os_exceptions(): self._init_append() # Create new file elif self._exists() == 0: with handle_os_exceptions(): self._create() else: raise PermissionError( "Insufficient permission to check if file already " "exists.") # Checks if object exists, # and raise if it is the case elif 'x' in mode and self._exists() == 1: raise FileExistsError elif 'x' in mode and self._exists() == -1: raise PermissionError( "Insufficient permission to check if file already " "exists.") # Create new file else: with handle_os_exceptions(): self._create() # Configure read mode else: # Get header and checks files exists with handle_os_exceptions(): self._head()
def write(self, b): """ Write the given bytes-like object, b, to the underlying raw stream, and return the number of bytes written. Args: b (bytes-like object): Bytes to write. Returns: int: The number of bytes written. """ if not self._writable: raise UnsupportedOperation('write') size = len(b) b_view = memoryview(b) size_left = size buffer_size = self._buffer_size max_buffers = self._max_buffers with self._seek_lock: end = self._buffer_seek buffer_view = memoryview(self._write_buffer) while size_left > 0: # Get range to copy start = end end = start + size_left if end > buffer_size: # End of buffer, need flush after copy end = buffer_size flush = True else: flush = False buffer_range = end - start # Update not remaining data size b_start = size - size_left size_left -= buffer_range # Copy data buffer_view[start:end] = b_view[b_start:b_start + buffer_range] # Flush buffer if needed if flush: # Update buffer seek # Needed to write the good amount of data self._buffer_seek = end # Update global seek, this is the number # of buffer flushed self._seek += 1 # Block flush based on maximum number of # buffers in flush progress if max_buffers: futures = self._write_futures flush_wait = self._FLUSH_WAIT while sum(1 for future in futures if not future.done()) >= max_buffers: sleep(flush_wait) # Flush with handle_os_exceptions(): self._flush() # Clear buffer self._write_buffer = bytearray(buffer_size) buffer_view = memoryview(self._write_buffer) end = 0 # Update buffer seek self._buffer_seek = end return size
def readinto(self, b): """ Read bytes into a pre-allocated, writable bytes-like object b, and return the number of bytes read. Args: b (bytes-like object): buffer. Returns: int: number of bytes read """ if not self._readable: raise UnsupportedOperation('read') with self._seek_lock: # Gets seek seek = self._seek # Initializes queue queue = self._read_queue if seek == 0: # Starts initial preloading on first call self._preload_range() # Initializes read data buffer size = len(b) if size: # Preallocated buffer: # Use memory view to avoid copies b_view = memoryview(b) size_left = size else: # Dynamic buffer: # Can't avoid copy, read until EOF b_view = b size_left = -1 b_end = 0 # Starts reading buffer_size = self._buffer_size while size_left > 0 or size_left == -1: # Finds buffer position in queue and buffer seek start = seek % buffer_size queue_index = seek - start # Gets preloaded buffer try: buffer = queue[queue_index] except KeyError: # EOF break # Get buffer from future with handle_os_exceptions(): try: queue[queue_index] = buffer = buffer.result() # Already evaluated except AttributeError: pass buffer_view = memoryview(buffer) data_size = len(buffer) # Checks if end of file reached if not data_size: break # Gets theoretical range to copy if size_left != -1: end = start + size_left else: end = data_size - start # Checks for end of buffer if end >= data_size: # Adjusts range to copy end = data_size # Removes consumed buffer from queue del queue[queue_index] # Append another buffer preload at end of queue index = queue_index + buffer_size * self._max_buffers if index < self._size: queue[index] = self._workers.submit( self._read_range, index, index + buffer_size) # Gets read size, updates seek and updates size left read_size = end - start if size_left != -1: size_left -= read_size seek += read_size # Defines read buffer range b_start = b_end b_end = b_start + read_size # Copy data from preload buffer to read buffer b_view[b_start:b_end] = buffer_view[start:end] # Updates seek and sync raw self._seek = seek self._raw.seek(seek) # Returns read size return b_end