def test_handle_os_exceptions():
"""Tests pycosio._core.exceptions.handle_os_exceptions"""
from pycosio._core.exceptions import (handle_os_exceptions,
ObjectNotFoundError,
ObjectPermissionError)
with pytest.raises(FileNotFoundError):
with handle_os_exceptions():
raise ObjectNotFoundError('error')
with pytest.raises(PermissionError):
with handle_os_exceptions():
raise ObjectPermissionError('error')
def _copy(src, dst, src_is_storage, dst_is_storage):
"""
Copies file from source to destination
Args:
src (str or file-like object): Source file.
dst (str or file-like object): Destination file.
src_is_storage (bool): Source is storage.
dst_is_storage (bool): Destination is storage.
"""
with handle_os_exceptions():
# If both storage: Tries to perform same storage direct copy
if src_is_storage and dst_is_storage:
system_src = get_instance(src)
system_dst = get_instance(dst)
# Same storage copy
if system_src is system_dst:
# Checks if same file
if system_src.relpath(src) == system_dst.relpath(dst):
raise SameFileError("'%s' and '%s' are the same file" %
(src, dst))
# Tries to copy
try:
return system_dst.copy(src, dst)
except (UnsupportedOperation, ObjectException):
pass
# Copy from compatible storage using "copy_from_<src_storage>" or
# "copy_to_<src_storage>" method if any
for caller, called, method in ((system_dst, system_src,
'copy_from_%s'),
(system_src, system_dst,
'copy_to_%s')):
if hasattr(caller, method % called.storage):
try:
return getattr(caller,
method % called.storage)(src, dst,
called)
except (UnsupportedOperation, ObjectException):
continue
# At least one storage object: copies streams
with cos_open(src, 'rb') as fsrc:
with cos_open(dst, 'wb') as fdst:
# Get stream buffer size
for stream in (fsrc, fdst):
try:
buffer_size = getattr(stream, '_buffer_size')
break
except AttributeError:
continue
else:
buffer_size = COPY_BUFSIZE
# Read and write
copyfileobj(fsrc, fdst, buffer_size)
def flush(self):
"""
Flush the write buffers of the stream if applicable and
save the object on the cloud.
"""
if self._writable:
with handle_os_exceptions():
self._flush(self._get_buffer())
def read(self, size=-1):
"""
Read and return up to size bytes,
with at most one call to the underlying raw stream’s.
Use at most one call to the underlying raw stream’s read method.
Args:
size (int): Number of bytes to read. -1 to read the
stream until end.
Returns:
bytes: Object content
"""
if not self._readable:
raise UnsupportedOperation('read')
# Checks if EOF
if self._seek == self._size:
return b''
# Returns existing buffer with no copy
if size == self._buffer_size:
queue_index = self._seek
# Starts initial preloading on first call
if queue_index == 0:
self._preload_range()
# Get buffer from future
with handle_os_exceptions():
buffer = self._read_queue.pop(queue_index).result()
# Append another buffer preload at end of queue
buffer_size = self._buffer_size
index = queue_index + buffer_size * self._max_buffers
if index < self._size:
self._read_queue[index] = self._workers.submit(
self._read_range, index, index + buffer_size)
# Update seek and return buffer
self._seek += len(buffer)
return buffer
# Uses a prealocated buffer
if size != -1:
buffer = bytearray(size)
# Uses a mutable buffer
else:
buffer = bytearray()
read_size = self.readinto(buffer)
return memoryview(buffer)[:read_size].tobytes()
def close(self):
"""
Flush the write buffers of the stream if applicable and
close the object.
"""
if self._writable and not self._closed:
self._closed = True
with self._seek_lock:
self._flush_raw_or_buffered()
if self._seek:
with handle_os_exceptions():
self._close_writable()
def decorated(path, *args, **kwargs):
"""Decorated function"""
# Handles path-like objects
path = fsdecode(path).replace('\\', '/')
# Storage object: Handle with Cloud object storage
# function
if is_storage(path):
with handle_os_exceptions():
return cos_function(path, *args, **kwargs)
# Local file: Redirect to standard function
return std_function(path, *args, **kwargs)
def _peek(self, size=-1):
"""
Return bytes from the stream without advancing the position.
Args:
size (int): Number of bytes to read. -1 to read the full
stream.
Returns:
bytes: bytes read
"""
with self._seek_lock:
seek = self._seek
with handle_os_exceptions():
return self._read_range(seek, seek + size)
def _scandir_generator(is_bytes, scandir_path, system):
"""
scandir generator
Args:
is_bytes (bool): True if DirEntry must handle path as bytes.
scandir_path (str): Path.
system (pycosio._core.io_system.SystemBase subclass):
Storage system.
Yields:
DirEntry: Directory entries
"""
with handle_os_exceptions():
for name, header in system.list_objects(scandir_path, first_level=True):
yield DirEntry(
scandir_path=scandir_path, system=system, name=name,
header=header, bytes_path=is_bytes)
def flush(self):
"""
Flush the write buffers of the stream if applicable and
save the object on the cloud.
"""
if self._writable:
with self._seek_lock:
buffer = self._get_buffer()
# Flush that part of the file
end = self._seek
start = end - len(buffer)
# Clear buffer
self._write_buffer = bytearray()
# Flush content
with handle_os_exceptions():
self._flush(buffer, start, end)
def _flush_raw_or_buffered(self):
"""
Flush using raw of buffered methods.
"""
# Flush only if bytes written
# This avoid no required process/thread
# creation and network call.
# This step is performed by raw stream.
if self._buffer_seek and self._seek:
self._seek += 1
with handle_os_exceptions():
self._flush()
# If data lower than buffer size
# flush data with raw stream to reduce IO calls
elif self._buffer_seek:
self._raw._write_buffer = self._get_buffer()
self._raw._seek = self._buffer_seek
self._raw.flush()
def readall(self):
"""
Read and return all the bytes from the stream until EOF.
Returns:
bytes: Object content
"""
if not self._readable:
raise UnsupportedOperation('read')
with self._seek_lock:
# Get data starting from seek
with handle_os_exceptions():
if self._seek and self._seekable:
data = self._read_range(self._seek)
# Get all data
else:
data = self._readall()
# Update seek
self._seek += len(data)
return data
def readinto(self, b):
"""
Read bytes into a pre-allocated, writable bytes-like object b,
and return the number of bytes read.
Args:
b (bytes-like object): buffer.
Returns:
int: number of bytes read
"""
if not self._readable:
raise UnsupportedOperation('read')
# Get and update stream positions
size = len(b)
with self._seek_lock:
start = self._seek
end = start + size
self._seek = end
# Read data range
with handle_os_exceptions():
read_data = self._read_range(start, end)
# Copy to bytes-like object
read_size = len(read_data)
if read_size:
memoryview(b)[:read_size] = read_data
# Update stream position if end of file
if read_size != size:
with self._seek_lock:
self._seek = start + read_size
# Return read size
return read_size
def __init__(self, name, mode='r', storage_parameters=None, **kwargs):
RawIOBase.__init__(self)
ObjectIOBase.__init__(self, name, mode=mode)
if storage_parameters is not None:
storage_parameters = storage_parameters.copy()
# Try to get cached head for this file
try:
self._cache['_head'] = storage_parameters.pop(
'pycosio.raw_io._head')
except (AttributeError, KeyError):
pass
# Initializes system
try:
# Try to get cached system
self._system = storage_parameters.pop('pycosio.system_cached')
except (AttributeError, KeyError):
self._system = None
if not self._system:
# If none cached, create a new system
self._system = self._SYSTEM_CLASS(
storage_parameters=storage_parameters, **kwargs)
# Gets storage local path from URL
self._path = self._system.relpath(name)
self._client_kwargs = self._system.get_client_kwargs(name)
# Mark as standalone RAW to avoid flush conflicts on close
self._is_raw_of_buffered = False
# Configures write mode
if self._writable:
self._write_buffer = bytearray()
# Initializes starting data
if 'a' in mode:
# Initialize with existing file content
if self._exists() == 1:
with handle_os_exceptions():
self._init_append()
# Create new file
elif self._exists() == 0:
with handle_os_exceptions():
self._create()
else:
raise PermissionError(
"Insufficient permission to check if file already "
"exists.")
# Checks if object exists,
# and raise if it is the case
elif 'x' in mode and self._exists() == 1:
raise FileExistsError
elif 'x' in mode and self._exists() == -1:
raise PermissionError(
"Insufficient permission to check if file already "
"exists.")
# Create new file
else:
with handle_os_exceptions():
self._create()
# Configure read mode
else:
# Get header and checks files exists
with handle_os_exceptions():
self._head()
def write(self, b):
"""
Write the given bytes-like object, b, to the underlying raw stream,
and return the number of bytes written.
Args:
b (bytes-like object): Bytes to write.
Returns:
int: The number of bytes written.
"""
if not self._writable:
raise UnsupportedOperation('write')
size = len(b)
b_view = memoryview(b)
size_left = size
buffer_size = self._buffer_size
max_buffers = self._max_buffers
with self._seek_lock:
end = self._buffer_seek
buffer_view = memoryview(self._write_buffer)
while size_left > 0:
# Get range to copy
start = end
end = start + size_left
if end > buffer_size:
# End of buffer, need flush after copy
end = buffer_size
flush = True
else:
flush = False
buffer_range = end - start
# Update not remaining data size
b_start = size - size_left
size_left -= buffer_range
# Copy data
buffer_view[start:end] = b_view[b_start:b_start + buffer_range]
# Flush buffer if needed
if flush:
# Update buffer seek
# Needed to write the good amount of data
self._buffer_seek = end
# Update global seek, this is the number
# of buffer flushed
self._seek += 1
# Block flush based on maximum number of
# buffers in flush progress
if max_buffers:
futures = self._write_futures
flush_wait = self._FLUSH_WAIT
while sum(1 for future in futures
if not future.done()) >= max_buffers:
sleep(flush_wait)
# Flush
with handle_os_exceptions():
self._flush()
# Clear buffer
self._write_buffer = bytearray(buffer_size)
buffer_view = memoryview(self._write_buffer)
end = 0
# Update buffer seek
self._buffer_seek = end
return size
def readinto(self, b):
"""
Read bytes into a pre-allocated, writable bytes-like object b,
and return the number of bytes read.
Args:
b (bytes-like object): buffer.
Returns:
int: number of bytes read
"""
if not self._readable:
raise UnsupportedOperation('read')
with self._seek_lock:
# Gets seek
seek = self._seek
# Initializes queue
queue = self._read_queue
if seek == 0:
# Starts initial preloading on first call
self._preload_range()
# Initializes read data buffer
size = len(b)
if size:
# Preallocated buffer:
# Use memory view to avoid copies
b_view = memoryview(b)
size_left = size
else:
# Dynamic buffer:
# Can't avoid copy, read until EOF
b_view = b
size_left = -1
b_end = 0
# Starts reading
buffer_size = self._buffer_size
while size_left > 0 or size_left == -1:
# Finds buffer position in queue and buffer seek
start = seek % buffer_size
queue_index = seek - start
# Gets preloaded buffer
try:
buffer = queue[queue_index]
except KeyError:
# EOF
break
# Get buffer from future
with handle_os_exceptions():
try:
queue[queue_index] = buffer = buffer.result()
# Already evaluated
except AttributeError:
pass
buffer_view = memoryview(buffer)
data_size = len(buffer)
# Checks if end of file reached
if not data_size:
break
# Gets theoretical range to copy
if size_left != -1:
end = start + size_left
else:
end = data_size - start
# Checks for end of buffer
if end >= data_size:
# Adjusts range to copy
end = data_size
# Removes consumed buffer from queue
del queue[queue_index]
# Append another buffer preload at end of queue
index = queue_index + buffer_size * self._max_buffers
if index < self._size:
queue[index] = self._workers.submit(
self._read_range, index, index + buffer_size)
# Gets read size, updates seek and updates size left
read_size = end - start
if size_left != -1:
size_left -= read_size
seek += read_size
# Defines read buffer range
b_start = b_end
b_end = b_start + read_size
# Copy data from preload buffer to read buffer
b_view[b_start:b_end] = buffer_view[start:end]
# Updates seek and sync raw
self._seek = seek
self._raw.seek(seek)
# Returns read size
return b_end