def _copy(src, dst, src_is_storage, dst_is_storage):
"""
Copies file from source to destination
Args:
src (str or file-like object): Source file.
dst (str or file-like object): Destination file.
src_is_storage (bool): Source is storage.
dst_is_storage (bool): Destination is storage.
"""
with handle_os_exceptions():
# If both storage: Tries to perform same storage direct copy
if src_is_storage and dst_is_storage:
system_src = get_instance(src)
system_dst = get_instance(dst)
# Same storage copy
if system_src is system_dst:
# Checks if same file
if system_src.relpath(src) == system_dst.relpath(dst):
raise SameFileError("'%s' and '%s' are the same file" %
(src, dst))
# Tries to copy
try:
return system_dst.copy(src, dst)
except (UnsupportedOperation, ObjectException):
pass
# Copy from compatible storage using "copy_from_<src_storage>" or
# "copy_to_<src_storage>" method if any
for caller, called, method in ((system_dst, system_src,
'copy_from_%s'),
(system_src, system_dst,
'copy_to_%s')):
if hasattr(caller, method % called.storage):
try:
return getattr(caller,
method % called.storage)(src, dst,
called)
except (UnsupportedOperation, ObjectException):
continue
# At least one storage object: copies streams
with cos_open(src, 'rb') as fsrc:
with cos_open(dst, 'wb') as fdst:
# Get stream buffer size
for stream in (fsrc, fdst):
try:
buffer_size = getattr(stream, '_buffer_size')
break
except AttributeError:
continue
else:
buffer_size = COPY_BUFSIZE
# Read and write
copyfileobj(fsrc, fdst, buffer_size)
def scandir(path='.'):
"""
Return an iterator of os.DirEntry objects corresponding to the entries in
the directory given by path. The entries are yielded in arbitrary order,
and the special entries '.' and '..' are not included.
Equivalent to "os.scandir".
Args:
path (path-like object): Path or URL.
If path is of type bytes (directly or indirectly through the
PathLike interface), the type of the name and path attributes
of each os.DirEntry will be bytes; in all other circumstances,
they will be of type str.
Returns:
Generator of os.DirEntry: Entries information.
"""
# Handles path-like objects
scandir_path = fsdecode(path).replace('\\', '/')
if not is_storage(scandir_path):
return os_scandir(scandir_path)
return _scandir_generator(
is_bytes=isinstance(fspath(path), (bytes, bytearray)),
scandir_path=scandir_path, system=get_instance(scandir_path))
def makedirs(name, mode=0o777, exist_ok=False):
"""
Super-mkdir; create a leaf directory and all intermediate ones.
Works like mkdir, except that any intermediate path segment
(not just the rightmost) will be created if it does not exist.
Equivalent to "os.makedirs".
Args:
name (path-like object): Path or URL.
mode (int): The mode parameter is passed to os.mkdir();
see the os.mkdir() description for how it is interpreted.
Not supported on cloud storage objects.
exist_ok (bool): Don't raises error if target directory already
exists.
Raises:
FileExistsError: if exist_ok is False and if the target directory
already exists.
"""
system = get_instance(name)
# Checks if directory not already exists
if not exist_ok and system.isdir(system.ensure_dir_path(name)):
raise ObjectExistsError("File exists: '%s'" % name)
# Create directory
system.make_dir(name)
def rmdir(path, dir_fd=None):
"""
Remove a directory.
Equivalent to "os.rmdir".
Args:
path (path-like object): Path or URL.
dir_fd: directory descriptors;
see the os.rmdir() description for how it is interpreted.
Not supported on cloud storage objects.
"""
system = get_instance(path)
system.remove(system.ensure_dir_path(path))
def listdir(path='.'):
"""
Return a list containing the names of the entries in the directory given by
path.
Equivalent to "os.listdir".
Args:
path (path-like object): Path or URL.
Returns:
list of str: Entries names.
"""
return [name.rstrip('/') for name, _ in
get_instance(path).list_objects(path, first_level=True)]
def lstat(path, dir_fd=None):
"""
Get the status of a file or a file descriptor.
Perform the equivalent of a "lstat()" system call on the given path.
Equivalent to "os.lstat".
On cloud object, may return extra storage specific attributes in
"os.stat_result".
Args:
path (path-like object): Path or URL.
dir_fd: directory descriptors;
see the os.rmdir() description for how it is interpreted.
Not supported on cloud storage objects.
Returns:
os.stat_result: stat result.
"""
return get_instance(path).stat(path)
def remove(path, dir_fd=None):
"""
Remove a file.
Equivalent to "os.remove" and "os.unlink".
Args:
path (path-like object): Path or URL.
dir_fd: directory descriptors;
see the os.remove() description for how it is interpreted.
Not supported on cloud storage objects.
"""
system = get_instance(path)
# Only support files
if system.is_locator(path) or path[-1] == '/':
raise IsADirectoryError("Is a directory: '%s'" % path)
# Remove
system.remove(path)
def mkdir(path, mode=0o777, dir_fd=None):
"""
Create a directory named path with numeric mode mode.
Equivalent to "os.mkdir".
Args:
path (path-like object): Path or URL.
mode (int): The mode parameter is passed to os.mkdir();
see the os.mkdir() description for how it is interpreted.
Not supported on cloud storage objects.
dir_fd: directory descriptors;
see the os.remove() description for how it is interpreted.
Not supported on cloud storage objects.
Raises:
FileExistsError : Directory already exists.
FileNotFoundError: Parent directory not exists.
"""
system = get_instance(path)
relative = system.relpath(path)
# Checks if parent directory exists
parent_dir = dirname(relative.rstrip('/'))
if parent_dir:
parent = path.rsplit(relative, 1)[0] + parent_dir + '/'
if not system.isdir(parent):
raise ObjectNotFoundError(
"No such file or directory: '%s'" % parent)
# Checks if directory not already exists
if system.isdir(system.ensure_dir_path(path)):
raise ObjectExistsError("File exists: '%s'" % path)
# Create directory
system.make_dir(relative, relative=True)
def cos_open(file,
mode='r',
buffering=-1,
encoding=None,
errors=None,
newline=None,
storage=None,
storage_parameters=None,
unsecure=None,
**kwargs):
"""
Open file and return a corresponding file object.
Equivalent to "io.open" or builtin "open".
File can also be binary opened file-like object.
Args:
file (path-like object or file-like object): File path, object URL or
opened file-like object.
mode (str): mode in which the file is opened (default to 'rb').
see "io.open" for all possible modes. Note that all modes may
not be supported by all kind of file and storage.
buffering (int): Set the buffering policy.
-1 to use default behavior,
0 to switch buffering off,
1 to select line buffering (only usable in text mode),
and an integer > 1 to indicate the size in bytes of a
fixed-size chunk buffer.
See "io.open" for more information.
encoding (str): The name of the encoding used to
decode or encode the file. This should only be used in text mode.
See "io.open" for more information.
errors (str): Specifies how encoding and decoding errors
are to be handled.
This should only be used in text mode.
See "io.open" for more information.
newline (str): Controls how universal newlines mode works.
This should only be used in text mode.
See "io.open" for more information.
storage (str): Storage name.
storage_parameters (dict): Storage configuration parameters.
Generally, client configuration and credentials.
unsecure (bool): If True, disables TLS/SSL to improves
transfer performance. But makes connection unsecure.
Default to False.
kwargs: Other arguments to pass to opened object.
Note that theses arguments may not be compatible with
all kind of file and storage.
Returns:
file-like object: opened file.
Raises:
OSError: If the file cannot be opened.
FileExistsError: File open in 'x' mode already exists.
"""
# Handles file-like objects:
if hasattr(file, 'read'):
with _text_io_wrapper(file, mode, encoding, errors,
newline) as wrapped:
yield wrapped
return
# Handles path-like objects
file = fsdecode(file).replace('\\', '/')
# Storage object
if is_storage(file, storage):
with get_instance(name=file,
cls='raw' if buffering == 0 else 'buffered',
storage=storage,
storage_parameters=storage_parameters,
mode=mode,
unsecure=unsecure,
**kwargs) as stream:
with _text_io_wrapper(stream,
mode=mode,
encoding=encoding,
errors=errors,
newline=newline) as wrapped:
yield wrapped
# Local file: Redirect to "io.open"
else:
with io_open(file, mode, buffering, encoding, errors, newline,
**kwargs) as stream:
yield stream
def test_mount():
"""Tests pycosio._core.storage_manager.mount and get_instance"""
from pycosio._core.storage_manager import (
mount, MOUNTED, get_instance, _compare_root)
from pycosio.storage.http import (
HTTPRawIO, _HTTPSystem, HTTPBufferedIO)
import requests
# Get HTTP as storage to mount
roots = _HTTPSystem().roots
storage_parameters = {'arg1': 1, 'arg2': 2}
storage_parameters_2 = {'arg1': 1, 'arg2': 3}
expected_info = dict(
raw=HTTPRawIO, system=_HTTPSystem,
buffered=HTTPBufferedIO,
system_parameters={'storage_parameters': storage_parameters})
https = 'https://path'
http = 'http://path'
# Mock requests
class Response:
"""Fake response"""
status_code = 200
headers = {'Accept-Ranges': 'bytes',
'Content-Length': '100'}
class Session:
"""Fake Session"""
def __init__(self, *_, **__):
"""Do nothing"""
@staticmethod
def request(*_, **__):
"""Returns fake result"""
return Response()
requests_session = requests.Session
requests.Session = Session
# Tests mount:
try:
for mount_kwargs in (
# With storage
dict(storage='http',
storage_parameters=storage_parameters),
# With name
dict(name=http,
storage_parameters=storage_parameters),
# With both
dict(storage='http', name=http,
storage_parameters=storage_parameters),
# Lazy registration
None):
# Unmount if already mounted
for root in roots:
try:
del MOUNTED[root]
except KeyError:
continue
# Add dummy storage to mounted
MOUNTED['aaaa'] = {}
MOUNTED['zzzz'] = {}
MOUNTED[re.compile('bbbbbb')] = {}
# mount
if mount_kwargs:
# Using mount function
mount(**mount_kwargs)
else:
# Using lazy registration
get_instance(name=http,
storage_parameters=storage_parameters)
# Check registration
for root in roots:
# Test infos
for key, value in expected_info.items():
assert MOUNTED[root][key] == value
# Tests cached system
assert isinstance(
MOUNTED[root]['system_cached'], _HTTPSystem)
# Tests get_instance cached system
assert get_instance(
name=root) is MOUNTED[root]['system_cached']
assert get_instance(
storage_parameters=storage_parameters,
name=root) is MOUNTED[root]['system_cached']
assert get_instance(
storage_parameters=storage_parameters_2,
name=root) is not MOUNTED[root]['system_cached']
# Test get_instance other classes with cached system
raw = get_instance(name=https, cls='raw')
assert isinstance(raw, HTTPRawIO)
assert raw._system is MOUNTED[root]['system_cached']
buffered = get_instance(name=http, cls='buffered')
assert isinstance(buffered, HTTPBufferedIO)
assert buffered._raw._system is MOUNTED[root]['system_cached']
buffered = get_instance(
name=http, cls='buffered',
storage_parameters=storage_parameters_2)
assert isinstance(buffered, HTTPBufferedIO)
assert (buffered._raw._system is not
MOUNTED[root]['system_cached'])
# Test mount order
assert tuple(MOUNTED) == tuple(reversed(sorted(
MOUNTED, key=_compare_root)))
# Cleanup
del MOUNTED['aaaa']
del MOUNTED['zzzz']
for root in roots:
del MOUNTED[root]
# Tests extra root
extra = 'extra_http://'
mount(storage='http', extra_root=extra,
storage_parameters=storage_parameters),
assert MOUNTED[extra] == MOUNTED[roots[0]]
for root in roots:
del MOUNTED[root]
del MOUNTED[extra]
# Tests not as arguments to define storage
with pytest.raises(ValueError):
mount(name='path')
# Restore mocked functions
finally:
requests.Session = requests_session