def makedir(self, path, recursive=False, allow_recreate=False):
path = normpath(path)
if path in ('', '/'):
return
if recursive:
created = False
for path_part in recursepath(path):
if not self.isdir(path_part):
self.conn.mkdir(self.smb_path(path_part))
created = True
else:
if self.isfile(path_part):
raise ResourceInvalidError(path_part)
if not created and not allow_recreate:
raise DestinationExistsError(path)
else:
base = dirname(path)
if not self.exists(base):
raise ParentDirectoryMissingError(path)
if not allow_recreate:
if self.exists(path):
if self.isfile(path):
raise ResourceInvalidError(path)
raise DestinationExistsError(path)
self.conn.mkdir(self.smb_path(path))
else:
if not self.isdir(path):
self.conn.mkdir(self.smb_path(path))
def copyfile_non_atomic(src_fs,
src_path,
dst_fs,
dst_path,
overwrite=True,
chunk_size=64 * 1024):
"""A non atomic version of copyfile (will not block other threads using src_fs or dst_fst)
:param src_fs: Source filesystem object
:param src_path: -- Source path
:param dst_fs: Destination filesystem object
:param dst_path: Destination filesystem object
:param chunk_size: Size of chunks to move if system copyfile is not available (default 64K)
"""
if not overwrite and dst_fs.exists(dst_path):
raise DestinationExistsError(dst_path)
src = None
dst = None
try:
src = src_fs.open(src_path, 'rb')
dst = dst_fs.open(dst_path, 'wb')
write = dst.write
read = src.read
chunk = read(chunk_size)
while chunk:
write(chunk)
chunk = read(chunk_size)
finally:
if src is not None:
src.close()
if dst is not None:
dst.close()
def copy(self, src, dst, overwrite=False, chunk_size=1024 * 1024):
src = normpath(src)
if not self.isfile(src):
if self.isdir(src):
raise ResourceInvalidError(
src, msg="Source is not a file: %(path)s")
raise ResourceNotFoundError(src)
dst = normpath(dst)
if not overwrite and self.exists(dst):
raise DestinationExistsError(dst)
src_file = None
dst_file = None
try:
src_file = self.open(src, 'r')
dst_file = self.open(dst, 'w')
while 1:
copy_buffer = src_file.read(chunk_size)
if copy_buffer:
dst_file.write(copy_buffer)
else:
break
finally:
if src_file is not None:
src_file.close()
if dst_file is not None:
dst_file.close()
def copy(self, src, dst, overwrite=False, chunk_size=65536):
if self.isdir(src):
raise ResourceInvalidError(src)
if not self.isfile(src):
if not self.isdir(dirname(src)):
raise ParentDirectoryMissingError(src)
raise ResourceNotFoundError(src)
if self.isdir(dst):
raise ResourceInvalidError(dst)
if self.isfile(dst):
if overwrite:
self.remove(dst)
else:
raise DestinationExistsError(dst)
else:
if not self.isdir(dirname(dst)):
raise ParentDirectoryMissingError(dst)
parent_path = self._ids[dirname(dst)]
copy_fh = {'title': basename(dst), 'parents': [{'id': parent_path}]}
copy_fh = self.client.auth.service.files() \
.copy(fileId=self._ids[src], body=copy_fh) \
.execute()
self._ids[dst] = copy_fh['id']
def makedir(self, path, recursive=False, allow_recreate=False):
"""Creates a file with mimeType _folder_mimetype
which acts as a folder in GoogleDrive."""
if self.isdir(path):
if allow_recreate:
return
else:
raise DestinationExistsError(path)
if self.isfile(path):
raise ResourceInvalidError(path)
if not recursive and not self.isdir(dirname(path)):
raise ParentDirectoryMissingError(path)
if recursive:
self.makedir(dirname(path),
recursive=recursive,
allow_recreate=True)
parent_id = self._ids[dirname(path)]
fh = self.client.CreateFile({
'title': basename(path),
'mimeType': self._folder_mimetype,
'parents': [{
'id': parent_id
}]
})
fh.Upload()
self._ids[path] = fh['id']
def _move(self, src, dst, overwrite=False):
"""Move source to destination with support for overwriting destination.
Used by ``XRootDPyFS.move()``, ``XRootDPyFS.movedir()`` and
``XRootDPyFS.rename()``.
.. warning::
It is the responsibility of the caller of this method to check that
the source exists.
If ``overwrite`` is ``True``, this method will first remove any
destination directory/file if it exists, and then try to move the
source. Hence, if the source doesn't exists, it will remove the
destination and then fail.
"""
if self.exists(dst):
if not overwrite:
raise DestinationExistsError(dst)
if self.isfile(dst):
self.remove(dst)
elif self.isdir(dst):
self.removedir(dst, force=True)
status, dummy = self._client.mv(src, dst)
if not status.ok:
self._raise_status(dst, status)
return True
def copydir(self, src, dst, overwrite=False, parallel=True):
"""Copy a directory from source to destination.
By default the copy is done by recreating the source directory
structure at the destination, and then copy files in parallel from
source to destination.
:param src: Source directory path.
:type src: str
:param dst: Destination directory path.
:type dst: str
:param overwrite: If True then any existing files in the destination
directory will be overwritten.
:type overwrite: bool
:param parallel: If True (default), the copy will be done in parallel.
:type parallel: bool
"""
if not self.isdir(src):
if self.isfile(src):
raise ResourceInvalidError(
src, msg="Source is not a directory: %(path)s")
raise ResourceNotFoundError(src)
if self.exists(dst):
if overwrite:
if self.isdir(dst):
self.removedir(dst, force=True)
elif self.isfile(dst):
self.remove(dst)
else:
raise DestinationExistsError(dst)
if parallel:
process = CopyProcess()
def process_copy(src, dst, overwrite=False):
process.add_job(src, dst)
copyfile = process_copy
else:
copyfile = self.copy
self.makedir(dst, allow_recreate=True)
for src_dirpath, filenames in self.walk(src):
dst_dirpath = pathcombine(dst, frombase(src, src_dirpath))
self.makedir(dst_dirpath, allow_recreate=True, recursive=True)
for filename in filenames:
src_filename = pathjoin(src_dirpath, filename)
dst_filename = pathjoin(dst_dirpath, filename)
copyfile(src_filename, dst_filename, overwrite=overwrite)
if parallel:
process.prepare()
process.run()
return True
def _raise_status(self, path, status):
"""Raise error based on status."""
if status.errno == 3006:
raise DestinationExistsError(path=path, details=status)
elif status.errno == 3005:
raise DirectoryNotEmptyError(path=path, details=status)
elif status.errno == 3011:
raise ResourceInvalidError(path=path, details=status)
else:
raise FSError(details=status)
def movefile(src_fs,
src_path,
dst_fs,
dst_path,
overwrite=True,
chunk_size=64 * 1024):
"""Move a file from one filesystem to another. Will use system copyfile, if both files have a syspath.
Otherwise file will be copied a chunk at a time.
:param src_fs: Source filesystem object
:param src_path: Source path
:param dst_fs: Destination filesystem object
:param dst_path: Destination filesystem object
:param chunk_size: Size of chunks to move if system copyfile is not available (default 64K)
"""
src_syspath = src_fs.getsyspath(src_path, allow_none=True)
dst_syspath = dst_fs.getsyspath(dst_path, allow_none=True)
if not overwrite and dst_fs.exists(dst_path):
raise DestinationExistsError(dst_path)
if src_fs is dst_fs:
src_fs.move(src_path, dst_path, overwrite=overwrite)
return
# System copy if there are two sys paths
if src_syspath is not None and dst_syspath is not None:
FS._shutil_movefile(src_syspath, dst_syspath)
return
src_lock = getattr(src_fs, '_lock', None)
if src_lock is not None:
src_lock.acquire()
try:
src = None
try:
# Chunk copy
src = src_fs.open(src_path, 'rb')
dst_fs.setcontents(dst_path, src, chunk_size=chunk_size)
except:
raise
else:
src_fs.remove(src_path)
finally:
if src is not None:
src.close()
finally:
if src_lock is not None:
src_lock.release()
def rename(self, src, dst):
if not self.exists(src):
raise ResourceNotFoundError(src)
if self.exists(dst):
raise DestinationExistsError(dst)
if isprefix(src, dst):
raise ResourceInvalidError(dst)
fh = self.client.CreateFile({
'id': self._ids[src],
'title': basename(dst)
})
fh.Upload()
self._ids[dst] = self._ids.pop(src)
def move(self, src, dst, overwrite=False, chunk_size=16384):
if self.isfile(src):
src = self._prepare_normpath(src)
dst = self._prepare_normpath(dst)
if self.isdir(dst):
dst = '/'.join([dst, basename(src)])
if not overwrite and self.exists(dst):
raise DestinationExistsError(dst)
self.rename(src, dst)
else:
raise ResourceInvalidError(src,
msg="Source is not a file: %(path)s")
def _raise_status(self, path, status):
"""Raise error based on status."""
if status.errno in [3006, 17]:
raise DestinationExistsError(path=path, details=status)
elif status.errno == 3005:
# Unfortunately only way to determine if the error is due to a
# directory not being empty, or that a resource is not a directory:
if status.message.strip().endswith("not a directory"):
raise ResourceInvalidError(path=path, details=status)
else:
raise DirectoryNotEmptyError(path=path, details=status)
elif status.errno == 3011:
raise ResourceNotFoundError(path=path, details=status)
else:
raise ResourceError(path=path, details=status)
def makedir(self, path, recursive=False, allow_recreate=False):
if not path and not allow_recreate:
raise PathError(path)
path = normpath(path)
if path in ('', '/'):
if allow_recreate:
return
raise DestinationExistsError(path)
parent_path, dirname = pathsplit(path)
parent_box_id = self._root_id
for name in iteratepath(parent_path):
children_items = self._get_children_items(parent_box_id)
child_item = children_items.get(name)
if not child_item:
if recursive:
child_item = self._makedir(parent_box_id, name)
else:
raise ParentDirectoryMissingError(path)
if child_item['type'] != _ITEM_TYPE_FOLDER:
raise ResourceInvalidError(path)
parent_box_id = child_item['id']
# Check if an item with required name already exists.
children_items = self._get_children_items(parent_box_id)
child_item = children_items.get(dirname)
if child_item:
if allow_recreate and child_item['type'] == _ITEM_TYPE_FOLDER:
return
else:
raise DestinationExistsError(path)
self._makedir(parent_box_id, dirname)
def file_create_folder(self, path):
"""Add newly created directory to cache."""
try:
metadata = super(DropboxClient, self).file_create_folder(path)
except rest.ErrorResponse as e:
if e.status == 403:
raise DestinationExistsError(path)
if e.status == 400:
raise OperationFailedError(opname='file_create_folder',
msg=str(e))
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired or user"
" credentials are invalid.")
self.cache.set(path, metadata)
return metadata['path']
def inner(*args, **kwargs):
try:
return outer(*args, **kwargs)
except DestinationExistsError:
raise DestinationExistsError(args[2])
except ResourceNotFoundError:
# Parent directory does not exist or is not a directory.
src, dst = args[1:3]
fs = args[0]
for p in (src, dst):
if not fs.exists(p):
root = dirname(p)
if not fs.isdir(root):
if fs.isfile(root):
raise ResourceInvalidError(p)
else:
raise ParentDirectoryMissingError(p)
else:
raise ResourceNotFoundError(p)
def file_copy(self, src, dst):
"""Copy a file to another location."""
try:
metadata = super(DropboxClient, self).file_copy(src, dst)
except rest.ErrorResponse as e:
if e.status == 404:
raise ResourceNotFoundError(src)
if e.status == 403:
raise DestinationExistsError(dst)
if e.status == 503:
raise OperationFailedError(opname='file_copy',
msg="User over storage quota")
raise OperationFailedError(opname='file_copy', msg=str(e))
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired or user"
" credentials are invalid.")
self.cache.set(dst, metadata)
return metadata['path']
def makedir(self, path, recursive=False, allow_recreate=False):
# Create a directory from the top downwards depending upon the flags.
paths = recursepath(path) if recursive else (path, )
for p in paths:
if p == '/':
continue
# Try to create a directory first then ask for forgiveness.
try:
self._create_dir(p)
except DestinationExistsError as e:
if self.isfile(p):
raise ResourceInvalidError(path)
elif self.isdir(p):
if not recursive and not allow_recreate:
raise DestinationExistsError(path)
except ResourceNotFoundError as e:
if not recursive and not self.isdir(dirname(p)):
raise ParentDirectoryMissingError(path)
e.path = path
raise
except FSError as e:
e.path = path
raise
def inner(*args, **kwargs):
try:
return outer(*args, **kwargs)
except socket.gaierror as e:
if e.errno == socket.EAI_NONAME:
raise RemoteConnectionError(str(e), details=e)
raise
except socket.error as e:
if e.errno in (errno.ECONNREFUSED, errno.EPIPE, errno.ETIMEDOUT):
raise RemoteConnectionError(str(e), details=e)
raise
except OperationFailure as e:
# Cycle through each message and map the first one to PyFilesystem
# that is not a successful status (0x00000000).
share = args[0]
path = args[1]
for msg in e.smb_messages:
# Protocol versions expose the error values differently.
if msg.protocol == 1:
msg_status = msg.status.internal_value
else:
msg_status = msg.status
if msg_status == 0x0:
continue
elif msg_status == 0x103:
# Unknown error, but message says it is not found.
raise ResourceNotFoundError(path=path)
elif msg_status == 0xc000000f:
raise ResourceNotFoundError(path=path)
elif msg_status == 0xc0000022:
raise PermissionDeniedError('access', path=path)
elif msg_status == 0xc0000033:
raise ResourceInvalidError(path=path)
elif msg_status == 0xc0000034:
raise ResourceNotFoundError(path=path)
elif msg_status == 0xc0000035:
raise DestinationExistsError(path=path)
elif msg_status == 0xc000003a:
raise ResourceNotFoundError(path=path)
elif msg_status == 0xc0000056:
raise DeletePendingError(path=path)
elif msg_status == 0xc000006f:
raise AccountTimeRestrictedError(path=path)
elif msg_status == 0xc0000071:
raise PasswordExpiredError(path=path)
elif msg_status == 0xc00000ba:
raise ResourceInvalidError(path=path)
elif msg_status == 0xc00000cc:
# Share does not exist.
raise ResourceInvalidError(path=share)
elif msg_status == 0xc00000d0:
raise ResourceInvalidError(path=path)
elif msg_status == 0xc0000101:
raise DirectoryNotEmptyError(path=path)
elif msg_status == 0xc0000103:
raise ResourceInvalidError(path=path)
elif msg_status == 0xc0000193:
raise AccountExpiredError(path=path)
elif msg_status == 0xc0000224:
raise PasswordChangeRequiredError(path=path)
elif msg_status == 0xc0000234:
raise AccountLockedError(path=path)
else:
raise Exception('Unhandled SMB error: {0}'.format(
hex(msg_status)))
raise
except NotConnectedError as e:
# Connection does not exist or was disconnected. Using the wrong
# NetBIOS name can cause this against a Windows server while Samba
# will ignore it.
raise RemoteConnectionError(details=e)
except NotReadyError as e:
# Connection has not been successfully authenticated.
raise RemoteConnectionError(details=e)