def __init__(self, path, mode='r', buffering=-1, encoding=None,
errors=None, newline=None, line_buffering=False,
buffer_size=None, **kwargs):
"""The XRootDPyFile constructor.
Raises PathError if the given path isn't a valid XRootD URL,
and InvalidPathError if it isn't a valid XRootD file path.
"""
if not is_valid_url(path):
raise PathError(path)
xpath = spliturl(path)[1]
if not is_valid_path(xpath):
raise InvalidPathError(xpath)
if newline not in [None, '', '\n', '\r', '\r\n']:
raise UnsupportedError(
"Newline character {0} not supported".format(newline))
if line_buffering is not False:
raise NotImplementedError("Line buffering for writing is not "
"supported.")
buffering = int(buffering)
if buffering == 1 and 'b' in mode:
raise UnsupportedError(
"Line buffering is not supported for "
"binary files.")
# PyFS attributes
self.mode = mode
# XRootD attributes & internals
self.path = path
self.encoding = encoding or sys.getdefaultencoding()
self.errors = errors or 'strict'
self.buffer_size = buffer_size or 64 * 1024
self.buffering = buffering
self._file = File()
self._ipp = 0
self._size = -1
self._iterator = None
self._newline = newline or b("\n")
self._buffer = b('')
self._buffer_pos = 0
# flag translation
self._flags = translate_file_mode_to_flags(mode)
statmsg, response = self._file.open(path, flags=self._flags)
if not statmsg.ok:
self._raise_status(self.path, statmsg,
"instantiating file ({0})".format(path))
# Deal with the modes
if 'a' in self.mode:
self.seek(self.size, SEEK_SET)
def getinfo(self, path):
"""Return information for a path as a dictionary.
The exact content of this dictionary will vary depending on the
implementation, but will likely include a few common values. The
following values will be found in info dictionaries for most
implementations:
* "size" - Number of bytes used to store the file or directory
* "created_time" - A datetime object containing the time the resource
was created
* "accessed_time" - A datetime object containing the time the resource
was last accessed
* "modified_time" - A datetime object containing the time the resource
was modified
:param path: a path to retrieve information for
:type path: string
:rtype: dict
:raises `fs.errors.ParentDirectoryMissingError`: if an intermediate
directory is missing
:raises `fs.errors.ResourceInvalidError`: if the path is not a
directory
:raises `fs.errors.ResourceNotFoundError`: if the path does not exist
"""
raise UnsupportedError("get resource info")
def open(self, path, mode='r', **kwargs):
if 'w' in mode or '+' in mode or 'a' in mode:
logging.error('cannot use httpfs.open() in write mode: %s' % path)
raise UnsupportedError('open', path=path)
url = self._build_url(path)
for attempt in (1, 2, 3):
try:
response = requests.get(url)
except requests.RequestException as error:
logging.warning('open attempt %d: %s %s' %
(attempt, url, error))
else:
break
else:
raise RemoteConnectionError('getinfo', path)
if response.status_code == 200:
return StringIO(response.content)
elif response.status_code == 404:
raise ResourceNotFoundError(path)
else:
logging.warning(
'open status %d for %s assumed as connection error.' %
(response.status_code, url))
raise RemoteConnectionError('open', path)
def open(self, path, mode="r"):
if '+' in mode or 'w' in mode or 'a' in mode:
raise UnsupportedError('write')
url = self._make_url(path)
try:
f = urlopen(url)
except URLError, e:
raise ResourceNotFoundError(path, details=e)
def movedir(self,
src,
dst,
overwrite=False,
ignore_errors=False,
chunk_size=16384):
"""
@attention: onedrive API doesn't allow to move folders
"""
raise UnsupportedError("move a directory")
def _query(self, flag, arg, parse=True):
"""Query an xrootd server."""
status, res = self._client.query(flag, arg)
if not status.ok:
if status.errno == 3013:
raise UnsupportedError(opname="calcualte checksum",
details=status)
raise FSError(details=status)
return parse_qs(res) if parse else res
def rename(self, src, dst):
"""
@param src: Id of the file to be renamed
@param dst: New title of the file
@raise UnsupportedError: If trying to rename the root directory
@return: Id of the renamed file
"""
if self.is_root(path=src):
raise UnsupportedError("Can't rename the root directory")
return self.client.update_file(src, {"name": dst})
def move(self, src, dst, overwrite=False, chunk_size=16384):
"""
@param src: id of the file to be moved
@param dst: id of the folder in which the file will be moved
@param overwrite: for Sky drive it is always false
@param chunk_size: if using chunk upload
@return: Id of the moved file
"""
if self.isdir(src):
raise UnsupportedError("move a directory")
return self.client.file_move(src, dst)
def remove(self, path):
"""
@param path: id of the folder to be deleted
@return: None if removal was successful
"""
path = self._normpath(path)
if self.is_root(path=path):
raise UnsupportedError("Can't remove the root directory")
if self.isdir(path=path):
raise ResourceInvalidError("Specified path is a directory")
return self.client.file_delete(path)
def open(self, path, mode='r', buffering=-1, encoding=None, errors=None, newline=None, line_buffering=False, **kwargs):
""" Only permit read access """
if 'w' in mode or 'a' in mode or '+' in mode:
raise UnsupportedError('write')
return super(ReadOnlyFS, self).open(path,
mode=mode,
buffering=buffering,
encoding=encoding,
errors=errors,
newline=newline,
line_buffering=line_buffering,
**kwargs)
def removedir(self, path):
"""Remove a directory of a given path.
:param path: id of the folder to be deleted
:return: None if removal was successful
"""
path = self._normpath(path)
if not self.isdir(path):
raise ResourceInvalidError("Specified path is not a directory")
if self.is_root(path=path):
raise UnsupportedError("remove the root directory")
self.client.file_delete(path)
def rename(self, src, dst):
"""Rename a file of a given path.
:param src: Id of the file to be renamed
:param dst: New title of the file
:raises UnsupportedError: If trying to rename the root directory
:return: Id of the renamed file
"""
if self.is_root(path=src):
raise UnsupportedError("Can't rename the root directory")
f = self.client.metadata(src)
f['title'] = dst
return self.client.update_file(src, f)['id']
def remove(self, path):
"""Remove a file of a given path.
:param path: id of the file to be deleted
:return: None if removal was successful
"""
path = self._normpath(path)
if self.is_root(path=path):
raise UnsupportedError("Can't remove the root directory")
if self.isdir(path=path):
raise ResourceInvalidError("Specified path is a directory. "
"Please use removedir.")
self.client.file_delete(path)
def removedir(self, path, *args, **kwargs):
"""Remove a directory of a given path.
:param path: path to the file to be deleted
:return: None if removal was successful
"""
path = abspath(normpath(path))
if self.is_root(path=path):
raise UnsupportedError("Can't remove the root directory")
if self.isfile(path=path):
raise ResourceInvalidError(
"Specified path is a directory. Please use removedir.")
self.client.file_delete(path)
def open(self, path, mode="r"):
path = normpath(path)
mode = mode.lower()
if self.isdir(path):
raise ResourceInvalidError(path)
if 'a' in mode or '+' in mode:
raise UnsupportedError('write')
if 'r' in mode:
if not self.isfile(path):
raise ResourceNotFoundError(path)
f = HttpApiFSFile(self, normpath(path), mode)
return f
def open(self, path, mode="r"):
path = (path)
mode = mode.lower()
if self.isdir(path):
raise ResourceInvalidError(path)
if 'a' in mode or '+' in mode:
raise UnsupportedError('write')
#if 'r' in mode:
# if not self.isfile(path):
# raise ResourceNotFoundError(path)
f = _DropBoxFSFile(self, (path), mode)
return f
def makedir(self, path, recursive=False, allow_recreate=False):
"""Create a directory of a given path.
:param path: path to the folder to be created.
If only the new folder is specified
it will be created in the root directory
:param recursive: allows recursive creation of directories
:param allow_recreate: dropbox currently doesn't support
allow_recreate, so if a folder exists it will
:return: Id of the created directory
"""
if not self._checkRecursive(recursive, path):
raise UnsupportedError("Recursively create specified folder.")
path = abspath(normpath(path))
return self.client.file_create_folder(path)
def media(self, path):
"""Media."""
try:
info = super(DropboxClient, self).media(path)
return info.get('url', None)
except rest.ErrorResponse as e:
if e.status == 400:
raise UnsupportedError("create a link to a folder")
if e.status == 404:
raise ResourceNotFoundError(path)
raise OperationFailedError(opname='file_copy', msg=str(e))
except:
raise RemoteConnectionError(
"Most probable reasons: access token has expired or user"
" credentials are invalid.")
def rename(self, src, dst):
"""Rename a file or directory.
:param src: path to rename
:type src: string
:param dst: new name
:type dst: string
:raises ParentDirectoryMissingError: if a containing directory is
missing
:raises ResourceInvalidError: if the path or a parent path is not a
directory or src is a parent of dst or one of src or dst is a dir
and the other don't
:raises ResourceNotFoundError: if the src path does not exist
"""
raise UnsupportedError("rename resource")
def open(self,
path,
mode='r',
buffering=-1,
encoding=None,
errors=None,
newline=None,
line_buffering=False,
**kwargs):
if '+' in mode or 'w' in mode or 'a' in mode:
raise UnsupportedError('write')
url = self._make_url(path)
try:
f = urlopen(url)
except URLError, e:
raise ResourceNotFoundError(path, details=e)
def checksum(self, chunk_size=None, progress_callback=None):
"""Compute checksum of file.
Queries the XRootD server to get checksum if possible, otherwise falls
back to default Python implementation. The checksum algorithm used
will be the one configured on the XRootD server.
"""
try:
fs, path = self._get_fs()
if not hasattr(fs, 'xrd_checksum'):
raise UnsupportedError()
algo, val = fs.xrd_checksum(path)
return u'{0}:{1}'.format(self.checksum_algo or algo, val)
except UnsupportedError:
return super(XRootDFileStorage, self).checksum(
chunk_size=chunk_size,
progress_callback=progress_callback,
)
def open(self,
path,
mode='r',
buffering=-1,
encoding=None,
errors=None,
newline=None,
line_buffering=False,
**kwargs):
logger.debug("open: %s mode %s", path, mode)
if 'w' in mode or 'a' in mode:
raise UnsupportedError()
# the isfile() call will also raise an exception if the file does not exist
if not self.isfile(path):
logger.debug("ResourceInvalidError: %s", path)
raise ResourceInvalidError(path)
return SRFPFile(self, filename2proto(path))
def removedir(self, path, recursive=False, force=False):
"""Remove a directory from the filesystem.
:param path: Path of the directory to remove.
:type path: str
:param recursive: Unsupported by XRootDPyFS implementation.
:type recursive: bool
:param force: If True, any directory contents will be removed
(recursively). Note that this can be very expensive as the xrootd
protocol does not support recursive deletes - i.e. the library
will do a full recursive listing of the directory and send a
network request per file/directory.
:type force: bool
:raises: `fs.errors.DirectoryNotEmptyError` if the directory is not
empty and force is `False`.
:raises: `fs.errors.ResourceInvalidError` if the path is not a
directory.
:raises: `fs.errors.ResourceNotFoundError` if the path does not exist.
"""
if recursive:
raise UnsupportedError("recursive parameter is not supported.")
status, res = self._client.rmdir(self._p(path))
if not status.ok:
if force and status.errno == 3005:
# xrootd does not support recursive removal so do we have to
# do it ourselves.
for d, filenames in self.walk(path, search="depth"):
for filename in filenames:
relpath = pathjoin(d, filename)
status, res = self._client.rm(self._p(relpath))
if not status.ok:
self._raise_status(relpath, status)
status, res = self._client.rmdir(self._p(d))
if not status.ok:
self._raise_status(path, status)
return True
self._raise_status(path, status)
return True
def makedir(self, path, recursive=False, allow_recreate=False):
"""
@param path: path to the folder you want to create.
it has to be in one of the following forms:
- parent_id/new_folder_name (when recursive is False)
- parent_id/new_folder1/new_folder2... (when recursive is True)
- /new_folder_name to create a new folder in root directory
- /new_folder1/new_folder2... to recursively create a new folder in root
@param recursive: allows recursive creation of directories
@param allow_recreate: for OneDrive this param is always False, it will
never recreate a directory
"""
parts = path.split("/")
if parts[0] == "":
parent_id = self._root
elif len(parts) >= 2:
parent_id = parts[0]
if not self.exists(parent_id):
raise ResourceNotFoundError(
"parent with the id '%s' doesn't exist" % parent_id)
if len(parts) > 2:
if recursive:
for i in range(len(parts) - 1):
title = parts[i + 1]
resp = self.client.file_create_folder(parent_id, title)
parent_id = resp
else:
raise UnsupportedError("recursively create a folder")
return resp
else:
if len(parts) == 1:
title = parts[0]
parent_id = self._root
else:
title = parts[1]
return self.client.file_create_folder(parent_id, title)
def makedir(self, path, recursive=False, allow_recreate=False):
"""Create a directory of a given path.
:param path: path to the folder you want to create.
It has to be in one of the following forms:
``parent_id/new_folder_name`` when recursive is False,
``parent_id/new_folder1/new_folder2`` when recursive is True,
``/new_folder_name`` to create a new folder in root directory,
``/new_folder1/new_folder2`` to recursively create a new folder
in root directory.
:param recursive: allows recursive creation of directories
:param allow_recreate: for google drive this param is
always False, it will never recreate a directory with
the same id ( same names are allowed )
:return: Id of the created directory
"""
path = self._normpath(path)
if '/' in path:
parts = path.split("/")
else:
parts = path
if len(parts) == 1 and parts[0] == self._root:
raise ResourceNotFoundError("Please, specify a folder name.")
elif len(parts) == 1:
return self.client.file_create_folder(self._root, parts[0])['id']
else:
if recursive:
resp = self.client.file_create_folder(self._root, parts[0])
for folder in parts[1:]:
if folder != '':
resp = self.client.file_create_folder(resp['id'],
folder)
return resp['id']
else:
raise UnsupportedError("Recursively create a folder.")
def makedir(self, path, recursive=False, allow_recreate=False):
raise UnsupportedError()
def rename(self, src, dst):
raise UnsupportedError()
def removedir(self, path, recursive=False, force=False):
raise UnsupportedError()
def remove(self, path):
raise UnsupportedError()
def _no_can_do(self, *args, **kwargs):
""" Replacement method for methods that can modify the file system """
raise UnsupportedError('write')