Exemple #1
0
    def __init__(self, path, mode='r', buffering=-1, encoding=None,
                 errors=None, newline=None, line_buffering=False,
                 buffer_size=None, **kwargs):
        """The XRootDPyFile constructor.

        Raises PathError if the given path isn't a valid XRootD URL,
        and InvalidPathError if it isn't a valid XRootD file path.
        """
        if not is_valid_url(path):
            raise PathError(path)

        xpath = spliturl(path)[1]

        if not is_valid_path(xpath):
            raise InvalidPathError(xpath)

        if newline not in [None, '', '\n', '\r', '\r\n']:
            raise UnsupportedError(
                "Newline character {0} not supported".format(newline))

        if line_buffering is not False:
            raise NotImplementedError("Line buffering for writing is not "
                                      "supported.")

        buffering = int(buffering)
        if buffering == 1 and 'b' in mode:
            raise UnsupportedError(
                "Line buffering is not supported for "
                "binary files.")

        # PyFS attributes
        self.mode = mode

        # XRootD attributes & internals
        self.path = path
        self.encoding = encoding or sys.getdefaultencoding()
        self.errors = errors or 'strict'
        self.buffer_size = buffer_size or 64 * 1024
        self.buffering = buffering
        self._file = File()
        self._ipp = 0
        self._size = -1
        self._iterator = None
        self._newline = newline or b("\n")
        self._buffer = b('')
        self._buffer_pos = 0

        # flag translation
        self._flags = translate_file_mode_to_flags(mode)

        statmsg, response = self._file.open(path, flags=self._flags)

        if not statmsg.ok:
            self._raise_status(self.path, statmsg,
                               "instantiating file ({0})".format(path))

        # Deal with the modes
        if 'a' in self.mode:
            self.seek(self.size, SEEK_SET)
Exemple #2
0
    def getinfo(self, path):
        """Return information for a path as a dictionary.

        The exact content of this dictionary will vary depending on the
        implementation, but will likely include a few common values. The
        following values will be found in info dictionaries for most
        implementations:

         * "size" - Number of bytes used to store the file or directory
         * "created_time" - A datetime object containing the time the resource
            was created
         * "accessed_time" - A datetime object containing the time the resource
            was last accessed
         * "modified_time" - A datetime object containing the time the resource
            was modified

        :param path: a path to retrieve information for
        :type path: string

        :rtype: dict

        :raises `fs.errors.ParentDirectoryMissingError`: if an intermediate
            directory is missing
        :raises `fs.errors.ResourceInvalidError`: if the path is not a
            directory
        :raises `fs.errors.ResourceNotFoundError`: if the path does not exist
        """
        raise UnsupportedError("get resource info")
    def open(self, path, mode='r', **kwargs):

        if 'w' in mode or '+' in mode or 'a' in mode:
            logging.error('cannot use httpfs.open() in write mode: %s' % path)
            raise UnsupportedError('open', path=path)

        url = self._build_url(path)

        for attempt in (1, 2, 3):
            try:
                response = requests.get(url)
            except requests.RequestException as error:
                logging.warning('open attempt %d: %s %s' %
                                (attempt, url, error))
            else:
                break
        else:
            raise RemoteConnectionError('getinfo', path)

        if response.status_code == 200:
            return StringIO(response.content)
        elif response.status_code == 404:
            raise ResourceNotFoundError(path)
        else:
            logging.warning(
                'open status %d for %s assumed as connection error.' %
                (response.status_code, url))
            raise RemoteConnectionError('open', path)
Exemple #4
0
    def open(self, path, mode="r"):

        if '+' in mode or 'w' in mode or 'a' in mode:
            raise UnsupportedError('write')

        url = self._make_url(path)
        try:
            f = urlopen(url)
        except URLError, e:
            raise ResourceNotFoundError(path, details=e)
Exemple #5
0
 def movedir(self,
             src,
             dst,
             overwrite=False,
             ignore_errors=False,
             chunk_size=16384):
     """
     @attention: onedrive API doesn't allow to move folders
     """
     raise UnsupportedError("move a directory")
Exemple #6
0
    def _query(self, flag, arg, parse=True):
        """Query an xrootd server."""
        status, res = self._client.query(flag, arg)

        if not status.ok:
            if status.errno == 3013:
                raise UnsupportedError(opname="calcualte checksum",
                                       details=status)
            raise FSError(details=status)
        return parse_qs(res) if parse else res
Exemple #7
0
    def rename(self, src, dst):
        """
        @param src: Id of the file to be renamed
        @param dst: New title of the file
        @raise UnsupportedError: If trying to rename the root directory
        @return: Id of the renamed file
        """
        if self.is_root(path=src):
            raise UnsupportedError("Can't rename the root directory")

        return self.client.update_file(src, {"name": dst})
Exemple #8
0
 def move(self, src, dst, overwrite=False, chunk_size=16384):
     """
     @param src: id of the file to be moved
     @param dst: id of the folder in which the file will be moved
     @param overwrite: for Sky drive it is always false
     @param chunk_size: if using chunk upload
     @return: Id of the moved file
     """
     if self.isdir(src):
         raise UnsupportedError("move a directory")
     return self.client.file_move(src, dst)
Exemple #9
0
    def remove(self, path):
        """
        @param path: id of the folder to be deleted
        @return: None if removal was successful
        """
        path = self._normpath(path)
        if self.is_root(path=path):
            raise UnsupportedError("Can't remove the root directory")
        if self.isdir(path=path):
            raise ResourceInvalidError("Specified path is a directory")

        return self.client.file_delete(path)
Exemple #10
0
 def open(self, path, mode='r', buffering=-1, encoding=None, errors=None, newline=None, line_buffering=False, **kwargs):
     """ Only permit read access """
     if 'w' in mode or 'a' in mode or '+' in mode:
         raise UnsupportedError('write')
     return super(ReadOnlyFS, self).open(path,
                                         mode=mode,
                                         buffering=buffering,
                                         encoding=encoding,
                                         errors=errors,
                                         newline=newline,
                                         line_buffering=line_buffering,
                                         **kwargs)
    def removedir(self, path):
        """Remove a directory of a given path.

        :param path: id of the folder to be deleted
        :return: None if removal was successful
        """
        path = self._normpath(path)

        if not self.isdir(path):
            raise ResourceInvalidError("Specified path is not a directory")
        if self.is_root(path=path):
            raise UnsupportedError("remove the root directory")
        self.client.file_delete(path)
    def rename(self, src, dst):
        """Rename a file of a given path.

        :param src: Id of the file to be renamed
        :param dst: New title of the file
        :raises UnsupportedError: If trying to rename the root directory
        :return: Id of the renamed file
        """
        if self.is_root(path=src):
            raise UnsupportedError("Can't rename the root directory")
        f = self.client.metadata(src)
        f['title'] = dst
        return self.client.update_file(src, f)['id']
    def remove(self, path):
        """Remove a file of a given path.

        :param path: id of the file to be deleted
        :return: None if removal was successful
        """
        path = self._normpath(path)

        if self.is_root(path=path):
            raise UnsupportedError("Can't remove the root directory")
        if self.isdir(path=path):
            raise ResourceInvalidError("Specified path is a directory. "
                                       "Please use removedir.")
        self.client.file_delete(path)
Exemple #14
0
    def removedir(self, path, *args, **kwargs):
        """Remove a directory of a given path.

        :param path: path to the file to be deleted
        :return: None if removal was successful
        """
        path = abspath(normpath(path))

        if self.is_root(path=path):
            raise UnsupportedError("Can't remove the root directory")
        if self.isfile(path=path):
            raise ResourceInvalidError(
                "Specified path is a directory. Please use removedir.")

        self.client.file_delete(path)
Exemple #15
0
    def open(self, path, mode="r"):

        path = normpath(path)
        mode = mode.lower()
        if self.isdir(path):
            raise ResourceInvalidError(path)
        if 'a' in mode or '+' in mode:
            raise UnsupportedError('write')

        if 'r' in mode:
            if not self.isfile(path):
                raise ResourceNotFoundError(path)

        f = HttpApiFSFile(self, normpath(path), mode)
        return f
Exemple #16
0
    def open(self, path, mode="r"):

        path = (path)
        mode = mode.lower()
        if self.isdir(path):
            raise ResourceInvalidError(path)
        if 'a' in mode or '+' in mode:
            raise UnsupportedError('write')

        #if 'r' in mode:
        # if not self.isfile(path):
        # raise ResourceNotFoundError(path)

        f = _DropBoxFSFile(self, (path), mode)

        return f
Exemple #17
0
    def makedir(self, path, recursive=False, allow_recreate=False):
        """Create a directory of a given path.

        :param path: path to the folder to be created.
            If only the new folder is specified
            it will be created in the root directory
        :param recursive: allows recursive creation of directories
        :param allow_recreate: dropbox currently doesn't support
            allow_recreate, so if a folder exists it will
        :return: Id of the created directory
        """
        if not self._checkRecursive(recursive, path):
            raise UnsupportedError("Recursively create specified folder.")

        path = abspath(normpath(path))
        return self.client.file_create_folder(path)
Exemple #18
0
    def media(self, path):
        """Media."""
        try:
            info = super(DropboxClient, self).media(path)
            return info.get('url', None)
        except rest.ErrorResponse as e:
            if e.status == 400:
                raise UnsupportedError("create a link to a folder")
            if e.status == 404:
                raise ResourceNotFoundError(path)

            raise OperationFailedError(opname='file_copy', msg=str(e))
        except:
            raise RemoteConnectionError(
                "Most probable reasons: access token has expired or user"
                " credentials are invalid.")
Exemple #19
0
    def rename(self, src, dst):
        """Rename a file or directory.

        :param src: path to rename
        :type src: string
        :param dst: new name
        :type dst: string

        :raises ParentDirectoryMissingError: if a containing directory is
            missing
        :raises ResourceInvalidError: if the path or a parent path is not a
            directory or src is a parent of dst or one of src or dst is a dir
            and the other don't
        :raises ResourceNotFoundError: if the src path does not exist

        """
        raise UnsupportedError("rename resource")
Exemple #20
0
    def open(self,
             path,
             mode='r',
             buffering=-1,
             encoding=None,
             errors=None,
             newline=None,
             line_buffering=False,
             **kwargs):

        if '+' in mode or 'w' in mode or 'a' in mode:
            raise UnsupportedError('write')

        url = self._make_url(path)
        try:
            f = urlopen(url)
        except URLError, e:
            raise ResourceNotFoundError(path, details=e)
Exemple #21
0
    def checksum(self, chunk_size=None, progress_callback=None):
        """Compute checksum of file.

        Queries the XRootD server to get checksum if possible, otherwise falls
        back to default Python implementation. The checksum algorithm used
        will be the one configured on the XRootD server.
        """
        try:
            fs, path = self._get_fs()
            if not hasattr(fs, 'xrd_checksum'):
                raise UnsupportedError()
            algo, val = fs.xrd_checksum(path)
            return u'{0}:{1}'.format(self.checksum_algo or algo, val)
        except UnsupportedError:
            return super(XRootDFileStorage, self).checksum(
                chunk_size=chunk_size,
                progress_callback=progress_callback,
            )
Exemple #22
0
    def open(self,
             path,
             mode='r',
             buffering=-1,
             encoding=None,
             errors=None,
             newline=None,
             line_buffering=False,
             **kwargs):
        logger.debug("open: %s mode %s", path, mode)
        if 'w' in mode or 'a' in mode:
            raise UnsupportedError()

        # the isfile() call will also raise an exception if the file does not exist
        if not self.isfile(path):
            logger.debug("ResourceInvalidError: %s", path)
            raise ResourceInvalidError(path)

        return SRFPFile(self, filename2proto(path))
Exemple #23
0
    def removedir(self, path, recursive=False, force=False):
        """Remove a directory from the filesystem.

        :param path: Path of the directory to remove.
        :type path: str
        :param recursive: Unsupported by XRootDPyFS implementation.
        :type recursive: bool
        :param force: If True, any directory contents will be removed
            (recursively). Note that this can be very expensive as the xrootd
            protocol does not support recursive deletes - i.e. the library
            will do a full recursive listing of the directory and send a
            network request per file/directory.
        :type force: bool

        :raises: `fs.errors.DirectoryNotEmptyError` if the directory is not
            empty and force is `False`.
        :raises: `fs.errors.ResourceInvalidError` if the path is not a
            directory.
        :raises: `fs.errors.ResourceNotFoundError` if the path does not exist.
        """
        if recursive:
            raise UnsupportedError("recursive parameter is not supported.")

        status, res = self._client.rmdir(self._p(path))

        if not status.ok:
            if force and status.errno == 3005:
                # xrootd does not support recursive removal so do we have to
                # do it ourselves.
                for d, filenames in self.walk(path, search="depth"):
                    for filename in filenames:
                        relpath = pathjoin(d, filename)
                        status, res = self._client.rm(self._p(relpath))
                        if not status.ok:
                            self._raise_status(relpath, status)
                    status, res = self._client.rmdir(self._p(d))
                    if not status.ok:
                        self._raise_status(path, status)
                return True
            self._raise_status(path, status)
        return True
Exemple #24
0
    def makedir(self, path, recursive=False, allow_recreate=False):
        """
        @param path: path to the folder you want to create.
            it has to be in one of the following forms:
                - parent_id/new_folder_name  (when recursive is False)
                - parent_id/new_folder1/new_folder2...  (when recursive is True)
                - /new_folder_name to create a new folder in root directory
                - /new_folder1/new_folder2... to recursively create a new folder in root
        @param recursive: allows recursive creation of directories
        @param allow_recreate: for OneDrive this param is always False, it will
            never recreate a directory
        """
        parts = path.split("/")

        if parts[0] == "":
            parent_id = self._root
        elif len(parts) >= 2:
            parent_id = parts[0]
            if not self.exists(parent_id):
                raise ResourceNotFoundError(
                    "parent with the id '%s' doesn't exist" % parent_id)

        if len(parts) > 2:
            if recursive:
                for i in range(len(parts) - 1):
                    title = parts[i + 1]
                    resp = self.client.file_create_folder(parent_id, title)
                    parent_id = resp
            else:
                raise UnsupportedError("recursively create a folder")
            return resp
        else:
            if len(parts) == 1:
                title = parts[0]
                parent_id = self._root
            else:
                title = parts[1]
            return self.client.file_create_folder(parent_id, title)
    def makedir(self, path, recursive=False, allow_recreate=False):
        """Create a directory of a given path.

        :param path: path to the folder you want to create.
            It has to be in one of the following forms:
            ``parent_id/new_folder_name`` when recursive is False,
            ``parent_id/new_folder1/new_folder2`` when recursive is True,
            ``/new_folder_name`` to create a new folder in root directory,
            ``/new_folder1/new_folder2`` to recursively create a new folder
            in root directory.
        :param recursive: allows recursive creation of directories
        :param allow_recreate: for google drive this param is
            always False, it will never recreate a directory with
            the same id ( same names are allowed )
        :return: Id of the created directory
        """
        path = self._normpath(path)
        if '/' in path:
            parts = path.split("/")
        else:
            parts = path

        if len(parts) == 1 and parts[0] == self._root:
            raise ResourceNotFoundError("Please, specify a folder name.")
        elif len(parts) == 1:
            return self.client.file_create_folder(self._root, parts[0])['id']
        else:
            if recursive:
                resp = self.client.file_create_folder(self._root, parts[0])
                for folder in parts[1:]:
                    if folder != '':
                        resp = self.client.file_create_folder(resp['id'],
                                                              folder)
                return resp['id']
            else:
                raise UnsupportedError("Recursively create a folder.")
Exemple #26
0
 def makedir(self, path, recursive=False, allow_recreate=False):
     raise UnsupportedError()
Exemple #27
0
 def rename(self, src, dst):
     raise UnsupportedError()
Exemple #28
0
 def removedir(self, path, recursive=False, force=False):
     raise UnsupportedError()
Exemple #29
0
 def remove(self, path):
     raise UnsupportedError()
Exemple #30
0
 def _no_can_do(self, *args, **kwargs):
     """ Replacement method for methods that can modify the file system """
     raise UnsupportedError('write')