def __init__(self, path, mode='r', buffering=-1, encoding=None, errors=None, newline=None, line_buffering=False, buffer_size=None, **kwargs): """The XRootDPyFile constructor. Raises PathError if the given path isn't a valid XRootD URL, and InvalidPathError if it isn't a valid XRootD file path. """ if not is_valid_url(path): raise PathError(path) xpath = spliturl(path)[1] if not is_valid_path(xpath): raise InvalidPathError(xpath) if newline not in [None, '', '\n', '\r', '\r\n']: raise UnsupportedError( "Newline character {0} not supported".format(newline)) if line_buffering is not False: raise NotImplementedError("Line buffering for writing is not " "supported.") buffering = int(buffering) if buffering == 1 and 'b' in mode: raise UnsupportedError( "Line buffering is not supported for " "binary files.") # PyFS attributes self.mode = mode # XRootD attributes & internals self.path = path self.encoding = encoding or sys.getdefaultencoding() self.errors = errors or 'strict' self.buffer_size = buffer_size or 64 * 1024 self.buffering = buffering self._file = File() self._ipp = 0 self._size = -1 self._iterator = None self._newline = newline or b("\n") self._buffer = b('') self._buffer_pos = 0 # flag translation self._flags = translate_file_mode_to_flags(mode) statmsg, response = self._file.open(path, flags=self._flags) if not statmsg.ok: self._raise_status(self.path, statmsg, "instantiating file ({0})".format(path)) # Deal with the modes if 'a' in self.mode: self.seek(self.size, SEEK_SET)
def __init__(self, path, bufsize=1024 * 64, mode='r'): """.""" super(XRootDFile, self).__init__(bufsize) # set .__file to empty xrootd.client.File-object. self.__file = XFile() status, response = self.__file.open(path, mode=mode)
class XRootDFile(fs.filelike.FileLikeBase): """.""" def __init__(self, path, bufsize=1024 * 64, mode='r'): """.""" super(XRootDFile, self).__init__(bufsize) # set .__file to empty xrootd.client.File-object. self.__file = XFile() status, response = self.__file.open(path, mode=mode) # todo: raise appropriate errors def seek(self, offset, whence=0): """.""" self._ifp = offset def read(self, size=0, offset=None, timeout=0, callback=None): """.""" if offset is None: offset = self._ifp return self._file.read(offset, size, timeout, callback) def tell(): """.""" pass def truncate(size=None): """.""" pass def write(string): """.""" pass
def __init__(self, path, mode='r', buffering=-1, encoding=None, errors=None, newline=None, line_buffering=False, buffer_size=None, **kwargs): """XRootDPyFile constructor. Raises PathError if the given path isn't a valid XRootD URL, and InvalidPathError if it isn't a valid XRootD file path. """ if not is_valid_url(path): raise PathError(path) xpath = spliturl(path)[1] if not is_valid_path(xpath): raise InvalidPathError(xpath) if newline not in [None, '', '\n', '\r', '\r\n']: raise UnsupportedError( "Newline character {0} not supported".format(newline)) if line_buffering is not False: raise NotImplementedError("Line buffering for writing is not " "supported.") buffering = int(buffering) if buffering == 1 and 'b' in mode: raise UnsupportedError( "Line buffering is not supported for " "binary files.") # PyFS attributes self.mode = mode # XRootD attributes & internals self.path = path self.encoding = encoding or sys.getdefaultencoding() self.errors = errors or 'strict' self.buffer_size = buffer_size or 64*1024 self.buffering = buffering self._file = File() self._ipp = 0 self._size = -1 self._iterator = None self._newline = newline or b("\n") self._buffer = b('') self._buffer_pos = 0 # flag translation self._flags = translate_file_mode_to_flags(mode) statmsg, response = self._file.open(path, flags=self._flags) if not statmsg.ok: self._raise_status(self.path, statmsg, "instantiating file ({0})".format(path)) # Deal with the modes if 'a' in self.mode: self.seek(self.size, SEEK_SET)
class XRootDPyFile(object): r"""File-like interface for working with files over XRootD protocol. This class understands and will accept the following mode strings, with any additional characters being ignored: * ``r`` - Open the file for reading only. * ``r+`` - Open the file for reading and writing. * ``r-`` - Open the file for streamed reading; do not allow seek/tell. * ``w`` - Open the file for writing only; create the file if it doesn't exist; truncate it to zero length. * ``w+`` - Open the file for reading and writing; create the file if it doesn't exist; truncate it to zero length. * ``w-`` - Open the file for streamed writing; do not allow seek/tell. * ``a`` - Open the file for writing only; create the file if it doesn't exist; place pointer at end of file. * ``a+`` - Open the file for reading and writing; create the file if it doesn't exist; place pointer at end of file. .. note:: Streamed reading/writing modes has no performance advantages over non-streamed reading/writing for XRootD. :param path: Path to file that should be opened. :type path: string :param mode: Mode of file to open, identical to the mode string used in 'file' and 'open' builtins. :type mode: string :param buffering: An optional integer used to set the buffering policy. Pass 0 to switch buffering off (only allowed in binary mode), 1 to select line buffering (only usable in text mode), and an integer > 1 to indicate the size of a fixed-size chunk buffer. :param encoding: Determines encoding used when writing unicode data. :param errors: An optional string that specifies how encoding and decoding errors are to be handled (e.g. ``strict``, ``ignore`` or ``replace``). :param newline: Newline character to use (either ``\\n``, ``\\r``, ``\\r\\n``, ``''`` or ``None``). :param line_buffering: Unsupported. Anything by False will raise and error. :param buffer_size: Buffer size used when reading files (defaults to 64K). This can likely be optimized to chunks up to 2MB depending on your desired memory usage. """ def __init__(self, path, mode='r', buffering=-1, encoding=None, errors=None, newline=None, line_buffering=False, buffer_size=None, **kwargs): """XRootDPyFile constructor. Raises PathError if the given path isn't a valid XRootD URL, and InvalidPathError if it isn't a valid XRootD file path. """ if not is_valid_url(path): raise PathError(path) xpath = spliturl(path)[1] if not is_valid_path(xpath): raise InvalidPathError(xpath) if newline not in [None, '', '\n', '\r', '\r\n']: raise UnsupportedError( "Newline character {0} not supported".format(newline)) if line_buffering is not False: raise NotImplementedError("Line buffering for writing is not " "supported.") buffering = int(buffering) if buffering == 1 and 'b' in mode: raise UnsupportedError( "Line buffering is not supported for " "binary files.") # PyFS attributes self.mode = mode # XRootD attributes & internals self.path = path self.encoding = encoding or sys.getdefaultencoding() self.errors = errors or 'strict' self.buffer_size = buffer_size or 64*1024 self.buffering = buffering self._file = File() self._ipp = 0 self._size = -1 self._iterator = None self._newline = newline or b("\n") self._buffer = b('') self._buffer_pos = 0 # flag translation self._flags = translate_file_mode_to_flags(mode) statmsg, response = self._file.open(path, flags=self._flags) if not statmsg.ok: self._raise_status(self.path, statmsg, "instantiating file ({0})".format(path)) # Deal with the modes if 'a' in self.mode: self.seek(self.size, SEEK_SET) def _raise_status(self, path, status, source=None): """Raise error based on status.""" if status.errno == 3011: raise ResourceNotFoundError(path) else: if source: errstr = "XRootD error {0}file: {1}".format( source+' ', status.message) raise IOError(errstr) def __del__(self): """Close file on object deletion.""" self.close() def __iter__(self): """Initialize the internal iterator.""" self._next_func = self.read self._next_args = ([], dict(sizehint=self.buffer_size)) if self.buffering == 1 or \ (self.buffering == -1 and 'b' not in self.mode): self._next_func = self.readline self._next_args = ([], dict()) elif self.buffering > 1: self._next_args = ([], dict(sizehint=self.buffering)) return self def __enter__(self): """Enter context manager method.""" return self def __exit__(self, exc_type, exc_value, traceback): """Exit context manager method.""" self.close() def next(self): """Return next item for file iteration.""" item = self._next_func(*self._next_args[0], **self._next_args[1]) if not item: raise StopIteration return item def read(self, sizehint=-1): """Read ``sizehint`` bytes from the file object. If no ``sizehint`` is provided the entire file is read! Multiple calls to this method after EOF as been reached, will return an empty string. :oaram sizehint: Number of bytes to read from file object. """ if self.closed: raise ValueError("I/O operation on closed file.") self._assert_mode("r-") chunksize = sizehint if sizehint > 0 else self.size # Read data statmsg, res = self._file.read( offset=self._ipp, size=chunksize, ) if not statmsg.ok: self._raise_status(self.path, statmsg, "reading") # Increment internal file pointer. self._ipp = min( self._ipp + chunksize, self.size if self.size > self._ipp else self._ipp ) return res def readline(self): """Read one entire line from the file. A trailing newline character is kept in the string (but may be absent when a file ends with an incomplete line). """ bits = [self._buffer if self._buffer_pos == self.tell() else b("")] indx = bits[-1].find(self._newline) if indx == -1: # Read chunks until first newline is found or entire file is read. while indx == -1: bit = self.read(self.buffer_size) bits.append(bit) if not bit: break indx = bit.find(self._newline) if indx == -1: return b("").join(bits) indx += len(self._newline) extra = bits[-1][indx:] bits[-1] = bits[-1][:indx] self._buffer = extra self._buffer_pos = self.tell() return b("").join(bits) def readlines(self): """Read until EOF using readline(). .. warning:: This methods reads the entire file into memory! You are probably better off using either ``xreadlines`` or just normal iteration over the file object. """ return list(self.xreadlines()) def xreadlines(self, sizehint=-1): """Get an iterator over number of lines.""" line = True while line: line = self.readline() if not line: break yield line def write(self, data, flushing=False): """Write the given string to the file. If the keyword argument 'flushing' is true, it indicates that the internal write buffers are being flushed, and *all* the given data is expected to be written to the file. """ self._assert_mode("w-") if 'a' in self.mode: self.seek(0, SEEK_END) if not isinstance(data, binary_type): if isinstance(data, bytearray): data = bytes(data) elif isinstance(data, text_type): data = data.encode(self.encoding, self.errors) statmsg, res = self._file.write(data, offset=self._ipp) if not statmsg.ok: self._raise_status(self.path, statmsg, "writing") self._ipp += len(data) self._size = max(self.size, self.tell()) if flushing: self.flush() def writelines(self, sequence): """Write an sequence of lines to file.""" for s in sequence: self.write(s) def seek(self, offset, whence=SEEK_SET): """Set the file's internal position pointer, approximately. The possible values of whence and their meaning are defined in the Linux man pages for `lseek()`: http://man7.org/linux/man-pages/man2/lseek.2.html ``SEEK_SET`` The internal position pointer is set to offset bytes. ``SEEK_CUR`` The ipp is set to its current position plus offset bytes. ``SEEK_END`` The ipp is set to the size of the file plus offset bytes. """ if "-" in self.mode: raise IOError("File is not seekable.") # Convert to integer by rounding down/omitting everything after # the decimal point offset = int(offset) if offset < 0: raise IOError("Invalid argument.") if whence == SEEK_SET: self._ipp = offset elif whence == SEEK_CUR: self._ipp += offset elif whence == SEEK_END: self._ipp = self.size + offset else: raise NotImplementedError(whence) def tell(self): """Get the location of the file's internal position pointer.""" return self._ipp def truncate(self, size=None): """Truncate the file's size to ``size``. Note that ``size`` will never be None; if it was not specified by the user the current file position is used. """ self._assert_mode('w') if size is None: size = self.tell() statmsg = self._file.truncate(size)[0] if not statmsg.ok: self._raise_status(self.path, statmsg, "truncating") self._size = size def close(self): """Close the file, including flushing the write buffers. The file may not be accessed further once it is closed. """ if not self.closed: self._file.close() def flush(self): """Flush write buffers.""" if not self.closed: statmsg, dummy = self._file.sync() if not statmsg.ok: self._raise_status(self.path, statmsg, "flushing write buffer") def seekable(self): """Check if file is seekable.""" return '-' not in self.mode def readable(self): """Check if file is readable.""" return 'r' in self.mode or '+' in self.mode def writable(self): """Check if file is writable.""" return 'w' in self.mode or '+' in self.mode or 'a' in self.mode def isatty(self): """Check if file is a TTY (false always). Added for ``io`` module compatibility. """ return False def fileno(self): """Get the underlying file descriptor. Unsupported by XRootDPyFS (added for ``io`` module compatibility). """ raise IOError("File descriptor is unsupported by xrootd.") @property def name(self): """Get filename.""" return basename(self.path) @property def closed(self): """Check if file is closed.""" return not self._file.is_open() @property def size(self): """Get file size.""" if self._size == -1: statmsg, res = self._file.stat() if not statmsg.ok: self._raise_status(self.path, statmsg, "retrieving size") self._size = res.size return self._size def _assert_mode(self, mode, mstr=None): """Check whether the file may be accessed in the given mode.""" if mstr is None: try: mstr = self.mode except AttributeError: raise AttributeError("Mode attribute missing -- " "was it deleted? " "Close and re-open the file.") if "+" in mstr: return True if "-" in mstr and "-" not in mode: raise IOError("File does not support seeking.") if "r" in mode: if "r" not in mstr: raise IOError("File not opened for reading") if "w" in mode: if "w" not in mstr and "a" not in mstr: raise IOError("File not opened for writing") return True
class XRootDPyFile(object): r"""File-like interface for working with files over XRootD protocol. This class understands and will accept the following mode strings, with any additional characters being ignored: * ``r`` - Open the file for reading only. * ``r+`` - Open the file for reading and writing. * ``r-`` - Open the file for streamed reading; do not allow seek/tell. * ``w`` - Open the file for writing only; create the file if it doesn't exist; truncate it to zero length. * ``w+`` - Open the file for reading and writing; create the file if it doesn't exist; truncate it to zero length. * ``w-`` - Open the file for streamed writing; do not allow seek/tell. * ``a`` - Open the file for writing only; create the file if it doesn't exist; place pointer at end of file. * ``a+`` - Open the file for reading and writing; create the file if it doesn't exist; place pointer at end of file. .. note:: Streamed reading/writing modes has no performance advantages over non-streamed reading/writing for XRootD. :param path: Path to file that should be opened. :type path: string :param mode: Mode of file to open, identical to the mode string used in 'file' and 'open' builtins. :type mode: string :param buffering: An optional integer used to set the buffering policy. Pass 0 to switch buffering off (only allowed in binary mode), 1 to select line buffering (only usable in text mode), and an integer > 1 to indicate the size of a fixed-size chunk buffer. :param encoding: Determines encoding used when writing unicode data. :param errors: An optional string that specifies how encoding and decoding errors are to be handled (e.g. ``strict``, ``ignore`` or ``replace``). :param newline: Newline character to use (either ``\\n``, ``\\r``, ``\\r\\n``, ``''`` or ``None``). :param line_buffering: Unsupported. Anything by False will raise and error. :param buffer_size: Buffer size used when reading files (defaults to 64K). This can likely be optimized to chunks up to 2MB depending on your desired memory usage. """ def __init__(self, path, mode='r', buffering=-1, encoding=None, errors=None, newline=None, line_buffering=False, buffer_size=None, **kwargs): """The XRootDPyFile constructor. Raises PathError if the given path isn't a valid XRootD URL, and InvalidPathError if it isn't a valid XRootD file path. """ if not is_valid_url(path): raise PathError(path) xpath = spliturl(path)[1] if not is_valid_path(xpath): raise InvalidPathError(xpath) if newline not in [None, '', '\n', '\r', '\r\n']: raise UnsupportedError( "Newline character {0} not supported".format(newline)) if line_buffering is not False: raise NotImplementedError("Line buffering for writing is not " "supported.") buffering = int(buffering) if buffering == 1 and 'b' in mode: raise UnsupportedError( "Line buffering is not supported for " "binary files.") # PyFS attributes self.mode = mode # XRootD attributes & internals self.path = path self.encoding = encoding or sys.getdefaultencoding() self.errors = errors or 'strict' self.buffer_size = buffer_size or 64 * 1024 self.buffering = buffering self._file = File() self._ipp = 0 self._size = -1 self._iterator = None self._newline = newline or b("\n") self._buffer = b('') self._buffer_pos = 0 # flag translation self._flags = translate_file_mode_to_flags(mode) statmsg, response = self._file.open(path, flags=self._flags) if not statmsg.ok: self._raise_status(self.path, statmsg, "instantiating file ({0})".format(path)) # Deal with the modes if 'a' in self.mode: self.seek(self.size, SEEK_SET) def _raise_status(self, path, status, source=None): """Raise error based on status.""" if status.errno == 3011: raise ResourceNotFoundError(path) else: if source: errstr = "XRootD error {0}file: {1}".format( source + ' ', status.message) raise IOError(errstr) def __del__(self): """Close file on object deletion.""" self.close() def __iter__(self): """Initialize the internal iterator.""" self._next_func = self.read self._next_args = ([], dict(sizehint=self.buffer_size)) if self.buffering == 1 or \ (self.buffering == -1 and 'b' not in self.mode): self._next_func = self.readline self._next_args = ([], dict()) elif self.buffering > 1: self._next_args = ([], dict(sizehint=self.buffering)) return self def __enter__(self): """Enter context manager method.""" return self def __exit__(self, exc_type, exc_value, traceback): """Exit context manager method.""" self.close() def next(self): """Return next item for file iteration.""" item = self._next_func(*self._next_args[0], **self._next_args[1]) if not item: raise StopIteration return item def read(self, sizehint=-1): """Read ``sizehint`` bytes from the file object. If no ``sizehint`` is provided the entire file is read! Multiple calls to this method after EOF as been reached, will return an empty string. :oaram sizehint: Number of bytes to read from file object. """ if self.closed: raise ValueError("I/O operation on closed file.") self._assert_mode("r-") chunksize = sizehint if sizehint > 0 else self.size # Read data statmsg, res = self._file.read( offset=self._ipp, size=chunksize, ) if not statmsg.ok: self._raise_status(self.path, statmsg, "reading") # Increment internal file pointer. self._ipp = min( self._ipp + chunksize, self.size if self.size > self._ipp else self._ipp ) return res def readline(self): """Read one entire line from the file. A trailing newline character is kept in the string (but may be absent when a file ends with an incomplete line). """ bits = [self._buffer if self._buffer_pos == self.tell() else b("")] indx = bits[-1].find(self._newline) if indx == -1: # Read chunks until first newline is found or entire file is read. while indx == -1: bit = self.read(self.buffer_size) bits.append(bit) if not bit: break indx = bit.find(self._newline) if indx == -1: return b("").join(bits) indx += len(self._newline) extra = bits[-1][indx:] bits[-1] = bits[-1][:indx] self._buffer = extra self._buffer_pos = self.tell() return b("").join(bits) def readlines(self): """Read until EOF using readline(). .. warning:: This methods reads the entire file into memory! You are probably better off using either ``xreadlines`` or just normal iteration over the file object. """ return list(self.xreadlines()) def xreadlines(self, sizehint=-1): """Get an iterator over number of lines.""" line = True while line: line = self.readline() if not line: break yield line def write(self, data, flushing=False): """Write the given string to the file. If the keyword argument 'flushing' is true, it indicates that the internal write buffers are being flushed, and *all* the given data is expected to be written to the file. """ self._assert_mode("w-") if 'a' in self.mode: self.seek(0, SEEK_END) if not isinstance(data, binary_type): if isinstance(data, bytearray): data = bytes(data) elif isinstance(data, text_type): data = data.encode(self.encoding, self.errors) statmsg, res = self._file.write(data, offset=self._ipp) if not statmsg.ok: self._raise_status(self.path, statmsg, "writing") self._ipp += len(data) self._size = max(self.size, self.tell()) if flushing: self.flush() def writelines(self, sequence): """Write an sequence of lines to file.""" for s in sequence: self.write(s) def seek(self, offset, whence=SEEK_SET): """Set the file's internal position pointer, approximately. The possible values of whence and their meaning are defined in the Linux man pages for `lseek()`: http://man7.org/linux/man-pages/man2/lseek.2.html ``SEEK_SET`` The internal position pointer is set to offset bytes. ``SEEK_CUR`` The ipp is set to its current position plus offset bytes. ``SEEK_END`` The ipp is set to the size of the file plus offset bytes. """ if "-" in self.mode: raise IOError("File is not seekable.") # Convert to integer by rounding down/omitting everything after # the decimal point offset = int(offset) # If not in binary mode and seeking from the end, forbid negative # offsets if not ('b' in self.mode and whence == SEEK_END) and offset < 0: raise IOError("Invalid argument.") if whence == SEEK_SET: self._ipp = offset elif whence == SEEK_CUR: self._ipp += offset elif whence == SEEK_END: self._ipp = self.size + offset else: raise NotImplementedError(whence) def tell(self): """Get the location of the file's internal position pointer.""" return self._ipp def truncate(self, size=None): """Truncate the file's size to ``size``. Note that ``size`` will never be None; if it was not specified by the user the current file position is used. """ self._assert_mode('w') if size is None: size = self.tell() statmsg = self._file.truncate(size)[0] if not statmsg.ok: self._raise_status(self.path, statmsg, "truncating") self._size = size def close(self): """Close the file, including flushing the write buffers. The file may not be accessed further once it is closed. """ if not self.closed: self._file.close() def flush(self): """Flush write buffers.""" if not self.closed: statmsg, dummy = self._file.sync() if not statmsg.ok: self._raise_status(self.path, statmsg, "flushing write buffer") def seekable(self): """Check if file is seekable.""" return '-' not in self.mode def readable(self): """Check if file is readable.""" return 'r' in self.mode or '+' in self.mode def writable(self): """Check if file is writable.""" return 'w' in self.mode or '+' in self.mode or 'a' in self.mode def isatty(self): """Check if file is a TTY (false always). Added for ``io`` module compatibility. """ return False def fileno(self): """Get the underlying file descriptor. Unsupported by XRootDPyFS (added for ``io`` module compatibility). """ raise IOError("File descriptor is unsupported by xrootd.") @property def name(self): """Get filename.""" return basename(self.path) @property def closed(self): """Check if file is closed.""" return not self._file.is_open() @property def size(self): """Get file size.""" if self._size == -1: statmsg, res = self._file.stat() if not statmsg.ok: self._raise_status(self.path, statmsg, "retrieving size") self._size = res.size return self._size def _assert_mode(self, mode, mstr=None): """Check whether the file may be accessed in the given mode.""" if mstr is None: try: mstr = self.mode except AttributeError: raise AttributeError("Mode attribute missing -- " "was it deleted? " "Close and re-open the file.") if "+" in mstr: return True if "-" in mstr and "-" not in mode: raise IOError("File does not support seeking.") if "r" in mode: if "r" not in mstr: raise IOError("File not opened for reading") if "w" in mode: if "w" not in mstr and "a" not in mstr: raise IOError("File not opened for writing") return True