def openbin(self,
path: str,
mode: str = "r",
buffering: int = -1,
**options) -> "GCSFile":
_mode = Mode(mode)
_mode.validate_bin()
self.check()
_path = self.validatepath(path)
_key = self._path_to_key(_path)
def on_close(gcs_file):
if _mode.create or _mode.writing:
gcs_file.raw.seek(0)
blob = self._get_blob(_key)
if not blob:
blob = self.bucket.blob(_key)
blob.upload_from_file(gcs_file.raw)
gcs_file.raw.close()
if _mode.create:
dir_path = dirname(_path)
if dir_path != "/":
_dir_key = self._path_to_dir_key(dir_path)
if not self.bucket.get_blob(_dir_key):
raise errors.ResourceNotFound(path)
try:
info = self.getinfo(path)
except errors.ResourceNotFound:
pass
else:
if _mode.exclusive:
raise errors.FileExists(path)
if info.is_dir:
raise errors.FileExpected(path)
gcs_file = GCSFile.factory(path, _mode, on_close=on_close)
if _mode.appending:
blob = self._get_blob(_key)
if blob: # in case there is an existing blob in GCS, we download it and seek until the end of the stream
gcs_file.seek(0, os.SEEK_END)
blob.download_to_file(gcs_file.raw)
return gcs_file
if self.strict:
info = self.getinfo(path)
if info.is_dir:
raise errors.FileExpected(path)
gcs_file = GCSFile.factory(path, _mode, on_close=on_close)
blob = self._get_blob(_key)
if not blob:
raise errors.ResourceNotFound
blob.download_to_file(gcs_file.raw)
gcs_file.seek(0)
return gcs_file
def openbin(self, path, mode="r", buffering=-1, **options):
"""
Open file under `path` in binary mode.
:param str path: Path pointing to a file.
:param str mode: Text representation of open mode e.g. "rw+"
:param int buffering: Whether the BaseIO instance should be buffered
or not
:param map options: Additional PyFilesystem options
"""
# type: (Text, Text, int, **Any) -> BinaryIO
path = ensureUnicode(path)
_mode = Mode(mode)
_mode.validate_bin()
_path = toAscii(self.validatepath(path))
with self._lock:
try:
info = self.getinfo(path)
except errors.ResourceNotFound:
if _mode.reading:
raise errors.ResourceNotFound(path)
if _mode.writing and not self.isdir(dirname(_path)):
raise errors.ResourceNotFound(path)
else:
if info.is_dir:
raise errors.FileExpected(path)
if _mode.exclusive:
raise errors.FileExists(path)
# TODO support mode
handle = self._odfs.open(_path)
onedata_file = OnedataFile(self._odfs, handle, path, mode)
return onedata_file # type: ignore
def openbin(self, path, mode="r", buffering=-1, **options):
_mode = Mode(mode)
_mode.validate_bin()
self.check()
_path = self.validatepath(path)
_key = self._path_to_key(_path)
info = None
try:
info = self.getinfo(path)
except errors.ResourceNotFound:
pass
else:
if info.is_dir:
raise errors.FileExpected(path)
if _mode.create:
try:
dir_path = dirname(_path)
if dir_path != "/":
self.getinfo(dir_path)
except errors.ResourceNotFound:
raise errors.ResourceNotFound(path)
if info and _mode.exclusive:
raise errors.FileExists(path)
# AzureDLFile does not support exclusive mode, but we mimic it
dlkfile = self.dlk.open(_key, str(_mode).replace("x", ""))
return dlkfile
def dlkerrors(path):
""" Translate Datalake errors to FSErrors.
FS errors: https://docs.pyfilesystem.org/en/latest/reference/errors.html
DLK errors: https://docs.pyfilesystem.org/en/latest/reference/errors.html
"""
try:
yield
except client_error.FileNotFoundError as error:
raise errors.ResourceNotFound(path, exc=error)
except client_error.FileExistsError as error:
raise errors.FileExists(path, exc=error)
except client_error.PermissionError as error:
raise errors.PermissionDenied(path, exc=error)
except client_error.DatalakeBadOffsetException as error:
raise errors.RemoteConnectionError(path,
exc=error,
msg="DatalakeBadOffsetException")
except client_error.DatalakeIncompleteTransferException as error:
raise errors.RemoteConnectionError(
path, exc=error, msg="DatalakeIncompleteTransferException")
except client_error.DatalakeRESTException as error:
raise errors.RemoteConnectionError(path,
exc=error,
msg="DatalakeRESTException")
def openbin(self, path, mode="r", buffering=-1, **options):
path = self.fix_path(path)
_mode = Mode(mode)
mode = _mode
_mode.validate_bin()
_path = self.validatepath(path)
log.debug("openbin: %s, %s", path, mode)
with self._lock:
try:
info = self.getinfo(_path)
log.debug("Info: %s", info)
except errors.ResourceNotFound:
if not _mode.create:
raise errors.ResourceNotFound(path)
# Check the parent is an existing directory
if not self.getinfo(self.get_parent(_path)).is_dir:
raise errors.DirectoryExpected(path)
else:
if info.is_dir:
raise errors.FileExpected(path)
if _mode.exclusive:
raise errors.FileExists(path)
return DropboxFile(self.dropbox, path, mode)
def openbin(self, path, mode="r", buffering=-1, **options):
_mode = Mode(mode)
_mode.validate_bin()
self.check()
_path = self.validatepath(path)
_key = self._path_to_key(_path)
if _mode.create:
def on_close(s3file):
try:
s3file.raw.seek(0)
with s3errors(path):
self.client.upload_fileobj(s3file.raw,
self._bucket_name, _key)
finally:
s3file.raw.close()
try:
info = self.getinfo(path)
except errors.ResourceNotFound:
pass
else:
if _mode.exclusive:
raise errors.FileExists(path)
if info.is_dir:
raise errors.FileExpected(path)
s3file = S3File.factory(path, _mode, on_close=on_close)
if _mode.appending:
try:
with s3errors(path):
self.client.download_fileobj(self._bucket_name, _key,
s3file.raw)
except errors.ResourceNotFound:
pass
else:
s3file.seek(0, os.SEEK_END)
return s3file
info = self.getinfo(path)
if info.is_dir:
raise errors.FileExpected(path)
def on_close(s3file):
try:
if _mode.writing:
s3file.raw.seek(0, os.SEEK_SET)
with s3errors(path):
self.client.upload_fileobj(s3file.raw,
self._bucket_name, _key)
finally:
s3file.raw.close()
s3file = S3File.factory(path, _mode, on_close=on_close)
with s3errors(path):
self.client.download_fileobj(self._bucket_name, _key, s3file.raw)
s3file.seek(0, os.SEEK_SET)
return s3file
def openbin(self, path, mode="r", buffering=-1, **options):
path = self.fix_path(path)
_mode = Mode(mode)
mode = _mode
_mode.validate_bin()
_path = self.validatepath(path)
log.debug("openbin: %s, %s", path, mode)
with self._lock:
try:
info = self.getinfo(_path)
except errors.ResourceNotFound:
if not _mode.create:
raise
# Target doesn't exist and we're in create mode. Ensure the
# parent is an existing directory before we try to create a file
# in it.
parent_path = self.get_parent(_path)
# Can't use self.isdir() because it doesn't crash if the
# directory doesn't exist, and we don't want to stat a file twice
# if we can avoid it.
info = self.getinfo(parent_path)
if not info.is_dir:
raise errors.DirectoryExpected(parent_path)
return DropboxFile(self.dropbox, path, mode)
# Target exists.
if info.is_dir:
raise errors.FileExpected(path)
if _mode.exclusive:
raise errors.FileExists(path)
return DropboxFile(self.dropbox, path, mode)
def move(self, src_path, dst_path, overwrite=False):
"""
Rename file from `src_path` to `dst_path`.
:param str src_path: The old file path
:param str dst_path: The new file path
:param bool overwrite: When `True`, existing file at `dst_path` will be
replaced by contents of file at `src_path`
"""
# type: (Text, Text, bool) -> None
src_path = ensureUnicode(src_path)
dst_path = ensureUnicode(dst_path)
if not overwrite and self.exists(dst_path):
raise errors.FileExists(dst_path)
self._odfs.rename(toAscii(src_path), toAscii(dst_path))
def openbin(self, path, mode='r', buffering=-1, **options): # noqa: D102
"""Open a binary file-like object.
Arguments:
path (str): A path on the filesystem.
mode (str): Mode to open the file (must be a valid, non-text mode).
Since this method only opens binary files, the ``b`` in the mode
is implied.
buffering (int): the buffering policy (-1 to use default buffering,
0 to disable completely, 1 to enable line based buffering, or
any larger positive integer for a custom buffer size).
Keyword Arguments:
pipelined (bool): Set the transfer in pipelined mode (should
improve transfer speed). Defaults to ``True``.
Raises:
fs.errors.FileExpected: if the path if not a file.
fs.errors.FileExists: if the file already exists and
*exclusive mode* is specified (``x`` in the mode).
fs.errors.ResourceNotFound: if the path does not exist.
Returns:
io.IOBase: a file handle.
"""
self.check()
_path = self.validatepath(path)
_mode = Mode(mode)
_mode.validate_bin()
with self._lock:
if _mode.exclusive and self.exists(_path):
raise errors.FileExists(path)
elif _mode.reading and not _mode.create and not self.exists(_path):
raise errors.ResourceNotFound(path)
elif self.isdir(_path):
raise errors.FileExpected(path)
with convert_sshfs_errors('openbin', path):
_sftp = self._client.open_sftp()
handle = _sftp.open(_path,
mode=_mode.to_platform_bin(),
bufsize=buffering)
handle.set_pipelined(options.get("pipelined", True))
return SSHFile(handle)
def openbin(self, path, mode='r', buffering=-1, **options):
_mode = Mode(mode)
_mode.validate_bin()
_path = self.validatepath(path)
log.debug("openbin: %s, %s", path, mode)
with self._lock:
try:
info = self.getinfo(_path)
log.debug("Info: %s", info)
except errors.ResourceNotFound:
if not _mode.create:
raise errors.ResourceNotFound(path)
else:
if info.is_dir:
raise errors.FileExpected(path)
if _mode.exclusive:
raise errors.FileExists(path)
return WebDAVFile(self, _path, _mode)
def openbin(self, path, mode="r", buffering=-1, **options):
_mode = Mode(mode)
_mode.validate_bin()
self.check()
_path = self.validatepath(path)
_key = self._path_to_key(_path)
if _mode.appending:
raise errors.ResourceError(path,
msg="append mode is not supported")
if _mode.create:
if self.strict:
try:
dir_path = dirname(_path)
if dir_path != "/":
_dir_key = self._path_to_dir_key(dir_path)
self._get_object(dir_path, _dir_key)
except errors.ResourceNotFound:
raise errors.ResourceNotFound(path)
try:
info = self._getinfo(path)
except errors.ResourceNotFound:
pass
else:
if _mode.exclusive:
raise errors.FileExists(path)
if self.strict and info.is_dir:
raise errors.FileExpected(path)
obj = self.s3.Object(self._bucket_name, _key)
return S3OutputFile(obj, upload_kwargs=self._get_upload_args(_key))
if self.strict:
info = self.getinfo(path)
if info.is_dir:
raise errors.FileExpected(path)
obj = self.s3.Object(self._bucket_name, _key)
return S3InputFile(obj)
def openbin(self, path, mode='r', buffering=-1, **options):
_mode = Mode(mode)
_mode.validate_bin()
_path = self.validatepath(path)
log.debug("openbin: %s, %s", path, mode)
with self._lock:
try:
info = self.getinfo(_path)
log.debug("Info: %s", info)
except errors.ResourceNotFound:
if not _mode.create:
raise errors.ResourceNotFound(path)
# Check the parent is an existing directory
if self.gettype(dirname(_path)) is not ResourceType.directory:
raise errors.DirectoryExpected(dirname(path))
else:
if info.is_dir:
raise errors.FileExpected(path)
if _mode.exclusive:
raise errors.FileExists(path)
return WebDAVFile(self, _path, _mode)
def openbin(self, path: Text, mode: Text = "r", buffering: int = -1, **options: Any) -> BinaryIO:
""" Open a binary file. """
npath = self.normalize_path(path)
if self._closed:
raise errors.FilesystemClosed
if not self.exists(dirname(npath)):
raise errors.ResourceNotFound(dirname(path))
if "t" in mode:
raise ValueError
if "b" not in mode:
mode += "b"
file_mode = Mode(mode)
exists = self.exists(npath)
if file_mode.exclusive and exists:
raise errors.FileExists(path)
if file_mode.reading and not exists:
raise errors.ResourceNotFound(path)
if (file_mode.reading or (file_mode.writing and exists)) and self.isdir(path):
raise errors.FileExpected(path)
data = gzip.compress(b"")
if file_mode.create and not exists:
cursor = self.connection.cursor()
cursor.execute(
"INSERT INTO sqlar (name, mode, mtime, sz, data) VALUES (?, ?, ?, ?, ?)",
(npath, 0o700, datetime.utcnow().timestamp(), 0, data)
)
cursor.close()
elif file_mode.truncate:
cursor = self.connection.cursor()
cursor.execute("UPDATE sqlar SET data = ? WHERE name = ?", (data, npath))
cursor.close()
return SQLiteFile(self, npath, file_mode)
def openbin(self, path, mode="r", buffering=-1, **options):
_mode = Mode(mode)
_mode.validate_bin()
_bucket_name_copy = self._bucket_name
self.check()
_path = self.validatepath(path)
_key = self._path_to_key(_path)
if _mode.create:
def on_close_create(s3file):
"""Called when the S3 file closes, to upload data."""
try:
s3file.raw.seek(0)
with s3errors(path):
self.client.upload_fileobj(
s3file.raw,
s3file._bucket_name,
_key,
ExtraArgs=self._get_upload_args(_key),
)
finally:
s3file.raw.close()
try:
dir_path = dirname(_path)
if dir_path != "/":
_dir_key = self._path_to_dir_key(dir_path)
self._get_object(dir_path, _dir_key)
except errors.ResourceNotFound:
raise errors.ResourceNotFound(path)
try:
info = self._getinfo(path)
except errors.ResourceNotFound:
pass
else:
if _mode.exclusive:
raise errors.FileExists(path)
if info.is_dir:
raise errors.FileExpected(path)
s3file = S3File.factory(_bucket_name_copy, path, _mode, on_close=on_close_create)
if _mode.appending:
try:
with s3errors(path):
self.client.download_fileobj(
s3file._bucket_name,
_key,
s3file.raw,
ExtraArgs=self.download_args,
)
except errors.ResourceNotFound:
pass
else:
s3file.seek(0, os.SEEK_END)
return s3file
if self.strict:
info = self.getinfo(path)
if info.is_dir:
raise errors.FileExpected(path)
def on_close(s3file):
"""Called when the S3 file closes, to upload the data."""
try:
if _mode.writing:
s3file.raw.seek(0, os.SEEK_SET)
with s3errors(path):
self.client.upload_fileobj(
s3file.raw,
s3file._bucket_name,
_key,
ExtraArgs=self._get_upload_args(_key),
)
finally:
s3file.raw.close()
s3file = S3File.factory(_bucket_name_copy, path, _mode, on_close=on_close)
with s3errors(path):
self.client.download_fileobj(
s3file._bucket_name, _key, s3file.raw, ExtraArgs=self.download_args
)
s3file.seek(0, os.SEEK_SET)
return s3file
def openbin(
self,
path, # type: Text
mode="r", # type: Text
buffering=-1, # type: int
**options # type: Any
):
# type: (...) -> BinaryIO
"""Open a binary file-like object.
Arguments:
path (str): A path on the filesystem.
mode (str): Mode to open file (must be a valid non-text mode,
defaults to *r*). Since this method only opens binary files,
the ``b`` in the mode string is implied.
buffering (int): Buffering policy (-1 to use default buffering,
0 to disable buffering, or any positive integer to indicate
a buffer size).
**options: keyword arguments for any additional information
required by the filesystem (if any).
Returns:
io.IOBase: a *file-like* object.
Raises:
fs.errors.FileExpected: If the path is not a file.
fs.errors.FileExists: If the file exists, and *exclusive mode*
is specified (``x`` in the mode).
fs.errors.ResourceNotFound: If the path does not exist.
"""
_mode = Mode(mode)
_mode.validate_bin()
_path = self.validatepath(path)
dir_path, file_name = split(_path)
if not file_name:
raise errors.FileExpected(path)
with self._lock:
_dir_res = self._getresource(dir_path)
if not _dir_res or not _dir_res.is_collection:
raise errors.ResourceNotFound(path)
if _mode.create:
if file_name in _dir_res.get_member_names():
if _mode.exclusive:
raise errors.FileExists(path)
_res = self._getresource(path)
if not _res or _res.is_collection:
raise errors.FileExpected(path)
stream = io.BufferedWriter(_res.begin_write())
io_object = RawWrapper(stream, mode=mode, name=path)
return io_object
_res = _dir_res.create_empty_resource(file_name)
stream = io.BufferedWriter(_res.begin_write())
io_object = RawWrapper(stream, mode=mode, name=path)
return io_object
if file_name not in _dir_res.get_member_names():
raise errors.ResourceNotFound(path)
_res = self._getresource(path)
if not _res or _res.is_collection:
raise errors.FileExpected(path)
if _mode.appending:
# stream.seek(0, 2) # io.SEEK_END
raise NotImplementedError("Appending is not supported")
if _mode.updating:
raise NotImplementedError("Updating is not supported")
if _mode.reading:
stream = io.BufferedReader(_res.get_content())
io_object = RawWrapper(stream, mode=mode, name=path)
return io_object
stream = io.BufferedWriter(_res.begin_write())
io_object = RawWrapper(stream, mode=mode, name=path)
return io_object
def openbin(
self,
path, # type: Text
mode="r", # type: Text
buffering=-1, # type: int
**options # type: Any
):
# type: (...) -> BinaryIO
"""Open a binary file-like object.
Arguments:
path (str): A path on the filesystem.
mode (str): Mode to open file (must be a valid non-text mode,
defaults to *r*). Since this method only opens binary files,
the ``b`` in the mode string is implied.
buffering (int): Buffering policy (-1 to use default buffering,
0 to disable buffering, or any positive integer to indicate
a buffer size).
**options: keyword arguments for any additional information
required by the filesystem (if any).
Returns:
io.IOBase: a *file-like* object.
Raises:
fs.errors.FileExpected: If the path is not a file.
fs.errors.FileExists: If the file exists, and *exclusive mode*
is specified (``x`` in the mode).
fs.errors.ResourceNotFound: If the path does not exist.
"""
_mode = Mode(mode)
_mode.validate_bin()
_path = self.validatepath(path)
dir_path, file_name = split(_path)
if not file_name:
raise errors.FileExpected(path)
with self._lock:
_dir_res = self._getresource(dir_path)
if not _dir_res or not _dir_res.isdir():
raise errors.ResourceNotFound(path)
if _mode.create:
if file_name in _dir_res.listdir():
if _mode.exclusive:
raise errors.FileExists(path)
_res = self._getresource(path)
if not _res or not _res.isfile():
raise errors.FileExpected(path)
return self._btopen(_res, mode)
return self._btopen(self._prep_path(_path), mode)
if file_name not in _dir_res.listdir():
raise errors.ResourceNotFound(path)
_res = self._getresource(path)
if not _res or not _res.isfile():
raise errors.FileExpected(path)
return self._btopen(_res, mode)
