def remove(self, path):
"""
Remove a file.
Raises OSError on error.
"""
path = self.abspath(path)
logging.debug("remove %r" % path)
logging.info("remove %r" % path)
container, name = parse_fspath(path)
if not name:
raise IOSError(EACCES, "Can't remove a container")
if self.isdir(path):
raise IOSError(EACCES,
"Can't remove a directory (use rmdir instead)")
meta = self.conn.head_object(container, name)
if 'x-object-manifest' in meta:
self._remove_path_folder_files(
u'/' + unicode(unquote(meta['x-object-manifest']), "utf-8"))
self.conn.delete_object(container, name)
self._listdir_cache.flush(posixpath.dirname(path))
return not name
def rmdir(self, path):
"""
Remove a directory.
Eaise OSError on error.
"""
path = self.abspath(path)
logging.debug("rmdir %r" % path)
container, obj = parse_fspath(path)
if not self.isdir(path):
if self.isfile(path):
raise IOSError(ENOTDIR, "Not a directory")
raise IOSError(ENOENT, 'No such file or directory')
if self.listdir(path):
raise IOSError(ENOTEMPTY, "Directory not empty: %s" % path)
if obj:
self._listdir_cache.flush(posixpath.dirname(path))
logging.debug("Removing directory %r in %r" % (obj, container))
self.conn.delete_object(container, obj)
else:
self._listdir_cache.flush("/")
logging.debug("Removing container %r" % (container,))
self.conn.delete_container(container)
def rename(self, src, dst):
"""
Rename a file/directory from src to dst.
Raises OSError on error.
"""
src = self.abspath(src)
dst = self.abspath(dst)
logging.debug("rename %r -> %r" % (src, dst))
self._listdir_cache.flush()
# Check not renaming to itself
if src == dst:
logging.debug("Renaming %r to itself - doing nothing" % src)
return
# If dst is an existing directory, copy src inside it
if self.isdir(dst):
if dst:
dst += "/"
dst += posixpath.basename(src)
# Check constraints for renaming a directory
if self.isdir(src):
if self.listdir(src):
raise IOSError(ENOTEMPTY, "Can't rename non-empty directory: %s" % src)
if self.isfile(dst):
raise IOSError(ENOTDIR, "Can't rename directory to file")
# Check not renaming to itself
if src == dst:
logging.debug("Renaming %r to itself - doing nothing" % src)
return
# Parse the paths now
src_container_name, src_path = parse_fspath(src)
dst_container_name, dst_path = parse_fspath(dst)
logging.debug("`.. %r/%r -> %r/%r" % (src_container_name, src_path, dst_container_name, dst_path))
# Check if we are renaming containers
if not src_path and not dst_path and src_container_name and dst_container_name:
return self._rename_container(src_container_name, dst_container_name)
# ...otherwise can't deal with root stuff
if not src_container_name or not src_path or not dst_container_name or not dst_path:
raise IOSError(EACCES, "Can't rename to / from root")
# Check destination directory exists
if not self.isdir(posixpath.split(dst)[0]):
raise IOSError(ENOENT, "Can't copy %r to %r, destination directory doesn't exist" % (src, dst))
# check dst container
self._container_exists(dst_container_name)
# Do the rename of the file/dir
meta = self.conn.head_object(src_container_name, src_path)
if 'x-object-manifest' in meta:
# a manifest file
headers = { 'x-object-manifest': meta['x-object-manifest'] }
else:
# regular file
headers = { 'x-copy-from': "/%s/%s" % (src_container_name, src_path) }
self.conn.put_object(dst_container_name, dst_path, headers=headers, contents=None)
# Delete src
self.conn.delete_object(src_container_name, src_path)
self._listdir_cache.flush(posixpath.dirname(src))
self._listdir_cache.flush(posixpath.dirname(dst))
def rename(self, src, dst):
'''Rename a file/directory from src to dst, raise OSError on error'''
src = self.abspath(src)
dst = self.abspath(dst)
logging.debug("rename %r -> %r" % (src, dst))
self._listdir_cache.flush()
# Check not renaming to itself
if src == dst:
logging.debug("Renaming %r to itself - doing nothing" % src)
return
# If dst is an existing directory, copy src inside it
if self.isdir(dst):
if dst:
dst += "/"
dst += posixpath.basename(src)
# Check constraints for renaming a directory
if self.isdir(src):
if self.listdir(src):
raise IOSError(ENOTEMPTY,
"Can't rename non-empty directory: %s" % src)
if self.isfile(dst):
raise IOSError(ENOTDIR, "Can't rename directory to file")
# Check not renaming to itself
if src == dst:
logging.debug("Renaming %r to itself - doing nothing" % src)
return
# Parse the paths now
src_container_name, src_path = parse_fspath(src)
dst_container_name, dst_path = parse_fspath(dst)
logging.debug(
"`.. %r/%r -> %r/%r" %
(src_container_name, src_path, dst_container_name, dst_path))
# Check if we are renaming containers
if not src_path and not dst_path and src_container_name and dst_container_name:
return self._rename_container(src_container_name,
dst_container_name)
# ...otherwise can't deal with root stuff
if not src_container_name or not src_path or not dst_container_name or not dst_path:
raise IOSError(EACCES, "Can't rename to / from root")
# Check destination directory exists
if not self.isdir(posixpath.split(dst)[0]):
raise IOSError(
ENOENT,
"Can't copy %r to %r, destination directory doesn't exist" %
(src, dst))
# Do the rename of the file/dir
src_container = self._get_container(src_container_name)
dst_container = self._get_container(dst_container_name)
src_obj = src_container.get_object(src_path)
# Copy src -> dst
src_obj.copy_to(dst_container_name, dst_path)
# Delete src
src_container.delete_object(src_path)
self._listdir_cache.flush(posixpath.dirname(src))
self._listdir_cache.flush(posixpath.dirname(dst))
def chdir(self, path):
"""Change current directory, raise OSError on error"""
path = self.abspath(path)
logging.debug("chdir %r" % path)
if not path.startswith("/"):
raise IOSError(ENOENT, 'Failed to change directory.')
container, obj = parse_fspath(path)
if not container:
logging.debug("cd to /")
else:
logging.debug("cd to container %r directory %r" % (container, obj))
if not self.isdir(path):
raise IOSError(ENOTDIR, "Can't cd to a file")
self._cwd = path
def stat(self, path):
"""
Returns an os.stat_result for path or raises IOSError.
Returns the information from the cache if possible.
"""
path = path.rstrip("/") or "/"
logging.debug("stat path %r" % (path))
directory, leaf = posixpath.split(path)
# Refresh the cache it if is old, or wrong
if not self.valid(directory):
logging.debug("invalid cache for %r (path: %r)" %
(directory, self.path))
self.listdir(directory)
if path != "/":
try:
stat_info = self.cache[smart_str(leaf)]
except KeyError:
logging.debug("Didn't find %r in directory listing" % leaf)
# it can be a container and the user doesn't have
# permissions to list the root
if directory == '/' and leaf:
try:
container = self.conn.head_container(leaf)
if (self.cffs.storage_policy is not None
and container['x-storage-policy'] !=
self.cffs.storage_policy):
raise IOSError(
ENOENT, 'No such file or directory %s' % leaf)
except ClientException:
raise IOSError(ENOENT,
'No such file or directory %s' % leaf)
logging.debug(
"Accessing %r container without root listing" % leaf)
stat_info = self._make_stat(
count=int(container["x-container-object-count"]),
bytes=int(container["x-container-bytes-used"]),
)
else:
raise IOSError(ENOENT,
'No such file or directory %s' % leaf)
else:
# Root directory size is sum of containers, count is containers
bytes = sum(stat_info.st_size for stat_info in self.cache.values())
count = len(self.cache)
stat_info = self._make_stat(count=count, bytes=bytes)
logging.debug("stat path: %r" % stat_info)
return stat_info
def md5(self, path):
'''Return the object MD5 for path, raise OSError on error'''
path = self.abspath(path)
logging.debug("md5 %r" % path)
container, name = parse_fspath(path)
if not name:
raise IOSError(EACCES, "Can't return the MD5 of a container")
if self.isdir(path):
# this is only 100% accurate for virtual directories
raise IOSError(EACCES, "Can't return the MD5 of a directory")
container = self._get_container(container)
obj = container.get_object(name)
return obj.etag
def __init__(self, cffs, container, obj, mode):
self.cffs = cffs
self.container = container
self.name = obj
self.mode = mode
self.closed = False
self.total_size = 0
self.part_size = 0
self.part = 0
self.headers = dict()
self.content_type = mimetypes.guess_type(self.name)[0]
self.pending_copy_task = None
self.obj = None
# this is only used by `seek`, so we delay the HEAD request until is required
self.size = None
if not all([container, obj]):
self.closed = True
raise IOSError(EPERM, 'Container and object required')
logging.debug("ObjectStorageFD object: %r (mode: %r)" % (obj, mode))
if 'r' in self.mode:
logging.debug("read fd %r" % self.name)
else: # write
logging.debug("write fd %r" % self.name)
self.obj = ChunkObject(self.conn, self.container, self.name, content_type=self.content_type)
def write(self, data):
"""Write data to the object."""
if 'r' in self.mode:
raise IOSError(EPERM, "File is opened for read")
# large file support
if self.split_size:
# data can be of any size, so we need to split it in split_size chunks
offs = 0
while offs < len(data):
if self.part_size + len(data) - offs > self.split_size:
current_size = self.split_size-self.part_size
logging.debug("data is to large (%r), using %s" % (len(data), current_size))
else:
current_size = len(data)-offs
self.part_size += current_size
if not self.obj:
self.obj = ChunkObject(self.conn, self.container, self.part_name, content_type=self.content_type)
self.obj.send_chunk(data[offs:offs+current_size])
offs += current_size
if self.part_size == self.split_size:
logging.debug("current size is %r, split_file is %r" % (self.part_size, self.split_size))
self.obj.finish_chunk()
# this obj is not valid anymore, will create a new one if a new part is required
self.obj = None
# make it the first part
if self.part == 0:
self._start_copy_task()
self.part_size = 0
self.part += 1
else:
self.obj.send_chunk(data)
def _container_exists(self, container):
# verify the container exsists
try:
self.conn.head_container(container)
except ClientException, e:
if e.http_status == 404:
raise IOSError(ENOTDIR, "Container not found")
raise
def wrapper(*args, **kwargs):
name = getattr(fn, "func_name", "unknown")
log = lambda msg: logging.warning("At %s: %s" % (name, msg))
try:
return fn(*args, **kwargs)
except (cloudfiles.errors.NoSuchContainer,
cloudfiles.errors.NoSuchObject), e:
raise IOSError(ENOENT, 'Not found: %s' % e)
def remove(self, path):
'''Remove a file, raise OSError on error'''
path = self.abspath(path)
logging.debug("remove %r" % path)
container, name = parse_fspath(path)
if not name:
raise IOSError(EACCES, "Can't remove a container")
if self.isdir(path):
raise IOSError(EACCES,
"Can't remove a directory (use rmdir instead)")
container = self._get_container(container)
obj = container.get_object(name)
container.delete_object(obj)
self._listdir_cache.flush(posixpath.dirname(path))
return not name
def mkstemp(self, suffix='', prefix='', dir=None, mode='wb'):
"""
A wrapper around tempfile.mkstemp creating a file with a unique name.
Unlike mkstemp it returns an object with a file-like interface.
"""
e = "mkstemp suffix=%r prefix=%r, dir=%r mode=%r - not implemented" % (suffix, prefix, dir, mode)
logging.debug(e)
raise IOSError(EPERM, 'Operation not permitted: %s' % e)
def readlink(self, path):
"""
Return a string representing the path to which a symbolic link points.
We never return that we have a symlink in stat, so this should
never be called.
"""
e = "readlink %r - not implemented" % path
logging.debug(e)
raise IOSError(EPERM, 'Operation not permitted: %s' % e)
def md5(self, path):
"""
Return the object MD5 for path.
Raise OSError on error.
"""
path = self.abspath(path)
logging.debug("md5 %r" % path)
container, name = parse_fspath(path)
if not name:
raise IOSError(EACCES, "Can't return the MD5 of a container")
if self.isdir(path):
# this is only 100% accurate for virtual directories
raise IOSError(EACCES, "Can't return the MD5 of a directory")
meta = self.conn.head_object(container, name)
return meta["etag"]
def close(self):
"""Close the object and finish the data transfer."""
if 'r' in self.mode:
return
if self.pending_copy_task:
logging.debug("waiting for a pending copy task...")
self.pending_copy_task.join()
logging.debug("wait is over")
if self.pending_copy_task.exitcode != 0:
raise IOSError(EIO, 'Failed to store the file')
if self.obj is not None:
self.obj.finish_chunk()
def stat(self, path):
'''Returns an os.stat_result for path or raises IOSError
Returns the information from the cache if possible
'''
path = path.rstrip("/") or "/"
logging.debug("stat path %r" % (path))
directory, leaf = posixpath.split(path)
# Refresh the cache it if is old, or wrong
if not self.valid(directory):
logging.debug("invalid cache for %r (path: %r)" %
(directory, self.path))
self.listdir(directory)
if path != "/":
try:
stat_info = self.cache[leaf]
except KeyError:
logging.debug("Didn't find %r in directory listing" % leaf)
# it can be a container and the user doesn't have
# permissions to list the root
if directory == '/' and leaf:
try:
container = self.cffs.connection.get_container(leaf)
except cloudfiles.errors.ResponseError:
raise IOSError(ENOENT,
'No such file or directory %s' % leaf)
logging.debug(
"Accessing %r container without root listing" % leaf)
stat_info = self._make_stat(count=container.object_count,
bytes=container.size_used)
else:
raise IOSError(ENOENT,
'No such file or directory %s' % leaf)
else:
# Root directory size is sum of containers, count is containers
bytes = sum(stat_info.st_size for stat_info in self.cache.values())
count = len(self.cache)
stat_info = self._make_stat(count=count, bytes=bytes)
return stat_info
def parse_fspath(path):
'''Returns a (container, path) tuple. For shorter paths
replaces not provided values with empty strings.
May raise IOSError for invalid paths
'''
if not path.startswith('/'):
logging.warning(
'parse_fspath: You have to provide an absolute path: %r' % path)
raise IOSError(ENOENT, 'Absolute path needed')
parts = path.split('/', 2)[1:]
while len(parts) < 2:
parts.append('')
return tuple(parts)
def seek(self, offset, whence=None):
"""
Seek in the object.
It's supported only for read operations because of object storage limitations.
"""
logging.debug("seek offset=%s, whence=%s" % (str(offset), str(whence)))
if 'r' in self.mode:
if self.size is None:
meta = self.conn.head_object(self.container, self.name)
try:
self.size = int(meta["content-length"])
except ValueError:
raise IOSError(EPERM, "Invalid file size")
if not whence:
offs = offset
elif whence == 1:
offs = self.total_size + offset
elif whence == 2:
offs = self.size - offset
else:
raise IOSError(EPERM, "Invalid file offset")
if offs < 0 or offs > self.size:
raise IOSError(EPERM, "Invalid file offset")
# we need to start over after a seek call
if self.obj is not None:
del self.obj # GC the generator
self.obj = None
self.total_size = offs
else:
raise IOSError(EPERM, "Seek not available for write operations")
def translate_cloudfiles_error(fn):
"""
Decorator to catch cloudfiles errors and translating them into IOSError.
Other exceptions are not caught.
"""
@wraps(fn)
def wrapper(*args, **kwargs):
name = getattr(fn, "func_name", "unknown")
log = lambda msg: logging.warning("At %s: %s" % (name, msg))
try:
return fn(*args, **kwargs)
except (cloudfiles.errors.NoSuchContainer,
cloudfiles.errors.NoSuchObject), e:
raise IOSError(ENOENT, 'Not found: %s' % e)
except cloudfiles.errors.ContainerNotEmpty, e:
raise IOSError(ENOTEMPTY, 'Directory not empty: %s' % e)
def wrapper(*args,**kwargs):
name = getattr(fn, "func_name", "unknown")
log = lambda msg: logging.debug("At %s: %s" % (name, msg))
try:
return fn(*args, **kwargs)
except ClientException, e:
# some errno mapping
if e.http_status == 404:
err = ENOENT
elif e.http_status == 400:
err = EPERM
elif e.http_status == 403:
err = EACCES
else:
err = EIO
msg = "%s: %s" % (smart_str(e.msg), smart_str(e.http_reason))
log(msg)
raise IOSError(err, msg)
def __init__(self, cffs, container, obj, mode):
self.cffs = cffs
self.container = container
self.name = obj
self.mode = mode
self.closed = False
self.total_size = 0
self.stream = None
if not all([container, obj]):
self.closed = True
raise IOSError(EPERM, 'Container and object required')
self.container = self.cffs._get_container(self.container)
if 'r' in self.mode:
self.obj = self.container.get_object(self.name)
logging.debug("read fd obj.name=%r obj.size=%r" %
(self.obj.name, self.obj.size))
else: #write
self.obj = ChunkObject(self.container, obj)
self.obj.content_type = mimetypes.guess_type(obj)[0]
self.obj.prepare_chunk()
def write(self, data):
'''Write data to the object'''
if 'r' in self.mode:
raise IOSError(EPERM, "Can't write to stream opened for read")
self.obj.send_chunk(data)
def seek(self, *kargs, **kwargs):
'''Seek in the object: FIXME doesn't work and raises an error'''
logging.debug("seek args=%s, kargs=%s" % (str(kargs), str(kwargs)))
raise IOSError(EPERM, "Seek not implemented")
def chmod(self, path, mode):
"""Change file/directory mode"""
e = "chmod %03o %r - not implemented" % (mode, path)
logging.debug(e)
raise IOSError(EPERM, 'Operation not permitted: %s' % e)
"""
@wraps(fn)
def wrapper(*args, **kwargs):
name = getattr(fn, "func_name", "unknown")
log = lambda msg: logging.warning("At %s: %s" % (name, msg))
try:
return fn(*args, **kwargs)
except (cloudfiles.errors.NoSuchContainer,
cloudfiles.errors.NoSuchObject), e:
raise IOSError(ENOENT, 'Not found: %s' % e)
except cloudfiles.errors.ContainerNotEmpty, e:
raise IOSError(ENOTEMPTY, 'Directory not empty: %s' % e)
except cloudfiles.errors.ResponseError, e:
log("Response error: %s" % e)
# FIXME make some attempt to raise different errors on e.status
raise IOSError(EPERM, 'Operation not permitted: %s' % e)
except (cloudfiles.errors.AuthenticationError,
cloudfiles.errors.AuthenticationFailed,
cloudfiles.errors.ContainerNotPublic), e:
log("Authentication error: %s" % e)
raise IOSError(EPERM, 'Operation not permitted: %s' % e)
except (cloudfiles.errors.CDNNotEnabled,
cloudfiles.errors.IncompleteSend,
cloudfiles.errors.InvalidContainerName,
cloudfiles.errors.InvalidMetaName,
cloudfiles.errors.InvalidMetaValue,
cloudfiles.errors.InvalidObjectName,
cloudfiles.errors.InvalidObjectSize,
cloudfiles.errors.InvalidUrl), e:
log("Unexpected cloudfiles error: %s" % e)
raise IOSError(EIO, 'Unexpected cloudfiles error')