def __init__(self, path, device, partition, account, container, obj,
logger, keep_data_fp=False, disk_chunk_size=65536,
uid=DEFAULT_UID, gid=DEFAULT_GID):
self.disk_chunk_size = disk_chunk_size
device = account
#Don't support obj_name ending/begining with '/', like /a, a/, /a/b/ etc
obj = obj.strip('/')
if '/' in obj:
self.obj_path, self.obj = obj.rsplit('/', 1)
else:
self.obj_path = ''
self.obj = obj
if self.obj_path:
self.name = '/'.join((container, self.obj_path))
else:
self.name = container
#Absolute path for obj directory.
self.datadir = os.path.join(path, device, self.name)
self.device_path = os.path.join(path, device)
self.container_path = os.path.join(path, device, container)
self.tmpdir = os.path.join(path, device, 'tmp')
self.logger = logger
self.metadata = {}
self.data_file = None
self.fp = None
self.iter_etag = None
self.started_at_0 = False
self.read_to_eof = False
self.quarantined_dir = None
self.keep_cache = False
self.is_dir = False
self.is_valid = True
self.uid = int(uid)
self.gid = int(gid)
if not os.path.exists(self.datadir + '/' + self.obj):
return
self.data_file = os.path.join(self.datadir, self.obj)
self.metadata = read_metadata(self.datadir + '/' + self.obj)
if not self.metadata:
create_object_metadata(self.datadir + '/' + self.obj)
self.metadata = read_metadata(self.datadir + '/' + self.obj)
if not validate_object(self.metadata):
create_object_metadata(self.datadir + '/' + self.obj)
self.metadata = read_metadata(self.datadir + '/' +
self.obj)
self.filter_metadata()
if os.path.isdir(self.datadir + '/' + self.obj):
self.is_dir = True
else:
self.fp = do_open(self.data_file, 'rb')
if not keep_data_fp:
self.close(verify_file=False)
def _create_dir_object(self, dir_path):
#TODO: if object already exists???
if os.path.exists(dir_path) and not os.path.isdir(dir_path):
self.logger.error("Deleting file %s", dir_path)
do_unlink(dir_path)
#If dir aleady exist just override metadata.
mkdirs(dir_path)
do_chown(dir_path, self.uid, self.gid)
create_object_metadata(dir_path)
def open(self):
"""
Open the object.
This implementation opens the data file representing the object, reads
the associated metadata in the extended attributes, additionally
combining metadata from fast-POST `.meta` files.
.. note::
An implementation is allowed to raise any of the following
exceptions, but is only required to raise `DiskFileNotExist` when
the object representation does not exist.
:raises DiskFileNotExist: if the object does not exist
:raises DiskFileExpired: if the object has expired
:returns: itself for use as a context manager
"""
# Writes are always performed to a temporary file
try:
fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC)
except GlusterFileSystemOSError as err:
if err.errno in (errno.ENOENT, errno.ENOTDIR):
# If the file does exist, or some part of the path does not
# exist, raise the expected DiskFileNotExist
raise DiskFileNotExist
raise
else:
stats = do_fstat(fd)
if not stats:
return
self._is_dir = stat.S_ISDIR(stats.st_mode)
obj_size = stats.st_size
self._metadata = read_metadata(fd)
if not validate_object(self._metadata):
create_object_metadata(fd)
self._metadata = read_metadata(fd)
assert self._metadata is not None
self._filter_metadata()
if self._is_dir:
do_close(fd)
obj_size = 0
self._fd = -1
else:
if self._is_object_expired(self._metadata):
raise DiskFileExpired(metadata=self._metadata)
self._fd = fd
self._obj_size = obj_size
return self
def open(self):
"""
Open the object.
This implementation opens the data file representing the object, reads
the associated metadata in the extended attributes, additionally
combining metadata from fast-POST `.meta` files.
.. note::
An implementation is allowed to raise any of the following
exceptions, but is only required to raise `DiskFileNotExist` when
the object representation does not exist.
:raises DiskFileNotExist: if the object does not exist
:raises DiskFileExpired: if the object has expired
:returns: itself for use as a context manager
"""
# Writes are always performed to a temporary file
try:
fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC)
except GlusterFileSystemOSError as err:
if err.errno in (errno.ENOENT, errno.ENOTDIR):
# If the file does exist, or some part of the path does not
# exist, raise the expected DiskFileNotExist
raise DiskFileNotExist
raise
else:
stats = do_fstat(fd)
if not stats:
return
self._is_dir = stat.S_ISDIR(stats.st_mode)
obj_size = stats.st_size
self._metadata = read_metadata(fd)
if not validate_object(self._metadata):
create_object_metadata(fd)
self._metadata = read_metadata(fd)
assert self._metadata is not None
self._filter_metadata()
if self._is_dir:
do_close(fd)
obj_size = 0
self._fd = -1
else:
if self._is_object_expired(self._metadata):
raise DiskFileExpired(metadata=self._metadata)
self._fd = fd
self._obj_size = obj_size
return self
def list_objects_iter(self):
self.update_object_count()
objects, object_count, bytes_used = self.object_info
if objects:
objects.sort()
container_list = []
if objects:
for obj in objects:
list_item = []
list_item.append(obj)
obj_path = os.path.join(self.datadir, obj)
metadata = read_metadata(obj_path)
if not metadata or not validate_object(metadata):
metadata = create_object_metadata(obj_path)
if metadata:
# list_item.append(metadata[X_TIMESTAMP])
list_item.append(int(metadata[X_CONTENT_LENGTH]))
# list_item.append(metadata[X_CONTENT_TYPE])
list_item.append(metadata[X_ETAG])
container_list.append(list_item)
return container_list
def list_objects_iter(self, limit, marker, end_marker,
prefix, delimiter, path):
"""
Returns tuple of name, created_at, size, content_type, etag.
"""
if path:
prefix = path = path.rstrip('/') + '/'
delimiter = '/'
if delimiter and not prefix:
prefix = ''
self.update_object_count()
objects, object_count, bytes_used = self.object_info
if objects:
objects.sort()
if objects and prefix:
objects = self.filter_prefix(objects, prefix)
if objects and delimiter:
objects = self.filter_delimiter(objects, delimiter, prefix)
if objects and marker:
objects = self.filter_marker(objects, marker)
if objects and end_marker:
objects = self.filter_end_marker(objects, end_marker)
if objects and limit:
if len(objects) > limit:
objects = self.filter_limit(objects, limit)
container_list = []
if objects:
for obj in objects:
list_item = []
list_item.append(obj)
obj_path = os.path.join(self.datadir, obj)
metadata = read_metadata(obj_path)
if not metadata or not validate_object(metadata):
metadata = create_object_metadata(obj_path)
if metadata:
# list_item.append(metadata[X_TIMESTAMP])
list_item.append(int(metadata[X_CONTENT_LENGTH]))
# list_item.append(metadata[X_CONTENT_TYPE])
list_item.append(metadata[X_ETAG])
container_list.append(list_item)
return container_list
def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter,
path):
"""
Returns tuple of name, created_at, size, content_type, etag.
"""
if path:
prefix = path = path.rstrip('/') + '/'
delimiter = '/'
if delimiter and not prefix:
prefix = ''
self.update_object_count()
objects, object_count, bytes_used = self.object_info
if objects:
objects.sort()
if objects and prefix:
objects = self.filter_prefix(objects, prefix)
if objects and delimiter:
objects = self.filter_delimiter(objects, delimiter, prefix)
if objects and marker:
objects = self.filter_marker(objects, marker)
if objects and end_marker:
objects = self.filter_end_marker(objects, end_marker)
if objects and limit:
if len(objects) > limit:
objects = self.filter_limit(objects, limit)
container_list = []
if objects:
for obj in objects:
list_item = []
list_item.append(obj)
obj_path = os.path.join(self.datadir, obj)
metadata = read_metadata(obj_path)
if not metadata or not validate_object(metadata):
metadata = create_object_metadata(obj_path)
if metadata:
list_item.append(metadata[X_TIMESTAMP])
list_item.append(int(metadata[X_CONTENT_LENGTH]))
list_item.append(metadata[X_CONTENT_TYPE])
list_item.append(metadata[X_ETAG])
container_list.append(list_item)
return container_list
def test_create_object_metadata_file(self):
tf = tempfile.NamedTemporaryFile()
tf.file.write('4567'); tf.file.flush()
r_md = utils.create_object_metadata(tf.name)
xkey = _xkey(tf.name, utils.METADATA_KEY)
assert len(_xattrs.keys()) == 1
assert xkey in _xattrs
assert _xattr_op_cnt['get'] == 1
assert _xattr_op_cnt['set'] == 1
md = pickle.loads(_xattrs[xkey])
assert r_md == md
for key in self.obj_keys:
assert key in md, "Expected key %s in %r" % (key, md)
assert md[utils.X_TYPE] == utils.OBJECT
assert md[utils.X_OBJECT_TYPE] == utils.FILE
assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE
assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name)
assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(tf.name))
assert md[utils.X_ETAG] == utils._get_etag(tf.name)
def test_create_object_metadata_dir(self):
td = tempfile.mkdtemp()
try:
r_md = utils.create_object_metadata(td)
xkey = _xkey(td, utils.METADATA_KEY)
assert len(_xattrs.keys()) == 1
assert xkey in _xattrs
assert _xattr_op_cnt['set'] == 1
md = deserialize_metadata(_xattrs[xkey])
assert r_md == md
for key in self.obj_keys:
assert key in md, "Expected key %s in %r" % (key, md)
assert md[utils.X_TYPE] == utils.OBJECT
assert md[utils.X_OBJECT_TYPE] == utils.DIR_NON_OBJECT
assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE
assert md[utils.X_CONTENT_LENGTH] == 0
assert md[utils.X_TIMESTAMP] == utils.normalize_timestamp(os.path.getctime(td))
assert md[utils.X_ETAG] == hashlib.md5().hexdigest()
finally:
os.rmdir(td)
def test_create_object_metadata_dir(self):
td = tempfile.mkdtemp()
try:
r_md = utils.create_object_metadata(td)
xkey = _xkey(td, utils.METADATA_KEY)
assert len(_xattrs.keys()) == 1
assert xkey in _xattrs
assert _xattr_op_cnt['get'] == 1
assert _xattr_op_cnt['set'] == 1
md = pickle.loads(_xattrs[xkey])
assert r_md == md
for key in self.obj_keys:
assert key in md, "Expected key %s in %r" % (key, md)
assert md[utils.X_TYPE] == utils.OBJECT
assert md[utils.X_OBJECT_TYPE] == utils.DIR
assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE
assert md[utils.X_CONTENT_LENGTH] == 0
assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(td))
assert md[utils.X_ETAG] == hashlib.md5().hexdigest()
finally:
os.rmdir(td)
def open(self):
"""
Open the object.
This implementation opens the data file representing the object, reads
the associated metadata in the extended attributes, additionally
combining metadata from fast-POST `.meta` files.
.. note::
An implementation is allowed to raise any of the following
exceptions, but is only required to raise `DiskFileNotExist` when
the object representation does not exist.
:raises DiskFileNotExist: if the object does not exist
:raises DiskFileExpired: if the object has expired
:returns: itself for use as a context manager
"""
# Writes are always performed to a temporary file
try:
self._fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC)
except GlusterFileSystemOSError as err:
if err.errno in (errno.ENOENT, errno.ENOTDIR):
# If the file does exist, or some part of the path does not
# exist, raise the expected DiskFileNotExist
raise DiskFileNotExist
raise
try:
if not self._stat:
self._stat = do_fstat(self._fd)
obj_size = self._stat.st_size
if not self._metadata:
self._metadata = read_metadata(self._fd)
if not validate_object(self._metadata, self._stat):
self._metadata = create_object_metadata(self._fd, self._stat,
self._metadata)
assert self._metadata is not None
self._filter_metadata()
if stat.S_ISDIR(self._stat.st_mode):
do_close(self._fd)
obj_size = 0
self._fd = -1
else:
if self._is_object_expired(self._metadata):
raise DiskFileExpired(metadata=self._metadata)
self._obj_size = obj_size
except (OSError, IOError, DiskFileExpired) as err:
# Something went wrong. Context manager will not call
# __exit__. So we close the fd manually here.
self._close_fd()
if hasattr(err, 'errno') and \
err.errno in (errno.ENOENT, errno.ESTALE):
# Handle races: ENOENT/ESTALE can be raised by read_metadata()
# call in GlusterFS if file gets deleted by another
# client after do_open() succeeds
logging.warn("open(%s) succeeded but one of the subsequent "
"syscalls failed with ENOENT/ESTALE. Raising "
"DiskFileNotExist." % (self._data_file))
raise DiskFileNotExist
else:
# Re-raise the original exception after fd has been closed
raise
self._disk_file_open = True
return self
def __init__(self, path, device, partition, account, container, obj,
logger, keep_data_fp=False,
disk_chunk_size=DEFAULT_DISK_CHUNK_SIZE,
uid=DEFAULT_UID, gid=DEFAULT_GID, iter_hook=None):
self.disk_chunk_size = disk_chunk_size
self.iter_hook = iter_hook
obj = obj.strip(os.path.sep)
if os.path.sep in obj:
self._obj_path, self._obj = os.path.split(obj)
else:
self._obj_path = ''
self._obj = obj
if self._obj_path:
self.name = os.path.join(container, self._obj_path)
else:
self.name = container
# Absolute path for object directory.
self.datadir = os.path.join(path, device, self.name)
self.device_path = os.path.join(path, device)
self._container_path = os.path.join(path, device, container)
if _use_put_mount:
self.put_datadir = os.path.join(self.device_path + '_PUT',
self.name)
else:
self.put_datadir = self.datadir
self._is_dir = False
self.tmppath = None
self.logger = logger
self.metadata = {}
self.meta_file = None
self.fp = None
self.iter_etag = None
self.started_at_0 = False
self.read_to_eof = False
self.quarantined_dir = None
self.keep_cache = False
self.uid = int(uid)
self.gid = int(gid)
self.suppress_file_closing = False
# Don't store a value for data_file until we know it exists.
self.data_file = None
data_file = os.path.join(self.put_datadir, self._obj)
try:
stats = do_stat(data_file)
except OSError as err:
if err.errno == errno.ENOTDIR:
return
else:
if not stats:
return
self.data_file = data_file
self._is_dir = stat.S_ISDIR(stats.st_mode)
self.metadata = read_metadata(data_file)
if not self.metadata:
create_object_metadata(data_file)
self.metadata = read_metadata(data_file)
if not validate_object(self.metadata):
create_object_metadata(data_file)
self.metadata = read_metadata(data_file)
self.filter_metadata()
if not self._is_dir and keep_data_fp:
# The caller has an assumption that the "fp" field of this
# object is an file object if keep_data_fp is set. However,
# this implementation of the DiskFile object does not need to
# open the file for internal operations. So if the caller
# requests it, we'll just open the file for them.
self.fp = do_open(data_file, 'rb')
def list_objects_iter(self, limit, marker, end_marker,
prefix, delimiter, path=None):
"""
Returns tuple of name, created_at, size, content_type, etag.
"""
assert limit >= 0
assert not delimiter or (len(delimiter) == 1 and ord(delimiter) <= 254)
if path is not None:
if path:
prefix = path = path.rstrip('/') + '/'
else:
prefix = path
delimiter = '/'
elif delimiter and not prefix:
prefix = ''
container_list = []
objects = self._update_object_count()
if objects:
objects.sort()
else:
return container_list
if end_marker:
objects = filter_end_marker(objects, end_marker)
if marker and marker >= prefix:
objects = filter_marker(objects, marker)
elif prefix:
objects = filter_prefix_as_marker(objects, prefix)
if prefix is None:
# No prefix, we don't need to apply the other arguments, we just
# return what we have.
pass
else:
# We have a non-None (for all intents and purposes it is a string)
# prefix.
if not delimiter:
if not prefix:
# We have nothing more to do
pass
else:
objects = filter_prefix(objects, prefix)
else:
objects = filter_delimiter(objects, delimiter, prefix, marker,
path)
count = 0
for obj in objects:
obj_path = os.path.join(self.datadir, obj)
metadata = read_metadata(obj_path)
if not metadata or not validate_object(metadata):
if delimiter == '/' and obj_path[-1] == delimiter:
clean_obj_path = obj_path[:-1]
else:
clean_obj_path = obj_path
try:
metadata = create_object_metadata(clean_obj_path)
except OSError as e:
# FIXME - total hack to get upstream swift ported unit
# test cases working for now.
if e.errno != errno.ENOENT:
raise
if Glusterfs.OBJECT_ONLY and metadata \
and metadata[X_CONTENT_TYPE] == DIR_TYPE \
and not dir_is_object(metadata):
continue
list_item = []
list_item.append(obj)
if metadata:
list_item.append(metadata[X_TIMESTAMP])
list_item.append(int(metadata[X_CONTENT_LENGTH]))
list_item.append(metadata[X_CONTENT_TYPE])
list_item.append(metadata[X_ETAG])
container_list.append(list_item)
count += 1
if count >= limit:
break
return container_list
def list_objects_iter(self,
limit,
marker,
end_marker,
prefix,
delimiter,
path=None):
"""
Returns tuple of name, created_at, size, content_type, etag.
"""
assert limit >= 0
assert not delimiter or (len(delimiter) == 1 and ord(delimiter) <= 254)
if path is not None:
if path:
prefix = path = path.rstrip('/') + '/'
else:
prefix = path
delimiter = '/'
elif delimiter and not prefix:
prefix = ''
container_list = []
objects = self._update_object_count()
if objects:
objects.sort()
else:
return container_list
if end_marker:
objects = filter_end_marker(objects, end_marker)
if marker and marker >= prefix:
objects = filter_marker(objects, marker)
elif prefix:
objects = filter_prefix_as_marker(objects, prefix)
if prefix is None:
# No prefix, we don't need to apply the other arguments, we just
# return what we have.
pass
else:
# We have a non-None (for all intents and purposes it is a string)
# prefix.
if not delimiter:
if not prefix:
# We have nothing more to do
pass
else:
objects = filter_prefix(objects, prefix)
else:
objects = filter_delimiter(objects, delimiter, prefix, marker,
path)
count = 0
for obj in objects:
obj_path = os.path.join(self.datadir, obj)
metadata = read_metadata(obj_path)
if not metadata or not validate_object(metadata):
if delimiter == '/' and obj_path[-1] == delimiter:
clean_obj_path = obj_path[:-1]
else:
clean_obj_path = obj_path
try:
metadata = create_object_metadata(clean_obj_path)
except OSError as e:
# FIXME - total hack to get upstream swift ported unit
# test cases working for now.
if e.errno != errno.ENOENT:
raise
if not Glusterfs._implicit_dir_objects and metadata \
and metadata[X_CONTENT_TYPE] == DIR_TYPE \
and not dir_is_object(metadata):
continue
list_item = []
list_item.append(obj)
if metadata:
list_item.append(metadata[X_TIMESTAMP])
list_item.append(int(metadata[X_CONTENT_LENGTH]))
list_item.append(metadata[X_CONTENT_TYPE])
list_item.append(metadata[X_ETAG])
container_list.append(list_item)
count += 1
if count >= limit:
break
return container_list
def list_objects_iter(self, limit, marker, end_marker,
prefix, delimiter, path=None,
storage_policy_index=0,
out_content_type=None, reverse=False):
"""
Returns tuple of name, created_at, size, content_type, etag.
"""
assert limit >= 0
assert not delimiter or (len(delimiter) == 1 and ord(delimiter) <= 254)
if path is not None:
if path:
prefix = path = path.rstrip('/') + '/'
else:
prefix = path
delimiter = '/'
elif delimiter and not prefix:
prefix = ''
container_list = []
if self.account == 'gsexpiring':
objects = list_objects_gsexpiring_container(self.datadir)
else:
objects = self._update_object_count()
if objects:
objects.sort()
else:
# No objects in container , return empty list
return container_list
if marker and end_marker and reverse:
marker, end_marker = end_marker, marker
if end_marker:
objects = filter_end_marker(objects, end_marker)
if marker and marker >= prefix:
objects = filter_marker(objects, marker)
elif prefix:
objects = filter_prefix_as_marker(objects, prefix)
if prefix is None:
# No prefix, we don't need to apply the other arguments, we just
# return what we have.
pass
else:
# We have a non-None (for all intents and purposes it is a string)
# prefix.
if not delimiter:
if not prefix:
# We have nothing more to do
pass
else:
objects = filter_prefix(objects, prefix)
else:
objects = filter_delimiter(objects, delimiter, prefix, marker,
path)
if out_content_type == 'text/plain' or \
self.account == 'gsexpiring':
# When out_content_type == 'text/plain':
#
# The client is only asking for a plain list of objects and NOT
# asking for any extended information about objects such as
# bytes used or etag.
#
# When self.account == 'gsexpiring':
#
# This is a JSON request sent by the object expirer to list
# tracker objects in a container in gsexpiring volume.
# When out_content_type is 'application/json', the caller
# expects each record entry to have the following ordered
# fields: (name, timestamp, size, content_type, etag)
for obj in objects:
container_list.append((obj, '0', 0, 'text/plain', ''))
if len(container_list) >= limit:
break
if reverse:
container_list.reverse()
return container_list
count = 0
for obj in objects:
obj_path = os.path.join(self.datadir, obj)
try:
metadata = read_metadata(obj_path)
except GlusterFileSystemIOError as err:
if err.errno in (errno.ENOENT, errno.ESTALE):
# obj might have been deleted by another process
# since the objects list was originally built
continue
else:
raise err
if not metadata or not validate_object(metadata):
if delimiter == '/' and obj_path[-1] == delimiter:
clean_obj_path = obj_path[:-1]
else:
clean_obj_path = obj_path
try:
metadata = create_object_metadata(clean_obj_path)
except OSError as e:
# FIXME - total hack to get upstream swift ported unit
# test cases working for now.
if e.errno not in (errno.ENOENT, errno.ESTALE):
raise
if not Glusterfs._implicit_dir_objects and metadata \
and metadata[X_CONTENT_TYPE] == DIR_TYPE \
and not dir_is_object(metadata):
continue
list_item = []
list_item.append(obj)
if metadata:
list_item.append(metadata[X_TIMESTAMP])
list_item.append(int(metadata[X_CONTENT_LENGTH]))
list_item.append(metadata[X_CONTENT_TYPE])
list_item.append(metadata[X_ETAG])
container_list.append(list_item)
count += 1
if count >= limit:
break
if reverse:
container_list.reverse()
return container_list
def __init__(self, path, device, partition, account, container, obj,
logger, keep_data_fp=False,
disk_chunk_size=DEFAULT_DISK_CHUNK_SIZE,
uid=DEFAULT_UID, gid=DEFAULT_GID, iter_hook=None):
self.disk_chunk_size = disk_chunk_size
self.iter_hook = iter_hook
# Don't support obj_name ending/begining with '/', like /a, a/, /a/b/,
# etc.
obj = obj.strip(os.path.sep)
if os.path.sep in obj:
self._obj_path, self._obj = os.path.split(obj)
else:
self._obj_path = ''
self._obj = obj
if self._obj_path:
self.name = os.path.join(container, self._obj_path)
else:
self.name = container
# Absolute path for object directory.
self.datadir = os.path.join(path, device, self.name)
self.device_path = os.path.join(path, device)
self._container_path = os.path.join(path, device, container)
self._is_dir = False
self.tmppath = None
self.logger = logger
self.metadata = {}
self.meta_file = None
self.fp = None
self.iter_etag = None
self.started_at_0 = False
self.read_to_eof = False
self.quarantined_dir = None
self.keep_cache = False
self.uid = int(uid)
self.gid = int(gid)
# Don't store a value for data_file until we know it exists.
self.data_file = None
data_file = os.path.join(self.datadir, self._obj)
if not os.path.exists(data_file):
return
self.data_file = os.path.join(data_file)
self.metadata = read_metadata(data_file)
if not self.metadata:
create_object_metadata(data_file)
self.metadata = read_metadata(data_file)
if not validate_object(self.metadata):
create_object_metadata(data_file)
self.metadata = read_metadata(data_file)
self.filter_metadata()
if os.path.isdir(data_file):
self._is_dir = True
else:
if keep_data_fp:
# The caller has an assumption that the "fp" field of this
# object is an file object if keep_data_fp is set. However,
# this implementation of the DiskFile object does not need to
# open the file for internal operations. So if the caller
# requests it, we'll just open the file for them.
self.fp = do_open(data_file, 'rb')
def __init__(self,
path,
device,
partition,
account,
container,
obj,
logger,
keep_data_fp=False,
disk_chunk_size=65536,
uid=DEFAULT_UID,
gid=DEFAULT_GID):
self.disk_chunk_size = disk_chunk_size
device = account
#Don't support obj_name ending/begining with '/', like /a, a/, /a/b/ etc
obj = obj.strip('/')
if '/' in obj:
self.obj_path, self.obj = obj.rsplit('/', 1)
else:
self.obj_path = ''
self.obj = obj
if self.obj_path:
self.name = '/'.join((container, self.obj_path))
else:
self.name = container
#Absolute path for obj directory.
self.datadir = os.path.join(path, device, self.name)
self.device_path = os.path.join(path, device)
self.container_path = os.path.join(path, device, container)
self.tmpdir = os.path.join(path, device, 'tmp')
self.logger = logger
self.metadata = {}
self.data_file = None
self.fp = None
self.iter_etag = None
self.started_at_0 = False
self.read_to_eof = False
self.quarantined_dir = None
self.keep_cache = False
self.is_dir = False
self.is_valid = True
self.uid = int(uid)
self.gid = int(gid)
if not os.path.exists(self.datadir + '/' + self.obj):
return
self.data_file = os.path.join(self.datadir, self.obj)
self.metadata = read_metadata(self.datadir + '/' + self.obj)
if not self.metadata:
create_object_metadata(self.datadir + '/' + self.obj)
self.metadata = read_metadata(self.datadir + '/' + self.obj)
if not validate_object(self.metadata):
create_object_metadata(self.datadir + '/' + self.obj)
self.metadata = read_metadata(self.datadir + '/' + self.obj)
self.filter_metadata()
if os.path.isdir(self.datadir + '/' + self.obj):
self.is_dir = True
else:
self.fp = do_open(self.data_file, 'rb')
if not keep_data_fp:
self.close(verify_file=False)
def open(self):
"""
Open the object.
This implementation opens the data file representing the object, reads
the associated metadata in the extended attributes, additionally
combining metadata from fast-POST `.meta` files.
.. note::
An implementation is allowed to raise any of the following
exceptions, but is only required to raise `DiskFileNotExist` when
the object representation does not exist.
:raises DiskFileNotExist: if the object does not exist
:raises DiskFileExpired: if the object has expired
:returns: itself for use as a context manager
"""
# Writes are always performed to a temporary file
try:
self._fd = do_open(self._data_file, os.O_RDONLY | O_CLOEXEC)
except GlusterFileSystemOSError as err:
if err.errno in (errno.ENOENT, errno.ENOTDIR):
# If the file does exist, or some part of the path does not
# exist, raise the expected DiskFileNotExist
raise DiskFileNotExist
raise
try:
if not self._stat:
self._stat = do_fstat(self._fd)
obj_size = self._stat.st_size
if not self._metadata:
self._metadata = read_metadata(self._fd)
if not validate_object(self._metadata, self._stat):
self._metadata = create_object_metadata(self._fd, self._stat,
self._metadata)
assert self._metadata is not None
self._filter_metadata()
if stat.S_ISDIR(self._stat.st_mode):
do_close(self._fd)
obj_size = 0
self._fd = -1
else:
if self._is_object_expired(self._metadata):
raise DiskFileExpired(metadata=self._metadata)
self._obj_size = obj_size
except (OSError, IOError, DiskFileExpired) as err:
# Something went wrong. Context manager will not call
# __exit__. So we close the fd manually here.
self._close_fd()
if hasattr(err, 'errno') and \
err.errno in (errno.ENOENT, errno.ESTALE):
# Handle races: ENOENT/ESTALE can be raised by read_metadata()
# call in GlusterFS if file gets deleted by another
# client after do_open() succeeds
logging.warn("open(%s) succeeded but one of the subsequent "
"syscalls failed with ENOENT/ESTALE. Raising "
"DiskFileNotExist." % (self._data_file))
raise DiskFileNotExist
else:
# Re-raise the original exception after fd has been closed
raise
self._disk_file_open = True
return self