def _construct_exception_from_ts_file(self, ts_file):
"""
If a tombstone is present it means the object is considered
deleted. We just need to pull the metadata from the tombstone file
which has the timestamp to construct the deleted exception. If there
was no tombstone, just report it does not exist.
:param ts_file: the tombstone file name found on disk
:returns: DiskFileDeleted if the ts_file was provided, else
DiskFileNotExist
"""
if not ts_file:
exc = DiskFileNotExist()
else:
try:
metadata = self._failsafe_read_metadata(ts_file, ts_file)
except DiskFileQuarantined:
# If the tombstone's corrupted, quarantine it and pretend it
# wasn't there
exc = DiskFileNotExist()
else:
# All well and good that we have found a tombstone file, but
# we don't have a data file so we are just going to raise an
# exception that we could not find the object, providing the
# tombstone's timestamp.
exc = DiskFileDeleted()
exc.timestamp = metadata['X-Timestamp']
return exc
def read_metadata(self, fd, obj_path):
"""
Helper function to read the pickled metadata from an object file.
:param fd: file descriptor or filename to load the metadata from
:param filename: full path of the file
:returns: dictionary of metadata
"""
metadata = ''
key = 0
try:
while True:
metadata += xattr.getxattr(
fd, '%s%s' % (SWIFT_METADATA_KEY, (key or '')))
key += 1
except (IOError, OSError) as e:
if metadata == '':
return False
for err in 'ENOTSUP', 'EOPNOTSUPP':
if hasattr(errno, err) and e.errno == getattr(errno, err):
msg = "Filesystem at %s does not support xattr" % \
obj_path
logging.exception(msg)
raise DiskFileXattrNotSupported(e)
if e.errno == errno.ENOENT:
raise DiskFileNotExist()
return pickle.loads(metadata)
def get_data_file_size(self):
"""
Returns the os.path.getsize for the file. Raises an exception if this
file does not match the Content-Length stored in the metadata. Or if
self.data_file does not exist.
:returns: file size as an int
:raises DiskFileError: on file size mismatch.
:raises DiskFileNotExist: on file not existing (including deleted)
"""
#Marker directory.
if self._is_dir:
return 0
try:
file_size = 0
if self.data_file:
file_size = os.path.getsize(self.data_file)
if X_CONTENT_LENGTH in self.metadata:
metadata_size = int(self.metadata[X_CONTENT_LENGTH])
if file_size != metadata_size:
self.metadata[X_CONTENT_LENGTH] = file_size
write_metadata(self.data_file, self.metadata)
return file_size
except OSError as err:
if err.errno != errno.ENOENT:
raise
raise DiskFileNotExist('Data File does not exist.')
def _verify_data_file(self, fp, current_time):
"""
Verify the metadata's name value matches what we think the object is
named.
:raises DiskFileCollision: if the metadata stored name does not match
the referenced name of the file
:raises DiskFileNotExist: if the object has expired
:raises DiskFileQuarantined: if data inconsistencies were detected
between the metadata and the file-system
metadata
"""
try:
mname = self._metadata['name']
except KeyError:
raise self._quarantine(self._name, "missing name metadata")
else:
if mname != self._name:
raise DiskFileCollision('Client path does not match path '
'stored in object metadata')
try:
x_delete_at = int(self._metadata['X-Delete-At'])
except KeyError:
pass
except ValueError:
# Quarantine, the x-delete-at key is present but not an
# integer.
raise self._quarantine(
self._name, "bad metadata x-delete-at value %s" %
(self._metadata['X-Delete-At']))
else:
if current_time is None:
current_time = time.time()
if x_delete_at <= current_time:
raise DiskFileNotExist('Expired')
try:
metadata_size = int(self._metadata['Content-Length'])
except KeyError:
raise self._quarantine(self._name,
"missing content-length in metadata")
except ValueError:
# Quarantine, the content-length key is present but not an
# integer.
raise self._quarantine(
self._name, "bad metadata content-length value %s" %
(self._metadata['Content-Length']))
try:
fp.seek(0, 2)
obj_size = fp.tell()
fp.seek(0, 0)
except OSError as err:
# Quarantine, we can't successfully stat the file.
raise self._quarantine(self._name, "not stat-able: %s" % err)
if obj_size != metadata_size:
raise self._quarantine(
self._name, "metadata content-length %s does"
" not match actual object size %s" % (metadata_size, obj_size))
return fp
def _construct_exception_from_ts_file(self, ts_file):
"""
If a tombstone is present it means the object is considered
deleted. We just need to pull the metadata from the tombstone file
which has the timestamp to construct the deleted exception. If there
was no tombstone, just report it does not exist.
:param ts_file: the tombstone file name found on disk
:returns: DiskFileDeleted if the ts_file was provided, else
DiskFileNotExist
"""
if not ts_file:
exc = DiskFileNotExist()
else:
try:
metadata = self._failsafe_read_metadata(ts_file, ts_file)
except DiskFileQuarantined:
# If the tombstone's corrupted, quarantine it and pretend it
# wasn't there
exc = DiskFileNotExist()
else:
# All well and good that we have found a tombstone file, but
# we don't have a data file so we are just going to raise an
# exception that we could not find the object, providing the
# tombstone's timestamp.
exc = DiskFileDeleted()
exc.timestamp = metadata['X-Timestamp']
return exc
def _verify_data_file(self):
"""
Verify the metadata's name value matches what we think the object is
named.
:raises DiskFileCollision: if the metadata stored name does not match
the referenced name of the file
:raises DiskFileNotExist: if the object has expired
:raises DiskFileQuarantined: if data inconsistencies were detected
between the metadata and the file-system
metadata
"""
try:
mname = self._metadata['name']
except KeyError:
self._quarantine("missing name metadata")
else:
if mname != self._name:
raise DiskFileCollision('Client path does not match path '
'stored in object metadata')
try:
x_delete_at = int(self._metadata['X-Delete-At'])
except KeyError:
pass
except ValueError:
# Quarantine, the x-delete-at key is present but not an
# integer.
self._quarantine("bad metadata x-delete-at value %s" %
(self._metadata['X-Delete-At']))
else:
if x_delete_at <= time.time():
raise DiskFileNotExist('Expired')
try:
metadata_size = int(self._metadata['Content-Length'])
except KeyError:
self._quarantine("missing content-length in metadata")
except ValueError:
# Quarantine, the content-length key is present but not an
# integer.
self._quarantine("bad metadata content-length value %s" %
(self._metadata['Content-Length']))
obj_size = self._fs_inst.size(self._name)
if obj_size != metadata_size:
self._quarantine("metadata content-length %s does"
" not match actual object size %s" %
(metadata_size, obj_size))
def open(self):
"""
Open the file and read the metadata.
This method must populate the _metadata attribute.
:raises DiskFileCollision: on name mis-match with metadata
:raises DiskFileNotExist: if it does not exist
:raises DiskFileQuarantined: if while reading metadata of the file
some data did not pass cross checks
"""
self._fs_inst = self._fs.open()
self._metadata = self._fs_inst.get_metadata(self._name)
if self._metadata is None:
raise DiskFileNotExist()
self._verify_data_file()
self._metadata = self._metadata or {}
return self
def _get_data_file_size(self):
# ensure file is opened
metadata = self.get_metadata()
try:
file_size = 0
if self.data_file:
file_size = self.threadpool.run_in_thread(
getsize, self.data_file)
if 'Content-Length' in metadata:
metadata_size = int(metadata['Content-Length'])
if file_size != metadata_size:
raise DiskFileError(
'Content-Length of %s does not match file size '
'of %s' % (metadata_size, file_size))
return file_size
except OSError as err:
if err.errno != errno.ENOENT:
raise
raise DiskFileNotExist('Data File does not exist.')
quarantine_renamer, self.device_path, self.data_file)
self.logger.increment('quarantines')
return self.quarantined_dir
def get_data_file_size(self):
"""
Returns the os.path.getsize for the file. Raises an exception if this
file does not match the Content-Length stored in the metadata. Or if
self.data_file does not exist.
:returns: file size as an int
:raises DiskFileError: on file size mismatch.
:raises DiskFileNotExist: on file not existing (including deleted)
"""
try:
file_size = 0
if self.data_file:
file_size = self.threadpool.run_in_thread(
getsize, self.data_file)
if 'Content-Length' in self.metadata:
metadata_size = int(self.metadata['Content-Length'])
if file_size != metadata_size:
raise DiskFileError(
'Content-Length of %s does not match file size '
'of %s' % (metadata_size, file_size))
return file_size
except OSError, err:
if err.errno != errno.ENOENT:
raise
raise DiskFileNotExist('Data File does not exist.')
def _verify_data_file(self, data_file, fp):
"""
Verify the metadata's name value matches what we think the object is
named.
:param data_file: data file name being consider, used when quarantines
occur
:param fp: open file pointer so that we can `fstat()` the file to
verify the on-disk size with Content-Length metadata value
:raises DiskFileCollision: if the metadata stored name does not match
the referenced name of the file
:raises DiskFileNotExist: if the object has expired
:raises DiskFileQuarantined: if data inconsistencies were detected
between the metadata and the file-system
metadata
"""
try:
mname = self._metadata['name']
except KeyError:
self._quarantine(data_file, "missing name metadata")
else:
if mname != self._name:
self._logger.error(
_('Client path %(client)s does not match '
'path stored in object metadata %(meta)s'), {
'client': self._name,
'meta': mname
})
raise DiskFileCollision('Client path does not match path '
'stored in object metadata')
try:
x_delete_at = int(self._metadata['X-Delete-At'])
except KeyError:
pass
except ValueError:
# Quarantine, the x-delete-at key is present but not an
# integer.
self._quarantine(
data_file, "bad metadata x-delete-at value %s" %
(self._metadata['X-Delete-At']))
else:
if x_delete_at <= time.time():
raise DiskFileNotExist('Expired')
try:
metadata_size = int(self._metadata['Content-Length'])
except KeyError:
self._quarantine(data_file, "missing content-length in metadata")
except ValueError:
# Quarantine, the content-length key is present but not an
# integer.
self._quarantine(
data_file, "bad metadata content-length value %s" %
(self._metadata['Content-Length']))
fd = fp.fileno()
try:
statbuf = os.fstat(fd)
except OSError as err:
# Quarantine, we can't successfully stat the file.
self._quarantine(data_file, "not stat-able: %s" % err)
else:
obj_size = statbuf.st_size
if metadata_size is not None and obj_size != metadata_size:
self._quarantine(
data_file, "metadata content-length %s does"
" not match actual object size %s" %
(metadata_size, statbuf.st_size))
return obj_size