def _construct_exception_from_ts_file(self, ts_file): """ If a tombstone is present it means the object is considered deleted. We just need to pull the metadata from the tombstone file which has the timestamp to construct the deleted exception. If there was no tombstone, just report it does not exist. :param ts_file: the tombstone file name found on disk :returns: DiskFileDeleted if the ts_file was provided, else DiskFileNotExist """ if not ts_file: exc = DiskFileNotExist() else: try: metadata = self._failsafe_read_metadata(ts_file, ts_file) except DiskFileQuarantined: # If the tombstone's corrupted, quarantine it and pretend it # wasn't there exc = DiskFileNotExist() else: # All well and good that we have found a tombstone file, but # we don't have a data file so we are just going to raise an # exception that we could not find the object, providing the # tombstone's timestamp. exc = DiskFileDeleted() exc.timestamp = metadata['X-Timestamp'] return exc
def read_metadata(self, fd, obj_path): """ Helper function to read the pickled metadata from an object file. :param fd: file descriptor or filename to load the metadata from :param filename: full path of the file :returns: dictionary of metadata """ metadata = '' key = 0 try: while True: metadata += xattr.getxattr( fd, '%s%s' % (SWIFT_METADATA_KEY, (key or ''))) key += 1 except (IOError, OSError) as e: if metadata == '': return False for err in 'ENOTSUP', 'EOPNOTSUPP': if hasattr(errno, err) and e.errno == getattr(errno, err): msg = "Filesystem at %s does not support xattr" % \ obj_path logging.exception(msg) raise DiskFileXattrNotSupported(e) if e.errno == errno.ENOENT: raise DiskFileNotExist() return pickle.loads(metadata)
def get_data_file_size(self): """ Returns the os.path.getsize for the file. Raises an exception if this file does not match the Content-Length stored in the metadata. Or if self.data_file does not exist. :returns: file size as an int :raises DiskFileError: on file size mismatch. :raises DiskFileNotExist: on file not existing (including deleted) """ #Marker directory. if self._is_dir: return 0 try: file_size = 0 if self.data_file: file_size = os.path.getsize(self.data_file) if X_CONTENT_LENGTH in self.metadata: metadata_size = int(self.metadata[X_CONTENT_LENGTH]) if file_size != metadata_size: self.metadata[X_CONTENT_LENGTH] = file_size write_metadata(self.data_file, self.metadata) return file_size except OSError as err: if err.errno != errno.ENOENT: raise raise DiskFileNotExist('Data File does not exist.')
def _verify_data_file(self, fp, current_time): """ Verify the metadata's name value matches what we think the object is named. :raises DiskFileCollision: if the metadata stored name does not match the referenced name of the file :raises DiskFileNotExist: if the object has expired :raises DiskFileQuarantined: if data inconsistencies were detected between the metadata and the file-system metadata """ try: mname = self._metadata['name'] except KeyError: raise self._quarantine(self._name, "missing name metadata") else: if mname != self._name: raise DiskFileCollision('Client path does not match path ' 'stored in object metadata') try: x_delete_at = int(self._metadata['X-Delete-At']) except KeyError: pass except ValueError: # Quarantine, the x-delete-at key is present but not an # integer. raise self._quarantine( self._name, "bad metadata x-delete-at value %s" % (self._metadata['X-Delete-At'])) else: if current_time is None: current_time = time.time() if x_delete_at <= current_time: raise DiskFileNotExist('Expired') try: metadata_size = int(self._metadata['Content-Length']) except KeyError: raise self._quarantine(self._name, "missing content-length in metadata") except ValueError: # Quarantine, the content-length key is present but not an # integer. raise self._quarantine( self._name, "bad metadata content-length value %s" % (self._metadata['Content-Length'])) try: fp.seek(0, 2) obj_size = fp.tell() fp.seek(0, 0) except OSError as err: # Quarantine, we can't successfully stat the file. raise self._quarantine(self._name, "not stat-able: %s" % err) if obj_size != metadata_size: raise self._quarantine( self._name, "metadata content-length %s does" " not match actual object size %s" % (metadata_size, obj_size)) return fp
def _construct_exception_from_ts_file(self, ts_file): """ If a tombstone is present it means the object is considered deleted. We just need to pull the metadata from the tombstone file which has the timestamp to construct the deleted exception. If there was no tombstone, just report it does not exist. :param ts_file: the tombstone file name found on disk :returns: DiskFileDeleted if the ts_file was provided, else DiskFileNotExist """ if not ts_file: exc = DiskFileNotExist() else: try: metadata = self._failsafe_read_metadata(ts_file, ts_file) except DiskFileQuarantined: # If the tombstone's corrupted, quarantine it and pretend it # wasn't there exc = DiskFileNotExist() else: # All well and good that we have found a tombstone file, but # we don't have a data file so we are just going to raise an # exception that we could not find the object, providing the # tombstone's timestamp. exc = DiskFileDeleted() exc.timestamp = metadata['X-Timestamp'] return exc
def _verify_data_file(self): """ Verify the metadata's name value matches what we think the object is named. :raises DiskFileCollision: if the metadata stored name does not match the referenced name of the file :raises DiskFileNotExist: if the object has expired :raises DiskFileQuarantined: if data inconsistencies were detected between the metadata and the file-system metadata """ try: mname = self._metadata['name'] except KeyError: self._quarantine("missing name metadata") else: if mname != self._name: raise DiskFileCollision('Client path does not match path ' 'stored in object metadata') try: x_delete_at = int(self._metadata['X-Delete-At']) except KeyError: pass except ValueError: # Quarantine, the x-delete-at key is present but not an # integer. self._quarantine("bad metadata x-delete-at value %s" % (self._metadata['X-Delete-At'])) else: if x_delete_at <= time.time(): raise DiskFileNotExist('Expired') try: metadata_size = int(self._metadata['Content-Length']) except KeyError: self._quarantine("missing content-length in metadata") except ValueError: # Quarantine, the content-length key is present but not an # integer. self._quarantine("bad metadata content-length value %s" % (self._metadata['Content-Length'])) obj_size = self._fs_inst.size(self._name) if obj_size != metadata_size: self._quarantine("metadata content-length %s does" " not match actual object size %s" % (metadata_size, obj_size))
def open(self): """ Open the file and read the metadata. This method must populate the _metadata attribute. :raises DiskFileCollision: on name mis-match with metadata :raises DiskFileNotExist: if it does not exist :raises DiskFileQuarantined: if while reading metadata of the file some data did not pass cross checks """ self._fs_inst = self._fs.open() self._metadata = self._fs_inst.get_metadata(self._name) if self._metadata is None: raise DiskFileNotExist() self._verify_data_file() self._metadata = self._metadata or {} return self
def _get_data_file_size(self): # ensure file is opened metadata = self.get_metadata() try: file_size = 0 if self.data_file: file_size = self.threadpool.run_in_thread( getsize, self.data_file) if 'Content-Length' in metadata: metadata_size = int(metadata['Content-Length']) if file_size != metadata_size: raise DiskFileError( 'Content-Length of %s does not match file size ' 'of %s' % (metadata_size, file_size)) return file_size except OSError as err: if err.errno != errno.ENOENT: raise raise DiskFileNotExist('Data File does not exist.')
quarantine_renamer, self.device_path, self.data_file) self.logger.increment('quarantines') return self.quarantined_dir def get_data_file_size(self): """ Returns the os.path.getsize for the file. Raises an exception if this file does not match the Content-Length stored in the metadata. Or if self.data_file does not exist. :returns: file size as an int :raises DiskFileError: on file size mismatch. :raises DiskFileNotExist: on file not existing (including deleted) """ try: file_size = 0 if self.data_file: file_size = self.threadpool.run_in_thread( getsize, self.data_file) if 'Content-Length' in self.metadata: metadata_size = int(self.metadata['Content-Length']) if file_size != metadata_size: raise DiskFileError( 'Content-Length of %s does not match file size ' 'of %s' % (metadata_size, file_size)) return file_size except OSError, err: if err.errno != errno.ENOENT: raise raise DiskFileNotExist('Data File does not exist.')
def _verify_data_file(self, data_file, fp): """ Verify the metadata's name value matches what we think the object is named. :param data_file: data file name being consider, used when quarantines occur :param fp: open file pointer so that we can `fstat()` the file to verify the on-disk size with Content-Length metadata value :raises DiskFileCollision: if the metadata stored name does not match the referenced name of the file :raises DiskFileNotExist: if the object has expired :raises DiskFileQuarantined: if data inconsistencies were detected between the metadata and the file-system metadata """ try: mname = self._metadata['name'] except KeyError: self._quarantine(data_file, "missing name metadata") else: if mname != self._name: self._logger.error( _('Client path %(client)s does not match ' 'path stored in object metadata %(meta)s'), { 'client': self._name, 'meta': mname }) raise DiskFileCollision('Client path does not match path ' 'stored in object metadata') try: x_delete_at = int(self._metadata['X-Delete-At']) except KeyError: pass except ValueError: # Quarantine, the x-delete-at key is present but not an # integer. self._quarantine( data_file, "bad metadata x-delete-at value %s" % (self._metadata['X-Delete-At'])) else: if x_delete_at <= time.time(): raise DiskFileNotExist('Expired') try: metadata_size = int(self._metadata['Content-Length']) except KeyError: self._quarantine(data_file, "missing content-length in metadata") except ValueError: # Quarantine, the content-length key is present but not an # integer. self._quarantine( data_file, "bad metadata content-length value %s" % (self._metadata['Content-Length'])) fd = fp.fileno() try: statbuf = os.fstat(fd) except OSError as err: # Quarantine, we can't successfully stat the file. self._quarantine(data_file, "not stat-able: %s" % err) else: obj_size = statbuf.st_size if metadata_size is not None and obj_size != metadata_size: self._quarantine( data_file, "metadata content-length %s does" " not match actual object size %s" % (metadata_size, statbuf.st_size)) return obj_size