def _get_restore_point(self, base_name, backup_id): """Get restore point snapshot name for incremental backup. If the backup was not incremental (determined by the fact that the base has no snapshots/restore points), None is returned. Otherwise, the restore point associated with backup_id is returned. """ with rbd_driver.RADOSClient(self, self._ceph_backup_pool) as client: base_rbd = self.rbd.Image(client.ioctx, base_name, read_only=True) try: restore_point = self._get_backup_snap_name(base_rbd, base_name, backup_id) finally: base_rbd.close() return restore_point
def _restore_metadata(self, backup, volume_id): """Restore volume metadata from backup. If this backup has associated metadata, save it to the restore target otherwise do nothing. """ try: with rbd_driver.RADOSClient(self) as client: meta_bak = VolumeMetadataBackup(client, backup['id']) meta = meta_bak.get() if meta is not None: self.put_metadata(volume_id, meta) else: LOG.debug(_("Volume has no backed up metadata")) except exception.BackupMetadataUnsupportedVersion: msg = _("Metadata restore failed due to incompatible version") LOG.error(msg) raise exception.BackupOperationError(msg)
def test_create_volume(self): name = u'volume-00000001' size = 1 volume = dict(name=name, size=size) mock_client = self.mox.CreateMockAnything() self.mox.StubOutWithMock(driver, 'RADOSClient') driver.RADOSClient(self.driver).AndReturn(mock_client) mock_client.__enter__().AndReturn(mock_client) self.rbd.RBD_FEATURE_LAYERING = 1 _mock_rbd = self.mox.CreateMockAnything() self.rbd.RBD().AndReturn(_mock_rbd) _mock_rbd.create(mox.IgnoreArg(), str(name), size * units.GiB, old_format=False, features=self.rbd.RBD_FEATURE_LAYERING) mock_client.__exit__(None, None, None).AndReturn(None) self.mox.ReplayAll() self.driver.create_volume(volume)
def test_delete_volume(self): name = u'volume-00000001' volume = dict(name=name) mock_client = self.mox.CreateMockAnything() self.mox.StubOutWithMock(driver, 'RADOSClient') self.stubs.Set(self.driver, '_get_backup_snaps', lambda *args: None) driver.RADOSClient(self.driver).AndReturn(mock_client) mock_client.__enter__().AndReturn(mock_client) mock_image = self.mox.CreateMockAnything() self.rbd.Image(mox.IgnoreArg(), str(name)).AndReturn(mock_image) mock_image.close() mock_rbd = self.mox.CreateMockAnything() self.rbd.RBD().AndReturn(mock_rbd) mock_rbd.remove(mox.IgnoreArg(), str(name)) mock_client.__exit__(None, None, None).AndReturn(None) self.mox.ReplayAll() self.driver.delete_volume(volume)
def _backup_metadata(self, backup): """Backup volume metadata. NOTE(dosaboy): the metadata we are backing up is obtained from a versioned api so we should not alter it in any way here. We must also be sure that the service that will perform the restore is compatible with version used. """ json_meta = self.get_metadata(backup['volume_id']) if not json_meta: LOG.debug("No volume metadata to backup") return LOG.debug("Backing up volume metadata") try: with rbd_driver.RADOSClient(self) as client: vol_meta_backup = VolumeMetadataBackup(client, backup['id']) vol_meta_backup.set(json_meta) except exception.VolumeMetadataBackupExists as e: msg = _("Failed to backup volume metadata - %s") % (str(e)) raise exception.BackupOperationError(msg)
def _full_restore(self, backup_id, volume_id, dest_file, dest_name, length, src_snap=None): """Restore volume using full copy i.e. all extents. This will result in all extents being copied from source to destination. """ with rbd_driver.RADOSClient(self, self._ceph_backup_pool) as client: # If a source snapshot is provided we assume the base is diff # format. if src_snap: diff_format = True else: diff_format = False backup_name = self._get_backup_base_name(volume_id, backup_id=backup_id, diff_format=diff_format) # Retrieve backup volume src_rbd = self.rbd.Image(client.ioctx, backup_name, snapshot=src_snap, read_only=True) try: rbd_meta = rbd_driver.RBDImageMetadata(src_rbd, self._ceph_backup_pool, self._ceph_backup_user, self._ceph_backup_conf) rbd_fd = rbd_driver.RBDImageIOWrapper(rbd_meta) self._transfer_data(rbd_fd, backup_name, dest_file, dest_name, length) finally: src_rbd.close()
def delete(self, backup): """Delete the given backup from Ceph object store.""" backup_id = backup['id'] LOG.debug(_('Delete started for backup=%s') % backup['id']) delete_failed = False try: self._try_delete_base_image(backup['id'], backup['volume_id']) except self.rbd.ImageNotFound: msg = _("RBD image not found but continuing anyway so that we can " "attempt to delete metadata backup and db entry can be " "removed") LOG.warning(msg) delete_failed = True with rbd_driver.RADOSClient(self) as client: VolumeMetadataBackup(client, backup['id']).remove_if_exists() if delete_failed: LOG.info(_("Delete '%s' finished with warning") % (backup_id)) else: LOG.debug(_("Delete '%s' finished") % (backup_id))
def test_update_volume_stats_error(self): self.stubs.Set(self.driver.configuration, 'safe_get', lambda x: 'RBD') mock_client = self.mox.CreateMockAnything() self.mox.StubOutWithMock(driver, 'RADOSClient') driver.RADOSClient(self.driver).AndReturn(mock_client) mock_client.__enter__().AndReturn(mock_client) self.mox.StubOutWithMock(mock_client, 'cluster') self.stubs.Set(self.rados, 'Error', test.TestingException) mock_client.cluster.get_cluster_stats().AndRaise(test.TestingException) mock_client.__exit__(test.TestingException, mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(None) self.mox.ReplayAll() expected = dict(volume_backend_name='RBD', vendor_name='Open Source', driver_version=self.driver.VERSION, storage_protocol='ceph', total_capacity_gb='unknown', free_capacity_gb='unknown', reserved_percentage=0) actual = self.driver.get_volume_stats(True) self.assertDictMatch(expected, actual)
def _restore_volume(self, backup, volume, volume_file): """Restore volume from backup using diff transfer if possible. Attempts a differential restore and reverts to full copy if diff fails. """ volume_name = volume['name'] backup_id = backup['id'] backup_volume_id = backup['volume_id'] length = int(volume['size']) * units.GiB base_name = self._get_backup_base_name(backup['volume_id'], diff_format=True) with rbd_driver.RADOSClient(self, self._ceph_backup_pool) as client: diff_allowed, restore_point = \ self._diff_restore_allowed(base_name, backup, volume, volume_file, client) do_full_restore = True if diff_allowed: # Attempt diff try: self._diff_restore_rbd(base_name, volume_file, volume_name, restore_point, length) do_full_restore = False except exception.BackupRBDOperationFailed: LOG.debug(_("Forcing full restore")) if do_full_restore: # Otherwise full copy self._full_restore(backup_id, backup_volume_id, volume_file, volume_name, length, src_snap=restore_point)
def _backup_rbd(self, backup_id, volume_id, volume_file, volume_name, length): """Create a incremental backup from an RBD image.""" rbd_user = volume_file.rbd_user rbd_pool = volume_file.rbd_pool rbd_conf = volume_file.rbd_conf source_rbd_image = volume_file.rbd_image # Identify our --from-snap point (if one exists) from_snap = self._get_most_recent_snap(source_rbd_image) LOG.debug(_("Using --from-snap '%s'") % from_snap) base_name = self._get_backup_base_name(volume_id, diff_format=True) image_created = False with rbd_driver.RADOSClient(self, self._ceph_backup_pool) as client: # If from_snap does not exist at the destination (and the # destination exists), this implies a previous backup has failed. # In this case we will force a full backup. # # TODO(dosaboy): find a way to repair the broken backup # if base_name not in self.rbd.RBD().list(ioctx=client.ioctx): # If a from_snap is defined but the base does not exist, we # ignore it since it is stale and waiting to be cleaned up. if from_snap: LOG.debug( _("Source snapshot '%s' is stale so deleting") % (from_snap)) source_rbd_image.remove_snap(from_snap) from_snap = None # Create new base image self._create_base_image(base_name, length, client) image_created = True else: # If a from_snap is defined but does not exist in the back base # then we cannot proceed (see above) if not self._snap_exists(base_name, from_snap, client): errmsg = (_("Snapshot='%(snap)s' does not exist in base " "image='%(base)s' - aborting incremental " "backup") % { 'snap': from_snap, 'base': base_name }) LOG.info(errmsg) # Raise this exception so that caller can try another # approach raise exception.BackupRBDOperationFailed(errmsg) # Snapshot source volume so that we have a new point-in-time new_snap = self._get_new_snap_name(backup_id) LOG.debug(_("Creating backup snapshot='%s'") % (new_snap)) source_rbd_image.create_snap(new_snap) # Attempt differential backup. If this fails, perhaps because librbd # or Ceph cluster version does not support it, do a full backup # instead. # # TODO(dosaboy): find a way to determine if the operation is supported # rather than brute force approach. try: before = time.time() self._rbd_diff_transfer(volume_name, rbd_pool, base_name, self._ceph_backup_pool, src_user=rbd_user, src_conf=rbd_conf, dest_user=self._ceph_backup_user, dest_conf=self._ceph_backup_conf, src_snap=new_snap, from_snap=from_snap) LOG.debug( _("Differential backup transfer completed in %.4fs") % (time.time() - before)) # We don't need the previous snapshot (if there was one) anymore so # delete it. if from_snap: source_rbd_image.remove_snap(from_snap) except exception.BackupRBDOperationFailed: LOG.debug(_("Differential backup transfer failed")) # Clean up if image was created as part of this operation if image_created: self._try_delete_base_image(backup_id, volume_id, base_name=base_name) # Delete snapshot LOG.debug(_("Deleting backup snapshot='%s'") % (new_snap)) source_rbd_image.remove_snap(new_snap) # Re-raise the exception so that caller can try another approach raise
def _try_delete_base_image(self, backup_id, volume_id, base_name=None): """Try to delete backup RBD image. If the rbd image is a base image for incremental backups, it may have snapshots. Delete the snapshot associated with backup_id and if the image has no more snapshots, delete it. Otherwise return. If no base name is provided try normal (full) format then diff format image name. If a base name is provided but does not exist, ImageNotFound will be raised. If the image is busy, a number of retries will be performed if ImageBusy is received, after which the exception will be propagated to the caller. """ retries = 3 delay = 5 try_diff_format = False if base_name is None: try_diff_format = True base_name = self._get_backup_base_name(volume_id, backup_id) LOG.debug( _("Trying diff format name format basename='%s'") % (base_name)) with rbd_driver.RADOSClient(self) as client: rbd_exists, base_name = \ self._rbd_image_exists(base_name, volume_id, client, try_diff_format=try_diff_format) if not rbd_exists: raise self.rbd.ImageNotFound( _("image %s not found") % (base_name)) while retries >= 0: # First delete associated snapshot from base image (if exists) snap, rem = self._delete_backup_snapshot( client, base_name, backup_id) if rem: msg = (_("Base image still has %s snapshots so skipping " "base image delete") % (rem)) LOG.info(msg) return LOG.info(_("Deleting base image='%s'") % (base_name)) # Delete base if no more snapshots try: self.rbd.RBD().remove(client.ioctx, base_name) except self.rbd.ImageBusy as exc: # Allow a retry if the image is busy if retries > 0: LOG.info((_("Image busy, retrying %(retries)s " "more time(s) in %(delay)ss") % { 'retries': retries, 'delay': delay })) eventlet.sleep(delay) else: LOG.error(_("Max retries reached - raising error")) raise exc else: LOG.debug( _("Base backup image='%s' deleted)") % (base_name)) retries = 0 finally: retries -= 1 # Since we have deleted the base image we can delete the source # volume backup snapshot. src_name = strutils.safe_encode(volume_id) if src_name in self.rbd.RBD().list(client.ioctx): LOG.debug(_("Deleting source snapshot '%s'") % snap) src_rbd = self.rbd.Image(client.ioctx, src_name) try: src_rbd.remove_snap(snap) finally: src_rbd.close()
def _try_delete_base_image(self, backup_id, volume_id, base_name=None): """Try to delete backup RBD image. If the rbd image is a base image for incremental backups, it may have snapshots. Delete the snapshot associated with backup_id and if the image has no more snapshots, delete it. Otherwise return. If no base name is provided try normal (full) format then diff format image name. If a base name is provided but does not exist, ImageNotFound will be raised. If the image is busy, a number of retries will be performed if ImageBusy is received, after which the exception will be propagated to the caller. """ retries = 3 delay = 5 try_diff_format = False if base_name is None: try_diff_format = True base_name = self._get_backup_base_name(volume_id, backup_id) LOG.debug("Trying diff format basename='%(basename)s' for " "backup base image of volume %(volume)s.", {'basename': base_name, 'volume': volume_id}) with rbd_driver.RADOSClient(self) as client: rbd_exists, base_name = \ self._rbd_image_exists(base_name, volume_id, client, try_diff_format=try_diff_format) if not rbd_exists: raise self.rbd.ImageNotFound(_("image %s not found") % base_name) while retries >= 0: # First delete associated snapshot from base image (if exists) snap, rem = self._delete_backup_snapshot(client, base_name, backup_id) if rem: LOG.info( _LI("Backup base image of volume %(volume)s still " "has %(snapshots)s snapshots so skipping base " "image delete."), {'snapshots': rem, 'volume': volume_id}) return LOG.info(_LI("Deleting backup base image='%(basename)s' of " "volume %(volume)s."), {'basename': base_name, 'volume': volume_id}) # Delete base if no more snapshots try: self.rbd.RBD().remove(client.ioctx, base_name) except self.rbd.ImageBusy: # Allow a retry if the image is busy if retries > 0: LOG.info(_LI("Backup image of volume %(volume)s is " "busy, retrying %(retries)s more time(s) " "in %(delay)ss."), {'retries': retries, 'delay': delay, 'volume': volume_id}) eventlet.sleep(delay) else: LOG.error(_LE("Max retries reached deleting backup " "%(basename)s image of volume " "%(volume)s."), {'volume': volume_id, 'basename': base_name}) raise else: LOG.debug("Base backup image='%(basename)s' of volume " "%(volume)s deleted.", {'basename': base_name, 'volume': volume_id}) retries = 0 finally: retries -= 1 # Since we have deleted the base image we can delete the source # volume backup snapshot. src_name = utils.convert_str(volume_id) if src_name in self.rbd.RBD().list(client.ioctx): LOG.debug("Deleting source volume snapshot '%(snapshot)s' " "for backup %(basename)s.", {'snapshot': snap, 'basename': base_name}) src_rbd = self.rbd.Image(client.ioctx, src_name) try: src_rbd.remove_snap(snap) finally: src_rbd.close()