Beispiel #1
0
    def _get_restore_point(self, base_name, backup_id):
        """Get restore point snapshot name for incremental backup.

        If the backup was not incremental (determined by the fact that the
        base has no snapshots/restore points), None is returned. Otherwise, the
        restore point associated with backup_id is returned.
        """
        with rbd_driver.RADOSClient(self, self._ceph_backup_pool) as client:
            base_rbd = self.rbd.Image(client.ioctx, base_name, read_only=True)
            try:
                restore_point = self._get_backup_snap_name(base_rbd, base_name,
                                                           backup_id)
            finally:
                base_rbd.close()

        return restore_point
Beispiel #2
0
    def _restore_metadata(self, backup, volume_id):
        """Restore volume metadata from backup.

        If this backup has associated metadata, save it to the restore target
        otherwise do nothing.
        """
        try:
            with rbd_driver.RADOSClient(self) as client:
                meta_bak = VolumeMetadataBackup(client, backup['id'])
                meta = meta_bak.get()
                if meta is not None:
                    self.put_metadata(volume_id, meta)
                else:
                    LOG.debug(_("Volume has no backed up metadata"))
        except exception.BackupMetadataUnsupportedVersion:
            msg = _("Metadata restore failed due to incompatible version")
            LOG.error(msg)
            raise exception.BackupOperationError(msg)
Beispiel #3
0
    def test_create_volume(self):
        name = u'volume-00000001'
        size = 1
        volume = dict(name=name, size=size)
        mock_client = self.mox.CreateMockAnything()
        self.mox.StubOutWithMock(driver, 'RADOSClient')

        driver.RADOSClient(self.driver).AndReturn(mock_client)
        mock_client.__enter__().AndReturn(mock_client)
        self.rbd.RBD_FEATURE_LAYERING = 1
        _mock_rbd = self.mox.CreateMockAnything()
        self.rbd.RBD().AndReturn(_mock_rbd)
        _mock_rbd.create(mox.IgnoreArg(), str(name), size * units.GiB,
                         old_format=False,
                         features=self.rbd.RBD_FEATURE_LAYERING)
        mock_client.__exit__(None, None, None).AndReturn(None)

        self.mox.ReplayAll()

        self.driver.create_volume(volume)
Beispiel #4
0
    def test_delete_volume(self):
        name = u'volume-00000001'
        volume = dict(name=name)
        mock_client = self.mox.CreateMockAnything()
        self.mox.StubOutWithMock(driver, 'RADOSClient')
        self.stubs.Set(self.driver, '_get_backup_snaps', lambda *args: None)

        driver.RADOSClient(self.driver).AndReturn(mock_client)
        mock_client.__enter__().AndReturn(mock_client)
        mock_image = self.mox.CreateMockAnything()
        self.rbd.Image(mox.IgnoreArg(), str(name)).AndReturn(mock_image)
        mock_image.close()
        mock_rbd = self.mox.CreateMockAnything()
        self.rbd.RBD().AndReturn(mock_rbd)
        mock_rbd.remove(mox.IgnoreArg(), str(name))
        mock_client.__exit__(None, None, None).AndReturn(None)

        self.mox.ReplayAll()

        self.driver.delete_volume(volume)
Beispiel #5
0
    def _backup_metadata(self, backup):
        """Backup volume metadata.

        NOTE(dosaboy): the metadata we are backing up is obtained from a
                       versioned api so we should not alter it in any way here.
                       We must also be sure that the service that will perform
                       the restore is compatible with version used.
        """
        json_meta = self.get_metadata(backup['volume_id'])
        if not json_meta:
            LOG.debug("No volume metadata to backup")
            return

        LOG.debug("Backing up volume metadata")
        try:
            with rbd_driver.RADOSClient(self) as client:
                vol_meta_backup = VolumeMetadataBackup(client, backup['id'])
                vol_meta_backup.set(json_meta)
        except exception.VolumeMetadataBackupExists as e:
            msg = _("Failed to backup volume metadata - %s") % (str(e))
            raise exception.BackupOperationError(msg)
Beispiel #6
0
    def _full_restore(self,
                      backup_id,
                      volume_id,
                      dest_file,
                      dest_name,
                      length,
                      src_snap=None):
        """Restore volume using full copy i.e. all extents.

        This will result in all extents being copied from source to
        destination.
        """
        with rbd_driver.RADOSClient(self, self._ceph_backup_pool) as client:
            # If a source snapshot is provided we assume the base is diff
            # format.
            if src_snap:
                diff_format = True
            else:
                diff_format = False

            backup_name = self._get_backup_base_name(volume_id,
                                                     backup_id=backup_id,
                                                     diff_format=diff_format)

            # Retrieve backup volume
            src_rbd = self.rbd.Image(client.ioctx,
                                     backup_name,
                                     snapshot=src_snap,
                                     read_only=True)
            try:
                rbd_meta = rbd_driver.RBDImageMetadata(src_rbd,
                                                       self._ceph_backup_pool,
                                                       self._ceph_backup_user,
                                                       self._ceph_backup_conf)
                rbd_fd = rbd_driver.RBDImageIOWrapper(rbd_meta)
                self._transfer_data(rbd_fd, backup_name, dest_file, dest_name,
                                    length)
            finally:
                src_rbd.close()
Beispiel #7
0
    def delete(self, backup):
        """Delete the given backup from Ceph object store."""
        backup_id = backup['id']
        LOG.debug(_('Delete started for backup=%s') % backup['id'])

        delete_failed = False
        try:
            self._try_delete_base_image(backup['id'], backup['volume_id'])
        except self.rbd.ImageNotFound:
            msg = _("RBD image not found but continuing anyway so that we can "
                    "attempt to delete metadata backup and db entry can be "
                    "removed")
            LOG.warning(msg)
            delete_failed = True

        with rbd_driver.RADOSClient(self) as client:
            VolumeMetadataBackup(client, backup['id']).remove_if_exists()

        if delete_failed:
            LOG.info(_("Delete '%s' finished with warning") % (backup_id))
        else:
            LOG.debug(_("Delete '%s' finished") % (backup_id))
Beispiel #8
0
    def test_update_volume_stats_error(self):
        self.stubs.Set(self.driver.configuration, 'safe_get', lambda x: 'RBD')
        mock_client = self.mox.CreateMockAnything()
        self.mox.StubOutWithMock(driver, 'RADOSClient')

        driver.RADOSClient(self.driver).AndReturn(mock_client)
        mock_client.__enter__().AndReturn(mock_client)
        self.mox.StubOutWithMock(mock_client, 'cluster')
        self.stubs.Set(self.rados, 'Error', test.TestingException)
        mock_client.cluster.get_cluster_stats().AndRaise(test.TestingException)
        mock_client.__exit__(test.TestingException, mox.IgnoreArg(),
                             mox.IgnoreArg()).AndReturn(None)

        self.mox.ReplayAll()

        expected = dict(volume_backend_name='RBD',
                        vendor_name='Open Source',
                        driver_version=self.driver.VERSION,
                        storage_protocol='ceph',
                        total_capacity_gb='unknown',
                        free_capacity_gb='unknown',
                        reserved_percentage=0)
        actual = self.driver.get_volume_stats(True)
        self.assertDictMatch(expected, actual)
Beispiel #9
0
    def _restore_volume(self, backup, volume, volume_file):
        """Restore volume from backup using diff transfer if possible.

        Attempts a differential restore and reverts to full copy if diff fails.
        """
        volume_name = volume['name']
        backup_id = backup['id']
        backup_volume_id = backup['volume_id']
        length = int(volume['size']) * units.GiB

        base_name = self._get_backup_base_name(backup['volume_id'],
                                               diff_format=True)

        with rbd_driver.RADOSClient(self, self._ceph_backup_pool) as client:
            diff_allowed, restore_point = \
                self._diff_restore_allowed(base_name, backup, volume,
                                           volume_file, client)

        do_full_restore = True
        if diff_allowed:
            # Attempt diff
            try:
                self._diff_restore_rbd(base_name, volume_file, volume_name,
                                       restore_point, length)
                do_full_restore = False
            except exception.BackupRBDOperationFailed:
                LOG.debug(_("Forcing full restore"))

        if do_full_restore:
            # Otherwise full copy
            self._full_restore(backup_id,
                               backup_volume_id,
                               volume_file,
                               volume_name,
                               length,
                               src_snap=restore_point)
Beispiel #10
0
    def _backup_rbd(self, backup_id, volume_id, volume_file, volume_name,
                    length):
        """Create a incremental backup from an RBD image."""
        rbd_user = volume_file.rbd_user
        rbd_pool = volume_file.rbd_pool
        rbd_conf = volume_file.rbd_conf
        source_rbd_image = volume_file.rbd_image

        # Identify our --from-snap point (if one exists)
        from_snap = self._get_most_recent_snap(source_rbd_image)
        LOG.debug(_("Using --from-snap '%s'") % from_snap)

        base_name = self._get_backup_base_name(volume_id, diff_format=True)
        image_created = False
        with rbd_driver.RADOSClient(self, self._ceph_backup_pool) as client:
            # If from_snap does not exist at the destination (and the
            # destination exists), this implies a previous backup has failed.
            # In this case we will force a full backup.
            #
            # TODO(dosaboy): find a way to repair the broken backup
            #
            if base_name not in self.rbd.RBD().list(ioctx=client.ioctx):
                # If a from_snap is defined but the base does not exist, we
                # ignore it since it is stale and waiting to be cleaned up.
                if from_snap:
                    LOG.debug(
                        _("Source snapshot '%s' is stale so deleting") %
                        (from_snap))
                    source_rbd_image.remove_snap(from_snap)
                    from_snap = None

                # Create new base image
                self._create_base_image(base_name, length, client)
                image_created = True
            else:
                # If a from_snap is defined but does not exist in the back base
                # then we cannot proceed (see above)
                if not self._snap_exists(base_name, from_snap, client):
                    errmsg = (_("Snapshot='%(snap)s' does not exist in base "
                                "image='%(base)s' - aborting incremental "
                                "backup") % {
                                    'snap': from_snap,
                                    'base': base_name
                                })
                    LOG.info(errmsg)
                    # Raise this exception so that caller can try another
                    # approach
                    raise exception.BackupRBDOperationFailed(errmsg)

        # Snapshot source volume so that we have a new point-in-time
        new_snap = self._get_new_snap_name(backup_id)
        LOG.debug(_("Creating backup snapshot='%s'") % (new_snap))
        source_rbd_image.create_snap(new_snap)

        # Attempt differential backup. If this fails, perhaps because librbd
        # or Ceph cluster version does not support it, do a full backup
        # instead.
        #
        # TODO(dosaboy): find a way to determine if the operation is supported
        #                rather than brute force approach.
        try:
            before = time.time()
            self._rbd_diff_transfer(volume_name,
                                    rbd_pool,
                                    base_name,
                                    self._ceph_backup_pool,
                                    src_user=rbd_user,
                                    src_conf=rbd_conf,
                                    dest_user=self._ceph_backup_user,
                                    dest_conf=self._ceph_backup_conf,
                                    src_snap=new_snap,
                                    from_snap=from_snap)

            LOG.debug(
                _("Differential backup transfer completed in %.4fs") %
                (time.time() - before))

            # We don't need the previous snapshot (if there was one) anymore so
            # delete it.
            if from_snap:
                source_rbd_image.remove_snap(from_snap)

        except exception.BackupRBDOperationFailed:
            LOG.debug(_("Differential backup transfer failed"))

            # Clean up if image was created as part of this operation
            if image_created:
                self._try_delete_base_image(backup_id,
                                            volume_id,
                                            base_name=base_name)

            # Delete snapshot
            LOG.debug(_("Deleting backup snapshot='%s'") % (new_snap))
            source_rbd_image.remove_snap(new_snap)

            # Re-raise the exception so that caller can try another approach
            raise
Beispiel #11
0
    def _try_delete_base_image(self, backup_id, volume_id, base_name=None):
        """Try to delete backup RBD image.

        If the rbd image is a base image for incremental backups, it may have
        snapshots. Delete the snapshot associated with backup_id and if the
        image has no more snapshots, delete it. Otherwise return.

        If no base name is provided try normal (full) format then diff format
        image name.

        If a base name is provided but does not exist, ImageNotFound will be
        raised.

        If the image is busy, a number of retries will be performed if
        ImageBusy is received, after which the exception will be propagated to
        the caller.
        """
        retries = 3
        delay = 5
        try_diff_format = False

        if base_name is None:
            try_diff_format = True

            base_name = self._get_backup_base_name(volume_id, backup_id)
            LOG.debug(
                _("Trying diff format name format basename='%s'") %
                (base_name))

        with rbd_driver.RADOSClient(self) as client:
            rbd_exists, base_name = \
                self._rbd_image_exists(base_name, volume_id, client,
                                       try_diff_format=try_diff_format)
            if not rbd_exists:
                raise self.rbd.ImageNotFound(
                    _("image %s not found") % (base_name))

            while retries >= 0:
                # First delete associated snapshot from base image (if exists)
                snap, rem = self._delete_backup_snapshot(
                    client, base_name, backup_id)
                if rem:
                    msg = (_("Base image still has %s snapshots so skipping "
                             "base image delete") % (rem))
                    LOG.info(msg)
                    return

                LOG.info(_("Deleting base image='%s'") % (base_name))
                # Delete base if no more snapshots
                try:
                    self.rbd.RBD().remove(client.ioctx, base_name)
                except self.rbd.ImageBusy as exc:
                    # Allow a retry if the image is busy
                    if retries > 0:
                        LOG.info((_("Image busy, retrying %(retries)s "
                                    "more time(s) in %(delay)ss") % {
                                        'retries': retries,
                                        'delay': delay
                                    }))
                        eventlet.sleep(delay)
                    else:
                        LOG.error(_("Max retries reached - raising error"))
                        raise exc
                else:
                    LOG.debug(
                        _("Base backup image='%s' deleted)") % (base_name))
                    retries = 0
                finally:
                    retries -= 1

            # Since we have deleted the base image we can delete the source
            # volume backup snapshot.
            src_name = strutils.safe_encode(volume_id)
            if src_name in self.rbd.RBD().list(client.ioctx):
                LOG.debug(_("Deleting source snapshot '%s'") % snap)
                src_rbd = self.rbd.Image(client.ioctx, src_name)
                try:
                    src_rbd.remove_snap(snap)
                finally:
                    src_rbd.close()
Beispiel #12
0
    def _try_delete_base_image(self, backup_id, volume_id, base_name=None):
        """Try to delete backup RBD image.

        If the rbd image is a base image for incremental backups, it may have
        snapshots. Delete the snapshot associated with backup_id and if the
        image has no more snapshots, delete it. Otherwise return.

        If no base name is provided try normal (full) format then diff format
        image name.

        If a base name is provided but does not exist, ImageNotFound will be
        raised.

        If the image is busy, a number of retries will be performed if
        ImageBusy is received, after which the exception will be propagated to
        the caller.
        """
        retries = 3
        delay = 5
        try_diff_format = False

        if base_name is None:
            try_diff_format = True

            base_name = self._get_backup_base_name(volume_id, backup_id)
            LOG.debug("Trying diff format basename='%(basename)s' for "
                      "backup base image of volume %(volume)s.",
                      {'basename': base_name, 'volume': volume_id})

        with rbd_driver.RADOSClient(self) as client:
            rbd_exists, base_name = \
                self._rbd_image_exists(base_name, volume_id, client,
                                       try_diff_format=try_diff_format)
            if not rbd_exists:
                raise self.rbd.ImageNotFound(_("image %s not found") %
                                             base_name)

            while retries >= 0:
                # First delete associated snapshot from base image (if exists)
                snap, rem = self._delete_backup_snapshot(client, base_name,
                                                         backup_id)
                if rem:
                    LOG.info(
                        _LI("Backup base image of volume %(volume)s still "
                            "has %(snapshots)s snapshots so skipping base "
                            "image delete."),
                        {'snapshots': rem, 'volume': volume_id})
                    return

                LOG.info(_LI("Deleting backup base image='%(basename)s' of "
                             "volume %(volume)s."),
                         {'basename': base_name, 'volume': volume_id})
                # Delete base if no more snapshots
                try:
                    self.rbd.RBD().remove(client.ioctx, base_name)
                except self.rbd.ImageBusy:
                    # Allow a retry if the image is busy
                    if retries > 0:
                        LOG.info(_LI("Backup image of volume %(volume)s is "
                                     "busy, retrying %(retries)s more time(s) "
                                     "in %(delay)ss."),
                                 {'retries': retries,
                                  'delay': delay,
                                  'volume': volume_id})
                        eventlet.sleep(delay)
                    else:
                        LOG.error(_LE("Max retries reached deleting backup "
                                      "%(basename)s image of volume "
                                      "%(volume)s."),
                                  {'volume': volume_id,
                                   'basename': base_name})
                        raise
                else:
                    LOG.debug("Base backup image='%(basename)s' of volume "
                              "%(volume)s deleted.",
                              {'basename': base_name, 'volume': volume_id})
                    retries = 0
                finally:
                    retries -= 1

            # Since we have deleted the base image we can delete the source
            # volume backup snapshot.
            src_name = utils.convert_str(volume_id)
            if src_name in self.rbd.RBD().list(client.ioctx):
                LOG.debug("Deleting source volume snapshot '%(snapshot)s' "
                          "for backup %(basename)s.",
                          {'snapshot': snap, 'basename': base_name})
                src_rbd = self.rbd.Image(client.ioctx, src_name)
                try:
                    src_rbd.remove_snap(snap)
                finally:
                    src_rbd.close()