def _get_restore_point(self, base_name, backup_id):
"""Get restore point snapshot name for incremental backup.
If the backup was not incremental (determined by the fact that the
base has no snapshots/restore points), None is returned. Otherwise, the
restore point associated with backup_id is returned.
"""
with rbd_driver.RADOSClient(self, self._ceph_backup_pool) as client:
base_rbd = self.rbd.Image(client.ioctx, base_name, read_only=True)
try:
restore_point = self._get_backup_snap_name(base_rbd, base_name,
backup_id)
finally:
base_rbd.close()
return restore_point
def _restore_metadata(self, backup, volume_id):
"""Restore volume metadata from backup.
If this backup has associated metadata, save it to the restore target
otherwise do nothing.
"""
try:
with rbd_driver.RADOSClient(self) as client:
meta_bak = VolumeMetadataBackup(client, backup['id'])
meta = meta_bak.get()
if meta is not None:
self.put_metadata(volume_id, meta)
else:
LOG.debug(_("Volume has no backed up metadata"))
except exception.BackupMetadataUnsupportedVersion:
msg = _("Metadata restore failed due to incompatible version")
LOG.error(msg)
raise exception.BackupOperationError(msg)
def test_create_volume(self):
name = u'volume-00000001'
size = 1
volume = dict(name=name, size=size)
mock_client = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(driver, 'RADOSClient')
driver.RADOSClient(self.driver).AndReturn(mock_client)
mock_client.__enter__().AndReturn(mock_client)
self.rbd.RBD_FEATURE_LAYERING = 1
_mock_rbd = self.mox.CreateMockAnything()
self.rbd.RBD().AndReturn(_mock_rbd)
_mock_rbd.create(mox.IgnoreArg(), str(name), size * units.GiB,
old_format=False,
features=self.rbd.RBD_FEATURE_LAYERING)
mock_client.__exit__(None, None, None).AndReturn(None)
self.mox.ReplayAll()
self.driver.create_volume(volume)
def test_delete_volume(self):
name = u'volume-00000001'
volume = dict(name=name)
mock_client = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(driver, 'RADOSClient')
self.stubs.Set(self.driver, '_get_backup_snaps', lambda *args: None)
driver.RADOSClient(self.driver).AndReturn(mock_client)
mock_client.__enter__().AndReturn(mock_client)
mock_image = self.mox.CreateMockAnything()
self.rbd.Image(mox.IgnoreArg(), str(name)).AndReturn(mock_image)
mock_image.close()
mock_rbd = self.mox.CreateMockAnything()
self.rbd.RBD().AndReturn(mock_rbd)
mock_rbd.remove(mox.IgnoreArg(), str(name))
mock_client.__exit__(None, None, None).AndReturn(None)
self.mox.ReplayAll()
self.driver.delete_volume(volume)
def _backup_metadata(self, backup):
"""Backup volume metadata.
NOTE(dosaboy): the metadata we are backing up is obtained from a
versioned api so we should not alter it in any way here.
We must also be sure that the service that will perform
the restore is compatible with version used.
"""
json_meta = self.get_metadata(backup['volume_id'])
if not json_meta:
LOG.debug("No volume metadata to backup")
return
LOG.debug("Backing up volume metadata")
try:
with rbd_driver.RADOSClient(self) as client:
vol_meta_backup = VolumeMetadataBackup(client, backup['id'])
vol_meta_backup.set(json_meta)
except exception.VolumeMetadataBackupExists as e:
msg = _("Failed to backup volume metadata - %s") % (str(e))
raise exception.BackupOperationError(msg)
def _full_restore(self,
backup_id,
volume_id,
dest_file,
dest_name,
length,
src_snap=None):
"""Restore volume using full copy i.e. all extents.
This will result in all extents being copied from source to
destination.
"""
with rbd_driver.RADOSClient(self, self._ceph_backup_pool) as client:
# If a source snapshot is provided we assume the base is diff
# format.
if src_snap:
diff_format = True
else:
diff_format = False
backup_name = self._get_backup_base_name(volume_id,
backup_id=backup_id,
diff_format=diff_format)
# Retrieve backup volume
src_rbd = self.rbd.Image(client.ioctx,
backup_name,
snapshot=src_snap,
read_only=True)
try:
rbd_meta = rbd_driver.RBDImageMetadata(src_rbd,
self._ceph_backup_pool,
self._ceph_backup_user,
self._ceph_backup_conf)
rbd_fd = rbd_driver.RBDImageIOWrapper(rbd_meta)
self._transfer_data(rbd_fd, backup_name, dest_file, dest_name,
length)
finally:
src_rbd.close()
def delete(self, backup):
"""Delete the given backup from Ceph object store."""
backup_id = backup['id']
LOG.debug(_('Delete started for backup=%s') % backup['id'])
delete_failed = False
try:
self._try_delete_base_image(backup['id'], backup['volume_id'])
except self.rbd.ImageNotFound:
msg = _("RBD image not found but continuing anyway so that we can "
"attempt to delete metadata backup and db entry can be "
"removed")
LOG.warning(msg)
delete_failed = True
with rbd_driver.RADOSClient(self) as client:
VolumeMetadataBackup(client, backup['id']).remove_if_exists()
if delete_failed:
LOG.info(_("Delete '%s' finished with warning") % (backup_id))
else:
LOG.debug(_("Delete '%s' finished") % (backup_id))
def test_update_volume_stats_error(self):
self.stubs.Set(self.driver.configuration, 'safe_get', lambda x: 'RBD')
mock_client = self.mox.CreateMockAnything()
self.mox.StubOutWithMock(driver, 'RADOSClient')
driver.RADOSClient(self.driver).AndReturn(mock_client)
mock_client.__enter__().AndReturn(mock_client)
self.mox.StubOutWithMock(mock_client, 'cluster')
self.stubs.Set(self.rados, 'Error', test.TestingException)
mock_client.cluster.get_cluster_stats().AndRaise(test.TestingException)
mock_client.__exit__(test.TestingException, mox.IgnoreArg(),
mox.IgnoreArg()).AndReturn(None)
self.mox.ReplayAll()
expected = dict(volume_backend_name='RBD',
vendor_name='Open Source',
driver_version=self.driver.VERSION,
storage_protocol='ceph',
total_capacity_gb='unknown',
free_capacity_gb='unknown',
reserved_percentage=0)
actual = self.driver.get_volume_stats(True)
self.assertDictMatch(expected, actual)
def _restore_volume(self, backup, volume, volume_file):
"""Restore volume from backup using diff transfer if possible.
Attempts a differential restore and reverts to full copy if diff fails.
"""
volume_name = volume['name']
backup_id = backup['id']
backup_volume_id = backup['volume_id']
length = int(volume['size']) * units.GiB
base_name = self._get_backup_base_name(backup['volume_id'],
diff_format=True)
with rbd_driver.RADOSClient(self, self._ceph_backup_pool) as client:
diff_allowed, restore_point = \
self._diff_restore_allowed(base_name, backup, volume,
volume_file, client)
do_full_restore = True
if diff_allowed:
# Attempt diff
try:
self._diff_restore_rbd(base_name, volume_file, volume_name,
restore_point, length)
do_full_restore = False
except exception.BackupRBDOperationFailed:
LOG.debug(_("Forcing full restore"))
if do_full_restore:
# Otherwise full copy
self._full_restore(backup_id,
backup_volume_id,
volume_file,
volume_name,
length,
src_snap=restore_point)
def _backup_rbd(self, backup_id, volume_id, volume_file, volume_name,
length):
"""Create a incremental backup from an RBD image."""
rbd_user = volume_file.rbd_user
rbd_pool = volume_file.rbd_pool
rbd_conf = volume_file.rbd_conf
source_rbd_image = volume_file.rbd_image
# Identify our --from-snap point (if one exists)
from_snap = self._get_most_recent_snap(source_rbd_image)
LOG.debug(_("Using --from-snap '%s'") % from_snap)
base_name = self._get_backup_base_name(volume_id, diff_format=True)
image_created = False
with rbd_driver.RADOSClient(self, self._ceph_backup_pool) as client:
# If from_snap does not exist at the destination (and the
# destination exists), this implies a previous backup has failed.
# In this case we will force a full backup.
#
# TODO(dosaboy): find a way to repair the broken backup
#
if base_name not in self.rbd.RBD().list(ioctx=client.ioctx):
# If a from_snap is defined but the base does not exist, we
# ignore it since it is stale and waiting to be cleaned up.
if from_snap:
LOG.debug(
_("Source snapshot '%s' is stale so deleting") %
(from_snap))
source_rbd_image.remove_snap(from_snap)
from_snap = None
# Create new base image
self._create_base_image(base_name, length, client)
image_created = True
else:
# If a from_snap is defined but does not exist in the back base
# then we cannot proceed (see above)
if not self._snap_exists(base_name, from_snap, client):
errmsg = (_("Snapshot='%(snap)s' does not exist in base "
"image='%(base)s' - aborting incremental "
"backup") % {
'snap': from_snap,
'base': base_name
})
LOG.info(errmsg)
# Raise this exception so that caller can try another
# approach
raise exception.BackupRBDOperationFailed(errmsg)
# Snapshot source volume so that we have a new point-in-time
new_snap = self._get_new_snap_name(backup_id)
LOG.debug(_("Creating backup snapshot='%s'") % (new_snap))
source_rbd_image.create_snap(new_snap)
# Attempt differential backup. If this fails, perhaps because librbd
# or Ceph cluster version does not support it, do a full backup
# instead.
#
# TODO(dosaboy): find a way to determine if the operation is supported
# rather than brute force approach.
try:
before = time.time()
self._rbd_diff_transfer(volume_name,
rbd_pool,
base_name,
self._ceph_backup_pool,
src_user=rbd_user,
src_conf=rbd_conf,
dest_user=self._ceph_backup_user,
dest_conf=self._ceph_backup_conf,
src_snap=new_snap,
from_snap=from_snap)
LOG.debug(
_("Differential backup transfer completed in %.4fs") %
(time.time() - before))
# We don't need the previous snapshot (if there was one) anymore so
# delete it.
if from_snap:
source_rbd_image.remove_snap(from_snap)
except exception.BackupRBDOperationFailed:
LOG.debug(_("Differential backup transfer failed"))
# Clean up if image was created as part of this operation
if image_created:
self._try_delete_base_image(backup_id,
volume_id,
base_name=base_name)
# Delete snapshot
LOG.debug(_("Deleting backup snapshot='%s'") % (new_snap))
source_rbd_image.remove_snap(new_snap)
# Re-raise the exception so that caller can try another approach
raise
def _try_delete_base_image(self, backup_id, volume_id, base_name=None):
"""Try to delete backup RBD image.
If the rbd image is a base image for incremental backups, it may have
snapshots. Delete the snapshot associated with backup_id and if the
image has no more snapshots, delete it. Otherwise return.
If no base name is provided try normal (full) format then diff format
image name.
If a base name is provided but does not exist, ImageNotFound will be
raised.
If the image is busy, a number of retries will be performed if
ImageBusy is received, after which the exception will be propagated to
the caller.
"""
retries = 3
delay = 5
try_diff_format = False
if base_name is None:
try_diff_format = True
base_name = self._get_backup_base_name(volume_id, backup_id)
LOG.debug(
_("Trying diff format name format basename='%s'") %
(base_name))
with rbd_driver.RADOSClient(self) as client:
rbd_exists, base_name = \
self._rbd_image_exists(base_name, volume_id, client,
try_diff_format=try_diff_format)
if not rbd_exists:
raise self.rbd.ImageNotFound(
_("image %s not found") % (base_name))
while retries >= 0:
# First delete associated snapshot from base image (if exists)
snap, rem = self._delete_backup_snapshot(
client, base_name, backup_id)
if rem:
msg = (_("Base image still has %s snapshots so skipping "
"base image delete") % (rem))
LOG.info(msg)
return
LOG.info(_("Deleting base image='%s'") % (base_name))
# Delete base if no more snapshots
try:
self.rbd.RBD().remove(client.ioctx, base_name)
except self.rbd.ImageBusy as exc:
# Allow a retry if the image is busy
if retries > 0:
LOG.info((_("Image busy, retrying %(retries)s "
"more time(s) in %(delay)ss") % {
'retries': retries,
'delay': delay
}))
eventlet.sleep(delay)
else:
LOG.error(_("Max retries reached - raising error"))
raise exc
else:
LOG.debug(
_("Base backup image='%s' deleted)") % (base_name))
retries = 0
finally:
retries -= 1
# Since we have deleted the base image we can delete the source
# volume backup snapshot.
src_name = strutils.safe_encode(volume_id)
if src_name in self.rbd.RBD().list(client.ioctx):
LOG.debug(_("Deleting source snapshot '%s'") % snap)
src_rbd = self.rbd.Image(client.ioctx, src_name)
try:
src_rbd.remove_snap(snap)
finally:
src_rbd.close()
def _try_delete_base_image(self, backup_id, volume_id, base_name=None):
"""Try to delete backup RBD image.
If the rbd image is a base image for incremental backups, it may have
snapshots. Delete the snapshot associated with backup_id and if the
image has no more snapshots, delete it. Otherwise return.
If no base name is provided try normal (full) format then diff format
image name.
If a base name is provided but does not exist, ImageNotFound will be
raised.
If the image is busy, a number of retries will be performed if
ImageBusy is received, after which the exception will be propagated to
the caller.
"""
retries = 3
delay = 5
try_diff_format = False
if base_name is None:
try_diff_format = True
base_name = self._get_backup_base_name(volume_id, backup_id)
LOG.debug("Trying diff format basename='%(basename)s' for "
"backup base image of volume %(volume)s.",
{'basename': base_name, 'volume': volume_id})
with rbd_driver.RADOSClient(self) as client:
rbd_exists, base_name = \
self._rbd_image_exists(base_name, volume_id, client,
try_diff_format=try_diff_format)
if not rbd_exists:
raise self.rbd.ImageNotFound(_("image %s not found") %
base_name)
while retries >= 0:
# First delete associated snapshot from base image (if exists)
snap, rem = self._delete_backup_snapshot(client, base_name,
backup_id)
if rem:
LOG.info(
_LI("Backup base image of volume %(volume)s still "
"has %(snapshots)s snapshots so skipping base "
"image delete."),
{'snapshots': rem, 'volume': volume_id})
return
LOG.info(_LI("Deleting backup base image='%(basename)s' of "
"volume %(volume)s."),
{'basename': base_name, 'volume': volume_id})
# Delete base if no more snapshots
try:
self.rbd.RBD().remove(client.ioctx, base_name)
except self.rbd.ImageBusy:
# Allow a retry if the image is busy
if retries > 0:
LOG.info(_LI("Backup image of volume %(volume)s is "
"busy, retrying %(retries)s more time(s) "
"in %(delay)ss."),
{'retries': retries,
'delay': delay,
'volume': volume_id})
eventlet.sleep(delay)
else:
LOG.error(_LE("Max retries reached deleting backup "
"%(basename)s image of volume "
"%(volume)s."),
{'volume': volume_id,
'basename': base_name})
raise
else:
LOG.debug("Base backup image='%(basename)s' of volume "
"%(volume)s deleted.",
{'basename': base_name, 'volume': volume_id})
retries = 0
finally:
retries -= 1
# Since we have deleted the base image we can delete the source
# volume backup snapshot.
src_name = utils.convert_str(volume_id)
if src_name in self.rbd.RBD().list(client.ioctx):
LOG.debug("Deleting source volume snapshot '%(snapshot)s' "
"for backup %(basename)s.",
{'snapshot': snap, 'basename': base_name})
src_rbd = self.rbd.Image(client.ioctx, src_name)
try:
src_rbd.remove_snap(snap)
finally:
src_rbd.close()