def transfer(self, data):
host_src = data['host_src']
path_src = data['path_src']
host_dst = data['host_dst']
path_dst = data['path_dst']
cmd = "{bbcp} {options} {src} {dst} 2>&1"
options = CONF.bbcp.options
additional_options = []
# -f: forces the copy by first unlinking the target file before
# copying.
# -S: command to start bbcp on the source node.
# -T: command to start bbcp on the target node.
forced_options = ['-f']
if CONF.migrate.copy_with_md5_verification:
# -e: error check data for transmission errors using md5 checksum.
forced_options.append('-e')
for o in forced_options:
if o not in options:
additional_options.append(o)
run_options = ['-T']
if CONF.migrate.direct_transfer:
src = path_src
bbcp = BBCP_PATH
runner = self.runner(host_src, 'src', data.get('gateway'))
run = runner.run
else:
run_options += ['-S']
src = '{user_src}@{host_src}:{path_src}'.format(
user_src=CONF.src.ssh_user,
host_src=host_src,
path_src=path_src,
)
bbcp = CONF.bbcp.path
run = local.run
dst = '{user_dst}@{host_dst}:{path_dst}'.format(
user_dst=CONF.dst.ssh_user,
host_dst=host_dst,
path_dst=path_dst,
)
for o in run_options:
if o not in options:
additional_options.append(o + " '{remote_bbcp}'")
remote_bbcp = "ssh {ssh_opts} %I -l %U %H {path}".format(
ssh_opts=ssh_util.default_ssh_options(), path=BBCP_PATH)
options += ' ' + ' '.join(additional_options).format(
remote_bbcp=remote_bbcp)
retrier = retrying.Retry(max_attempts=CONF.migrate.retry, timeout=0)
try:
retrier.run(run,
cmd.format(bbcp=bbcp,
options=options,
src=src,
dst=dst),
capture_output=False)
except retrying.MaxAttemptsReached:
self.clean_dst(host_dst, path_dst)
raise base.FileCopyError(**data)
def retry(func, *args, **kwargs):
"""
Call function passed as first argument passing any remaining arguments
and keyword arguments. If function fail with exception, it is called
again after waiting for few seconds (specified by
``OpenstackCloud.request_attempts`` configuration parameter). It stops
retrying after ``OpenstackCloud.request_failure_sleep`` unsuccessful
attempts were made.
:param func: function, bound method or some other callable
:param expected_exceptions: tuple/list of exceptions that are expected
and handled, e.g. there is no need to retry
if such exception is caught
:param returns_iterable: set it to True if func is expected to return
iterable
:return: whatever func returns
"""
# pylint: disable=protected-access
expected_exceptions = kwargs.pop('expected_exceptions', None)
returns_iterable = kwargs.pop('returns_iterable', False)
assert isinstance(func, ClientProxy)
cloud = func._cloud
retry_obj = retrying.Retry(
max_attempts=cloud.request_attempts,
timeout=cloud.request_failure_sleep,
expected_exceptions=expected_exceptions,
reraise_original_exception=True)
if returns_iterable:
return retry_obj.run(lambda *a, **kw: [x for x in func(*a, **kw)],
*args, **kwargs)
else:
return retry_obj.run(func, *args, **kwargs)
def _dst_images(self):
dst_images = {}
keystone = self.cloud.resources["identity"]
LOG.info("Retrieving list of images from destination to make sure "
"images are not migrated twice. May take awhile, please be "
"patient.")
for dst_image in self.get_image_list():
LOG.debug("Working on destination image '%s (%s)'",
dst_image.name, dst_image.id)
retryer = retrying.Retry(
max_attempts=self.config.migrate.retry,
reraise_original_exception=True)
try:
# Destination cloud sporadically fails with Unauthorized for
# random images, thus this logic; see CF-385
tenant_name = retryer.run(
keystone.try_get_tenant_name_by_id,
dst_image.owner,
default=self.cloud.cloud_config.cloud.tenant)
image_key = (dst_image.name, tenant_name, dst_image.checksum,
dst_image.is_public)
dst_images[image_key] = dst_image
except keystone_exceptions.Unauthorized:
LOG.warning("Authorization failed in destination keystone, "
"image '%s (%s)' may be migrated twice later!")
return dst_images
def run_repeat_on_errors(self, cmd, **kwargs):
retrier = retrying.Retry(
max_attempts=CONF.migrate.retry,
reraise_original_exception=True,
timeout=0,
)
return retrier.run(self.run, cmd, **kwargs)
def _create_volume(self, src_volume):
src_keystone = self.src_cloud.resources[utils.IDENTITY_RESOURCE]
dst_keystone = self.dst_cloud.resources[utils.IDENTITY_RESOURCE]
dst_cinder = self.dst_cloud.resources[utils.STORAGE_RESOURCE]
dst_tenant = keystone.get_dst_tenant_from_src_tenant_id(
src_keystone, dst_keystone, src_volume['project_id'])
if dst_tenant is None:
msg = ("Tenant '{}' does not exist in destination, make sure "
"you migrated tenants.").format(src_volume['project_id'])
LOG.warning(msg)
raise exception.TenantNotPresentInDestination(msg)
LOG.info("Creating volume of size %sG in tenant %s in destination",
src_volume['size'], dst_tenant.name)
dst_volume = dst_cinder.create_volume_from_volume(
src_volume, dst_tenant.id)
LOG.info("Volume created: %s", volume_name(dst_volume))
# It takes time to create volume object
timeout = self.cfg.migrate.storage_backend_timeout
retryer = retrying.Retry(max_time=timeout)
dst_volume_object = retryer.run(
self.dst_cinder_backend.get_volume_object, self.dst_cloud,
dst_volume.id)
LOG.debug("Backing file for volume in destination: %s",
dst_volume_object)
return dst_volume, dst_volume_object
def migrate_volume(self, src_volume):
"""Creates volume on destination and copies volume data from source"""
LOG.info("Checking if volume '%s' already present in destination",
volume_name(src_volume))
dst_cinder = self.dst_cloud.resources[utils.STORAGE_RESOURCE]
dst_volume = dst_cinder.get_migrated_volume(src_volume['id'])
volume_exists_in_destination = (dst_volume is not None
and dst_volume.status
in ['available', 'in-use'])
if not volume_exists_in_destination:
try:
src_volume_object = self.src_cinder_backend.get_volume_object(
self.src_cloud, src_volume['id'])
LOG.debug("Backing file for source volume: %s",
src_volume_object)
dst_volume = self._create_volume(src_volume)
# It takes time to create volume object
timeout = self.cfg.migrate.storage_backend_timeout
retryer = retrying.Retry(max_time=timeout)
dst_volume_object = retryer.run(
self.dst_cinder_backend.get_volume_object, self.dst_cloud,
dst_volume.id)
LOG.debug("Backing file for volume in destination: %s",
dst_volume_object)
LOG.info("Starting volume copy from %s to %s",
src_volume_object, dst_volume_object)
self.copy_volume_data(src_volume_object, dst_volume_object)
except (plugins.base.VolumeObjectNotFoundError,
retrying.TimeoutExceeded,
exception.TenantNotPresentInDestination,
cinder_exceptions.OverLimit,
copy_mechanisms.CopyFailed) as e:
LOG.warning("%(error)s, volume %(name)s will be skipped", {
'error': e.message,
'name': volume_name(src_volume)
})
if dst_volume is not None:
msg = ("Removing volume {name} from destination "
"since it didn't migrate properly".format(
name=volume_name(dst_volume)))
LOG.info(msg)
self.delete_volume(dst_volume)
finally:
if dst_volume is not None:
self.dst_cinder_backend.cleanup(self.cloud, dst_volume.id)
else:
LOG.info(
"Volume '%s' is already present in destination cloud, "
"skipping", src_volume['id'])
return dst_volume
def _retrying(self, func, *args, **kwargs):
retry = retrying.Retry(
max_attempts=self.settings.connection_attempts,
timeout=self.settings.attempt_failure_sleep,
reraise_original_exception=True,
expected_exceptions=(paramiko.BadHostKeyException,
paramiko.AuthenticationException,
RemoteFailure))
return retry.run(func, *args, **kwargs)
def run_scp(self, host_src, path_src, host_dst, path_dst, gateway=None):
"""
Copy a file from source host to destination by scp.
:param host_src: Source host.
:param path_src: Path to a file on source host.
:param host_dst: Destination host.
:param path_dst: Path to a file on destination host.
:raise: FileCopyError in case the file was not copied by any reasons.
"""
cmd = "scp {ssh_opts} {src} {dst}"
if CONF.migrate.direct_transfer:
src = path_src
runner = self.runner(host_src, 'src', gateway)
run = runner.run
else:
# -3: Copies between two remote hosts are transferred through
# the local host.
src = '-3 {src_user}@{src_host}:{src_path}'.format(
src_user=CONF.src.ssh_user,
src_host=host_src,
src_path=path_src,
)
run = local.run
dst = '{dst_user}@{dst_host}:{dst_path}'.format(
dst_user=CONF.dst.ssh_user,
dst_host=host_dst,
dst_path=path_dst,
)
kwargs = {
'host_src': host_src,
'path_src': path_src,
'host_dst': host_dst,
'path_dst': path_dst,
'gateway': gateway
}
retrier = retrying.Retry(
max_attempts=CONF.migrate.retry,
predicate=self.verify,
predicate_kwargs=kwargs,
)
LOG.info("Copying file '%s' to '%s'", path_src, host_dst)
try:
retrier.run(run,
cmd.format(ssh_opts=ssh_util.default_ssh_options(),
src=src,
dst=dst),
capture_output=False)
except retrying.MaxAttemptsReached:
self.clean_dst(host_dst, path_dst)
raise base.FileCopyError(host_src=host_src,
path_src=path_src,
host_dst=host_dst,
path_dst=path_dst)
def test_raises_expected_exception(self, sleep_mock):
retry = retrying.Retry(max_attempts=10, expected_exceptions=[KeyError])
@retry
def func():
raise KeyError()
self.assertRaises(KeyError, func)
self.assertEqual(retry.attempt, 1)
self.assertFalse(sleep_mock.called)
def test_raises_error_if_predicate_failed_after_timeout(self, sleep_mock):
def func():
return 0
retry = retrying.Retry(max_time=100,
predicate_retval_as_arg=True,
predicate=lambda rv: rv == 1)
self.assertRaises(retrying.TimeoutExceeded, retry.run, func)
self.assertTrue(retry.total_time >= retry.max_time)
self.assertTrue(sleep_mock.called)
def test_stops_if_retval_matches_predicate(self, sleep_mock):
def func():
return 0
retry = retrying.Retry(max_attempts=5,
predicate_retval_as_arg=True,
predicate=lambda rv: rv == 0)
retry.run(func)
self.assertEqual(retry.attempt, 1)
self.assertFalse(sleep_mock.called)
def test_raises_last_error_if_all_attempts_failed(self, sleep_mock):
retry = retrying.Retry(max_attempts=5,
wait_exponential=False,
reraise_original_exception=True)
@retry
def func():
raise ValueError()
self.assertRaises(ValueError, func)
self.assertEqual(retry.attempt, retry.max_attempts)
self.assertEqual(sleep_mock.call_count, retry.max_attempts)
def test_raises_timeout_error_if_timedout(self, sleep_mock):
retry = retrying.Retry(max_time=100,
wait_exponential=True,
reraise_original_exception=False)
@retry
def func():
raise RuntimeError()
self.assertRaises(retrying.TimeoutExceeded, func)
self.assertTrue(retry.total_time >= retry.max_time)
self.assertTrue(sleep_mock.called)
def wait_for_instance_to_be_deleted(nv_client, instance):
retryer = retrying.Retry(max_time=300,
retry_on_return_value=True,
return_value=instance,
expected_exceptions=[nova_exc.NotFound],
retry_message="Instance still exists")
try:
with proxy_client.expect_exception(nova_exc.NotFound):
retryer.run(nv_client.nova_client.servers.get, instance.id)
except nova_exc.NotFound:
LOG.info("Instance '%s' has been successfully deleted.",
instance.name)
def test_retries_until_timed_out(self, sleep_mock):
retry = retrying.Retry(max_time=100,
wait_exponential=True,
raise_error=False)
@retry
def func():
raise RuntimeError()
func()
self.assertTrue(retry.total_time >= retry.max_time)
self.assertTrue(sleep_mock.called)
def test_returns_object_returned_by_function(self, sleep_mock):
expected_rv = 0
def func():
return expected_rv
retry = retrying.Retry(max_attempts=10,
predicate_retval_as_arg=True,
predicate=lambda rv: rv == expected_rv)
actual_rv = retry.run(func)
self.assertEqual(1, retry.attempt)
self.assertEqual(expected_rv, actual_rv)
self.assertFalse(sleep_mock.called)
def create_volume_from_volume(self, volume, tenant_id):
"""Creates volume based on values from :param volume: and adds
metadata in order to not copy already copied volumes
:param volume: CF volume object (dict)
:raises: retrying.TimeoutExceeded if volume did not become available
in migrate.storage_backend_timeout time
"""
glance = self.cloud.resources[utils.IMAGE_RESOURCE]
compute = self.cloud.resources[utils.COMPUTE_RESOURCE]
az_mapper = compute.attr_override
metadata = volume.get('metadata', {})
metadata[MIGRATED_VOLUMES_METADATA_KEY] = volume['id']
image_id = None
if volume['bootable']:
image_metadata = volume['volume_image_metadata']
dst_image = glance.get_matching_image(
uuid=image_metadata['image_id'],
size=image_metadata['size'],
name=image_metadata['image_name'],
checksum=image_metadata['checksum'])
if dst_image:
image_id = dst_image.id
src_az = compute.get_availability_zone(volume['availability_zone'])
created_volume = self.create_volume(
size=volume['size'],
project_id=tenant_id,
display_name=volume['display_name'],
display_description=volume['display_description'],
availability_zone=src_az
or az_mapper.get_attr(volume, 'availability_zone'),
metadata=metadata,
imageRef=image_id)
timeout = self.config.migrate.storage_backend_timeout
retryer = retrying.Retry(max_time=timeout,
predicate=lambda v: v.status == 'available',
predicate_retval_as_arg=True,
retry_message="Volume is not available")
retryer.run(self.get_volume_by_id, created_volume.id)
return created_volume
def test_retries_on_invalid_return_value(self, sleep_mock):
bad_value = 10
retry = retrying.Retry(max_attempts=5,
retry_on_return_value=True,
return_value=bad_value,
raise_error=False)
@retry
def func():
return bad_value
func()
self.assertEqual(retry.attempt, retry.max_attempts)
self.assertEqual(sleep_mock.call_count, retry.max_attempts)
def get_client(self):
""" Getting keystone client using authentication with admin auth token.
:return: OpenStack Keystone Client instance
"""
def func():
auth_ref = self.get_client_by_creds().auth_ref
return keystone_client.Client(auth_ref=auth_ref,
endpoint=self.config.cloud.auth_url,
cacert=self.config.cloud.cacert,
insecure=self.config.cloud.insecure)
retrier = retrying.Retry(
max_attempts=cfglib.CONF.migrate.retry,
expected_exceptions=[ks_exceptions.Unauthorized],
reraise_original_exception=True)
return retrier.run(func)
def test_does_not_retry_if_predicate_succeeds(self, sleep_mock):
def always_succeeds():
return True
retry = retrying.Retry(max_attempts=5,
wait_exponential=True,
raise_error=False,
predicate=always_succeeds)
@retry
def func():
pass
func()
self.assertEqual(retry.attempt, 1)
self.assertFalse(sleep_mock.called)
def test_retries_if_predicate_fails(self, sleep_mock):
def always_fail():
return False
retry = retrying.Retry(max_attempts=5,
wait_exponential=True,
raise_error=False,
predicate=always_fail)
@retry
def func():
pass
func()
self.assertTrue(retry.attempt >= retry.max_attempts)
self.assertEqual(sleep_mock.call_count, retry.max_attempts)
