def wait_for_allow_access(self, access_ref):
starttime = time.time()
deadline = starttime + self.migration_wait_access_rules_timeout
tries = 0
rule = self.api.access_get(self.context, access_ref['id'])
while rule['state'] != constants.STATUS_ACTIVE:
tries += 1
now = time.time()
if rule['state'] == constants.STATUS_ERROR:
msg = _("Failed to allow access"
" on share %s") % self.share['id']
raise exception.ShareMigrationFailed(reason=msg)
elif now > deadline:
msg = _("Timeout trying to allow access"
" on share %(share_id)s. Timeout "
"was %(timeout)s seconds.") % {
'share_id': self.share['id'],
'timeout': self.migration_wait_access_rules_timeout
}
raise exception.ShareMigrationFailed(reason=msg)
else:
time.sleep(tries**2)
rule = self.api.access_get(self.context, access_ref['id'])
return rule
def wait_for_access_update(context, db, share_instance,
migration_wait_access_rules_timeout):
starttime = time.time()
deadline = starttime + migration_wait_access_rules_timeout
tries = 0
while True:
instance = db.share_instance_get(context, share_instance['id'])
if instance['access_rules_status'] == constants.STATUS_ACTIVE:
break
tries += 1
now = time.time()
if (instance['access_rules_status'] ==
constants.SHARE_INSTANCE_RULES_ERROR):
msg = _("Failed to update access rules"
" on share instance %s") % share_instance['id']
raise exception.ShareMigrationFailed(reason=msg)
elif now > deadline:
msg = _("Timeout trying to update access rules"
" on share instance %(share_id)s. Timeout "
"was %(timeout)s seconds.") % {
'share_id': share_instance['id'],
'timeout': migration_wait_access_rules_timeout
}
raise exception.ShareMigrationFailed(reason=msg)
else:
# 1.414 = square-root of 2
time.sleep(1.414**tries)
def wait_for_access_update(self, share_instance):
starttime = time.time()
deadline = starttime + self.migration_wait_access_rules_timeout
tries = 0
while True:
instance = self.db.share_instance_get(self.context,
share_instance['id'])
if instance['access_rules_status'] == constants.STATUS_ACTIVE:
break
tries += 1
now = time.time()
if instance['access_rules_status'] == constants.STATUS_ERROR:
msg = _("Failed to update access rules"
" on share instance %s") % share_instance['id']
raise exception.ShareMigrationFailed(reason=msg)
elif now > deadline:
msg = _("Timeout trying to update access rules"
" on share instance %(share_id)s. Timeout "
"was %(timeout)s seconds.") % {
'share_id': share_instance['id'],
'timeout': self.migration_wait_access_rules_timeout
}
raise exception.ShareMigrationFailed(reason=msg)
else:
time.sleep(tries**2)
def create_instance_and_wait(self, share, share_instance, host):
new_share_instance = self.api.create_instance(
self.context, share, share_instance['share_network_id'],
host['host'])
# Wait for new_share_instance to become ready
starttime = time.time()
deadline = starttime + self.migration_create_delete_share_timeout
new_share_instance = self.db.share_instance_get(
self.context, new_share_instance['id'], with_share_data=True)
tries = 0
while new_share_instance['status'] != constants.STATUS_AVAILABLE:
tries += 1
now = time.time()
if new_share_instance['status'] == constants.STATUS_ERROR:
msg = _("Failed to create new share instance"
" (from %(share_id)s) on "
"destination host %(host_name)s") % {
'share_id': share['id'], 'host_name': host['host']}
self.cleanup_new_instance(new_share_instance)
raise exception.ShareMigrationFailed(reason=msg)
elif now > deadline:
msg = _("Timeout creating new share instance "
"(from %(share_id)s) on "
"destination host %(host_name)s") % {
'share_id': share['id'], 'host_name': host['host']}
self.cleanup_new_instance(new_share_instance)
raise exception.ShareMigrationFailed(reason=msg)
else:
time.sleep(tries ** 2)
new_share_instance = self.db.share_instance_get(
self.context, new_share_instance['id'], with_share_data=True)
return new_share_instance
def deny_rules_and_wait(self, context, share, saved_rules):
api = share_api.API()
# Deny each one.
for instance in share.instances:
for access in saved_rules:
api.deny_access_to_instance(context, instance, access)
# Wait for all rules to be cleared.
starttime = time.time()
deadline = starttime + self.migration_wait_access_rules_timeout
tries = 0
rules = self.db.share_access_get_all_for_share(context, share['id'])
while len(rules) > 0:
tries += 1
now = time.time()
if now > deadline:
msg = _("Timeout removing access rules from share "
"%(share_id)s. Timeout was %(timeout)s seconds.") % {
'share_id': share['id'],
'timeout': self.migration_wait_access_rules_timeout
}
raise exception.ShareMigrationFailed(reason=msg)
else:
time.sleep(tries**2)
rules = self.db.share_access_get_all_for_share(
context, share['id'])
def test_copy_share_data_remote_access_exception(self):
fake_share = db_utils.create_share(
id='fakeid', status=constants.STATUS_AVAILABLE, host='fake_host')
fake_share_instance = {'id': 'fake_id', 'host': 'fake_host'}
share_driver = self._setup_mocks_copy_share_data()
remote = {'access': {'access_to': '192.168.0.1'},
'mount': 'fake_mount',
'umount': 'fake_umount'}
local = {'access': {'access_to': '192.168.1.1'},
'mount': 'fake_mount',
'umount': 'fake_umount'}
helper = migration.ShareMigrationHelper(None, None, None, None, None)
driver.CONF.set_default('migration_tmp_location', '/fake/path')
driver.CONF.set_default('migration_ignore_files', None)
self.mock_object(migration.ShareMigrationHelper,
'deny_migration_access')
self.mock_object(
migration.ShareMigrationHelper,
'allow_migration_access',
mock.Mock(side_effect=[None,
exception.ShareMigrationFailed(
reason='fake')]))
self.mock_object(migration.ShareMigrationHelper,
'cleanup_migration_access')
self.assertRaises(exception.ShareMigrationFailed,
share_driver.copy_share_data, 'ctx', helper,
fake_share, fake_share_instance, None,
fake_share_instance, None, local, remote)
args = ((None, local['access'], False),
(None, remote['access'], False))
migration.ShareMigrationHelper.deny_migration_access.assert_has_calls(
[mock.call(*a) for a in args])
def test_copy_share_data_mount_for_migration_exception2(self):
fake_share = db_utils.create_share(id='fakeid',
status=constants.STATUS_AVAILABLE,
host='fake_host')
fake_share_instance = {'id': 'fake_id', 'host': 'fake_host'}
share_driver = self._setup_mocks_copy_share_data()
remote = {
'access': {
'access_to': '192.168.0.1'
},
'mount': 'fake_mount',
'umount': 'fake_umount'
}
local = {
'access': {
'access_to': '192.168.1.1'
},
'mount': 'fake_mount',
'umount': 'fake_umount'
}
helper = migration.ShareMigrationHelper(None, None, None, None, None)
msg = ('Failed to mount temporary folder for migration of share '
'instance fake_id to fake_id')
driver.CONF.set_default('migration_tmp_location', '/fake/path')
self.mock_object(migration.ShareMigrationHelper,
'deny_migration_access')
self.mock_object(migration.ShareMigrationHelper,
'allow_migration_access',
mock.Mock(return_value='fake_access_ref'))
self.mock_object(migration.ShareMigrationHelper,
'cleanup_migration_access')
self.mock_object(migration.ShareMigrationHelper, 'cleanup_temp_folder')
self.mock_object(migration.ShareMigrationHelper,
'cleanup_unmount_temp_folder')
self.mock_object(
utils, 'execute',
mock.Mock(side_effect=[
None, None, None,
exception.ShareMigrationFailed(msg)
]))
self.assertRaises(exception.ShareMigrationFailed,
share_driver.copy_share_data, 'ctx', helper,
fake_share, fake_share_instance, None,
fake_share_instance, None, local, remote)
args = ((None, local['access'], False), (None, remote['access'],
False))
migration.ShareMigrationHelper.deny_migration_access.assert_has_calls(
[mock.call(*a) for a in args])
def add_rules_and_wait(self,
context,
share,
saved_rules,
access_level=None):
for access in saved_rules:
if access_level:
level = access_level
else:
level = access['access_level']
self.api.allow_access(context, share, access['access_type'],
access['access_to'], level)
starttime = time.time()
deadline = starttime + self.migration_wait_access_rules_timeout
rules_added = False
tries = 0
rules = self.db.share_access_get_all_for_share(context, share['id'])
while not rules_added:
rules_added = True
tries += 1
now = time.time()
for access in rules:
if access['state'] != constants.STATUS_ACTIVE:
rules_added = False
if rules_added:
break
if now > deadline:
msg = _("Timeout adding access rules for share "
"%(share_id)s. Timeout was %(timeout)s seconds.") % {
'share_id': share['id'],
'timeout': self.migration_wait_access_rules_timeout
}
raise exception.ShareMigrationFailed(reason=msg)
else:
time.sleep(tries**2)
rules = self.db.share_access_get_all_for_share(
context, share['id'])
def delete_instance_and_wait(self, context, share_instance):
self.api.delete_instance(context, share_instance, True)
# Wait for deletion.
starttime = time.time()
deadline = starttime + self.migration_create_delete_share_timeout
tries = -1
instance = "Something not None"
while instance:
try:
instance = self.db.share_instance_get(context,
share_instance['id'])
tries += 1
now = time.time()
if now > deadline:
msg = _("Timeout trying to delete instance "
"%s") % share_instance['id']
raise exception.ShareMigrationFailed(reason=msg)
except exception.NotFound:
instance = None
else:
time.sleep(tries**2)
def wait_for_deny_access(self, access_ref):
starttime = time.time()
deadline = starttime + self.migration_wait_access_rules_timeout
tries = -1
rule = "Something not None"
while rule:
try:
rule = self.api.access_get(self.context, access_ref['id'])
tries += 1
now = time.time()
if now > deadline:
msg = _("Timeout trying to deny access"
" on share %(share_id)s. Timeout "
"was %(timeout)s seconds.") % {
'share_id': self.share['id'],
'timeout':
self.migration_wait_access_rules_timeout
}
raise exception.ShareMigrationFailed(reason=msg)
except exception.NotFound:
rule = None
else:
time.sleep(tries**2)
def copy_share_data(self, context, helper, share, share_instance,
share_server, new_share_instance, new_share_server,
migration_info_src, migration_info_dest):
# NOTE(ganso): This method is here because it is debatable if it can
# be overridden by a driver or not. Personally I think it should not,
# else it would be possible to lose compatibility with generic
# migration between backends, but allows the driver to use it on its
# own implementation if it wants to.
migrated = False
mount_path = self.configuration.safe_get('migration_tmp_location')
src_access = migration_info_src['access']
dest_access = migration_info_dest['access']
if None in (src_access['access_to'], dest_access['access_to']):
msg = _("Access rules not appropriate for mounting share instances"
" for migration of share %(share_id)s,"
" source share access: %(src_ip)s, destination share"
" access: %(dest_ip)s. Aborting.") % {
'src_ip': src_access['access_to'],
'dest_ip': dest_access['access_to'],
'share_id': share['id']
}
raise exception.ShareMigrationFailed(reason=msg)
# NOTE(ganso): Removing any previously conflicting access rules, which
# would cause the following access_allow to fail for one instance.
helper.deny_migration_access(None, src_access, False)
helper.deny_migration_access(None, dest_access, False)
# NOTE(ganso): I would rather allow access to instances separately,
# but I require an access_id since it is a new access rule and
# destination manager must receive an access_id. I can either move
# this code to manager code so I can create the rule in DB manually,
# or ignore duplicate access rule errors for some specific scenarios.
try:
src_access_ref = helper.allow_migration_access(src_access)
except Exception as e:
LOG.error(
_LE("Share migration failed attempting to allow "
"access of %(access_to)s to share "
"instance %(instance_id)s.") % {
'access_to': src_access['access_to'],
'instance_id': share_instance['id']
})
msg = six.text_type(e)
LOG.exception(msg)
raise exception.ShareMigrationFailed(reason=msg)
try:
dest_access_ref = helper.allow_migration_access(dest_access)
except Exception as e:
LOG.error(
_LE("Share migration failed attempting to allow "
"access of %(access_to)s to share "
"instance %(instance_id)s.") % {
'access_to': dest_access['access_to'],
'instance_id': new_share_instance['id']
})
msg = six.text_type(e)
LOG.exception(msg)
helper.cleanup_migration_access(src_access_ref, src_access)
raise exception.ShareMigrationFailed(reason=msg)
# NOTE(ganso): From here we have the possibility of not cleaning
# anything when facing an error. At this moment, we have the
# destination instance in "inactive" state, while we are performing
# operations on the source instance. I think it is best to not clean
# the instance, leave it in "inactive" state, but try to clean
# temporary access rules, mounts, folders, etc, since no additional
# harm is done.
def _mount_for_migration(migration_info):
try:
utils.execute(*migration_info['mount'], run_as_root=True)
except Exception:
LOG.error(
_LE("Failed to mount temporary folder for "
"migration of share instance "
"%(share_instance_id)s "
"to %(new_share_instance_id)s") % {
'share_instance_id': share_instance['id'],
'new_share_instance_id': new_share_instance['id']
})
helper.cleanup_migration_access(src_access_ref, src_access)
helper.cleanup_migration_access(dest_access_ref, dest_access)
raise
utils.execute('mkdir', '-p', ''.join(
(mount_path, share_instance['id'])))
utils.execute('mkdir', '-p', ''.join(
(mount_path, new_share_instance['id'])))
# NOTE(ganso): mkdir command sometimes returns faster than it
# actually runs, so we better sleep for 1 second.
time.sleep(1)
try:
_mount_for_migration(migration_info_src)
except Exception as e:
LOG.error(
_LE("Share migration failed attempting to mount "
"share instance %s.") % share_instance['id'])
msg = six.text_type(e)
LOG.exception(msg)
helper.cleanup_temp_folder(share_instance, mount_path)
helper.cleanup_temp_folder(new_share_instance, mount_path)
raise exception.ShareMigrationFailed(reason=msg)
try:
_mount_for_migration(migration_info_dest)
except Exception as e:
LOG.error(
_LE("Share migration failed attempting to mount "
"share instance %s.") % new_share_instance['id'])
msg = six.text_type(e)
LOG.exception(msg)
helper.cleanup_unmount_temp_folder(share_instance,
migration_info_src)
helper.cleanup_temp_folder(share_instance, mount_path)
helper.cleanup_temp_folder(new_share_instance, mount_path)
raise exception.ShareMigrationFailed(reason=msg)
try:
ignore_list = self.configuration.safe_get('migration_ignore_files')
copy = share_utils.Copy(mount_path + share_instance['id'],
mount_path + new_share_instance['id'],
ignore_list)
copy.run()
if copy.get_progress()['total_progress'] == 100:
migrated = True
except Exception as e:
LOG.exception(six.text_type(e))
LOG.error(
_LE("Failed to copy files for "
"migration of share instance %(share_instance_id)s "
"to %(new_share_instance_id)s") % {
'share_instance_id': share_instance['id'],
'new_share_instance_id': new_share_instance['id']
})
# NOTE(ganso): For some reason I frequently get AMQP errors after
# copying finishes, which seems like is the service taking too long to
# copy while not replying heartbeat messages, so AMQP closes the
# socket. There is no impact, it just shows a big trace and AMQP
# reconnects after, although I would like to prevent this situation
# without the use of additional threads. Suggestions welcome.
utils.execute(*migration_info_src['umount'], run_as_root=True)
utils.execute(*migration_info_dest['umount'], run_as_root=True)
utils.execute('rmdir',
''.join((mount_path, share_instance['id'])),
check_exit_code=False)
utils.execute('rmdir',
''.join((mount_path, new_share_instance['id'])),
check_exit_code=False)
helper.deny_migration_access(src_access_ref, src_access)
helper.deny_migration_access(dest_access_ref, dest_access)
if not migrated:
msg = ("Copying from share instance %(instance_id)s "
"to %(new_instance_id)s did not succeed." % {
'instance_id': share_instance['id'],
'new_instance_id': new_share_instance['id']
})
raise exception.ShareMigrationFailed(reason=msg)
LOG.debug("Copying completed in migration for share %s." % share['id'])
