def wait_for_allow_access(self, access_ref): starttime = time.time() deadline = starttime + self.migration_wait_access_rules_timeout tries = 0 rule = self.api.access_get(self.context, access_ref['id']) while rule['state'] != constants.STATUS_ACTIVE: tries += 1 now = time.time() if rule['state'] == constants.STATUS_ERROR: msg = _("Failed to allow access" " on share %s") % self.share['id'] raise exception.ShareMigrationFailed(reason=msg) elif now > deadline: msg = _("Timeout trying to allow access" " on share %(share_id)s. Timeout " "was %(timeout)s seconds.") % { 'share_id': self.share['id'], 'timeout': self.migration_wait_access_rules_timeout } raise exception.ShareMigrationFailed(reason=msg) else: time.sleep(tries**2) rule = self.api.access_get(self.context, access_ref['id']) return rule
def wait_for_access_update(context, db, share_instance, migration_wait_access_rules_timeout): starttime = time.time() deadline = starttime + migration_wait_access_rules_timeout tries = 0 while True: instance = db.share_instance_get(context, share_instance['id']) if instance['access_rules_status'] == constants.STATUS_ACTIVE: break tries += 1 now = time.time() if (instance['access_rules_status'] == constants.SHARE_INSTANCE_RULES_ERROR): msg = _("Failed to update access rules" " on share instance %s") % share_instance['id'] raise exception.ShareMigrationFailed(reason=msg) elif now > deadline: msg = _("Timeout trying to update access rules" " on share instance %(share_id)s. Timeout " "was %(timeout)s seconds.") % { 'share_id': share_instance['id'], 'timeout': migration_wait_access_rules_timeout } raise exception.ShareMigrationFailed(reason=msg) else: # 1.414 = square-root of 2 time.sleep(1.414**tries)
def wait_for_access_update(self, share_instance): starttime = time.time() deadline = starttime + self.migration_wait_access_rules_timeout tries = 0 while True: instance = self.db.share_instance_get(self.context, share_instance['id']) if instance['access_rules_status'] == constants.STATUS_ACTIVE: break tries += 1 now = time.time() if instance['access_rules_status'] == constants.STATUS_ERROR: msg = _("Failed to update access rules" " on share instance %s") % share_instance['id'] raise exception.ShareMigrationFailed(reason=msg) elif now > deadline: msg = _("Timeout trying to update access rules" " on share instance %(share_id)s. Timeout " "was %(timeout)s seconds.") % { 'share_id': share_instance['id'], 'timeout': self.migration_wait_access_rules_timeout } raise exception.ShareMigrationFailed(reason=msg) else: time.sleep(tries**2)
def create_instance_and_wait(self, share, share_instance, host): new_share_instance = self.api.create_instance( self.context, share, share_instance['share_network_id'], host['host']) # Wait for new_share_instance to become ready starttime = time.time() deadline = starttime + self.migration_create_delete_share_timeout new_share_instance = self.db.share_instance_get( self.context, new_share_instance['id'], with_share_data=True) tries = 0 while new_share_instance['status'] != constants.STATUS_AVAILABLE: tries += 1 now = time.time() if new_share_instance['status'] == constants.STATUS_ERROR: msg = _("Failed to create new share instance" " (from %(share_id)s) on " "destination host %(host_name)s") % { 'share_id': share['id'], 'host_name': host['host']} self.cleanup_new_instance(new_share_instance) raise exception.ShareMigrationFailed(reason=msg) elif now > deadline: msg = _("Timeout creating new share instance " "(from %(share_id)s) on " "destination host %(host_name)s") % { 'share_id': share['id'], 'host_name': host['host']} self.cleanup_new_instance(new_share_instance) raise exception.ShareMigrationFailed(reason=msg) else: time.sleep(tries ** 2) new_share_instance = self.db.share_instance_get( self.context, new_share_instance['id'], with_share_data=True) return new_share_instance
def deny_rules_and_wait(self, context, share, saved_rules): api = share_api.API() # Deny each one. for instance in share.instances: for access in saved_rules: api.deny_access_to_instance(context, instance, access) # Wait for all rules to be cleared. starttime = time.time() deadline = starttime + self.migration_wait_access_rules_timeout tries = 0 rules = self.db.share_access_get_all_for_share(context, share['id']) while len(rules) > 0: tries += 1 now = time.time() if now > deadline: msg = _("Timeout removing access rules from share " "%(share_id)s. Timeout was %(timeout)s seconds.") % { 'share_id': share['id'], 'timeout': self.migration_wait_access_rules_timeout } raise exception.ShareMigrationFailed(reason=msg) else: time.sleep(tries**2) rules = self.db.share_access_get_all_for_share( context, share['id'])
def test_copy_share_data_remote_access_exception(self): fake_share = db_utils.create_share( id='fakeid', status=constants.STATUS_AVAILABLE, host='fake_host') fake_share_instance = {'id': 'fake_id', 'host': 'fake_host'} share_driver = self._setup_mocks_copy_share_data() remote = {'access': {'access_to': '192.168.0.1'}, 'mount': 'fake_mount', 'umount': 'fake_umount'} local = {'access': {'access_to': '192.168.1.1'}, 'mount': 'fake_mount', 'umount': 'fake_umount'} helper = migration.ShareMigrationHelper(None, None, None, None, None) driver.CONF.set_default('migration_tmp_location', '/fake/path') driver.CONF.set_default('migration_ignore_files', None) self.mock_object(migration.ShareMigrationHelper, 'deny_migration_access') self.mock_object( migration.ShareMigrationHelper, 'allow_migration_access', mock.Mock(side_effect=[None, exception.ShareMigrationFailed( reason='fake')])) self.mock_object(migration.ShareMigrationHelper, 'cleanup_migration_access') self.assertRaises(exception.ShareMigrationFailed, share_driver.copy_share_data, 'ctx', helper, fake_share, fake_share_instance, None, fake_share_instance, None, local, remote) args = ((None, local['access'], False), (None, remote['access'], False)) migration.ShareMigrationHelper.deny_migration_access.assert_has_calls( [mock.call(*a) for a in args])
def test_copy_share_data_mount_for_migration_exception2(self): fake_share = db_utils.create_share(id='fakeid', status=constants.STATUS_AVAILABLE, host='fake_host') fake_share_instance = {'id': 'fake_id', 'host': 'fake_host'} share_driver = self._setup_mocks_copy_share_data() remote = { 'access': { 'access_to': '192.168.0.1' }, 'mount': 'fake_mount', 'umount': 'fake_umount' } local = { 'access': { 'access_to': '192.168.1.1' }, 'mount': 'fake_mount', 'umount': 'fake_umount' } helper = migration.ShareMigrationHelper(None, None, None, None, None) msg = ('Failed to mount temporary folder for migration of share ' 'instance fake_id to fake_id') driver.CONF.set_default('migration_tmp_location', '/fake/path') self.mock_object(migration.ShareMigrationHelper, 'deny_migration_access') self.mock_object(migration.ShareMigrationHelper, 'allow_migration_access', mock.Mock(return_value='fake_access_ref')) self.mock_object(migration.ShareMigrationHelper, 'cleanup_migration_access') self.mock_object(migration.ShareMigrationHelper, 'cleanup_temp_folder') self.mock_object(migration.ShareMigrationHelper, 'cleanup_unmount_temp_folder') self.mock_object( utils, 'execute', mock.Mock(side_effect=[ None, None, None, exception.ShareMigrationFailed(msg) ])) self.assertRaises(exception.ShareMigrationFailed, share_driver.copy_share_data, 'ctx', helper, fake_share, fake_share_instance, None, fake_share_instance, None, local, remote) args = ((None, local['access'], False), (None, remote['access'], False)) migration.ShareMigrationHelper.deny_migration_access.assert_has_calls( [mock.call(*a) for a in args])
def add_rules_and_wait(self, context, share, saved_rules, access_level=None): for access in saved_rules: if access_level: level = access_level else: level = access['access_level'] self.api.allow_access(context, share, access['access_type'], access['access_to'], level) starttime = time.time() deadline = starttime + self.migration_wait_access_rules_timeout rules_added = False tries = 0 rules = self.db.share_access_get_all_for_share(context, share['id']) while not rules_added: rules_added = True tries += 1 now = time.time() for access in rules: if access['state'] != constants.STATUS_ACTIVE: rules_added = False if rules_added: break if now > deadline: msg = _("Timeout adding access rules for share " "%(share_id)s. Timeout was %(timeout)s seconds.") % { 'share_id': share['id'], 'timeout': self.migration_wait_access_rules_timeout } raise exception.ShareMigrationFailed(reason=msg) else: time.sleep(tries**2) rules = self.db.share_access_get_all_for_share( context, share['id'])
def delete_instance_and_wait(self, context, share_instance): self.api.delete_instance(context, share_instance, True) # Wait for deletion. starttime = time.time() deadline = starttime + self.migration_create_delete_share_timeout tries = -1 instance = "Something not None" while instance: try: instance = self.db.share_instance_get(context, share_instance['id']) tries += 1 now = time.time() if now > deadline: msg = _("Timeout trying to delete instance " "%s") % share_instance['id'] raise exception.ShareMigrationFailed(reason=msg) except exception.NotFound: instance = None else: time.sleep(tries**2)
def wait_for_deny_access(self, access_ref): starttime = time.time() deadline = starttime + self.migration_wait_access_rules_timeout tries = -1 rule = "Something not None" while rule: try: rule = self.api.access_get(self.context, access_ref['id']) tries += 1 now = time.time() if now > deadline: msg = _("Timeout trying to deny access" " on share %(share_id)s. Timeout " "was %(timeout)s seconds.") % { 'share_id': self.share['id'], 'timeout': self.migration_wait_access_rules_timeout } raise exception.ShareMigrationFailed(reason=msg) except exception.NotFound: rule = None else: time.sleep(tries**2)
def copy_share_data(self, context, helper, share, share_instance, share_server, new_share_instance, new_share_server, migration_info_src, migration_info_dest): # NOTE(ganso): This method is here because it is debatable if it can # be overridden by a driver or not. Personally I think it should not, # else it would be possible to lose compatibility with generic # migration between backends, but allows the driver to use it on its # own implementation if it wants to. migrated = False mount_path = self.configuration.safe_get('migration_tmp_location') src_access = migration_info_src['access'] dest_access = migration_info_dest['access'] if None in (src_access['access_to'], dest_access['access_to']): msg = _("Access rules not appropriate for mounting share instances" " for migration of share %(share_id)s," " source share access: %(src_ip)s, destination share" " access: %(dest_ip)s. Aborting.") % { 'src_ip': src_access['access_to'], 'dest_ip': dest_access['access_to'], 'share_id': share['id'] } raise exception.ShareMigrationFailed(reason=msg) # NOTE(ganso): Removing any previously conflicting access rules, which # would cause the following access_allow to fail for one instance. helper.deny_migration_access(None, src_access, False) helper.deny_migration_access(None, dest_access, False) # NOTE(ganso): I would rather allow access to instances separately, # but I require an access_id since it is a new access rule and # destination manager must receive an access_id. I can either move # this code to manager code so I can create the rule in DB manually, # or ignore duplicate access rule errors for some specific scenarios. try: src_access_ref = helper.allow_migration_access(src_access) except Exception as e: LOG.error( _LE("Share migration failed attempting to allow " "access of %(access_to)s to share " "instance %(instance_id)s.") % { 'access_to': src_access['access_to'], 'instance_id': share_instance['id'] }) msg = six.text_type(e) LOG.exception(msg) raise exception.ShareMigrationFailed(reason=msg) try: dest_access_ref = helper.allow_migration_access(dest_access) except Exception as e: LOG.error( _LE("Share migration failed attempting to allow " "access of %(access_to)s to share " "instance %(instance_id)s.") % { 'access_to': dest_access['access_to'], 'instance_id': new_share_instance['id'] }) msg = six.text_type(e) LOG.exception(msg) helper.cleanup_migration_access(src_access_ref, src_access) raise exception.ShareMigrationFailed(reason=msg) # NOTE(ganso): From here we have the possibility of not cleaning # anything when facing an error. At this moment, we have the # destination instance in "inactive" state, while we are performing # operations on the source instance. I think it is best to not clean # the instance, leave it in "inactive" state, but try to clean # temporary access rules, mounts, folders, etc, since no additional # harm is done. def _mount_for_migration(migration_info): try: utils.execute(*migration_info['mount'], run_as_root=True) except Exception: LOG.error( _LE("Failed to mount temporary folder for " "migration of share instance " "%(share_instance_id)s " "to %(new_share_instance_id)s") % { 'share_instance_id': share_instance['id'], 'new_share_instance_id': new_share_instance['id'] }) helper.cleanup_migration_access(src_access_ref, src_access) helper.cleanup_migration_access(dest_access_ref, dest_access) raise utils.execute('mkdir', '-p', ''.join( (mount_path, share_instance['id']))) utils.execute('mkdir', '-p', ''.join( (mount_path, new_share_instance['id']))) # NOTE(ganso): mkdir command sometimes returns faster than it # actually runs, so we better sleep for 1 second. time.sleep(1) try: _mount_for_migration(migration_info_src) except Exception as e: LOG.error( _LE("Share migration failed attempting to mount " "share instance %s.") % share_instance['id']) msg = six.text_type(e) LOG.exception(msg) helper.cleanup_temp_folder(share_instance, mount_path) helper.cleanup_temp_folder(new_share_instance, mount_path) raise exception.ShareMigrationFailed(reason=msg) try: _mount_for_migration(migration_info_dest) except Exception as e: LOG.error( _LE("Share migration failed attempting to mount " "share instance %s.") % new_share_instance['id']) msg = six.text_type(e) LOG.exception(msg) helper.cleanup_unmount_temp_folder(share_instance, migration_info_src) helper.cleanup_temp_folder(share_instance, mount_path) helper.cleanup_temp_folder(new_share_instance, mount_path) raise exception.ShareMigrationFailed(reason=msg) try: ignore_list = self.configuration.safe_get('migration_ignore_files') copy = share_utils.Copy(mount_path + share_instance['id'], mount_path + new_share_instance['id'], ignore_list) copy.run() if copy.get_progress()['total_progress'] == 100: migrated = True except Exception as e: LOG.exception(six.text_type(e)) LOG.error( _LE("Failed to copy files for " "migration of share instance %(share_instance_id)s " "to %(new_share_instance_id)s") % { 'share_instance_id': share_instance['id'], 'new_share_instance_id': new_share_instance['id'] }) # NOTE(ganso): For some reason I frequently get AMQP errors after # copying finishes, which seems like is the service taking too long to # copy while not replying heartbeat messages, so AMQP closes the # socket. There is no impact, it just shows a big trace and AMQP # reconnects after, although I would like to prevent this situation # without the use of additional threads. Suggestions welcome. utils.execute(*migration_info_src['umount'], run_as_root=True) utils.execute(*migration_info_dest['umount'], run_as_root=True) utils.execute('rmdir', ''.join((mount_path, share_instance['id'])), check_exit_code=False) utils.execute('rmdir', ''.join((mount_path, new_share_instance['id'])), check_exit_code=False) helper.deny_migration_access(src_access_ref, src_access) helper.deny_migration_access(dest_access_ref, dest_access) if not migrated: msg = ("Copying from share instance %(instance_id)s " "to %(new_instance_id)s did not succeed." % { 'instance_id': share_instance['id'], 'new_instance_id': new_share_instance['id'] }) raise exception.ShareMigrationFailed(reason=msg) LOG.debug("Copying completed in migration for share %s." % share['id'])