def validate_backend_id(self, backend_id): # Currently, VNX driver only supports 1 remote device. if self.active_backend_id: if backend_id != 'default': raise exception.InvalidReplicationTarget( reason=_('Invalid backend_id specified.')) elif backend_id not in ( common.ReplicationDeviceList.get_backend_ids(self.config)): raise exception.InvalidReplicationTarget( reason=_('Invalid backend_id specified.'))
def failover_host(self, context, volumes, secondary_id=None, groups=None): if secondary_id == 'default': if not self.active_backend_id: return None, [], [] volumes_update = huawei_flow.failback(volumes, self.local_cli, self.replication_rmt_cli, self.configuration) secondary_id = '' elif secondary_id in (None, self.configuration.replication['backend_id']): if (self.active_backend_id == self.configuration.replication['backend_id']): # Already failover, return success return self.active_backend_id, [], [] volumes_update = huawei_flow.failover(volumes, self.local_cli, self.replication_rmt_cli, self.configuration) secondary_id = self.configuration.replication['backend_id'] else: msg = "Invalid secondary id %s." % secondary_id raise exception.InvalidReplicationTarget(reason=msg) self.active_backend_id = secondary_id self._switch_replication_clients() return secondary_id, volumes_update, []
class ReplicationTestCase(base.BaseVolumeTestCase): def setUp(self): super(ReplicationTestCase, self).setUp() self.host = 'host@backend#pool' self.manager = manager.VolumeManager(host=self.host) @mock.patch('cinder.objects.VolumeList.get_all') @mock.patch('cinder.volume.driver.BaseVD.failover_host', side_effect=exception.InvalidReplicationTarget('')) @ddt.data(('backend2', 'default', fields.ReplicationStatus.FAILED_OVER), ('backend2', 'backend3', fields.ReplicationStatus.FAILED_OVER), (None, 'backend2', fields.ReplicationStatus.ENABLED), ('', 'backend2', fields.ReplicationStatus.ENABLED)) @ddt.unpack def test_failover_host_invalid_target(self, svc_backend, new_backend, expected, mock_failover, mock_getall): """Test replication failover_host with invalid_target. When failingover fails due to an invalid target exception we return replication_status to its previous status, and we decide what that is depending on the currect active backend. """ svc = utils.create_service( self.context, { 'host': self.host, 'binary': constants.VOLUME_BINARY, 'active_backend_id': svc_backend, 'replication_status': fields.ReplicationStatus.FAILING_OVER }) self.manager.failover_host(self.context, new_backend) mock_getall.assert_called_once_with(self.context, filters={'host': self.host}) mock_failover.assert_called_once_with(self.context, mock_getall.return_value, secondary_id=new_backend) db_svc = objects.Service.get_by_id(self.context, svc.id) self.assertEqual(expected, db_svc.replication_status) @mock.patch('cinder.volume.driver.BaseVD.failover_host', mock.Mock(side_effect=exception.VolumeDriverException(''))) def test_failover_host_driver_exception(self): svc = utils.create_service( self.context, { 'host': self.host, 'binary': constants.VOLUME_BINARY, 'active_backend_id': None, 'replication_status': fields.ReplicationStatus.FAILING_OVER }) self.manager.failover_host(self.context, mock.sentinel.backend_id) db_svc = objects.Service.get_by_id(self.context, svc.id) self.assertEqual(fields.ReplicationStatus.FAILOVER_ERROR, db_svc.replication_status)
def _failover_host(self, volumes, secondary_id=None): if secondary_id == self.backend_name: msg = _("Cannot failover to the same host as the primary.") raise exception.InvalidReplicationTarget(reason=msg) replication_targets = self.get_replication_backend_names( self.configuration) if not replication_targets: msg = _("No replication targets configured for backend " "%s. Cannot failover.") raise exception.InvalidReplicationTarget(reason=msg % self.host) elif secondary_id and secondary_id not in replication_targets: msg = _("%(target)s is not among replication targets configured " "for back end %(host)s. Cannot failover.") payload = { 'target': secondary_id, 'host': self.host, } raise exception.InvalidReplicationTarget(reason=msg % payload) flexvols = self.ssc_library.get_ssc_flexvol_names() try: active_backend_name, volume_updates = self._complete_failover( self.backend_name, replication_targets, flexvols, volumes, failover_target=secondary_id) except exception.NetAppDriverException as e: msg = _("Could not complete failover: %s") % e raise exception.UnableToFailOver(reason=msg) # Update the ZAPI client to the backend we failed over to self._update_zapi_client(active_backend_name) self.failed_over = True self.failed_over_backend_name = active_backend_name return active_backend_name, volume_updates
def failover_host(self, context, volumes, secondary_id=None, groups=None): if secondary_id not in self.failover_choices: msg = (_("Target %(target)s is not a valid choice. " "Valid choices: %(choices)s.") % { "target": secondary_id, "choices": ', '.join(self.failover_choices) }) LOG.error(msg) raise exception.InvalidReplicationTarget(reason=msg) is_failback = secondary_id == manager.VolumeManager.FAILBACK_SENTINEL self.active_backend_id = secondary_id volumes_updates, groups_updates = self.adapter.failover_host( volumes, groups, is_failback) return secondary_id, volumes_updates, groups_updates
def _pre_fail_check(self, vol, running_status_set, data_status_set=None): # check the replica_pair status vol_name = huawei_utils.encode_name(vol.id) vol_id = self.local_client.get_lun_id_by_name(vol_name) pair_info = self.local_client.get_pair_info_by_lun_id(vol_id) if pair_info: running_status = self.local_op.is_running_status( running_status_set, pair_info) data_status = self.local_op.is_data_status( data_status_set, pair_info) if data_status_set else True if not (running_status and data_status): msg = _('Replication pair %(id)s is not at the status ' 'failover/failback available, RUNNINGSTATUS: ' '%(run)s, SECRESDATASTATUS: %(sec)s.') % { 'id': pair_info['ID'], 'run': pair_info['RUNNINGSTATUS'], 'sec': pair_info['SECRESDATASTATUS'] } LOG.error(msg) raise exception.InvalidReplicationTarget(reason=msg)
def _pre_fail_check(self, volumes, statuc_check_func): normal_volumes = [] pair_ids = [] group_ids = set() volume_pair_infos = {} for v in volumes: drv_data = huawei_utils.to_dict(v.replication_driver_data) pair_id = drv_data.get('pair_id') if not pair_id: normal_volumes.append(v.id) continue pair_info = self.pair_op.get_info(pair_id) volume_pair_infos[v.id] = pair_info cg_id = pair_info.get('CGID') if cg_id: if cg_id not in group_ids: group_ids.add(cg_id) else: pair_ids.append(pair_id) for pair_info in six.itervalues(volume_pair_infos): if not statuc_check_func(pair_info): msg = _('Replication pair %(id)s is not at the status ' 'failover/failback available, RUNNINGSTATUS: %(run)s, ' 'SECRESDATASTATUS: %(sec)s.') % { 'id': pair_info['ID'], 'run': pair_info['RUNNINGSTATUS'], 'sec': pair_info['SECRESDATASTATUS'] } LOG.error(msg) raise exception.InvalidReplicationTarget(reason=msg) return normal_volumes, list(group_ids), pair_ids, volume_pair_infos
def failover_host(self, ctxt, volumes, secondary_id, groups=None): """Fail over the volume back and forth. if secondary_id is 'default', volumes will be failed back, otherwize failed over. """ volume_update_list = [] if secondary_id == strings.PRIMARY_BACKEND_ID: if not self._active_backend_id: LOG.info("Host has been failed back. doesn't need " "to fail back again.") return self._active_backend_id, volume_update_list, [] else: if self._active_backend_id: LOG.info("Host has been failed over to %s.", self._active_backend_id) return self._active_backend_id, volume_update_list, [] backend_id = self._replication._target_helper.backend['id'] if secondary_id is None: secondary_id = backend_id elif secondary_id != backend_id: raise exception.InvalidReplicationTarget( message=(_('Invalid secondary_backend_id specified. ' 'Valid backend id is %s.') % backend_id)) LOG.debug("Starting failover to %s.", secondary_id) replicated_luns = [] for volume in volumes: lun = Lun(volume) if lun.type_replication and lun.status == "available": replicated_luns.append(lun) else: volume_update = ( self._replication.failover_unreplicated_volume(lun)) volume_update_list.append(volume_update) if replicated_luns: try: if secondary_id != strings.PRIMARY_BACKEND_ID: self._replication.do_pprc_failover(replicated_luns, secondary_id) self._active_backend_id = secondary_id replicated_luns = self._switch_backend_connection( secondary_id, replicated_luns) else: self._replication.start_pprc_failback( replicated_luns, self._active_backend_id) self._active_backend_id = "" self._helper = self._replication._source_helper except restclient.APIException as e: raise exception.UnableToFailOver( reason=(_("Unable to failover host to %(id)s. " "Exception= %(ex)s") % {'id': secondary_id, 'ex': six.text_type(e)})) for lun in replicated_luns: volume_update = lun.get_volume_update() volume_update['replication_status'] = ( 'failed-over' if self._active_backend_id else 'enabled') model_update = {'volume_id': lun.os_id, 'updates': volume_update} volume_update_list.append(model_update) else: LOG.info("No volume has replication capability.") if secondary_id != strings.PRIMARY_BACKEND_ID: LOG.info("Switch to the target %s", secondary_id) self._switch_backend_connection(secondary_id) self._active_backend_id = secondary_id else: LOG.info("Switch to the primary %s", secondary_id) self._switch_backend_connection(self._active_backend_id) self._active_backend_id = "" return secondary_id, volume_update_list, []
class ReplicationTestCase(base.BaseVolumeTestCase): def setUp(self): super(ReplicationTestCase, self).setUp() self.host = 'host@backend#pool' self.manager = manager.VolumeManager(host=self.host) @mock.patch('cinder.objects.VolumeList.get_all') @mock.patch('cinder.volume.driver.BaseVD.failover_host', side_effect=exception.InvalidReplicationTarget('')) @ddt.data(('backend2', 'default', fields.ReplicationStatus.FAILED_OVER), ('backend2', 'backend3', fields.ReplicationStatus.FAILED_OVER), (None, 'backend2', fields.ReplicationStatus.ENABLED), ('', 'backend2', fields.ReplicationStatus.ENABLED)) @ddt.unpack def test_failover_host_invalid_target(self, svc_backend, new_backend, expected, mock_failover, mock_getall): """Test replication failover_host with invalid_target. When failingover fails due to an invalid target exception we return replication_status to its previous status, and we decide what that is depending on the currect active backend. """ svc = utils.create_service( self.context, {'host': self.host, 'binary': constants.VOLUME_BINARY, 'active_backend_id': svc_backend, 'replication_status': fields.ReplicationStatus.FAILING_OVER}) self.manager.failover_host(self.context, new_backend) mock_getall.assert_called_once_with(self.context, filters={'host': self.host}) mock_failover.assert_called_once_with(self.context, mock_getall.return_value, secondary_id=new_backend) db_svc = objects.Service.get_by_id(self.context, svc.id) self.assertEqual(expected, db_svc.replication_status) @mock.patch('cinder.volume.driver.BaseVD.failover_host', mock.Mock(side_effect=exception.VolumeDriverException(''))) def test_failover_host_driver_exception(self): svc = utils.create_service( self.context, {'host': self.host, 'binary': constants.VOLUME_BINARY, 'active_backend_id': None, 'replication_status': fields.ReplicationStatus.FAILING_OVER}) self.manager.failover_host(self.context, mock.sentinel.backend_id) db_svc = objects.Service.get_by_id(self.context, svc.id) self.assertEqual(fields.ReplicationStatus.FAILOVER_ERROR, db_svc.replication_status) @mock.patch('cinder.objects.Service.is_up', True) @mock.patch.object(volume_rpcapi.VolumeAPI, 'failover') @mock.patch.object(cinder.db, 'conditional_update') @mock.patch.object(objects.ServiceList, 'get_all') def test_failover(self, mock_get_all, mock_db_update, mock_failover): """Test replication failover.""" service = fake_service.fake_service_obj(self.context, binary='cinder-volume') mock_get_all.return_value = [service] mock_db_update.return_value = {'replication_status': 'enabled'} volume_api = cinder.volume.api.API() volume_api.failover(self.context, host=CONF.host, cluster_name=None) mock_failover.assert_called_once_with(self.context, service, None) @mock.patch.object(volume_rpcapi.VolumeAPI, 'failover') @mock.patch.object(cinder.db, 'conditional_update') @mock.patch.object(cinder.db, 'service_get_all') def test_failover_unexpected_status(self, mock_db_get_all, mock_db_update, mock_failover): """Test replication failover unexpected status.""" mock_db_get_all.return_value = [fake_service.fake_service_obj( self.context, binary='cinder-volume')] mock_db_update.return_value = None volume_api = cinder.volume.api.API() self.assertRaises(exception.InvalidInput, volume_api.failover, self.context, host=CONF.host, cluster_name=None) @mock.patch.object(volume_rpcapi.VolumeAPI, 'freeze_host') @mock.patch.object(cinder.db, 'conditional_update', return_value=1) @mock.patch.object(cinder.objects.ServiceList, 'get_all') def test_freeze_host(self, mock_get_all, mock_db_update, mock_freeze): """Test replication freeze_host.""" service = fake_service.fake_service_obj(self.context, binary='cinder-volume') mock_get_all.return_value = [service] mock_freeze.return_value = True volume_api = cinder.volume.api.API() volume_api.freeze_host(self.context, host=CONF.host, cluster_name=None) mock_freeze.assert_called_once_with(self.context, service) @mock.patch.object(volume_rpcapi.VolumeAPI, 'freeze_host') @mock.patch.object(cinder.db, 'conditional_update') @mock.patch.object(cinder.db, 'service_get_all') def test_freeze_host_unexpected_status(self, mock_get_all, mock_db_update, mock_freeze): """Test replication freeze_host unexpected status.""" mock_get_all.return_value = [fake_service.fake_service_obj( self.context, binary='cinder-volume')] mock_db_update.return_value = None volume_api = cinder.volume.api.API() self.assertRaises(exception.InvalidInput, volume_api.freeze_host, self.context, host=CONF.host, cluster_name=None) @mock.patch.object(volume_rpcapi.VolumeAPI, 'thaw_host') @mock.patch.object(cinder.db, 'conditional_update', return_value=1) @mock.patch.object(cinder.objects.ServiceList, 'get_all') def test_thaw_host(self, mock_get_all, mock_db_update, mock_thaw): """Test replication thaw_host.""" service = fake_service.fake_service_obj(self.context, binary='cinder-volume') mock_get_all.return_value = [service] mock_thaw.return_value = True volume_api = cinder.volume.api.API() volume_api.thaw_host(self.context, host=CONF.host, cluster_name=None) mock_thaw.assert_called_once_with(self.context, service) @mock.patch.object(volume_rpcapi.VolumeAPI, 'thaw_host') @mock.patch.object(cinder.db, 'conditional_update') @mock.patch.object(cinder.db, 'service_get_all') def test_thaw_host_unexpected_status(self, mock_get_all, mock_db_update, mock_thaw): """Test replication thaw_host unexpected status.""" mock_get_all.return_value = [fake_service.fake_service_obj( self.context, binary='cinder-volume')] mock_db_update.return_value = None volume_api = cinder.volume.api.API() self.assertRaises(exception.InvalidInput, volume_api.thaw_host, self.context, host=CONF.host, cluster_name=None) @mock.patch('cinder.volume.driver.BaseVD.failover_completed') def test_failover_completed(self, completed_mock): rep_field = fields.ReplicationStatus svc = objects.Service(self.context, host=self.volume.host, binary=constants.VOLUME_BINARY, replication_status=rep_field.ENABLED) svc.create() self.volume.failover_completed( self.context, {'active_backend_id': 'secondary', 'replication_status': rep_field.FAILED_OVER}) service = objects.Service.get_by_id(self.context, svc.id) self.assertEqual('secondary', service.active_backend_id) self.assertEqual('failed-over', service.replication_status) completed_mock.assert_called_once_with(self.context, 'secondary') @mock.patch('cinder.volume.driver.BaseVD.failover_completed', wraps=True) def test_failover_completed_driver_failure(self, completed_mock): rep_field = fields.ReplicationStatus svc = objects.Service(self.context, host=self.volume.host, binary=constants.VOLUME_BINARY, replication_status=rep_field.ENABLED) svc.create() self.volume.failover_completed( self.context, {'active_backend_id': 'secondary', 'replication_status': rep_field.FAILED_OVER}) service = objects.Service.get_by_id(self.context, svc.id) self.assertEqual('secondary', service.active_backend_id) self.assertEqual(rep_field.ERROR, service.replication_status) self.assertTrue(service.disabled) self.assertIsNotNone(service.disabled_reason) completed_mock.assert_called_once_with(self.context, 'secondary') @mock.patch('cinder.volume.rpcapi.VolumeAPI.failover_completed') def test_finish_failover_non_clustered(self, completed_mock): svc = mock.Mock(is_clustered=None) self.volume.finish_failover(self.context, svc, mock.sentinel.updates) svc.update.assert_called_once_with(mock.sentinel.updates) svc.save.assert_called_once_with() completed_mock.assert_not_called() @mock.patch('cinder.volume.rpcapi.VolumeAPI.failover_completed') def test_finish_failover_clustered(self, completed_mock): svc = mock.Mock(cluster_name='cluster_name') updates = {'status': 'error'} self.volume.finish_failover(self.context, svc, updates) completed_mock.assert_called_once_with(self.context, svc, updates) svc.cluster.status = 'error' svc.cluster.save.assert_called_once() @ddt.data(None, 'cluster_name') @mock.patch('cinder.volume.manager.VolumeManager.finish_failover') @mock.patch('cinder.volume.manager.VolumeManager._get_my_volumes') def test_failover_manager(self, cluster, get_vols_mock, finish_mock): """Test manager's failover method for clustered and not clustered.""" rep_field = fields.ReplicationStatus svc = objects.Service(self.context, host=self.volume.host, binary=constants.VOLUME_BINARY, cluster_name=cluster, replication_status=rep_field.ENABLED) svc.create() vol = objects.Volume(self.context, host=self.volume.host) vol.create() get_vols_mock.return_value = [vol] with mock.patch.object(self.volume, 'driver') as driver: called, not_called = driver.failover_host, driver.failover if cluster: called, not_called = not_called, called called.return_value = ('secondary', [{'volume_id': vol.id, 'updates': {'status': 'error'}}]) self.volume.failover(self.context, secondary_backend_id='secondary') not_called.assert_not_called() called.assert_called_once_with(self.context, [vol], secondary_id='secondary') expected_update = {'replication_status': rep_field.FAILED_OVER, 'active_backend_id': 'secondary', 'disabled': True, 'disabled_reason': 'failed-over'} finish_mock.assert_called_once_with(self.context, svc, expected_update) volume = objects.Volume.get_by_id(self.context, vol.id) self.assertEqual('error', volume.status) @ddt.data(('host1', None), (None, 'mycluster')) @ddt.unpack def test_failover_api_fail_multiple_results(self, host, cluster): """Fail if we try to failover multiple backends in the same request.""" rep_field = fields.ReplicationStatus clusters = [ objects.Cluster(self.context, name='mycluster@backend1', replication_status=rep_field.ENABLED, binary=constants.VOLUME_BINARY), objects.Cluster(self.context, name='mycluster@backend2', replication_status=rep_field.ENABLED, binary=constants.VOLUME_BINARY) ] clusters[0].create() clusters[1].create() services = [ objects.Service(self.context, host='host1@backend1', cluster_name=clusters[0].name, replication_status=rep_field.ENABLED, binary=constants.VOLUME_BINARY), objects.Service(self.context, host='host1@backend2', cluster_name=clusters[1].name, replication_status=rep_field.ENABLED, binary=constants.VOLUME_BINARY), ] services[0].create() services[1].create() self.assertRaises(exception.Invalid, self.volume_api.failover, self.context, host, cluster) def test_failover_api_not_found(self): self.assertRaises(exception.ServiceNotFound, self.volume_api.failover, self.context, 'host1', None) @mock.patch('cinder.volume.rpcapi.VolumeAPI.failover') def test_failover_api_success_multiple_results(self, failover_mock): """Succeed to failover multiple services for the same backend.""" rep_field = fields.ReplicationStatus cluster_name = 'mycluster@backend1' cluster = objects.Cluster(self.context, name=cluster_name, replication_status=rep_field.ENABLED, binary=constants.VOLUME_BINARY) cluster.create() services = [ objects.Service(self.context, host='host1@backend1', cluster_name=cluster_name, replication_status=rep_field.ENABLED, binary=constants.VOLUME_BINARY), objects.Service(self.context, host='host2@backend1', cluster_name=cluster_name, replication_status=rep_field.ENABLED, binary=constants.VOLUME_BINARY), ] services[0].create() services[1].create() self.volume_api.failover(self.context, None, cluster_name, mock.sentinel.secondary_id) for service in services + [cluster]: self.assertEqual(rep_field.ENABLED, service.replication_status) service.refresh() self.assertEqual(rep_field.FAILING_OVER, service.replication_status) failover_mock.assert_called_once_with(self.context, mock.ANY, mock.sentinel.secondary_id) self.assertEqual(services[0].id, failover_mock.call_args[0][1].id) @mock.patch('cinder.volume.rpcapi.VolumeAPI.failover') def test_failover_api_success_multiple_results_not_updated(self, failover_mock): """Succeed to failover even if a service is not updated.""" rep_field = fields.ReplicationStatus cluster_name = 'mycluster@backend1' cluster = objects.Cluster(self.context, name=cluster_name, replication_status=rep_field.ENABLED, binary=constants.VOLUME_BINARY) cluster.create() services = [ objects.Service(self.context, host='host1@backend1', cluster_name=cluster_name, replication_status=rep_field.ENABLED, binary=constants.VOLUME_BINARY), objects.Service(self.context, host='host2@backend1', cluster_name=cluster_name, replication_status=rep_field.ERROR, binary=constants.VOLUME_BINARY), ] services[0].create() services[1].create() self.volume_api.failover(self.context, None, cluster_name, mock.sentinel.secondary_id) for service in services[:1] + [cluster]: service.refresh() self.assertEqual(rep_field.FAILING_OVER, service.replication_status) services[1].refresh() self.assertEqual(rep_field.ERROR, services[1].replication_status) failover_mock.assert_called_once_with(self.context, mock.ANY, mock.sentinel.secondary_id) self.assertEqual(services[0].id, failover_mock.call_args[0][1].id) @mock.patch('cinder.volume.rpcapi.VolumeAPI.failover') def test_failover_api_fail_multiple_results_not_updated(self, failover_mock): """Fail if none of the services could be updated.""" rep_field = fields.ReplicationStatus cluster_name = 'mycluster@backend1' cluster = objects.Cluster(self.context, name=cluster_name, replication_status=rep_field.ENABLED, binary=constants.VOLUME_BINARY) cluster.create() down_time = timeutils.datetime.datetime(1970, 1, 1) services = [ # This service is down objects.Service(self.context, host='host1@backend1', cluster_name=cluster_name, replication_status=rep_field.ENABLED, created_at=down_time, updated_at=down_time, modified_at=down_time, binary=constants.VOLUME_BINARY), # This service is not with the right replication status objects.Service(self.context, host='host2@backend1', cluster_name=cluster_name, replication_status=rep_field.ERROR, binary=constants.VOLUME_BINARY), ] services[0].create() services[1].create() self.assertRaises(exception.InvalidInput, self.volume_api.failover, self.context, None, cluster_name, mock.sentinel.secondary_id) for service in services: svc = objects.Service.get_by_id(self.context, service.id) self.assertEqual(service.replication_status, svc.replication_status) cluster.refresh() self.assertEqual(rep_field.ENABLED, cluster.replication_status) failover_mock.assert_not_called()
class ReplicationTestCase(base.BaseVolumeTestCase): def setUp(self): super(ReplicationTestCase, self).setUp() self.host = 'host@backend#pool' self.manager = manager.VolumeManager(host=self.host) @mock.patch('cinder.objects.VolumeList.get_all') @mock.patch('cinder.volume.driver.BaseVD.failover_host', side_effect=exception.InvalidReplicationTarget('')) @ddt.data(('backend2', 'default', fields.ReplicationStatus.FAILED_OVER), ('backend2', 'backend3', fields.ReplicationStatus.FAILED_OVER), (None, 'backend2', fields.ReplicationStatus.ENABLED), ('', 'backend2', fields.ReplicationStatus.ENABLED)) @ddt.unpack def test_failover_host_invalid_target(self, svc_backend, new_backend, expected, mock_failover, mock_getall): """Test replication failover_host with invalid_target. When failingover fails due to an invalid target exception we return replication_status to its previous status, and we decide what that is depending on the currect active backend. """ svc = utils.create_service( self.context, {'host': self.host, 'binary': constants.VOLUME_BINARY, 'active_backend_id': svc_backend, 'replication_status': fields.ReplicationStatus.FAILING_OVER}) self.manager.failover_host(self.context, new_backend) mock_getall.assert_called_once_with(self.context, filters={'host': self.host}) mock_failover.assert_called_once_with(self.context, [], secondary_id=new_backend, groups=[]) db_svc = objects.Service.get_by_id(self.context, svc.id) self.assertEqual(expected, db_svc.replication_status) @mock.patch('cinder.volume.driver.BaseVD.failover_host', mock.Mock(side_effect=exception.VolumeDriverException(''))) def test_failover_host_driver_exception(self): svc = utils.create_service( self.context, {'host': self.host, 'binary': constants.VOLUME_BINARY, 'active_backend_id': None, 'replication_status': fields.ReplicationStatus.FAILING_OVER}) self.manager.failover_host(self.context, mock.sentinel.backend_id) db_svc = objects.Service.get_by_id(self.context, svc.id) self.assertEqual(fields.ReplicationStatus.FAILOVER_ERROR, db_svc.replication_status) @mock.patch('cinder.objects.Service.is_up', True) @mock.patch.object(volume_rpcapi.VolumeAPI, 'failover') @mock.patch.object(cinder.db, 'conditional_update') @mock.patch.object(objects.ServiceList, 'get_all') def test_failover(self, mock_get_all, mock_db_update, mock_failover): """Test replication failover.""" service = fake_service.fake_service_obj(self.context, binary='cinder-volume') mock_get_all.return_value = [service] mock_db_update.return_value = {'replication_status': 'enabled'} volume_api = cinder.volume.api.API() volume_api.failover(self.context, host=CONF.host, cluster_name=None) mock_failover.assert_called_once_with(self.context, service, None) @mock.patch.object(volume_rpcapi.VolumeAPI, 'failover') @mock.patch.object(cinder.db, 'conditional_update') @mock.patch.object(cinder.db, 'service_get_all') def test_failover_unexpected_status(self, mock_db_get_all, mock_db_update, mock_failover): """Test replication failover unexpected status.""" mock_db_get_all.return_value = [fake_service.fake_service_obj( self.context, binary=constants.VOLUME_BINARY)] mock_db_update.return_value = None volume_api = cinder.volume.api.API() self.assertRaises(exception.InvalidInput, volume_api.failover, self.context, host=CONF.host, cluster_name=None) @mock.patch.object(volume_rpcapi.VolumeAPI, 'freeze_host') @mock.patch.object(cinder.db, 'conditional_update', return_value=1) @mock.patch.object(cinder.objects.ServiceList, 'get_all') def test_freeze_host(self, mock_get_all, mock_db_update, mock_freeze): """Test replication freeze_host.""" service = fake_service.fake_service_obj(self.context, binary=constants.VOLUME_BINARY) mock_get_all.return_value = [service] mock_freeze.return_value = True volume_api = cinder.volume.api.API() volume_api.freeze_host(self.context, host=CONF.host, cluster_name=None) mock_freeze.assert_called_once_with(self.context, service) @mock.patch.object(volume_rpcapi.VolumeAPI, 'freeze_host') @mock.patch.object(cinder.db, 'conditional_update') @mock.patch.object(cinder.db, 'service_get_all') def test_freeze_host_unexpected_status(self, mock_get_all, mock_db_update, mock_freeze): """Test replication freeze_host unexpected status.""" mock_get_all.return_value = [fake_service.fake_service_obj( self.context, binary=constants.VOLUME_BINARY)] mock_db_update.return_value = None volume_api = cinder.volume.api.API() self.assertRaises(exception.InvalidInput, volume_api.freeze_host, self.context, host=CONF.host, cluster_name=None) @mock.patch.object(volume_rpcapi.VolumeAPI, 'thaw_host') @mock.patch.object(cinder.db, 'conditional_update', return_value=1) @mock.patch.object(cinder.objects.ServiceList, 'get_all') def test_thaw_host(self, mock_get_all, mock_db_update, mock_thaw): """Test replication thaw_host.""" service = fake_service.fake_service_obj(self.context, binary=constants.VOLUME_BINARY) mock_get_all.return_value = [service] mock_thaw.return_value = True volume_api = cinder.volume.api.API() volume_api.thaw_host(self.context, host=CONF.host, cluster_name=None) mock_thaw.assert_called_once_with(self.context, service) @mock.patch.object(volume_rpcapi.VolumeAPI, 'thaw_host') @mock.patch.object(cinder.db, 'conditional_update') @mock.patch.object(cinder.db, 'service_get_all') def test_thaw_host_unexpected_status(self, mock_get_all, mock_db_update, mock_thaw): """Test replication thaw_host unexpected status.""" mock_get_all.return_value = [fake_service.fake_service_obj( self.context, binary=constants.VOLUME_BINARY)] mock_db_update.return_value = None volume_api = cinder.volume.api.API() self.assertRaises(exception.InvalidInput, volume_api.thaw_host, self.context, host=CONF.host, cluster_name=None) @mock.patch('cinder.volume.driver.BaseVD.failover_completed') def test_failover_completed(self, completed_mock): rep_field = fields.ReplicationStatus svc = objects.Service(self.context, host=self.volume.host, binary=constants.VOLUME_BINARY, replication_status=rep_field.ENABLED) svc.create() self.volume.failover_completed( self.context, {'active_backend_id': 'secondary', 'replication_status': rep_field.FAILED_OVER}) service = objects.Service.get_by_id(self.context, svc.id) self.assertEqual('secondary', service.active_backend_id) self.assertEqual('failed-over', service.replication_status) completed_mock.assert_called_once_with(self.context, 'secondary') @mock.patch('cinder.volume.driver.BaseVD.failover_completed', wraps=True) def test_failover_completed_driver_failure(self, completed_mock): rep_field = fields.ReplicationStatus svc = objects.Service(self.context, host=self.volume.host, binary=constants.VOLUME_BINARY, replication_status=rep_field.ENABLED) svc.create() self.volume.failover_completed( self.context, {'active_backend_id': 'secondary', 'replication_status': rep_field.FAILED_OVER}) service = objects.Service.get_by_id(self.context, svc.id) self.assertEqual('secondary', service.active_backend_id) self.assertEqual(rep_field.ERROR, service.replication_status) self.assertTrue(service.disabled) self.assertIsNotNone(service.disabled_reason) completed_mock.assert_called_once_with(self.context, 'secondary') @mock.patch('cinder.volume.rpcapi.VolumeAPI.failover_completed') def test_finish_failover_non_clustered(self, completed_mock): svc = mock.Mock(is_clustered=None) self.volume.finish_failover(self.context, svc, mock.sentinel.updates) svc.update.assert_called_once_with(mock.sentinel.updates) svc.save.assert_called_once_with() completed_mock.assert_not_called() @mock.patch('cinder.volume.rpcapi.VolumeAPI.failover_completed') def test_finish_failover_clustered(self, completed_mock): svc = mock.Mock(cluster_name='cluster_name') updates = {'status': 'error'} self.volume.finish_failover(self.context, svc, updates) completed_mock.assert_called_once_with(self.context, svc, updates) svc.cluster.status = 'error' svc.cluster.save.assert_called_once() @ddt.data(None, 'cluster_name') @mock.patch('cinder.volume.manager.VolumeManager.finish_failover') @mock.patch('cinder.volume.manager.VolumeManager._get_my_volumes') def test_failover_manager(self, cluster, get_vols_mock, finish_mock): """Test manager's failover method for clustered and not clustered.""" rep_field = fields.ReplicationStatus svc = objects.Service(self.context, host=self.volume.host, binary=constants.VOLUME_BINARY, cluster_name=cluster, replication_status=rep_field.ENABLED) svc.create() vol = objects.Volume(self.context, host=self.volume.host) vol.create() get_vols_mock.return_value = [vol] with mock.patch.object(self.volume, 'driver') as driver: called, not_called = driver.failover_host, driver.failover if cluster: called, not_called = not_called, called called.return_value = ('secondary', [{'volume_id': vol.id, 'updates': {'status': 'error'}}], []) self.volume.failover(self.context, secondary_backend_id='secondary') not_called.assert_not_called() called.assert_called_once_with(self.context, [vol], secondary_id='secondary', groups=[]) expected_update = {'replication_status': rep_field.FAILED_OVER, 'active_backend_id': 'secondary', 'disabled': True, 'disabled_reason': 'failed-over'} finish_mock.assert_called_once_with(self.context, svc, expected_update) volume = objects.Volume.get_by_id(self.context, vol.id) self.assertEqual('error', volume.status) @ddt.data(('host1', None), (None, 'mycluster')) @ddt.unpack def test_failover_api_fail_multiple_results(self, host, cluster): """Fail if we try to failover multiple backends in the same request.""" rep_field = fields.ReplicationStatus clusters = [ objects.Cluster(self.context, name='mycluster@backend1', replication_status=rep_field.ENABLED, binary=constants.VOLUME_BINARY), objects.Cluster(self.context, name='mycluster@backend2', replication_status=rep_field.ENABLED, binary=constants.VOLUME_BINARY) ] clusters[0].create() clusters[1].create() services = [ objects.Service(self.context, host='host1@backend1', cluster_name=clusters[0].name, replication_status=rep_field.ENABLED, binary=constants.VOLUME_BINARY), objects.Service(self.context, host='host1@backend2', cluster_name=clusters[1].name, replication_status=rep_field.ENABLED, binary=constants.VOLUME_BINARY), ] services[0].create() services[1].create() self.assertRaises(exception.Invalid, self.volume_api.failover, self.context, host, cluster) def test_failover_api_not_found(self): self.assertRaises(exception.ServiceNotFound, self.volume_api.failover, self.context, 'host1', None) @mock.patch('cinder.volume.rpcapi.VolumeAPI.failover') def test_failover_api_success_multiple_results(self, failover_mock): """Succeed to failover multiple services for the same backend.""" rep_field = fields.ReplicationStatus cluster_name = 'mycluster@backend1' cluster = objects.Cluster(self.context, name=cluster_name, replication_status=rep_field.ENABLED, binary=constants.VOLUME_BINARY) cluster.create() services = [ objects.Service(self.context, host='host1@backend1', cluster_name=cluster_name, replication_status=rep_field.ENABLED, binary=constants.VOLUME_BINARY), objects.Service(self.context, host='host2@backend1', cluster_name=cluster_name, replication_status=rep_field.ENABLED, binary=constants.VOLUME_BINARY), ] services[0].create() services[1].create() self.volume_api.failover(self.context, None, cluster_name, mock.sentinel.secondary_id) for service in services + [cluster]: self.assertEqual(rep_field.ENABLED, service.replication_status) service.refresh() self.assertEqual(rep_field.FAILING_OVER, service.replication_status) failover_mock.assert_called_once_with(self.context, mock.ANY, mock.sentinel.secondary_id) self.assertEqual(services[0].id, failover_mock.call_args[0][1].id) @mock.patch('cinder.volume.rpcapi.VolumeAPI.failover') def test_failover_api_success_multiple_results_not_updated(self, failover_mock): """Succeed to failover even if a service is not updated.""" rep_field = fields.ReplicationStatus cluster_name = 'mycluster@backend1' cluster = objects.Cluster(self.context, name=cluster_name, replication_status=rep_field.ENABLED, binary=constants.VOLUME_BINARY) cluster.create() services = [ objects.Service(self.context, host='host1@backend1', cluster_name=cluster_name, replication_status=rep_field.ENABLED, binary=constants.VOLUME_BINARY), objects.Service(self.context, host='host2@backend1', cluster_name=cluster_name, replication_status=rep_field.ERROR, binary=constants.VOLUME_BINARY), ] services[0].create() services[1].create() self.volume_api.failover(self.context, None, cluster_name, mock.sentinel.secondary_id) for service in services[:1] + [cluster]: service.refresh() self.assertEqual(rep_field.FAILING_OVER, service.replication_status) services[1].refresh() self.assertEqual(rep_field.ERROR, services[1].replication_status) failover_mock.assert_called_once_with(self.context, mock.ANY, mock.sentinel.secondary_id) self.assertEqual(services[0].id, failover_mock.call_args[0][1].id) @mock.patch('cinder.volume.rpcapi.VolumeAPI.failover') def test_failover_api_fail_multiple_results_not_updated(self, failover_mock): """Fail if none of the services could be updated.""" rep_field = fields.ReplicationStatus cluster_name = 'mycluster@backend1' cluster = objects.Cluster(self.context, name=cluster_name, replication_status=rep_field.ENABLED, binary=constants.VOLUME_BINARY) cluster.create() down_time = timeutils.datetime.datetime(1970, 1, 1) services = [ # This service is down objects.Service(self.context, host='host1@backend1', cluster_name=cluster_name, replication_status=rep_field.ENABLED, created_at=down_time, updated_at=down_time, modified_at=down_time, binary=constants.VOLUME_BINARY), # This service is not with the right replication status objects.Service(self.context, host='host2@backend1', cluster_name=cluster_name, replication_status=rep_field.ERROR, binary=constants.VOLUME_BINARY), ] services[0].create() services[1].create() self.assertRaises(exception.InvalidInput, self.volume_api.failover, self.context, None, cluster_name, mock.sentinel.secondary_id) for service in services: svc = objects.Service.get_by_id(self.context, service.id) self.assertEqual(service.replication_status, svc.replication_status) cluster.refresh() self.assertEqual(rep_field.ENABLED, cluster.replication_status) failover_mock.assert_not_called() def _check_failover_db(self, get_method, expected_results): db_data = get_method.get_all(self.context, None) db_data = {e.id: e for e in db_data} for expected in expected_results: id_ = expected['id'] for key, value in expected.items(): self.assertEqual(value, getattr(db_data[id_], key), '(%s) ref=%s != act=%s' % ( key, expected, dict(db_data[id_]))) def _test_failover_model_updates(self, in_volumes, in_snapshots, driver_volumes, driver_result, out_volumes, out_snapshots, in_groups=None, out_groups=None, driver_group_result=None, secondary_id=None): host = vol_utils.extract_host(self.manager.host) utils.create_service(self.context, {'host': host, 'binary': constants.VOLUME_BINARY}) for volume in in_volumes: utils.create_volume(self.context, self.manager.host, **volume) for snapshot in in_snapshots: utils.create_snapshot(self.context, **snapshot) for group in in_groups: utils.create_group(self.context, self.manager.host, **group) with mock.patch.object( self.manager.driver, 'failover_host', return_value=(secondary_id, driver_result, driver_group_result)) as driver_mock: self.manager.failover_host(self.context, secondary_id) self.assertSetEqual(driver_volumes, {v.id for v in driver_mock.call_args[0][1]}) self._check_failover_db(objects.VolumeList, out_volumes) self._check_failover_db(objects.SnapshotList, out_snapshots) self._check_failover_db(objects.GroupList, out_groups) @mock.patch('cinder.volume.utils.is_group_a_type') def test_failover_host_model_updates(self, mock_group_type): status = fields.ReplicationStatus mock_group_type.return_value = True in_groups = [ {'id': str(uuid.uuid4()), 'status': 'available', 'group_type_id': fake.GROUP_TYPE_ID, 'volume_type_ids': [fake.VOLUME_TYPE_ID], 'replication_status': status.FAILOVER_ERROR}, {'id': str(uuid.uuid4()), 'status': 'available', 'group_type_id': fake.GROUP_TYPE_ID, 'volume_type_ids': [fake.VOLUME_TYPE_ID], 'replication_status': status.ENABLED}, ] driver_group_result = [ {'group_id': in_groups[0]['id'], 'updates': {'replication_status': status.FAILOVER_ERROR}}, {'group_id': in_groups[1]['id'], 'updates': {'replication_status': status.FAILED_OVER}}, ] out_groups = [ {'id': in_groups[0]['id'], 'status': 'error', 'replication_status': status.FAILOVER_ERROR}, {'id': in_groups[1]['id'], 'status': in_groups[1]['status'], 'replication_status': status.FAILED_OVER}, ] # test volumes in_volumes = [ {'id': str(uuid.uuid4()), 'status': 'available', 'replication_status': status.DISABLED}, {'id': str(uuid.uuid4()), 'status': 'in-use', 'replication_status': status.NOT_CAPABLE}, {'id': str(uuid.uuid4()), 'status': 'available', 'replication_status': status.FAILOVER_ERROR}, {'id': str(uuid.uuid4()), 'status': 'in-use', 'replication_status': status.ENABLED}, {'id': str(uuid.uuid4()), 'status': 'available', 'replication_status': status.FAILOVER_ERROR}, {'id': str(uuid.uuid4()), 'status': 'in-use', 'replication_status': status.ENABLED}, {'id': str(uuid.uuid4()), 'status': 'available', 'group_id': in_groups[0]['id'], 'replication_status': status.FAILOVER_ERROR}, {'id': str(uuid.uuid4()), 'status': 'available', 'group_id': in_groups[1]['id'], 'replication_status': status.ENABLED}, ] in_snapshots = [ {'id': v['id'], 'volume_id': v['id'], 'status': 'available'} for v in in_volumes ] driver_volumes = { v['id'] for v in in_volumes if v['replication_status'] not in (status.DISABLED, status.NOT_CAPABLE)} driver_result = [ {'volume_id': in_volumes[3]['id'], 'updates': {'status': 'error'}}, {'volume_id': in_volumes[4]['id'], 'updates': {'replication_status': status.FAILOVER_ERROR}}, {'volume_id': in_volumes[5]['id'], 'updates': {'replication_status': status.FAILED_OVER}}, {'volume_id': in_volumes[6]['id'], 'updates': {'replication_status': status.FAILOVER_ERROR}}, {'volume_id': in_volumes[7]['id'], 'updates': {'replication_status': status.FAILED_OVER}}, ] out_volumes = [ {'id': in_volumes[0]['id'], 'status': 'error', 'replication_status': status.NOT_CAPABLE, 'previous_status': in_volumes[0]['status']}, {'id': in_volumes[1]['id'], 'status': 'error', 'replication_status': status.NOT_CAPABLE, 'previous_status': in_volumes[1]['status']}, {'id': in_volumes[2]['id'], 'status': in_volumes[2]['status'], 'replication_status': status.FAILED_OVER}, {'id': in_volumes[3]['id'], 'status': 'error', 'previous_status': in_volumes[3]['status'], 'replication_status': status.FAILOVER_ERROR}, {'id': in_volumes[4]['id'], 'status': 'error', 'previous_status': in_volumes[4]['status'], 'replication_status': status.FAILOVER_ERROR}, {'id': in_volumes[5]['id'], 'status': in_volumes[5]['status'], 'replication_status': status.FAILED_OVER}, {'id': in_volumes[6]['id'], 'status': 'error', 'previous_status': in_volumes[6]['status'], 'replication_status': status.FAILOVER_ERROR}, {'id': in_volumes[7]['id'], 'status': in_volumes[7]['status'], 'replication_status': status.FAILED_OVER}, ] out_snapshots = [ {'id': ov['id'], 'status': 'error' if ov['status'] == 'error' else 'available'} for ov in out_volumes ] self._test_failover_model_updates(in_volumes, in_snapshots, driver_volumes, driver_result, out_volumes, out_snapshots, in_groups, out_groups, driver_group_result) def test_failback_host_model_updates(self): status = fields.ReplicationStatus # IDs will be overwritten with UUIDs, but they help follow the code in_volumes = [ {'id': 0, 'status': 'available', 'replication_status': status.DISABLED}, {'id': 1, 'status': 'in-use', 'replication_status': status.NOT_CAPABLE}, {'id': 2, 'status': 'available', 'replication_status': status.FAILOVER_ERROR}, {'id': 3, 'status': 'in-use', 'replication_status': status.ENABLED}, {'id': 4, 'status': 'available', 'replication_status': status.FAILOVER_ERROR}, {'id': 5, 'status': 'in-use', 'replication_status': status.FAILED_OVER}, ] # Generate real volume IDs for volume in in_volumes: volume['id'] = str(uuid.uuid4()) in_snapshots = [ {'id': in_volumes[0]['id'], 'volume_id': in_volumes[0]['id'], 'status': fields.SnapshotStatus.ERROR_DELETING}, {'id': in_volumes[1]['id'], 'volume_id': in_volumes[1]['id'], 'status': fields.SnapshotStatus.AVAILABLE}, {'id': in_volumes[2]['id'], 'volume_id': in_volumes[2]['id'], 'status': fields.SnapshotStatus.CREATING}, {'id': in_volumes[3]['id'], 'volume_id': in_volumes[3]['id'], 'status': fields.SnapshotStatus.DELETING}, {'id': in_volumes[4]['id'], 'volume_id': in_volumes[4]['id'], 'status': fields.SnapshotStatus.CREATING}, {'id': in_volumes[5]['id'], 'volume_id': in_volumes[5]['id'], 'status': fields.SnapshotStatus.CREATING}, ] driver_volumes = { v['id'] for v in in_volumes if v['replication_status'] not in (status.DISABLED, status.NOT_CAPABLE)} driver_result = [ {'volume_id': in_volumes[3]['id'], 'updates': {'status': 'error'}}, {'volume_id': in_volumes[4]['id'], 'updates': {'replication_status': status.FAILOVER_ERROR}}, {'volume_id': in_volumes[5]['id'], 'updates': {'replication_status': status.FAILED_OVER}}, ] out_volumes = [ {'id': in_volumes[0]['id'], 'status': in_volumes[0]['status'], 'replication_status': in_volumes[0]['replication_status'], 'previous_status': None}, {'id': in_volumes[1]['id'], 'status': in_volumes[1]['status'], 'replication_status': in_volumes[1]['replication_status'], 'previous_status': None}, {'id': in_volumes[2]['id'], 'status': in_volumes[2]['status'], 'replication_status': status.ENABLED}, {'id': in_volumes[3]['id'], 'status': 'error', 'previous_status': in_volumes[3]['status'], 'replication_status': status.FAILOVER_ERROR}, {'id': in_volumes[4]['id'], 'status': 'error', 'previous_status': in_volumes[4]['status'], 'replication_status': status.FAILOVER_ERROR}, {'id': in_volumes[5]['id'], 'status': in_volumes[5]['status'], 'replication_status': status.ENABLED}, ] # Snapshot status is preserved except for those that error the failback out_snapshots = in_snapshots[:] out_snapshots[3]['status'] = fields.SnapshotStatus.ERROR out_snapshots[4]['status'] = fields.SnapshotStatus.ERROR self._test_failover_model_updates(in_volumes, in_snapshots, driver_volumes, driver_result, out_volumes, out_snapshots, [], [], [], self.manager.FAILBACK_SENTINEL) @mock.patch('cinder.utils.log_unsupported_driver_warning', mock.Mock()) @mock.patch('cinder.utils.require_driver_initialized', mock.Mock()) def test_init_host_with_rpc_clustered_replication(self): # These are not OVOs but ORM instances cluster = utils.create_cluster(self.context) service = utils.create_service(self.context, {'cluster_name': cluster.name, 'binary': cluster.binary}) self.assertNotEqual(fields.ReplicationStatus.ENABLED, cluster.replication_status) self.assertNotEqual(fields.ReplicationStatus.ENABLED, service.replication_status) vol_manager = manager.VolumeManager( 'cinder.tests.fake_driver.FakeHAReplicatedLoggingVolumeDriver', host=service.host, cluster=cluster.name) vol_manager.driver = mock.Mock() vol_manager.driver.get_volume_stats.return_value = { 'replication_enabled': True } vol_manager.init_host_with_rpc() cluster_ovo = objects.Cluster.get_by_id(self.context, cluster.id) service_ovo = objects.Service.get_by_id(self.context, service.id) self.assertEqual(fields.ReplicationStatus.ENABLED, cluster_ovo.replication_status) self.assertEqual(fields.ReplicationStatus.ENABLED, service_ovo.replication_status)