def test_amphora_post_vip_plug(self, mock_loadbalancer_repo_update, mock_driver, mock_generate_uuid, mock_log, mock_get_session, mock_listener_repo_get, mock_listener_repo_update, mock_amphora_repo_update): amphorae_net_config_mock = mock.Mock() amphora_post_vip_plug_obj = amphora_driver_tasks.AmphoraPostVIPPlug() amphora_post_vip_plug_obj.execute(_amphora_mock, _LB_mock, amphorae_net_config_mock) mock_driver.post_vip_plug.assert_called_once_with( _amphora_mock, _LB_mock, amphorae_net_config_mock) # Test revert amp = amphora_post_vip_plug_obj.revert(None, _amphora_mock, _LB_mock) repo.AmphoraRepository.update.assert_called_once_with( _session_mock, id=AMP_ID, status=constants.ERROR) repo.LoadBalancerRepository.update.assert_called_once_with( _session_mock, id=LB_ID, provisioning_status=constants.ERROR) self.assertIsNone(amp) # Test revert with repo exceptions repo.AmphoraRepository.update.reset_mock() repo.LoadBalancerRepository.update.reset_mock() mock_amphora_repo_update.side_effect = Exception('fail') mock_loadbalancer_repo_update.side_effect = Exception('fail') amp = amphora_post_vip_plug_obj.revert(None, _amphora_mock, _LB_mock) repo.AmphoraRepository.update.assert_called_once_with( _session_mock, id=AMP_ID, status=constants.ERROR) repo.LoadBalancerRepository.update.assert_called_once_with( _session_mock, id=LB_ID, provisioning_status=constants.ERROR) self.assertIsNone(amp)
def _get_amp_net_subflow(self, sf_name): flows = [] flows.append(network_tasks.PlugVIPAmpphora( name=sf_name + '-' + constants.PLUG_VIP_AMPHORA, requires=(constants.LOADBALANCER, constants.AMPHORA, constants.SUBNET), provides=constants.AMP_DATA)) flows.append(network_tasks.ApplyQosAmphora( name=sf_name + '-' + constants.APPLY_QOS_AMP, requires=(constants.LOADBALANCER, constants.AMP_DATA, constants.UPDATE_DICT))) flows.append(database_tasks.UpdateAmphoraVIPData( name=sf_name + '-' + constants.UPDATE_AMPHORA_VIP_DATA, requires=constants.AMP_DATA)) flows.append(database_tasks.ReloadAmphora( name=sf_name + '-' + constants.RELOAD_AMP_AFTER_PLUG_VIP, requires=constants.AMPHORA, provides=constants.AMPHORA)) flows.append(database_tasks.ReloadLoadBalancer( name=sf_name + '-' + constants.RELOAD_LB_AFTER_PLUG_VIP, requires=constants.LOADBALANCER_ID, provides=constants.LOADBALANCER)) flows.append(network_tasks.GetAmphoraNetworkConfigs( name=sf_name + '-' + constants.GET_AMP_NETWORK_CONFIG, requires=(constants.LOADBALANCER, constants.AMPHORA), provides=constants.AMPHORA_NETWORK_CONFIG)) flows.append(amphora_driver_tasks.AmphoraPostVIPPlug( name=sf_name + '-' + constants.AMP_POST_VIP_PLUG, rebind={constants.AMPHORAE_NETWORK_CONFIG: constants.AMPHORA_NETWORK_CONFIG}, requires=(constants.LOADBALANCER, constants.AMPHORAE_NETWORK_CONFIG))) return flows
def test_amphora_post_vip_plug( self, mock_lb_get, mock_loadbalancer_repo_update, mock_driver, mock_generate_uuid, mock_log, mock_get_session, mock_listener_repo_get, mock_listener_repo_update, mock_amphora_repo_get, mock_amphora_repo_update): amphorae_net_config_mock = mock.MagicMock() mock_amphora_repo_get.return_value = _db_amphora_mock mock_lb_get.return_value = _db_load_balancer_mock amphora_post_vip_plug_obj = amphora_driver_tasks.AmphoraPostVIPPlug() amphora_post_vip_plug_obj.execute(_amphora_mock, _LB_mock, amphorae_net_config_mock) vip_subnet = network_data_models.Subnet( **amphorae_net_config_mock[AMP_ID]['vip_subnet']) vrrp_port = network_data_models.Port( **amphorae_net_config_mock[AMP_ID]['vrrp_port']) mock_driver.post_vip_plug.assert_called_once_with( _db_amphora_mock, _db_load_balancer_mock, amphorae_net_config_mock, vip_subnet=vip_subnet, vrrp_port=vrrp_port) # Test revert amp = amphora_post_vip_plug_obj.revert(None, _amphora_mock, _LB_mock) repo.AmphoraRepository.update.assert_called_once_with( _session_mock, id=AMP_ID, status=constants.ERROR) repo.LoadBalancerRepository.update.assert_called_once_with( _session_mock, id=LB_ID, provisioning_status=constants.ERROR) self.assertIsNone(amp) # Test revert with repo exceptions repo.AmphoraRepository.update.reset_mock() repo.LoadBalancerRepository.update.reset_mock() mock_amphora_repo_update.side_effect = Exception('fail') mock_loadbalancer_repo_update.side_effect = Exception('fail') amp = amphora_post_vip_plug_obj.revert(None, _amphora_mock, _LB_mock) repo.AmphoraRepository.update.assert_called_once_with( _session_mock, id=AMP_ID, status=constants.ERROR) repo.LoadBalancerRepository.update.assert_called_once_with( _session_mock, id=LB_ID, provisioning_status=constants.ERROR) self.assertIsNone(amp) # Test revert when this task failed repo.AmphoraRepository.update.reset_mock() amp = amphora_post_vip_plug_obj.revert( failure.Failure.from_exception(Exception('boom')), _amphora_mock, None) repo.AmphoraRepository.update.assert_not_called()
def get_failover_flow(self, role=constants.ROLE_STANDALONE, load_balancer=None): """Creates a flow to failover a stale amphora :returns: The flow for amphora failover """ failover_amphora_flow = linear_flow.Flow( constants.FAILOVER_AMPHORA_FLOW) failover_amphora_flow.add(lifecycle_tasks.AmphoraToErrorOnRevertTask( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA)) failover_amphora_flow.add(network_tasks.FailoverPreparationForAmphora( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA)) # Note: It seems intuitive to boot an amphora prior to deleting # the old amphora, however this is a complicated issue. # If the target host (due to anit-affinity) is resource # constrained, this will fail where a post-delete will # succeed. Since this is async with the API it would result # in the LB ending in ERROR though the amps are still alive. # Consider in the future making this a complicated # try-on-failure-retry flow, or move upgrade failovers to be # synchronous with the API. For now spares pool and act/stdby # will mitigate most of this delay. # Delete the old amphora failover_amphora_flow.add( database_tasks.MarkAmphoraPendingDeleteInDB( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA)) failover_amphora_flow.add( database_tasks.MarkAmphoraHealthBusy( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA)) failover_amphora_flow.add(compute_tasks.ComputeDelete( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA)) failover_amphora_flow.add(network_tasks.WaitForPortDetach( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA)) failover_amphora_flow.add(database_tasks.MarkAmphoraDeletedInDB( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA)) # If this is an unallocated amp (spares pool), we're done if not load_balancer: failover_amphora_flow.add( database_tasks.DisableAmphoraHealthMonitoring( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA)) return failover_amphora_flow # Save failed amphora details for later failover_amphora_flow.add( database_tasks.GetAmphoraDetails( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA, provides=constants.AMP_DATA)) # Get a new amphora # Note: Role doesn't matter here. We will update it later. get_amp_subflow = self.get_amphora_for_lb_subflow( prefix=constants.FAILOVER_AMPHORA_FLOW) failover_amphora_flow.add(get_amp_subflow) # Update the new amphora with the failed amphora details failover_amphora_flow.add(database_tasks.UpdateAmpFailoverDetails( requires=(constants.AMPHORA, constants.AMP_DATA))) # Update the data stored in the flow from the database failover_amphora_flow.add(database_tasks.ReloadLoadBalancer( requires=constants.LOADBALANCER_ID, provides=constants.LOADBALANCER)) failover_amphora_flow.add(database_tasks.ReloadAmphora( requires=constants.AMPHORA, provides=constants.AMPHORA)) # Prepare to reconnect the network interface(s) failover_amphora_flow.add(network_tasks.GetAmphoraeNetworkConfigs( requires=constants.LOADBALANCER, provides=constants.AMPHORAE_NETWORK_CONFIG)) failover_amphora_flow.add(database_tasks.GetListenersFromLoadbalancer( requires=constants.LOADBALANCER, provides=constants.LISTENERS)) failover_amphora_flow.add(database_tasks.GetAmphoraeFromLoadbalancer( requires=constants.LOADBALANCER, provides=constants.AMPHORAE)) # Plug the VIP ports into the new amphora # The reason for moving these steps here is the udp listeners want to # do some kernel configuration before Listener update for forbidding # failure during rebuild amphora. failover_amphora_flow.add(network_tasks.PlugVIPPort( requires=(constants.AMPHORA, constants.AMPHORAE_NETWORK_CONFIG))) failover_amphora_flow.add(amphora_driver_tasks.AmphoraPostVIPPlug( requires=(constants.AMPHORA, constants.LOADBALANCER, constants.AMPHORAE_NETWORK_CONFIG))) # Listeners update needs to be run on all amphora to update # their peer configurations. So parallelize this with an # unordered subflow. update_amps_subflow = unordered_flow.Flow( constants.UPDATE_AMPS_SUBFLOW) timeout_dict = { constants.CONN_MAX_RETRIES: CONF.haproxy_amphora.active_connection_max_retries, constants.CONN_RETRY_INTERVAL: CONF.haproxy_amphora.active_connection_rety_interval} # Setup parallel flows for each amp. We don't know the new amp # details at flow creation time, so setup a subflow for each # amp on the LB, they let the task index into a list of amps # to find the amphora it should work on. amp_index = 0 db_lb = self.lb_repo.get(db_apis.get_session(), id=load_balancer[constants.LOADBALANCER_ID]) for amp in db_lb.amphorae: if amp.status == constants.DELETED: continue update_amps_subflow.add( amphora_driver_tasks.AmpListenersUpdate( name=constants.AMP_LISTENER_UPDATE + '-' + str(amp_index), requires=(constants.LOADBALANCER, constants.AMPHORAE), inject={constants.AMPHORA_INDEX: amp_index, constants.TIMEOUT_DICT: timeout_dict})) amp_index += 1 failover_amphora_flow.add(update_amps_subflow) # Plug the member networks into the new amphora failover_amphora_flow.add(network_tasks.CalculateAmphoraDelta( requires=(constants.LOADBALANCER, constants.AMPHORA), provides=constants.DELTA)) failover_amphora_flow.add(network_tasks.HandleNetworkDelta( requires=(constants.AMPHORA, constants.DELTA), provides=constants.ADDED_PORTS)) failover_amphora_flow.add(amphora_driver_tasks.AmphoraePostNetworkPlug( requires=(constants.LOADBALANCER, constants.ADDED_PORTS))) failover_amphora_flow.add(database_tasks.ReloadLoadBalancer( name='octavia-failover-LB-reload-2', requires=constants.LOADBALANCER_ID, provides=constants.LOADBALANCER)) # Handle the amphora role and VRRP if necessary if role == constants.ROLE_MASTER: failover_amphora_flow.add(database_tasks.MarkAmphoraMasterInDB( name=constants.MARK_AMP_MASTER_INDB, requires=constants.AMPHORA)) vrrp_subflow = self.get_vrrp_subflow(role) failover_amphora_flow.add(vrrp_subflow) elif role == constants.ROLE_BACKUP: failover_amphora_flow.add(database_tasks.MarkAmphoraBackupInDB( name=constants.MARK_AMP_BACKUP_INDB, requires=constants.AMPHORA)) vrrp_subflow = self.get_vrrp_subflow(role) failover_amphora_flow.add(vrrp_subflow) elif role == constants.ROLE_STANDALONE: failover_amphora_flow.add( database_tasks.MarkAmphoraStandAloneInDB( name=constants.MARK_AMP_STANDALONE_INDB, requires=constants.AMPHORA)) failover_amphora_flow.add(amphora_driver_tasks.ListenersStart( requires=(constants.LOADBALANCER, constants.AMPHORA))) failover_amphora_flow.add( database_tasks.DisableAmphoraHealthMonitoring( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA)) return failover_amphora_flow
def get_amphora_for_lb_failover_subflow( self, prefix, role=constants.ROLE_STANDALONE, failed_amp_vrrp_port_id=None, is_vrrp_ipv6=False): """Creates a new amphora that will be used in a failover flow. :requires: loadbalancer_id, flavor, vip, vip_sg_id, loadbalancer :provides: amphora_id, amphora :param prefix: The flow name prefix to use on the flow and tasks. :param role: The role this amphora will have in the topology. :param failed_amp_vrrp_port_id: The base port ID of the failed amp. :param is_vrrp_ipv6: True if the base port IP is IPv6. :return: A Taskflow sub-flow that will create the amphora. """ sf_name = prefix + '-' + constants.CREATE_AMP_FOR_FAILOVER_SUBFLOW amp_for_failover_flow = linear_flow.Flow(sf_name) # Try to allocate or boot an amphora instance (unconfigured) amp_for_failover_flow.add(self.get_amphora_for_lb_subflow( prefix=prefix + '-' + constants.FAILOVER_LOADBALANCER_FLOW, role=role)) # Create the VIP base (aka VRRP) port for the amphora. amp_for_failover_flow.add(network_tasks.CreateVIPBasePort( name=prefix + '-' + constants.CREATE_VIP_BASE_PORT, requires=(constants.VIP, constants.VIP_SG_ID, constants.AMPHORA_ID), provides=constants.BASE_PORT)) # Attach the VIP base (aka VRRP) port to the amphora. amp_for_failover_flow.add(compute_tasks.AttachPort( name=prefix + '-' + constants.ATTACH_PORT, requires=(constants.AMPHORA, constants.PORT), rebind={constants.PORT: constants.BASE_PORT})) # Update the amphora database record with the VIP base port info. amp_for_failover_flow.add(database_tasks.UpdateAmpFailoverDetails( name=prefix + '-' + constants.UPDATE_AMP_FAILOVER_DETAILS, requires=(constants.AMPHORA, constants.VIP, constants.BASE_PORT))) # Update the amphora networking for the plugged VIP port amp_for_failover_flow.add(network_tasks.GetAmphoraNetworkConfigsByID( name=prefix + '-' + constants.GET_AMPHORA_NETWORK_CONFIGS_BY_ID, requires=(constants.LOADBALANCER_ID, constants.AMPHORA_ID), provides=constants.AMPHORAE_NETWORK_CONFIG)) # Disable the base (vrrp) port on the failed amphora # This prevents a DAD failure when bringing up the new amphora. # Keepalived will handle this for act/stdby. if (role == constants.ROLE_STANDALONE and failed_amp_vrrp_port_id and is_vrrp_ipv6): amp_for_failover_flow.add(network_tasks.AdminDownPort( name=prefix + '-' + constants.ADMIN_DOWN_PORT, inject={constants.PORT_ID: failed_amp_vrrp_port_id})) amp_for_failover_flow.add(amphora_driver_tasks.AmphoraPostVIPPlug( name=prefix + '-' + constants.AMPHORA_POST_VIP_PLUG, requires=(constants.AMPHORA, constants.LOADBALANCER, constants.AMPHORAE_NETWORK_CONFIG))) # Plug member ports amp_for_failover_flow.add(network_tasks.CalculateAmphoraDelta( name=prefix + '-' + constants.CALCULATE_AMPHORA_DELTA, requires=(constants.LOADBALANCER, constants.AMPHORA, constants.AVAILABILITY_ZONE, constants.VRRP_PORT), rebind={constants.VRRP_PORT: constants.BASE_PORT}, provides=constants.DELTA)) amp_for_failover_flow.add(network_tasks.HandleNetworkDelta( name=prefix + '-' + constants.HANDLE_NETWORK_DELTA, requires=(constants.AMPHORA, constants.DELTA), provides=constants.ADDED_PORTS)) amp_for_failover_flow.add(amphora_driver_tasks.AmphoraePostNetworkPlug( name=prefix + '-' + constants.AMPHORAE_POST_NETWORK_PLUG, requires=(constants.LOADBALANCER, constants.ADDED_PORTS))) return amp_for_failover_flow