def get_create_member_flow(self): """Create a flow to create a member :returns: The flow for creating a member """ create_member_flow = linear_flow.Flow(constants.CREATE_MEMBER_FLOW) create_member_flow.add( lifecycle_tasks.MemberToErrorOnRevertTask(requires=[ constants.MEMBER, constants.LISTENERS, constants.LOADBALANCER, constants.POOL ])) create_member_flow.add( database_tasks.MarkMemberPendingCreateInDB( requires=constants.MEMBER)) create_member_flow.add( network_tasks.CalculateDelta(requires=constants.LOADBALANCER, provides=constants.DELTAS)) create_member_flow.add( network_tasks.HandleNetworkDeltas(requires=constants.DELTAS, provides=constants.ADDED_PORTS)) create_member_flow.add( amphora_driver_tasks.AmphoraePostNetworkPlug( requires=(constants.LOADBALANCER, constants.ADDED_PORTS))) create_member_flow.add( amphora_driver_tasks.ListenersUpdate( requires=(constants.LOADBALANCER, constants.LISTENERS))) create_member_flow.add( database_tasks.MarkMemberActiveInDB(requires=constants.MEMBER)) create_member_flow.add( database_tasks.MarkPoolActiveInDB(requires=constants.POOL)) create_member_flow.add( database_tasks.MarkLBAndListenersActiveInDB( requires=(constants.LOADBALANCER, constants.LISTENERS))) return create_member_flow
def get_create_load_balancer_graph_flows(self, topology, prefix): allocate_amphorae_flow = self.get_create_load_balancer_flow(topology) f_name = constants.CREATE_LOADBALANCER_GRAPH_FLOW lb_create_graph_flow = linear_flow.Flow(f_name) lb_create_graph_flow.add( self.get_post_lb_amp_association_flow(prefix, topology)) lb_create_graph_flow.add( database_tasks.ReloadLoadBalancer( name=constants.RELOAD_LB_AFTER_AMP_ASSOC_FULL_GRAPH, requires=constants.LOADBALANCER_ID, provides=constants.LOADBALANCER)) lb_create_graph_flow.add( network_tasks.CalculateDelta(requires=constants.LOADBALANCER, provides=constants.DELTAS)) lb_create_graph_flow.add( network_tasks.HandleNetworkDeltas(requires=constants.DELTAS, provides=constants.ADDED_PORTS)) lb_create_graph_flow.add( amphora_driver_tasks.AmphoraePostNetworkPlug( requires=(constants.LOADBALANCER, constants.ADDED_PORTS))) lb_create_graph_flow.add( self.listener_flows.get_create_all_listeners_flow()) lb_create_graph_flow.add( database_tasks.MarkLBActiveInDB(mark_listeners=True, requires=constants.LOADBALANCER)) return allocate_amphorae_flow, lb_create_graph_flow
def _create_listeners_flow(self): flows = [] flows.append( database_tasks.ReloadLoadBalancer( name=constants.RELOAD_LB_AFTER_AMP_ASSOC_FULL_GRAPH, requires=constants.LOADBALANCER_ID, provides=constants.LOADBALANCER ) ) flows.append( network_tasks.CalculateDelta( requires=constants.LOADBALANCER, provides=constants.DELTAS ) ) flows.append( network_tasks.HandleNetworkDeltas( requires=constants.DELTAS, provides=constants.ADDED_PORTS ) ) flows.append( amphora_driver_tasks.AmphoraePostNetworkPlug( requires=(constants.LOADBALANCER, constants.ADDED_PORTS) ) ) flows.append( self.listener_flows.get_create_all_listeners_flow() ) flows.append( database_tasks.MarkLBActiveInDB( mark_subobjects=True, requires=constants.LOADBALANCER ) ) return flows
def get_failover_flow(self, role=constants.ROLE_STANDALONE, status=constants.AMPHORA_READY): """Creates a flow to failover a stale amphora :returns: The flow for amphora failover """ failover_amphora_flow = linear_flow.Flow( constants.FAILOVER_AMPHORA_FLOW) failover_amphora_flow.add(lifecycle_tasks.AmphoraToErrorOnRevertTask( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA)) # Delete the old amphora failover_amphora_flow.add( database_tasks.MarkAmphoraPendingDeleteInDB( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA)) failover_amphora_flow.add( database_tasks.MarkAmphoraHealthBusy( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA)) failover_amphora_flow.add(compute_tasks.ComputeDelete( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA)) failover_amphora_flow.add(network_tasks.WaitForPortDetach( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA)) failover_amphora_flow.add( database_tasks.DisableAmphoraHealthMonitoring( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA)) failover_amphora_flow.add(database_tasks.MarkAmphoraDeletedInDB( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA)) # If this is an unallocated amp (spares pool), we're done if status != constants.AMPHORA_ALLOCATED: return failover_amphora_flow # Save failed amphora details for later failover_amphora_flow.add( database_tasks.GetAmphoraDetails( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA, provides=constants.AMP_DATA)) # Get a new amphora # Note: Role doesn't matter here. We will update it later. get_amp_subflow = self.get_amphora_for_lb_subflow( prefix=constants.FAILOVER_AMPHORA_FLOW) failover_amphora_flow.add(get_amp_subflow) # Update the new amphora with the failed amphora details failover_amphora_flow.add(database_tasks.UpdateAmpFailoverDetails( requires=(constants.AMPHORA, constants.AMP_DATA))) failover_amphora_flow.add(database_tasks.ReloadLoadBalancer( requires=constants.LOADBALANCER_ID, provides=constants.LOADBALANCER)) failover_amphora_flow.add(network_tasks.GetAmphoraeNetworkConfigs( requires=constants.LOADBALANCER, provides=constants.AMPHORAE_NETWORK_CONFIG)) failover_amphora_flow.add(database_tasks.GetListenersFromLoadbalancer( requires=constants.LOADBALANCER, provides=constants.LISTENERS)) failover_amphora_flow.add(amphora_driver_tasks.ListenersUpdate( requires=(constants.LOADBALANCER, constants.LISTENERS))) # Plug the VIP ports into the new amphora failover_amphora_flow.add(network_tasks.PlugVIPPort( requires=(constants.AMPHORA, constants.AMPHORAE_NETWORK_CONFIG))) failover_amphora_flow.add(amphora_driver_tasks.AmphoraPostVIPPlug( requires=(constants.AMPHORA, constants.LOADBALANCER, constants.AMPHORAE_NETWORK_CONFIG))) # Plug the member networks into the new amphora failover_amphora_flow.add(network_tasks.CalculateDelta( requires=constants.LOADBALANCER, provides=constants.DELTAS)) failover_amphora_flow.add(network_tasks.HandleNetworkDeltas( requires=constants.DELTAS, provides=constants.ADDED_PORTS)) failover_amphora_flow.add(amphora_driver_tasks.AmphoraePostNetworkPlug( requires=(constants.LOADBALANCER, constants.ADDED_PORTS))) # Handle the amphora role and VRRP if necessary if role == constants.ROLE_MASTER: failover_amphora_flow.add(database_tasks.MarkAmphoraMasterInDB( name=constants.MARK_AMP_MASTER_INDB, requires=constants.AMPHORA)) vrrp_subflow = self.get_vrrp_subflow(role) failover_amphora_flow.add(vrrp_subflow) elif role == constants.ROLE_BACKUP: failover_amphora_flow.add(database_tasks.MarkAmphoraBackupInDB( name=constants.MARK_AMP_BACKUP_INDB, requires=constants.AMPHORA)) vrrp_subflow = self.get_vrrp_subflow(role) failover_amphora_flow.add(vrrp_subflow) elif role == constants.ROLE_STANDALONE: failover_amphora_flow.add( database_tasks.MarkAmphoraStandAloneInDB( name=constants.MARK_AMP_STANDALONE_INDB, requires=constants.AMPHORA)) failover_amphora_flow.add(amphora_driver_tasks.ListenersStart( requires=(constants.LOADBALANCER, constants.LISTENERS))) return failover_amphora_flow
def get_batch_update_members_flow(self, old_members, new_members, updated_members): """Create a flow to batch update members :returns: The flow for batch updating members """ batch_update_members_flow = linear_flow.Flow( constants.BATCH_UPDATE_MEMBERS_FLOW) unordered_members_flow = unordered_flow.Flow( constants.UNORDERED_MEMBER_UPDATES_FLOW) unordered_members_active_flow = unordered_flow.Flow( constants.UNORDERED_MEMBER_ACTIVE_FLOW) # Delete old members unordered_members_flow.add( lifecycle_tasks.MembersToErrorOnRevertTask( inject={constants.MEMBERS: old_members}, name='{flow}-deleted'.format( flow=constants.MEMBER_TO_ERROR_ON_REVERT_FLOW))) for m in old_members: unordered_members_flow.add( model_tasks.DeleteModelObject( inject={constants.OBJECT: m}, name='{flow}-{id}'.format( id=m.id, flow=constants.DELETE_MODEL_OBJECT_FLOW))) unordered_members_flow.add( database_tasks.DeleteMemberInDB( inject={constants.MEMBER: m}, name='{flow}-{id}'.format( id=m.id, flow=constants.DELETE_MEMBER_INDB))) unordered_members_flow.add( database_tasks.DecrementMemberQuota( inject={constants.MEMBER: m}, name='{flow}-{id}'.format( id=m.id, flow=constants.DECREMENT_MEMBER_QUOTA_FLOW))) # Create new members unordered_members_flow.add( lifecycle_tasks.MembersToErrorOnRevertTask( inject={constants.MEMBERS: new_members}, name='{flow}-created'.format( flow=constants.MEMBER_TO_ERROR_ON_REVERT_FLOW))) for m in new_members: unordered_members_active_flow.add( database_tasks.MarkMemberActiveInDB( inject={constants.MEMBER: m}, name='{flow}-{id}'.format( id=m.id, flow=constants.MARK_MEMBER_ACTIVE_INDB))) # Update existing members unordered_members_flow.add( lifecycle_tasks.MembersToErrorOnRevertTask( # updated_members is a list of (obj, dict), only pass `obj` inject={constants.MEMBERS: [m[0] for m in updated_members]}, name='{flow}-updated'.format( flow=constants.MEMBER_TO_ERROR_ON_REVERT_FLOW))) for m, um in updated_members: um.pop('id', None) unordered_members_flow.add( model_tasks.UpdateAttributes( inject={ constants.OBJECT: m, constants.UPDATE_DICT: um }, name='{flow}-{id}'.format( id=m.id, flow=constants.UPDATE_ATTRIBUTES_FLOW))) unordered_members_flow.add( database_tasks.UpdateMemberInDB( inject={ constants.MEMBER: m, constants.UPDATE_DICT: um }, name='{flow}-{id}'.format( id=m.id, flow=constants.UPDATE_MEMBER_INDB))) unordered_members_active_flow.add( database_tasks.MarkMemberActiveInDB( inject={constants.MEMBER: m}, name='{flow}-{id}'.format( id=m.id, flow=constants.MARK_MEMBER_ACTIVE_INDB))) batch_update_members_flow.add(unordered_members_flow) # Done, do real updates batch_update_members_flow.add( network_tasks.CalculateDelta(requires=constants.LOADBALANCER, provides=constants.DELTAS)) batch_update_members_flow.add( network_tasks.HandleNetworkDeltas(requires=constants.DELTAS, provides=constants.ADDED_PORTS)) batch_update_members_flow.add( amphora_driver_tasks.AmphoraePostNetworkPlug( requires=(constants.LOADBALANCER, constants.ADDED_PORTS))) # Update the Listener (this makes the changes active on the Amp) batch_update_members_flow.add( amphora_driver_tasks.ListenersUpdate( requires=(constants.LOADBALANCER, constants.LISTENERS))) # Mark all the members ACTIVE here, then pool then LB/Listeners batch_update_members_flow.add(unordered_members_active_flow) batch_update_members_flow.add( database_tasks.MarkPoolActiveInDB(requires=constants.POOL)) batch_update_members_flow.add( database_tasks.MarkLBAndListenersActiveInDB( requires=(constants.LOADBALANCER, constants.LISTENERS))) return batch_update_members_flow
def test_handle_network_deltas(self, mock_get_net_driver): mock_driver = mock.MagicMock() mock_get_net_driver.return_value = mock_driver def _interface(network_id): return [data_models.Interface(network_id=network_id)] net = network_tasks.HandleNetworkDeltas() net.execute({}) self.assertFalse(mock_driver.plug_network.called) delta = data_models.Delta(amphora_id=self.amphora_mock.id, compute_id=self.amphora_mock.compute_id, add_nics=[], delete_nics=[]) net.execute({self.amphora_mock.id: delta}) self.assertFalse(mock_driver.plug_network.called) delta = data_models.Delta(amphora_id=self.amphora_mock.id, compute_id=self.amphora_mock.compute_id, add_nics=_interface(1), delete_nics=[]) net.execute({self.amphora_mock.id: delta}) mock_driver.plug_network.assert_called_once_with(COMPUTE_ID, 1) # revert net.execute({self.amphora_mock.id: delta}) self.assertFalse(mock_driver.unplug_network.called) delta = data_models.Delta(amphora_id=self.amphora_mock.id, compute_id=self.amphora_mock.compute_id, add_nics=[], delete_nics=[]) net.execute({self.amphora_mock.id: delta}) self.assertFalse(mock_driver.unplug_network.called) delta = data_models.Delta(amphora_id=self.amphora_mock.id, compute_id=self.amphora_mock.compute_id, add_nics=_interface(1), delete_nics=[]) mock_driver.reset_mock() mock_driver.unplug_network.side_effect = net_base.NetworkNotFound mock_driver.reset_mock() mock_driver.unplug_network.side_effect = TestException('test') self.assertRaises(TestException, net.revert, mock.ANY, {self.amphora_mock.id: delta}) mock_driver.unplug_network.assert_called_once_with(COMPUTE_ID, 1) mock_driver.reset_mock() net.execute({}) self.assertFalse(mock_driver.unplug_network.called) delta = data_models.Delta(amphora_id=self.amphora_mock.id, compute_id=self.amphora_mock.compute_id, add_nics=[], delete_nics=[]) net.execute({self.amphora_mock.id: delta}) self.assertFalse(mock_driver.unplug_network.called) delta = data_models.Delta(amphora_id=self.amphora_mock.id, compute_id=self.amphora_mock.compute_id, add_nics=[], delete_nics=_interface(1)) net.execute({self.amphora_mock.id: delta}) mock_driver.unplug_network.assert_called_once_with(COMPUTE_ID, 1) mock_driver.reset_mock() mock_driver.unplug_network.side_effect = net_base.NetworkNotFound net.execute({self.amphora_mock.id: delta}) mock_driver.unplug_network.assert_called_once_with(COMPUTE_ID, 1) # Do a test with a general exception in case behavior changes mock_driver.reset_mock() mock_driver.unplug_network.side_effect = Exception() net.execute({self.amphora_mock.id: delta}) mock_driver.unplug_network.assert_called_once_with(COMPUTE_ID, 1)
def get_failover_flow(self, role=constants.ROLE_STANDALONE, load_balancer_id=None): """Creates a flow to failover a stale amphora :returns: The flow for amphora failover """ failover_amphora_flow = linear_flow.Flow( constants.FAILOVER_AMPHORA_FLOW) failover_amphora_flow.add( lifecycle_tasks.AmphoraToErrorOnRevertTask( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA)) # Note: It seems intuitive to boot an amphora prior to deleting # the old amphora, however this is a complicated issue. # If the target host (due to anit-affinity) is resource # constrained, this will fail where a post-delete will # succeed. Since this is async with the API it would result # in the LB ending in ERROR though the amps are still alive. # Consider in the future making this a complicated # try-on-failure-retry flow, or move upgrade failovers to be # synchronous with the API. For now spares pool and act/stdby # will mitigate most of this delay. # Delete the old amphora failover_amphora_flow.add( database_tasks.MarkAmphoraPendingDeleteInDB( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA)) failover_amphora_flow.add( database_tasks.MarkAmphoraHealthBusy( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA)) failover_amphora_flow.add( compute_tasks.ComputeDelete( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA)) failover_amphora_flow.add( network_tasks.WaitForPortDetach( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA)) failover_amphora_flow.add( database_tasks.MarkAmphoraDeletedInDB( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA)) # If this is an unallocated amp (spares pool), we're done if not load_balancer_id: failover_amphora_flow.add( database_tasks.DisableAmphoraHealthMonitoring( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA)) return failover_amphora_flow # Save failed amphora details for later failover_amphora_flow.add( database_tasks.GetAmphoraDetails( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA, provides=constants.AMP_DATA)) # Get a new amphora # Note: Role doesn't matter here. We will update it later. get_amp_subflow = self.get_amphora_for_lb_subflow( prefix=constants.FAILOVER_AMPHORA_FLOW) failover_amphora_flow.add(get_amp_subflow) # Update the new amphora with the failed amphora details failover_amphora_flow.add( database_tasks.UpdateAmpFailoverDetails( requires=(constants.AMPHORA, constants.AMP_DATA))) # Update the data stored in the flow from the database failover_amphora_flow.add( database_tasks.ReloadLoadBalancer( requires=constants.LOADBALANCER_ID, provides=constants.LOADBALANCER)) failover_amphora_flow.add( database_tasks.ReloadAmphora(requires=constants.AMPHORA_ID, provides=constants.AMPHORA)) # Prepare to reconnect the network interface(s) failover_amphora_flow.add( network_tasks.GetAmphoraeNetworkConfigs( requires=constants.LOADBALANCER, provides=constants.AMPHORAE_NETWORK_CONFIG)) failover_amphora_flow.add( database_tasks.GetListenersFromLoadbalancer( requires=constants.LOADBALANCER, provides=constants.LISTENERS)) failover_amphora_flow.add( amphora_driver_tasks.ListenersUpdate( requires=(constants.LOADBALANCER, constants.LISTENERS))) # Plug the VIP ports into the new amphora failover_amphora_flow.add( network_tasks.PlugVIPPort( requires=(constants.AMPHORA, constants.AMPHORAE_NETWORK_CONFIG))) failover_amphora_flow.add( amphora_driver_tasks.AmphoraPostVIPPlug( requires=(constants.AMPHORA, constants.LOADBALANCER, constants.AMPHORAE_NETWORK_CONFIG))) # Plug the member networks into the new amphora failover_amphora_flow.add( network_tasks.CalculateDelta(requires=constants.LOADBALANCER, provides=constants.DELTAS)) failover_amphora_flow.add( network_tasks.HandleNetworkDeltas(requires=constants.DELTAS, provides=constants.ADDED_PORTS)) failover_amphora_flow.add( amphora_driver_tasks.AmphoraePostNetworkPlug( requires=(constants.LOADBALANCER, constants.ADDED_PORTS))) # Handle the amphora role and VRRP if necessary if role == constants.ROLE_MASTER: failover_amphora_flow.add( database_tasks.MarkAmphoraMasterInDB( name=constants.MARK_AMP_MASTER_INDB, requires=constants.AMPHORA)) vrrp_subflow = self.get_vrrp_subflow(role) failover_amphora_flow.add(vrrp_subflow) elif role == constants.ROLE_BACKUP: failover_amphora_flow.add( database_tasks.MarkAmphoraBackupInDB( name=constants.MARK_AMP_BACKUP_INDB, requires=constants.AMPHORA)) vrrp_subflow = self.get_vrrp_subflow(role) failover_amphora_flow.add(vrrp_subflow) elif role == constants.ROLE_STANDALONE: failover_amphora_flow.add( database_tasks.MarkAmphoraStandAloneInDB( name=constants.MARK_AMP_STANDALONE_INDB, requires=constants.AMPHORA)) failover_amphora_flow.add( amphora_driver_tasks.ListenersStart( requires=(constants.LOADBALANCER, constants.LISTENERS))) failover_amphora_flow.add( database_tasks.DisableAmphoraHealthMonitoring( rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA)) return failover_amphora_flow