def test_maintenance_agents_for_local_backends_wo_layout(self):
"""
Validates the checkup maintenance agents for LOCAL ALBA Backends without a specific layout specified
Additionally test:
* Checkup maintenance agents for a specific ALBA Backend
* Downscale the required services
* Upscale the required services
"""
alba_structure = AlbaDalHelper.build_dal_structure(
structure={
'alba_nodes': [1, 2, 3, 4],
'alba_backends': [[1, 'LOCAL']],
'alba_abm_clusters': [1]
})
# Simulate every ALBA Node has 1 OSD for `alba_backend_1`
local_ip = '127.0.0.1'
alba_backend_1 = alba_structure['alba_backends'][1]
for index, alba_node in enumerate(
alba_structure['alba_nodes'].itervalues()):
port = 35000 + index
ManagerClientMockup.test_results[alba_node].update({
'get_service_status': {
'status': [None, 'active']
},
'add_maintenance_service':
'',
'remove_maintenance_service':
'',
'get_stack': {
alba_node.node_id: {
'osds': {
'osd_id_{0}'.format(index): {
'ips': [local_ip],
'port': port
}
}
}
}
})
VirtualAlbaBackend.data['{0}:{1}'.format(
local_ip, port)] = alba_backend_1.guid
# Since all ALBA Nodes (4) are suitable for a maintenance agent, we only deploy a default amount of 3
AlbaController.checkup_maintenance_agents()
# Example of ManagerClientMockup.maintenance_agents
# {
# <AlbaNode (guid: d43df79f-9c47-4059-bd84-0f3ef81733c2, at: 0x7f80028e4750)>: {'alba-maintenance_backend_1-RWvL8aCzwIBk6FaZ': ['node_3']},
# <AlbaNode (guid: 5dbf972d-2619-48d1-adcd-86ec5b6342f7, at: 0x7f80027ee2d0)>: {'alba-maintenance_backend_1-OgvznajMoRagKCVb': ['node_2']},
# <AlbaNode (guid: 79a762f7-3019-4b86-80d6-a5560c52b208, at: 0x7f80027ee0d0)>: {'alba-maintenance_backend_1-ZV9v2vtRfvaYBhhw': ['node_4']}
# }
self.assertEqual(
first=3, second=len(ManagerClientMockup.maintenance_agents)
) # 3 out of 4 ALBA Nodes should have a maintenance agent
for alba_node, maintenance_info in ManagerClientMockup.maintenance_agents.iteritems(
):
self.assertEqual(first=1, second=len(maintenance_info))
self.assertEqual(first=[alba_node.node_id],
second=ManagerClientMockup.
maintenance_agents[alba_node].values()[0])
# Downscale the required amount of services from 3 to 2
config_key = AlbaController.NR_OF_AGENTS_CONFIG_KEY.format(
alba_backend_1.guid)
Configuration.set(key=config_key, value=2)
nodes_w_agents = ManagerClientMockup.maintenance_agents.keys()
AlbaController.checkup_maintenance_agents()
self.assertEqual(
first=2, second=len(ManagerClientMockup.maintenance_agents)
) # 2 out of 4 ALBA Nodes should have a maintenance agent now
for alba_node, maintenance_info in ManagerClientMockup.maintenance_agents.iteritems(
):
self.assertEqual(first=1, second=len(maintenance_info))
self.assertEqual(first=[alba_node.node_id],
second=ManagerClientMockup.
maintenance_agents[alba_node].values()[0])
for alba_node in ManagerClientMockup.maintenance_agents:
self.assertIn(
member=alba_node, container=nodes_w_agents
) # 1 removed, rest should still be part of previously used ALBA Nodes
# Upscale the required amount of services from 2 to 3 again
Configuration.set(key=config_key, value=3)
AlbaController.checkup_maintenance_agents()
self.assertEqual(
first=3, second=len(ManagerClientMockup.maintenance_agents)
) # 3 out of 4 ALBA Nodes should again have a maintenance agent
for alba_node, maintenance_info in ManagerClientMockup.maintenance_agents.iteritems(
):
self.assertEqual(first=1, second=len(maintenance_info))
self.assertEqual(first=[alba_node.node_id],
second=ManagerClientMockup.
maintenance_agents[alba_node].values()[0])
# Create an additional ALBA Backend and verify that it is not processed when asking to checkup the previously created ALBA Backend
alba_structure = AlbaDalHelper.build_dal_structure(
structure={
'alba_backends': [[2, 'LOCAL']],
'alba_abm_clusters': [2]
},
previous_structure=alba_structure)
alba_backend_2 = alba_structure['alba_backends'][2]
AlbaController.checkup_maintenance_agents(
alba_backend_guid=alba_backend_1.guid
) # Run checkup for previously created ALBA Backend, nothing should change
self.assertEqual(
first=3, second=len(ManagerClientMockup.maintenance_agents)
) # 3 out of 4 ALBA Nodes should again have a maintenance agent
for alba_node, maintenance_info in ManagerClientMockup.maintenance_agents.iteritems(
):
self.assertEqual(first=1, second=len(maintenance_info))
self.assertEqual(first=[alba_node.node_id],
second=ManagerClientMockup.
maintenance_agents[alba_node].values()[0])
# Execute a general checkup maintenance agents and verify newly created ALBA Backend has 1 service (because not linked to any OSDs)
AlbaController.checkup_maintenance_agents()
services = []
for alba_node, maintenance_info in ManagerClientMockup.maintenance_agents.iteritems(
):
alba_node.invalidate_dynamics('maintenance_services')
if alba_backend_2.name in alba_node.maintenance_services:
services.append(maintenance_info)
self.assertEqual(first=1, second=len(maintenance_info))
self.assertEqual(first=[alba_node.node_id],
second=maintenance_info.values()[0])
self.assertEqual(
first=1, second=len(services)
) # Only 1 service should have been deployed for the 2nd ALBA Backend
def test_maintenance_agents_for_local_backends_w_layout(self):
"""
Validates the checkup maintenance agents for LOCAL ALBA Backends with a specific layout specified
Additionally test whether at least 1 maintenance agent gets deployed even though none of the ALBA Nodes is linked to the ALBA Backend
"""
alba_structure = AlbaDalHelper.build_dal_structure(
structure={
'alba_nodes': [1, 2, 3, 4],
'alba_backends': [[1, 'LOCAL']],
'alba_abm_clusters': [1]
})
alba_backend = alba_structure['alba_backends'][1]
config_key = AlbaController.AGENTS_LAYOUT_CONFIG_KEY.format(
alba_backend.guid)
unknown_node_name = 'non-existing-node'
# Mock some return values for some of the calls performed by `checkup_maintenance_agents`
for alba_node in alba_structure['alba_nodes'].itervalues():
ManagerClientMockup.test_results[alba_node].update({
'get_stack': {},
'get_service_status': {
'status': [None, 'active']
},
'add_maintenance_service':
'',
'remove_maintenance_service':
''
})
###############################
# Verify incorrect layout value
log_entry = 'Layout is not a list and will be ignored'
Configuration.set(key=config_key,
value=unknown_node_name) # Value should be a list
# Checkup maintenance agents will not find any suitable ALBA Nodes to deploy a maintenance agent on, because no ALBA Nodes are linked to the ALBA Backend yet,
# therefore it'll deploy a maintenance on a random ALBA Node
AlbaController.checkup_maintenance_agents()
self.assertIn(member=log_entry, container=Logger._logs['lib'].keys())
self.assertEqual(first='WARNING',
second=Logger._logs['lib'][log_entry])
# Example of ManagerClientMockup.maintenance_agents
# {<AlbaNode (guid: c015cf06-8bd0-46c5-811d-41ac6f521a63, at: 0x7f77cb0af390)>: {'alba-maintenance_backend_1-J6PMBcEk1Ej42udp': ['node_1']}}
self.assertEqual(
first=1, second=len(ManagerClientMockup.maintenance_agents.keys(
))) # Only 1 ALBA Node should have a maintenance agent running
self.assertEqual(
first=1,
second=len(ManagerClientMockup.maintenance_agents.values())
) # Only 1 maintenance agent should have been deployed on that 1 ALBA Node
alba_node_w_agent_1 = ManagerClientMockup.maintenance_agents.keys()[0]
self.assertEqual(
first=[alba_node_w_agent_1.node_id],
second=ManagerClientMockup.maintenance_agents[alba_node_w_agent_1].
values()[0]
) # Read preference must be the Node ID of the Node on which the maintenance was deployed
# 3 out of 4 ALBA Nodes do not have a maintenance agent yet
alba_nodes_wo_agent = [
an for an in alba_structure['alba_nodes'].itervalues()
if an != alba_node_w_agent_1
]
self.assertEqual(first=3, second=len(alba_nodes_wo_agent))
# Link an ALBA Node without agent to the ALBA Backend, forcing the previously deployed service to be removed and a new 1 to be created on this ALBA Node
alba_node = alba_nodes_wo_agent[0]
VirtualAlbaBackend.data['127.0.0.1:35001'] = alba_backend.guid
ManagerClientMockup.test_results[alba_node]['get_stack'] = {
alba_node.node_id: {
'osds': {
'osd_id_1': {
'ips': ['127.0.0.1'],
'port': 35001
}
}
}
}
alba_node.invalidate_dynamics('stack')
AlbaController.checkup_maintenance_agents()
self.assertEqual(
first=1, second=len(ManagerClientMockup.maintenance_agents.keys(
))) # Only 1 ALBA Node should have a maintenance agent running
self.assertEqual(
first=1,
second=len(ManagerClientMockup.maintenance_agents.values())
) # Only 1 maintenance agent should have been deployed on that 1 ALBA Node
self.assertNotEqual(
first=alba_node_w_agent_1, second=alba_node
) # The maintenance agent should have moved to the node linked to the ALBA Backend
self.assertEqual(
first=[alba_node.node_id],
second=ManagerClientMockup.maintenance_agents[alba_node].values()
[0]
) # Read preference must be the Node ID of the Node on which the maintenance was moved to
# Re-scheduling a checkup should not change anything at this point
AlbaController.checkup_maintenance_agents()
self.assertEqual(
first=1, second=len(ManagerClientMockup.maintenance_agents.keys(
))) # Only 1 ALBA Node should have a maintenance agent running
self.assertEqual(
first=1,
second=len(ManagerClientMockup.maintenance_agents.values())
) # Only 1 maintenance agent should have been deployed on that 1 ALBA Node
alba_node_w_agent_2 = ManagerClientMockup.maintenance_agents.keys()[0]
self.assertEqual(first=alba_node, second=alba_node_w_agent_2
) # The maintenance agent should not have moved
# Set 2 out of 4 ALBA Nodes in the layout key
alba_nodes_wo_agent = [
an for an in alba_structure['alba_nodes'].itervalues()
if an != alba_node_w_agent_2
]
self.assertEqual(first=3, second=len(alba_nodes_wo_agent))
node_1 = alba_nodes_wo_agent[0]
node_2 = alba_nodes_wo_agent[1]
Configuration.set(key=config_key,
value=[node_1.node_id, node_2.node_id])
AlbaController.checkup_maintenance_agents()
self.assertIn(member=node_1,
container=ManagerClientMockup.maintenance_agents
) # Specified in the layout
self.assertIn(member=node_2,
container=ManagerClientMockup.maintenance_agents
) # Specified in the layout
self.assertEqual(
first=2, second=len(ManagerClientMockup.maintenance_agents)
) # Only the 2 specified ALBA Nodes should be running a maintenance agent
self.assertEqual(first=1,
second=len(
ManagerClientMockup.maintenance_agents[node_1])
) # 1 Maintenance agent for this ALBA Node
self.assertEqual(first=1,
second=len(
ManagerClientMockup.maintenance_agents[node_2])
) # 1 Maintenance agent for this ALBA Node
self.assertEqual(
first=[node_1.node_id],
second=ManagerClientMockup.maintenance_agents[node_1].values()
[0]) # Validate the read preference
self.assertEqual(
first=[node_2.node_id],
second=ManagerClientMockup.maintenance_agents[node_2].values()
[0]) # Validate the read preference
#########################################
# Verify all ALBA Nodes unknown in layout
Logger._logs['lib'] = {}
log_entry = 'Layout does not contain any known/reachable nodes and will be ignored'
Configuration.set(key=config_key,
value=[unknown_node_name
]) # Only unknown Nodes in layout
AlbaController.checkup_maintenance_agents()
self.assertIn(member=log_entry, container=Logger._logs['lib'].keys())
self.assertEqual(first='WARNING',
second=Logger._logs['lib'][log_entry])
self.assertIn(
member=alba_node, container=ManagerClientMockup.maintenance_agents
) # The ALBA Node linked to the ALBA Backend should again have the maintenance agent
self.assertEqual(
first=1,
second=len(ManagerClientMockup.maintenance_agents[alba_node]
)) # Only 1 maintenance agent should have been deployed
self.assertEqual(
first=[alba_node.node_id],
second=ManagerClientMockup.maintenance_agents[alba_node].values()
[0]
) # Read preference must be the Node ID of the Node on which the maintenance was deployed
# 3 out of 4 ALBA Nodes do not have a maintenance agent yet
alba_nodes_wo_agent = [
an for an in alba_structure['alba_nodes'].itervalues()
if an != alba_node_w_agent_1
]
self.assertEqual(first=3, second=len(alba_nodes_wo_agent))
#############################################
# Verify at least 1 known ALBA Node in layout
Logger._logs['lib'] = {}
node_3 = alba_structure['alba_nodes'][3]
log_entry = 'Layout contains unknown/unreachable node {0}'.format(
unknown_node_name)
Configuration.set(key=config_key,
value=[unknown_node_name, node_3.node_id
]) # 1 known ALBA Node in layout
AlbaController.checkup_maintenance_agents()
self.assertIn(member=log_entry, container=Logger._logs['lib'].keys())
self.assertEqual(first='WARNING',
second=Logger._logs['lib'][log_entry])
self.assertIn(
member=node_3, container=ManagerClientMockup.maintenance_agents
) # The ALBA Node specified in the layout should have the maintenance agent
self.assertEqual(
first=1,
second=len(ManagerClientMockup.maintenance_agents[node_3]
)) # Only 1 maintenance agent should have been deployed
self.assertEqual(
first=[node_3.node_id],
second=ManagerClientMockup.maintenance_agents[node_3].values()[0]
) # Read preference must be the Node ID of the Node on which the maintenance was deployed
# 3 out of 4 ALBA Nodes do not have a maintenance agent yet
alba_nodes_wo_agent = [
an for an in alba_structure['alba_nodes'].itervalues()
if an != node_3
]
self.assertEqual(first=3, second=len(alba_nodes_wo_agent))
def checkup_maintenance_agents():
"""
Perform a maintenance agent checkup
:return: None
"""
AlbaController.checkup_maintenance_agents()
