def mark_storagerouter_reachable_for_ha(cls, storagerouter):
# type: (StorageRouter) -> None
"""
Update the node distance map to add the storagerouter back into the HA pool
:param storagerouter: Storagerouter to put back into the distance map
:type storagerouter: StorageRouter
:return: None
"""
cls.logger.info("Marking Storagerouter {} as available for HA".format(
storagerouter.name))
Configuration.delete(os.path.join(VPOOL_UPDATE_KEY,
storagerouter.guid))
# Trigger a complete reload of node distance maps
StorageDriverController.cluster_registry_checkup()
# Wait a few moment for the edge to catch up all the configs
sleep_time = cls.get_edge_sync_time()
cls.logger.info(
"Waiting {} to sync up all edge clients".format(sleep_time))
time.sleep(sleep_time)
def mark_storagerouter_unreachable_for_ha(cls, storagerouter):
"""
Update the node distance maps to
Current code paths that update the node distance map on the volumedriver side are:
- Update of domains
- Update of vpool layout (extend/shrink)
- cluster registry checkup (ran periodically)
:return: None
:rtype: NoneType
"""
cls.logger.info(
"Marking Storagerouter {} as unavailable for HA".format(
storagerouter.name))
# Set the value used in the storagedriver cluster node config path
# This holds for all mentioned paths in the docstrings
Configuration.set(os.path.join(VPOOL_UPDATE_KEY, storagerouter.guid),
0)
# Trigger a complete reload of node distance maps
StorageDriverController.cluster_registry_checkup()
# Wait a few moment for the edge to catch up all the configs
sleep_time = cls.get_edge_sync_time()
cls.logger.info(
"Waiting {} to sync up all edge clients".format(sleep_time))
time.sleep(sleep_time)
def test_node_config_checkup(self):
"""
Validates correct working of cluster registry checkup
"""
base_structure = {
'1': {
'vrouter_id': '1',
'message_host': '10.0.1.1',
'message_port': 1,
'xmlrpc_host': '10.0.0.1',
'xmlrpc_port': 2,
'failovercache_host': '10.0.1.1',
'failovercache_port': 3,
'network_server_uri': 'tcp://10.0.1.1:4',
'node_distance_map': None
},
'2': {
'vrouter_id': '2',
'message_host': '10.0.1.2',
'message_port': 1,
'xmlrpc_host': '10.0.0.2',
'xmlrpc_port': 2,
'failovercache_host': '10.0.1.2',
'failovercache_port': 3,
'network_server_uri': 'tcp://10.0.1.2:4',
'node_distance_map': None
}
}
def _validate_node_config(_config, _expected_map):
expected = copy.deepcopy(base_structure[_config.vrouter_id])
expected['node_distance_map'] = _expected_map[_config.vrouter_id]
self.assertDictEqual(
expected, {
'vrouter_id': _config.vrouter_id,
'message_host': _config.message_host,
'message_port': _config.message_port,
'xmlrpc_host': _config.xmlrpc_host,
'xmlrpc_port': _config.xmlrpc_port,
'failovercache_host': _config.failovercache_host,
'failovercache_port': _config.failovercache_port,
'network_server_uri': _config.network_server_uri,
'node_distance_map': _config.node_distance_map
})
structure = DalHelper.build_dal_structure({
'vpools': [1],
'domains': [1, 2],
'storagerouters': [1, 2],
'storagedrivers':
[(1, 1, 1), (2, 1, 2)], # (<id>, <vpool_id>, <storagerouter_id>)
'storagerouter_domains': [(1, 1, 1, False), (2, 2, 1, False)]
} # (id>, <storagerouter_id>, <domain_id>, <backup>)
)
storagerouters = structure['storagerouters']
vpool = structure['vpools'][1]
arakoon_installer = ArakoonInstaller(cluster_name='voldrv')
arakoon_installer.create_cluster(
cluster_type=ServiceType.ARAKOON_CLUSTER_TYPES.SD,
ip=storagerouters[1].ip,
base_dir='/tmp')
# Initial run, it will now be configured
StorageRouterClient.node_config_recordings = []
result = StorageDriverController.cluster_registry_checkup()
self.assertDictEqual(result,
{vpool.guid: {
'success': True,
'changes': True
}})
self.assertListEqual(
sorted(StorageRouterClient.node_config_recordings), ['1', '2'])
expected_map = {
'1': {
'2': StorageDriver.DISTANCES.NEAR
},
'2': {
'1': StorageDriver.DISTANCES.NEAR
}
}
configs = vpool.clusterregistry_client.get_node_configs()
for config in configs:
_validate_node_config(config, expected_map)
# Running it again should not change anything
StorageRouterClient.node_config_recordings = []
result = StorageDriverController.cluster_registry_checkup()
self.assertDictEqual(result,
{vpool.guid: {
'success': True,
'changes': False
}})
self.assertListEqual(
sorted(StorageRouterClient.node_config_recordings), [])
expected_map = {
'1': {
'2': StorageDriver.DISTANCES.NEAR
},
'2': {
'1': StorageDriver.DISTANCES.NEAR
}
}
configs = vpool.clusterregistry_client.get_node_configs()
for config in configs:
_validate_node_config(config, expected_map)
# Validate some error paths
domain = structure['domains'][2]
junction = structure['storagerouters'][1].domains[0]
junction.domain = domain
junction.save()
vpool_config_path = 'file://opt/OpenvStorage/config/framework.json?key=/ovs/vpools/{0}/hosts/1/config'.format(
vpool.guid)
StorageRouterClient.exceptions['server_revision'] = {
vpool_config_path: Exception('ClusterNotReachableException')
}
StorageRouterClient.node_config_recordings = []
result = StorageDriverController.cluster_registry_checkup()
self.assertDictEqual(result,
{vpool.guid: {
'success': True,
'changes': True
}})
self.assertListEqual(
sorted(StorageRouterClient.node_config_recordings), ['2'])
expected_map = {
'1': {
'2': StorageDriver.DISTANCES.INFINITE
},
'2': {
'1': StorageDriver.DISTANCES.INFINITE
}
}
configs = vpool.clusterregistry_client.get_node_configs()
for config in configs:
_validate_node_config(config, expected_map)
def test_node_config_checkup(self):
"""
Validates correct working of cluster registry checkup
"""
base_structure = {'1': {'vrouter_id': '1',
'message_host': '10.0.1.1',
'message_port': 1,
'xmlrpc_host': '10.0.0.1',
'xmlrpc_port': 2,
'failovercache_host': '10.0.1.1',
'failovercache_port': 3,
'network_server_uri': 'tcp://10.0.1.1:4',
'node_distance_map': None},
'2': {'vrouter_id': '2',
'message_host': '10.0.1.2',
'message_port': 1,
'xmlrpc_host': '10.0.0.2',
'xmlrpc_port': 2,
'failovercache_host': '10.0.1.2',
'failovercache_port': 3,
'network_server_uri': 'tcp://10.0.1.2:4',
'node_distance_map': None}}
def _validate_node_config(_config, _expected_map):
expected = copy.deepcopy(base_structure[_config.vrouter_id])
expected['node_distance_map'] = _expected_map[_config.vrouter_id]
self.assertDictEqual(expected, {'vrouter_id': _config.vrouter_id,
'message_host': _config.message_host,
'message_port': _config.message_port,
'xmlrpc_host': _config.xmlrpc_host,
'xmlrpc_port': _config.xmlrpc_port,
'failovercache_host': _config.failovercache_host,
'failovercache_port': _config.failovercache_port,
'network_server_uri': _config.network_server_uri,
'node_distance_map': _config.node_distance_map})
structure = Helper.build_service_structure(
{'vpools': [1],
'domains': [1, 2],
'storagerouters': [1, 2],
'storagedrivers': [(1, 1, 1), (2, 1, 2)], # (<id>, <vpool_id>, <storagerouter_id>)
'storagerouter_domains': [(1, 1, 1, False), (2, 2, 1, False)]} # (id>, <storagerouter_id>, <domain_id>, <backup>)
)
storagerouters = structure['storagerouters']
vpool = structure['vpools'][1]
System._machine_id = {storagerouters[1].ip: '1',
storagerouters[2].ip: '2'}
ArakoonInstaller.create_cluster('voldrv', ServiceType.ARAKOON_CLUSTER_TYPES.SD, storagerouters[1].ip, '/tmp')
# Initial run, it will now be configured
StorageRouterClient.node_config_recordings = []
result = StorageDriverController.cluster_registry_checkup()
self.assertDictEqual(result, {vpool.guid: {'success': True,
'changes': True}})
self.assertListEqual(sorted(StorageRouterClient.node_config_recordings), ['1', '2'])
expected_map = {'1': {'2': StorageDriver.DISTANCES.NEAR},
'2': {'1': StorageDriver.DISTANCES.NEAR}}
configs = vpool.clusterregistry_client.get_node_configs()
for config in configs:
_validate_node_config(config, expected_map)
# Running it again should not change anything
StorageRouterClient.node_config_recordings = []
result = StorageDriverController.cluster_registry_checkup()
self.assertDictEqual(result, {vpool.guid: {'success': True,
'changes': False}})
self.assertListEqual(sorted(StorageRouterClient.node_config_recordings), [])
expected_map = {'1': {'2': StorageDriver.DISTANCES.NEAR},
'2': {'1': StorageDriver.DISTANCES.NEAR}}
configs = vpool.clusterregistry_client.get_node_configs()
for config in configs:
_validate_node_config(config, expected_map)
# Validate some error paths
domain = structure['domains'][2]
junction = structure['storagerouters'][1].domains[0]
junction.domain = domain
junction.save()
vpool_config_path = 'file://opt/OpenvStorage/config/framework.json?key=/ovs/vpools/{0}/hosts/1/config'.format(vpool.guid)
StorageRouterClient.exceptions['server_revision'] = {vpool_config_path: Exception('ClusterNotReachableException')}
StorageRouterClient.node_config_recordings = []
result = StorageDriverController.cluster_registry_checkup()
self.assertDictEqual(result, {vpool.guid: {'success': True,
'changes': True}})
self.assertListEqual(sorted(StorageRouterClient.node_config_recordings), ['2'])
expected_map = {'1': {'2': StorageDriver.DISTANCES.INFINITE},
'2': {'1': StorageDriver.DISTANCES.INFINITE}}
configs = vpool.clusterregistry_client.get_node_configs()
for config in configs:
_validate_node_config(config, expected_map)
