def test_rebalance_stability_join(self):
num_services = 10
num_nodes = 10000
# Adding 1 service to a set of N should move 1/(N+1) of all nodes
# Eg, for a cluster of 10 nodes, adding one should move 1/11, or 9%
# We allow for 1/N to allow for rounding in tests.
redistribution_factor = 1.0 / num_services
nodes = [str(x) for x in range(num_nodes)]
services = [str(x) for x in range(num_services)]
new_services = services + ['new']
delta = self._compare_rings(
nodes, services, hashring.HashRing(services),
new_services, hashring.HashRing(new_services))
self.assertLess(len(delta), num_nodes * redistribution_factor)
def test_report_state_one_node_one_port(self, mock_conn, mock_ir_client):
self.agent = ironic_neutron_agent.BaremetalNeutronAgent()
with mock.patch.object(self.agent.state_rpc,
'report_state',
autospec=True) as mock_report_state:
self.agent.ironic_client = mock_conn
mock_conn.ports.return_value = iter([FakePort1()])
self.agent.agent_id = 'agent_id'
self.agent.member_manager.hashring = hashring.HashRing(
[self.agent.agent_id])
expected = {
'topic': n_const.L2_AGENT_TOPIC,
'start_flag': True,
'binary': constants.BAREMETAL_BINARY,
'host': '55555555-4444-3333-2222-111111111111',
'configurations': {
'bridge_mappings': {
'physnet1': 'yes'
},
'log_agent_heartbeats': False,
},
'agent_type': constants.BAREMETAL_AGENT_TYPE
}
self.agent._report_state()
mock_report_state.assert_called_with(self.agent.context, expected)
def _load_hash_rings(self):
rings = {}
d2c = self.dbapi.get_active_hardware_type_dict()
for driver_name, hosts in d2c.items():
rings[driver_name] = hashring.HashRing(
hosts, partitions=2**CONF.hash_partition_exponent)
return rings
def test_rebalance_stability_leave(self):
num_services = 10
num_nodes = 10000
# Removing 1 service from a set of N should move 1/(N) of all nodes
# Eg, for a cluster of 10 nodes, removing one should move 1/10, or 10%
# We allow for 1/(N-1) to allow for rounding in tests.
redistribution_factor = 1.0 / (num_services - 1)
nodes = [str(x) for x in range(num_nodes)]
services = [str(x) for x in range(num_services)]
new_services = services[:]
new_services.pop()
delta = self._compare_rings(
nodes, services, hashring.HashRing(services),
new_services, hashring.HashRing(new_services))
self.assertLess(len(delta), num_nodes * redistribution_factor)
def test_add_node_bytes(self):
nodes = {'foo', 'bar'}
ring = hashring.HashRing(nodes)
self.assertEqual(nodes, set(ring.nodes.keys()))
self.assertEqual(2 ** 5 * len(nodes), len(ring))
nodes.add(b'Z\xe2\xfa\x90\x17EC\xac\xae\x88\xa7[\xa1}:E')
ring.add_node(b'Z\xe2\xfa\x90\x17EC\xac\xae\x88\xa7[\xa1}:E')
self.assertEqual(nodes, set(ring.nodes.keys()))
self.assertEqual(2 ** 5 * len(nodes), len(ring))
def test_add_node_unicode(self):
nodes = {'foo', 'bar'}
ring = hashring.HashRing(nodes)
self.assertEqual(nodes, set(ring.nodes.keys()))
self.assertEqual(2 ** 5 * len(nodes), len(ring))
nodes.add(u'\u0634\u0628\u06a9\u0647')
ring.add_node(u'\u0634\u0628\u06a9\u0647')
self.assertEqual(nodes, set(ring.nodes.keys()))
self.assertEqual(2 ** 5 * len(nodes), len(ring))
def test_add_node_weight(self):
nodes = {'foo', 'bar'}
ring = hashring.HashRing(nodes)
self.assertEqual(nodes, set(ring.nodes.keys()))
self.assertEqual(2 ** 5 * len(nodes), len(ring))
nodes.add('baz')
ring.add_node('baz', weight=10)
self.assertEqual(nodes, set(ring.nodes.keys()))
self.assertEqual(2 ** 5 * 12, len(ring))
def test_remove_node(self):
nodes = {'foo', 'bar'}
ring = hashring.HashRing(nodes)
self.assertEqual(nodes, set(ring.nodes.keys()))
self.assertEqual(2 ** 5 * len(nodes), len(ring))
nodes.discard('bar')
ring.remove_node('bar')
self.assertEqual(nodes, set(ring.nodes.keys()))
self.assertEqual(2 ** 5 * len(nodes), len(ring))
def test_ignore_nodes(self):
nodes = ['foo', 'bar', 'baz']
ring = hashring.HashRing(nodes)
equals_bar_or_baz = matchers.MatchesAny(matchers.Equals({'bar'}),
matchers.Equals({'baz'}))
self.assertThat(ring.get_nodes(b'fake', ignore_nodes=['foo']),
equals_bar_or_baz)
self.assertThat(ring.get_nodes(b'fake', ignore_nodes=['foo', 'bar']),
equals_bar_or_baz)
self.assertEqual(set(), ring.get_nodes(b'fake', ignore_nodes=nodes))
def __init__(self, coordinator, group_id,
partitions=DEFAULT_PARTITION_NUMBER):
self.partitions = partitions
self.group_id = group_id
self._coord = coordinator
self._coord.watch_join_group(self.group_id, self._on_member_join)
self._coord.watch_leave_group(self.group_id, self._on_member_leave)
members = self._coord.get_members(self.group_id)
self.ring = hashring.HashRing(members.get(),
partitions=self.partitions)
def test_distribution_one_replica(self):
nodes = ['foo', 'bar', 'baz']
ring = hashring.HashRing(nodes)
fake_1_nodes = ring.get_nodes(b'fake')
fake_2_nodes = ring.get_nodes(b'fake-again')
# We should have one nodes for each thing
self.assertEqual(1, len(fake_1_nodes))
self.assertEqual(1, len(fake_2_nodes))
# And they must not be the same answers even on this simple data.
self.assertNotEqual(fake_1_nodes, fake_2_nodes)
def test_start_flag_true_on_update_after_config_change(
self, mock_conn, mock_ir_client):
self.agent = ironic_neutron_agent.BaremetalNeutronAgent()
with mock.patch.object(self.agent.state_rpc,
'report_state',
autospec=True) as mock_report_state:
self.agent.ironic_client = mock_conn
mock_conn.ports.return_value = iter([FakePort1()])
self.agent.agent_id = 'agent_id'
self.agent.member_manager.hashring = hashring.HashRing(
[self.agent.agent_id])
expected = {
'topic': n_const.L2_AGENT_TOPIC,
'start_flag': 'PLACEHOLDER',
'binary': constants.BAREMETAL_BINARY,
'host': '55555555-4444-3333-2222-111111111111',
'configurations': {
'bridge_mappings': {
'physnet1': 'yes'
},
'log_agent_heartbeats': False,
},
'agent_type': constants.BAREMETAL_AGENT_TYPE
}
# First time report start_flag is True
expected.update({'start_flag': True})
self.agent._report_state()
mock_report_state.assert_called_with(self.agent.context, expected)
# Subsequent times report start_flag is False
mock_conn.ports.return_value = iter([FakePort1()])
expected.update({'start_flag': False})
self.agent._report_state()
mock_report_state.assert_called_with(self.agent.context, expected)
# After bridge_mapping config change start_flag is True once
mock_conn.ports.return_value = iter(
[FakePort1(physnet='new_physnet')])
expected.update({
'configurations': {
'bridge_mappings': {
'new_physnet': 'yes'
},
'log_agent_heartbeats': False
}
})
expected.update({'start_flag': True})
self.agent._report_state()
mock_report_state.assert_called_with(self.agent.context, expected)
# Subsequent times report start_flag is False
mock_conn.ports.return_value = iter(
[FakePort1(physnet='new_physnet')])
expected.update({'start_flag': False})
self.agent._report_state()
mock_report_state.assert_called_with(self.agent.context, expected)
def test_hash2int_returns_int(self, mock_md5):
r1 = 32 * 'a'
r2 = 32 * 'b'
# 2**PARTITION_EXPONENT calls to md5.update per node
# PARTITION_EXPONENT is currently always 5, so 32 calls each here
mock_md5.return_value.hexdigest.side_effect = [r1] * 32 + [r2] * 32
nodes = ['foo', 'bar']
ring = hashring.HashRing(nodes)
self.assertIn(int(r1, 16), ring._ring)
self.assertIn(int(r2, 16), ring._ring)
def _load_hash_rings(self):
rings = {}
d2c = self.dbapi.get_active_hardware_type_dict(
use_groups=self.use_groups)
for driver_name, hosts in d2c.items():
rings[driver_name] = hashring.HashRing(
hosts,
partitions=2**CONF.hash_partition_exponent,
hash_function=CONF.hash_ring_algorithm)
return rings
def __init__(self, coordinator, group_id,
partitions=DEFAULT_PARTITION_NUMBER):
members = coordinator.get_members(group_id)
self.partitions = partitions
self.group_id = group_id
self._coord = coordinator
caps = [(m, self._coord.get_member_capabilities(self.group_id, m))
for m in members.get()]
self._coord.watch_join_group(self.group_id, self._on_member_join)
self._coord.watch_leave_group(self.group_id, self._on_member_leave)
self.ring = hashring.HashRing([], partitions=self.partitions)
for m_id, cap in caps:
self.ring.add_node(m_id, utils.loads(cap.get()).get("weight", 1))
def test_state_rpc_report_state_fail(self, mock_report_state, mock_log,
mock_conn, mock_ir_client):
self.agent = ironic_neutron_agent.BaremetalNeutronAgent()
self.agent.agent_id = 'agent_id'
self.agent.member_manager.hashring = hashring.HashRing(
[self.agent.agent_id])
self.agent.ironic_client = mock_conn
self.agent.state_rpc = mock_report_state
mock_conn.ports.return_value = iter([FakePort1(), FakePort2()])
mock_report_state.report_state.side_effect = Exception()
self.agent._report_state()
self.assertEqual(1, mock_log.call_count)
def test_distribution_more_replica(self):
nodes = ['foo', 'bar', 'baz']
ring = hashring.HashRing(nodes)
fake_1_nodes = ring.get_nodes(b'fake', replicas=2)
fake_2_nodes = ring.get_nodes(b'fake-again', replicas=2)
# We should have one nodes for each thing
self.assertEqual(2, len(fake_1_nodes))
self.assertEqual(2, len(fake_2_nodes))
fake_1_nodes = ring.get_nodes(b'fake', replicas=3)
fake_2_nodes = ring.get_nodes(b'fake-again', replicas=3)
# We should have one nodes for each thing
self.assertEqual(3, len(fake_1_nodes))
self.assertEqual(3, len(fake_2_nodes))
self.assertEqual(fake_1_nodes, fake_2_nodes)
def test_hash2int_returns_int(self, mock_md5):
r1 = 32 * 'a'
r2 = 32 * 'b'
# 2**PARTITION_EXPONENT calls to md5.update per node
# PARTITION_EXPONENT is currently always 5, so 32 calls each here
mock_md5.return_value.hexdigest.side_effect = [r1] * 32 + [r2] * 32
nodes = ['foo', 'bar']
ring = hashring.HashRing(nodes)
self.assertIn(int(r1, 16), ring._ring)
self.assertIn(int(r2, 16), ring._ring)
if self._fips_enabled:
mock_md5.assert_called_with(mock.ANY, usedforsecurity=False)
else:
mock_md5.assert_called_with(mock.ANY)
def test_partitioning(self):
all_resources = ['resource_%s' % i for i in range(1000)]
agents = ['agent_%s' % i for i in range(10)]
expected_resources = [list() for _ in range(len(agents))]
hr = hashring.HashRing(agents, partitions=100)
for r in all_resources:
encode = coordination.PartitionCoordinator.encode_task
key = agents.index(list(hr.get_nodes(encode(r)))[0])
expected_resources[key].append(r)
agents_kwargs = []
for i, agent in enumerate(agents):
agents_kwargs.append(dict(agent_id=agent,
group_id='group',
all_resources=all_resources,
expected_resources=expected_resources[i]))
self._usage_simulation(*agents_kwargs)
def _load_hash_ring(self, refresh=False):
cache_timeout = timeutils.utcnow() - datetime.timedelta(
seconds=constants.HASH_RING_CACHE_TIMEOUT)
# Refresh the cache if:
# - Refreshed is forced (refresh=True)
# - Service just started (_wait_startup_before_caching)
# - Hash Ring is not yet instantiated
# - Cache has timed out
if (refresh or self._wait_startup_before_caching
or self._hash_ring is None or not self._hash_ring.nodes
or cache_timeout >= self._last_time_loaded):
nodes = db_hash_ring.get_active_nodes(
constants.HASH_RING_NODES_TIMEOUT)
self._hash_ring = hashring.HashRing(
{node.node_uuid
for node in nodes})
self._last_time_loaded = timeutils.utcnow()
class HashRingMemberManagerNotificationEndpoint(object):
"""Class variables members and hashring is shared by all instances"""
filter_rule = oslo_messaging.NotificationFilter(
publisher_id='^ironic-neutron-agent.*')
members = []
hashring = hashring.HashRing([])
def info(self, ctxt, publisher_id, event_type, payload, metadata):
timestamp = timeutils.utcnow_ts()
# Add members or update timestamp for existing members
if not payload['id'] in [x['id'] for x in self.members]:
try:
LOG.info('Adding member id %s on host %s to hashring.',
payload['id'], payload['host'])
self.hashring.add_node(payload['id'])
self.members.append(payload)
except Exception:
LOG.exception('Failed to add member %s to hash ring!',
payload['id'])
else:
for member in self.members:
if payload['id'] == member['id']:
member['timestamp'] = payload['timestamp']
# Remove members that have not checked in for a while
for member in self.members:
if (timestamp -
member['timestamp']) > (CONF.AGENT.report_interval * 3):
try:
LOG.info('Removing member %s on host %s from hashring.',
member['id'], member['host'])
self.hashring.remove_node(member['id'])
self.members.remove(member)
except Exception:
LOG.exception('Failed to remove member %s from hash ring!',
member['id'])
return oslo_messaging.NotificationResult.HANDLED
def extract_my_subset(self, group_id, iterable):
"""Filters an iterable, returning only objects assigned to this agent.
We have a list of objects and get a list of active group members from
`tooz`. We then hash all the objects into buckets and return only
the ones that hashed into *our* bucket.
"""
try:
members = self._get_members(group_id)
hr = hashring.HashRing(members, partitions=100)
iterable = list(iterable)
filtered = [v for v in iterable
if self._my_id in hr.get_nodes(self.encode_task(v))]
LOG.debug('The universal set: %s, my subset: %s',
[six.text_type(f) for f in iterable],
[six.text_type(f) for f in filtered])
return filtered
except tooz.coordination.ToozError:
LOG.exception('Error getting group membership info from '
'coordination backend.')
return []
# Copyright (C) 2020 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from tooz import hashring
hashring = hashring.HashRing({'node1', 'node2', 'node3'})
# Returns set(['node2'])
nodes_for_foo = hashring[b'foo']
# Returns set(['node2', 'node3'])
nodes_for_foo_with_replicas = hashring.get_nodes(b'foo',
replicas=2)
# Returns set(['node1', 'node3'])
nodes_for_foo_with_replicas = hashring.get_nodes(b'foo',
replicas=2,
ignore_nodes={'node2'})
def test_create_ring(self):
nodes = {'foo', 'bar'}
ring = hashring.HashRing(nodes)
self.assertEqual(nodes, set(ring.nodes.keys()))
self.assertEqual(2 ** 5 * 2, len(ring))
def test_ignore_non_existent_node(self):
nodes = ['foo', 'bar']
ring = hashring.HashRing(nodes)
self.assertEqual({'foo'}, ring.get_nodes(b'fake',
ignore_nodes=['baz']))
# -*- encoding: utf-8 -*-
from tooz import hashring
NUMBER_OF_NODES = 16
# Step #1 16개의 노드로 해시 링 생성
hr = hashring.HashRing(["node%d" % i for i in range(NUMBER_OF_NODES)])
nodes = hr.get_nodes(b"some data")
print(nodes)
nodes = hr.get_nodes(b"some data", replicas=2)
print(nodes)
nodes = hr.get_nodes(b"some other data", replicas=3)
print(nodes)
nodes = hr.get_nodes(b"some other of my data", replicas=2)
print(nodes)
# Step #2 노드 하나를 제거
print("Removing node8")
hr.remove_node("node8")
nodes = hr.get_nodes(b"some data")
print(nodes)
nodes = hr.get_nodes(b"some data", replicas=2)
print(nodes)
nodes = hr.get_nodes(b"some other data", replicas=3)
print(nodes)
nodes = hr.get_nodes(b"some other of my data", replicas=2)
print(nodes)
# Step #3 새 노드 추가
print("Adding node17")
def test_remove_node_unknown(self):
nodes = ['foo', 'bar']
ring = hashring.HashRing(nodes)
self.assertRaises(
hashring.UnknownNode,
ring.remove_node, 'biz')
