def test_decommissioning_working_node(self):
self.start_redpanda(num_nodes=4)
topics = []
for partition_count in range(1, 5):
for replication_factor in (3, 3):
name = f"topic{len(topics)}"
spec = TopicSpec(name=name,
partition_count=partition_count,
replication_factor=replication_factor)
topics.append(spec)
for spec in topics:
self.client().create_topic(spec)
self.topic = spec.name
self.start_producer(1)
self.start_consumer(1)
self.await_startup()
admin = Admin(self.redpanda)
brokers = admin.get_brokers()
to_decommission = random.choice(brokers)
self.logger.info(f"decommissioning node: {to_decommission}", )
admin.decommission_broker(to_decommission['node_id'])
def node_removed():
brokers = admin.get_brokers()
for b in brokers:
if b['node_id'] == to_decommission['node_id']:
return False
return True
wait_until(node_removed, timeout_sec=120, backoff_sec=2)
self.run_validation(enable_idempotence=False, consumer_timeout_sec=45)
def decommission(node_id):
self.logger.info(f"decommissioning node: {node_id}")
admin = Admin(self.redpanda)
admin.decommission_broker(id=node_id)
def node_removed():
admin = Admin(self.redpanda)
brokers = admin.get_brokers()
for b in brokers:
if b['node_id'] == node_id:
return False
return True
wait_until(node_removed, timeout_sec=240, backoff_sec=2)
def test_decommissioning_working_node(self):
self.start_redpanda(num_nodes=4)
topics = []
for partition_count in range(1, 5):
for replication_factor in (3, 3):
name = f"topic{len(topics)}"
spec = TopicSpec(name=name,
partition_count=partition_count,
replication_factor=replication_factor)
topics.append(spec)
for spec in topics:
self.client().create_topic(spec)
self.topic = spec.name
self.start_producer(1)
self.start_consumer(1)
self.await_startup()
admin = Admin(self.redpanda)
brokers = admin.get_brokers()
to_decommission = random.choice(brokers)
self.logger.info(f"decommissioning node: {to_decommission}", )
admin.decommission_broker(to_decommission['node_id'])
# A node which isn't being decommed, to use when calling into
# the admin API from this point onwards.
survivor_node = [
n for n in self.redpanda.nodes
if self.redpanda.idx(n) != to_decommission['node_id']
][0]
self.logger.info(
f"Using survivor node {survivor_node.name} {self.redpanda.idx(survivor_node)}"
)
def node_removed():
brokers = admin.get_brokers(node=survivor_node)
for b in brokers:
if b['node_id'] == to_decommission['node_id']:
return False
return True
wait_until(node_removed, timeout_sec=120, backoff_sec=2)
self.run_validation(enable_idempotence=False, consumer_timeout_sec=45)
def decommissioned():
try:
admin = Admin(self.redpanda)
# if broker is already draining, it is suceess
brokers = admin.get_brokers()
for b in brokers:
if b['node_id'] == node_id and b[
'membership_status'] == 'draining':
return True
r = admin.decommission_broker(id=node_id)
return r.status_code == 200
except requests.exceptions.RetryError:
return False
except requests.exceptions.ConnectionError:
return False
except requests.exceptions.HTTPError:
return False