def test_v4_full_disk_testing():
# check how kea4 behave when disk is full, using memfile and logs to file
assert False, "this test may destroy your setup, remove this line if you really want to run it"
misc.test_setup()
_create_ramdisk()
srv_control.config_srv_subnet('192.168.50.0/24',
'192.168.50.1-192.168.50.150')
srv_control.add_hooks('libdhcp_legal_log.so')
srv_control.open_control_channel()
srv_control.agent_control_channel(world.f_cfg.mgmt_address)
# set memfile and log location to created ramdisk
world.dhcp_cfg["lease-database"] = {
"type": "memfile",
"name": "/tmp/kea_ram_disk/dhcp.leases"
}
world.dhcp_cfg["loggers"] = [{
"debuglevel":
99,
"name":
"kea-dhcp4",
"output_options": [{
"output": "/tmp/kea_ram_disk/kea.log"
}],
"severity":
"DEBUG"
}]
world.dhcp_cfg["hooks-libraries"][0].update({
"parameters": {
"path": "/tmp/kea_ram_disk/",
"base-name": "kea-forensic"
}
})
srv_control.add_hooks('libdhcp_lease_cmds.so')
# srv_control.print_cfg()
srv_control.build_and_send_config_files()
srv_control.start_srv('DHCP', 'started')
_allocate_disk_space()
print(_check_disk())
# we expected that kea will keep working properly, assigning addresses in memory,
# turned out that when it can't save lease to memfile/db it return NAK/NoAddrAvail code
# even if disk is reported with no empty space left kea is able to save multiple leases
# mostly it's 36.
# Kea will keep working when it can't write main logs
# Kea will keep working when it can't write forensic logs, and log error about this event in main logs
generate_leases(leases_count=100,
dhcp_version='v4',
iapd=0,
mac="01:02:0c:03:0a:00")
send_command(cmd={"command": "lease4-get-all"})
def test_HA_hot_standby_hold_state_always(dhcp_version):
# HA SERVER 1
misc.test_setup()
if dhcp_version == 'v6':
srv_control.config_srv_subnet('2001:db8:1::/64',
'2001:db8:1::1-2001:db8:1::1')
else:
srv_control.config_srv_subnet('192.168.50.0/24',
'192.168.50.1-192.168.50.1')
srv_control.open_control_channel()
srv_control.agent_control_channel(world.f_cfg.mgmt_address)
srv_control.add_hooks('libdhcp_lease_cmds.so')
srv_control.add_ha_hook('libdhcp_ha.so')
world.dhcp_cfg["hooks-libraries"][1].update(HOT_STANDBY)
srv_control.update_ha_hook_parameter({
"heartbeat-delay": 1000,
"max-ack-delay": 0,
"max-response-delay": 1100,
"max-unacked-clients": 0,
"this-server-name": "server1",
"state-machine": {
"states": [{
"state": "waiting",
"pause": "always"
}, {
"state": "syncing",
"pause": "always"
}, {
"state": "ready",
"pause": "always"
}, {
"state": "hot-standby",
"pause": "always"
}, {
"state": "partner-down",
"pause": "always"
}]
}
})
srv_control.build_and_send_config_files()
srv_control.start_srv('DHCP', 'started')
# HA SERVER 2
misc.test_setup()
srv_control.clear_some_data('all', dest=world.f_cfg.mgmt_address_2)
srv_control.start_srv('DHCP', 'stopped', dest=world.f_cfg.mgmt_address_2)
if dhcp_version == 'v6':
srv_control.config_srv_subnet('2001:db8:1::/64',
'2001:db8:1::1-2001:db8:1::1')
else:
srv_control.config_srv_subnet('192.168.50.0/24',
'192.168.50.1-192.168.50.1')
srv_control.open_control_channel()
srv_control.agent_control_channel(world.f_cfg.mgmt_address_2)
srv_control.add_hooks('libdhcp_lease_cmds.so')
srv_control.add_ha_hook('libdhcp_ha.so')
world.dhcp_cfg["hooks-libraries"][1].update(HOT_STANDBY)
srv_control.update_ha_hook_parameter({
"heartbeat-delay": 1000,
"max-ack-delay": 0,
"max-response-delay": 1100,
"max-unacked-clients": 0,
"this-server-name": "server2",
"state-machine": {
"states": []
}
})
srv_control.build_and_send_config_files(dest=world.f_cfg.mgmt_address_2)
srv_control.start_srv('DHCP', 'started', dest=world.f_cfg.mgmt_address_2)
srv_msg.forge_sleep(3, 'seconds')
_send_message(dhcp=dhcp_version, expect_answer=False)
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "waiting"
assert send_heartbeat(
dhcp_version=dhcp_version,
dest=world.f_cfg.mgmt_address_2)["arguments"]["state"] == "waiting"
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "waiting"
assert send_heartbeat(
dhcp_version=dhcp_version,
dest=world.f_cfg.mgmt_address_2)["arguments"]["state"] == "waiting"
# continue server1 from WAITING
assert send_command(dhcp_version=dhcp_version,
cmd={"command": "ha-continue"
})["text"] == 'HA state machine continues.'
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "syncing"
assert send_heartbeat(
dhcp_version=dhcp_version,
dest=world.f_cfg.mgmt_address_2)["arguments"]["state"] == "waiting"
# continue server1 from SYNCING
assert send_command(dhcp_version=dhcp_version,
cmd={"command": "ha-continue"
})["text"] == 'HA state machine continues.'
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "ready"
assert send_heartbeat(
dhcp_version=dhcp_version,
dest=world.f_cfg.mgmt_address_2)["arguments"]["state"] == "ready"
_send_message(dhcp=dhcp_version, expect_answer=False)
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "ready"
assert send_heartbeat(
dhcp_version=dhcp_version,
dest=world.f_cfg.mgmt_address_2)["arguments"]["state"] == "ready"
_send_message(dhcp=dhcp_version, expect_answer=False)
# continue server1 from READY
assert send_command(dhcp_version=dhcp_version,
cmd={"command": "ha-continue"
})["text"] == 'HA state machine continues.'
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "hot-standby"
assert send_heartbeat(
dhcp_version=dhcp_version,
dest=world.f_cfg.mgmt_address_2)["arguments"]["state"] == "hot-standby"
_send_message(dhcp=dhcp_version)
srv_control.start_srv('DHCP', 'stopped', dest=world.f_cfg.mgmt_address_2)
srv_msg.forge_sleep(3, 'seconds')
# server1 has to keep hot-standby
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "hot-standby"
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "hot-standby"
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "hot-standby"
# continue server1 from hot-standby
assert send_command(dhcp_version=dhcp_version,
cmd={"command": "ha-continue"
})["text"] == 'HA state machine continues.'
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "partner-down"
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "partner-down"
_send_message(dhcp=dhcp_version)
srv_control.start_srv('DHCP', 'started', dest=world.f_cfg.mgmt_address_2)
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "partner-down"
assert send_heartbeat(
dhcp_version=dhcp_version,
dest=world.f_cfg.mgmt_address_2)["arguments"]["state"] == "ready"
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "partner-down"
assert send_heartbeat(
dhcp_version=dhcp_version,
dest=world.f_cfg.mgmt_address_2)["arguments"]["state"] == "ready"
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "partner-down"
assert send_heartbeat(
dhcp_version=dhcp_version,
dest=world.f_cfg.mgmt_address_2)["arguments"]["state"] == "ready"
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "partner-down"
assert send_heartbeat(
dhcp_version=dhcp_version,
dest=world.f_cfg.mgmt_address_2)["arguments"]["state"] == "ready"
# continue server1 from partner-down
assert send_command(dhcp_version=dhcp_version,
cmd={"command": "ha-continue"
})["text"] == 'HA state machine continues.'
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "hot-standby"
assert send_heartbeat(
dhcp_version=dhcp_version,
dest=world.f_cfg.mgmt_address_2)["arguments"]["state"] == "hot-standby"
srv_control.start_srv('DHCP', 'stopped', dest=world.f_cfg.mgmt_address_2)
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "hot-standby"
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "hot-standby"
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "hot-standby"
# continue AGAIN from hot-standby
assert send_command(dhcp_version=dhcp_version,
cmd={"command": "ha-continue"
})["text"] == 'HA state machine continues.'
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "partner-down"
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "partner-down"
srv_control.start_srv('DHCP', 'started', dest=world.f_cfg.mgmt_address_2)
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "partner-down"
assert send_heartbeat(
dhcp_version=dhcp_version,
dest=world.f_cfg.mgmt_address_2)["arguments"]["state"] == "ready"
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "partner-down"
assert send_heartbeat(
dhcp_version=dhcp_version,
dest=world.f_cfg.mgmt_address_2)["arguments"]["state"] == "ready"
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "partner-down"
assert send_heartbeat(
dhcp_version=dhcp_version,
dest=world.f_cfg.mgmt_address_2)["arguments"]["state"] == "ready"
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "partner-down"
assert send_heartbeat(
dhcp_version=dhcp_version,
dest=world.f_cfg.mgmt_address_2)["arguments"]["state"] == "ready"
assert send_command(dhcp_version=dhcp_version,
cmd={"command": "ha-continue"
})["text"] == 'HA state machine continues.'
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "hot-standby"
assert send_heartbeat(
dhcp_version=dhcp_version,
dest=world.f_cfg.mgmt_address_2)["arguments"]["state"] == "hot-standby"
_send_message(dhcp=dhcp_version)
def test_HA_load_balancing_hold_state_once(dhcp_version):
# HA SERVER 1
misc.test_setup()
if dhcp_version == 'v6':
srv_control.config_srv_subnet('2001:db8:1::/64',
'2001:db8:1::1-2001:db8:1::1')
else:
srv_control.config_srv_subnet('192.168.50.0/24',
'192.168.50.1-192.168.50.1')
srv_control.open_control_channel()
srv_control.agent_control_channel(world.f_cfg.mgmt_address)
srv_control.add_hooks('libdhcp_lease_cmds.so')
srv_control.add_ha_hook('libdhcp_ha.so')
world.dhcp_cfg["hooks-libraries"][1].update(LOAD_BALANCING)
srv_control.update_ha_hook_parameter({
"heartbeat-delay": 1000,
"max-ack-delay": 0,
"max-response-delay": 1100,
"max-unacked-clients": 0,
"this-server-name": "server1",
"state-machine": {
"states": [{
"state": "waiting",
"pause": "once"
}, {
"state": "syncing",
"pause": "once"
}, {
"state": "ready",
"pause": "once"
}, {
"state": "load-balancing",
"pause": "once"
}, {
"state": "partner-down",
"pause": "once"
}]
}
})
srv_control.build_and_send_config_files()
srv_control.start_srv('DHCP', 'started')
# HA SERVER 2
misc.test_setup()
srv_control.clear_some_data('all', dest=world.f_cfg.mgmt_address_2)
srv_control.start_srv('DHCP', 'stopped', dest=world.f_cfg.mgmt_address_2)
if dhcp_version == 'v6':
srv_control.config_srv_subnet('2001:db8:1::/64',
'2001:db8:1::1-2001:db8:1::1')
else:
srv_control.config_srv_subnet('192.168.50.0/24',
'192.168.50.1-192.168.50.1')
srv_control.open_control_channel()
srv_control.agent_control_channel(world.f_cfg.mgmt_address_2)
srv_control.add_hooks('libdhcp_lease_cmds.so')
srv_control.add_ha_hook('libdhcp_ha.so')
world.dhcp_cfg["hooks-libraries"][1].update(LOAD_BALANCING)
srv_control.update_ha_hook_parameter({
"heartbeat-delay": 1000,
"max-ack-delay": 0,
"max-response-delay": 1100,
"max-unacked-clients": 0,
"this-server-name": "server2",
"state-machine": {
"states": []
}
})
srv_control.build_and_send_config_files(dest=world.f_cfg.mgmt_address_2)
srv_control.start_srv('DHCP', 'started', dest=world.f_cfg.mgmt_address_2)
srv_msg.forge_sleep(3, 'seconds')
# check that both servers keep waiting (server1 is paused, server2 is waiting for server1
_send_message(dhcp=dhcp_version, expect_answer=False)
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "waiting"
assert send_heartbeat(
dhcp_version=dhcp_version,
dest=world.f_cfg.mgmt_address_2)["arguments"]["state"] == "waiting"
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "waiting"
assert send_heartbeat(
dhcp_version=dhcp_version,
dest=world.f_cfg.mgmt_address_2)["arguments"]["state"] == "waiting"
# continue server1 from WAITING
assert send_command(dhcp_version=dhcp_version,
cmd={"command": "ha-continue"
})["text"] == 'HA state machine continues.'
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "syncing"
assert send_heartbeat(
dhcp_version=dhcp_version,
dest=world.f_cfg.mgmt_address_2)["arguments"]["state"] == "waiting"
# continue server1 from SYNCING
assert send_command(dhcp_version=dhcp_version,
cmd={"command": "ha-continue"
})["text"] == 'HA state machine continues.'
# keep in ready
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "ready"
assert send_heartbeat(
dhcp_version=dhcp_version,
dest=world.f_cfg.mgmt_address_2)["arguments"]["state"] == "ready"
_send_message(dhcp=dhcp_version, expect_answer=False)
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "ready"
assert send_heartbeat(
dhcp_version=dhcp_version,
dest=world.f_cfg.mgmt_address_2)["arguments"]["state"] == "ready"
_send_message(dhcp=dhcp_version, expect_answer=False)
# continue server1 from READY
assert send_command(dhcp_version=dhcp_version,
cmd={"command": "ha-continue"
})["text"] == 'HA state machine continues.'
wait_until_ha_state("load-balancing", dhcp_version=dhcp_version)
wait_until_ha_state("load-balancing",
dest=world.f_cfg.mgmt_address_2,
dhcp_version=dhcp_version)
_send_message(dhcp=dhcp_version)
# stop server2, server1 should not move to partner-down
srv_control.start_srv('DHCP', 'stopped', dest=world.f_cfg.mgmt_address_2)
srv_msg.forge_sleep(3, 'seconds')
# server1 has to keep load-balancing
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "load-balancing"
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "load-balancing"
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "load-balancing"
# continue server1 from load-balancing
assert send_command(dhcp_version=dhcp_version,
cmd={"command": "ha-continue"
})["text"] == 'HA state machine continues.'
wait_until_ha_state("partner-down", dhcp_version=dhcp_version)
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "partner-down"
_send_message(dhcp=dhcp_version)
# start second server, first should stay in partner-down
srv_control.start_srv('DHCP', 'started', dest=world.f_cfg.mgmt_address_2)
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "partner-down"
assert send_heartbeat(
dhcp_version=dhcp_version,
dest=world.f_cfg.mgmt_address_2)["arguments"]["state"] == "ready"
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "partner-down"
assert send_heartbeat(
dhcp_version=dhcp_version,
dest=world.f_cfg.mgmt_address_2)["arguments"]["state"] == "ready"
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "partner-down"
assert send_heartbeat(
dhcp_version=dhcp_version,
dest=world.f_cfg.mgmt_address_2)["arguments"]["state"] == "ready"
# continue from partner down
assert send_command(dhcp_version=dhcp_version,
cmd={"command": "ha-continue"
})["text"] == 'HA state machine continues.'
srv_msg.forge_sleep(3, 'seconds')
wait_until_ha_state("load-balancing", dhcp_version=dhcp_version)
wait_until_ha_state("load-balancing",
dhcp_version=dhcp_version,
dest=world.f_cfg.mgmt_address_2)
srv_control.start_srv('DHCP', 'stopped', dest=world.f_cfg.mgmt_address_2)
srv_msg.forge_sleep(3, 'seconds')
# this time - no paused states!
wait_until_ha_state("partner-down", dhcp_version=dhcp_version)
assert send_command(dhcp_version=dhcp_version,
cmd={"command": "ha-continue"
})["text"] == 'HA state machine is not paused.'
srv_msg.forge_sleep(3, 'seconds')
assert send_heartbeat(
dhcp_version=dhcp_version)["arguments"]["state"] == "partner-down"
assert send_command(dhcp_version=dhcp_version,
cmd={"command": "ha-continue"
})["text"] == 'HA state machine is not paused.'
srv_msg.forge_sleep(3, 'seconds')
srv_control.start_srv('DHCP', 'started', dest=world.f_cfg.mgmt_address_2)
wait_until_ha_state("load-balancing", dhcp_version=dhcp_version)
wait_until_ha_state("load-balancing",
dhcp_version=dhcp_version,
dest=world.f_cfg.mgmt_address_2)
_send_message(dhcp=dhcp_version)