def global_service_with_reconn_disconn_host(client, state):
# Pick one of the host and power down hosts
host_down = ha_host_list[0]
host_name = ha_host_list[0].hostname
print "power down- " + host_name
action_on_digital_ocean_machine(ha_droplets[0][host_name], "power_off")
wait_for_host_agent_state(client, host_down, state,
HOST_ACTIVE_DISCONN_TIMEOUT)
# Create service
launch_config = {"imageUuid": HEALTH_CHECK_IMAGE_UUID}
launch_config["labels"] = {"io.rancher.scheduler.global": "true"}
service, env = create_env_and_svc(client, launch_config)
service = service.activate()
service = client.wait_success(service, 300)
assert service.state == "active"
container_list = get_service_container_list(client, service)
assert len(container_list) == get_service_instance_count(client, service)
# Power on the host
action_on_digital_ocean_machine(ha_droplets[0][host_name], "power_on")
wait_for_host_agent_state(client, host_down, "active",
HOST_DISCONN_ACTIVE_TIMEOUT)
service = wait_success(client, service, 300)
container_list = get_service_container_list(client, service)
assert len(container_list) == get_service_instance_count(client, service)
instance_list = get_containers_on_host_for_service(
client, host_down, service)
assert len(instance_list) == 1
def host_down_with_lb_services(super_client, client, lb_port, host_down_count,
scale=2, lb_scale=2, globalf=False):
# Wait for hosts in "reconnecting" state to get to "active" state
check_hosts_state(client)
# Create environment with lb_service and 2 healthcheck enabled
# service targets
env, lb_service, service1, service2 = \
env_with_lb_service_with_health_check_enabled_targets(
super_client, client, lb_port, scale, lb_scale, globalf)
# Pick hosts (and collect instances that will fgo unhealthy) that need
# to be powered down
down_hosts = []
down_instances = []
for i in range(0, len(ha_host_list)):
host = ha_host_list[i]
instance_list = \
get_containers_on_host_for_service(client, host, lb_service)
if len(instance_list) > 0:
down_instances.extend(instance_list)
down_hosts.append(host)
if len(down_hosts) == host_down_count:
break
# Power Down hosts where lb service instances are running
for host in down_hosts:
host_name = host.hostname
print "power down- " + host_name
print ha_droplets[0]
action_on_digital_ocean_machine(ha_droplets[0][host_name], "power_off")
# Check for service reconcile
check_for_service_reconcile(
super_client, client, lb_service, down_instances, instance_list)
validate_lb_service(super_client, client, lb_service,
lb_port, [service1, service2])
# Power on hosts that were powered off
for host in down_hosts:
host_name = host.hostname
print "power on- " + host_name
action_on_digital_ocean_machine(ha_droplets[0][host_name], "power_on")
# if service is global, validate that new instances of the service gets
# created on the host that gets powered on
if (globalf):
check_hosts_state(client)
wait_for_scale_to_adjust(super_client, service1)
wait_for_scale_to_adjust(super_client, service2)
wait_for_scale_to_adjust(super_client, lb_service)
validate_lb_service(super_client, client, lb_service,
lb_port, [service1, service2])
delete_all(client, [env])
def host_down_with_healthcheck_services(super_client, client, host_down_count,
retainIp=False):
# Wait for hosts in "reconnecting" state to get to "active" state
check_hosts_state(client)
# Create service that is healthcheck enabled
scale = 10
env, service = service_with_healthcheck_enabled(
client, super_client, scale, retainIp=retainIp)
# Pick hosts (and collect instances that will fgo unhealthy) that need
# to be powered down
down_hosts = []
down_instances = []
for i in range(0, len(ha_host_list)):
host = ha_host_list[i]
instance_list = \
get_containers_on_host_for_service(client, host, service)
if len(instance_list) > 0:
down_instances.extend(instance_list)
down_hosts.append(host)
if len(down_hosts) == host_down_count:
break
# Power Down hosts where service instances are running
for host in down_hosts:
host_name = host.hostname
print "power down- " + host_name
print ha_droplets[0]
action_on_digital_ocean_machine(ha_droplets[0][host_name], "power_off")
# Check for service reconcile
check_for_service_reconcile(
super_client, client, service, down_instances, instance_list)
# If retainIp is turned on , make sure that ip address assigned to
# reconciled instances are the same
if (retainIp):
for con in down_instances:
container_name = con.name
containers = super_client.list_container(name=container_name,
removed_null=True)
assert len(containers) == 1
container = containers[0]
assert container.primaryIpAddress == con.primaryIpAddress
assert container.externalId != con.externalId
# Power on hosts that were powered off
delete_all(client, [env])
for host in down_hosts:
host_name = host.hostname
print "power on- " + host_name
action_on_digital_ocean_machine(ha_droplets[0][host_name], "power_on")
def check_host_state_power_on(client):
print "Power on hosts that are in disconnected or reconnecting state"
print ha_droplets
inactive_hosts = client.list_host(
kind='docker', removed_null=True, agentState="disconnected")
print "Disconnected hosts:"
print inactive_hosts
reconn_hosts = client.list_host(
kind='docker', removed_null=True, agentState="reconnecting")
print "Reconnecting hosts:"
print reconn_hosts
inactive_hosts_dropletids = []
inactive_hosts_list = []
# Get droplet Id and hosts from disconnected hosts
for host in inactive_hosts:
host_name = host.hostname
print host_name
droplet_id = ha_droplets[0][host_name]
inactive_hosts_dropletids.append(droplet_id)
inactive_hosts_list.append(host)
# Get droplet Id and hosts from reconnecting hosts
# and append to the inactive host/droplet lists
for host in reconn_hosts:
host_name = host.hostname
print host_name
droplet_id = ha_droplets[0][host_name]
inactive_hosts_dropletids.append(droplet_id)
inactive_hosts_list.append(host)
print "List of all disconnected/reconnecting hosts"
print inactive_hosts_list
print inactive_hosts_dropletids
# Power on the droplets
for dropletid in inactive_hosts_dropletids:
print "Power on droplet " + str(droplet_id)
action_on_digital_ocean_machine(dropletid, "power_on")
# Wait for the host agent state to become active
for host in inactive_hosts_list:
print "In host wait method"
wait_for_host_agent_state(client, host, "active", 600)
def host_down_with_services(client, host_down_count,
globalf=False):
# Wait for hosts in "reconnecting" state to get to "active" state
check_hosts_state(client)
# Create service
launch_config = {"imageUuid": HEALTH_CHECK_IMAGE_UUID}
if globalf:
launch_config["labels"] = {"io.rancher.scheduler.global": "true"}
scale = 0
else:
scale = 10
service, env = create_env_and_svc(client, launch_config, scale)
service = service.activate()
service = client.wait_success(service, 300)
assert service.state == "active"
container_list = get_service_container_list(client, service)
assert len(container_list) == get_service_instance_count(client, service)
# Pick hosts (and collect instances that will go unhealthy) that need
# to be powered down
down_hosts = []
down_instances = []
for i in range(0, len(ha_host_list)):
host = ha_host_list[i]
instance_list = \
get_containers_on_host_for_service(client, host, service)
if len(instance_list) > 0:
down_instances.extend(instance_list)
down_hosts.append(host)
if len(down_hosts) == host_down_count:
break
# Power Down hosts where service instances are running
for host in down_hosts:
host_name = host.hostname
print "power down- " + host_name
print ha_droplets[0]
action_on_digital_ocean_machine(ha_droplets[0][host_name], "power_off")
print "Waiting for the hosts to go to disconnected state"
for host in down_hosts:
wait_for_host_agent_state(client, host, "disconnected",
HOST_DISCONN_ACTIVE_TIMEOUT)
# There will be no service reconcile since the instances will continue
# to be "running" state in rancher-server
for con in down_instances:
assert con.state == "running"
service = client.reload(service)
assert service.state == "active"
# Power on hosts that were powered off .
# "stopped" state of the containers on the host will get synced and
# service reconcile will trigger containers to be started.
for host in down_hosts:
host_name = host.hostname
print "power on- " + host_name
action_on_digital_ocean_machine(ha_droplets[0][host_name], "power_on")
print "Waiting for the hosts to go to active state"
for host in down_hosts:
wait_for_host_agent_state(client, host, "active",
HOST_DISCONN_ACTIVE_TIMEOUT)
wait_for_condition(
client, service,
lambda x: x.state == 'active',
lambda x: 'State is: ' + x.state)
for con in down_instances:
assert con.state == "running"
delete_all(client, [env])
def host_down_with_healthcheck_services(client, host_down_count,
retainIp=False):
# Wait for hosts in "reconnecting" state to get to "active" state
check_hosts_state(client)
# Create service that is healthcheck enabled
scale = 10
env, service = service_with_healthcheck_enabled(
client, scale, retainIp=retainIp)
# Pick hosts (and collect instances that will fgo unhealthy) that need
# to be powered down
down_hosts = []
down_instances = []
for i in range(0, len(ha_host_list)):
host = ha_host_list[i]
instance_list = \
get_containers_on_host_for_service(client, host, service)
if len(instance_list) > 0:
down_instances.extend(instance_list)
down_hosts.append(host)
if len(down_hosts) == host_down_count:
break
# Power Down hosts where service instances are running
for host in down_hosts:
host_name = host.hostname
print "power down- " + host_name
print ha_droplets[0]
action_on_digital_ocean_machine(ha_droplets[0][host_name], "power_off")
print "Waiting for the hosts to go to disconnected state"
for host in down_hosts:
wait_for_host_agent_state(client, host, "disconnected",
HOST_ACTIVE_DISCONN_TIMEOUT)
# Check for service reconcile
check_for_service_reconcile(
client, service, down_instances, instance_list)
# If retainIp is turned on , make sure that ip address assigned to
# reconciled instances are the same
if (retainIp):
for con in down_instances:
container_name = con.name
containers = client.list_container(name=container_name,
removed_null=True)
assert len(containers) == 1
container = containers[0]
assert container.primaryIpAddress == con.primaryIpAddress
assert container.externalId != con.externalId
# Power on hosts that were powered off
delete_all(client, [env])
for host in down_hosts:
host_name = host.hostname
print "power on- " + host_name
action_on_digital_ocean_machine(ha_droplets[0][host_name], "power_on")
print "Waiting for the hosts to go to active state"
for host in down_hosts:
wait_for_host_agent_state(client, host, "active",
HOST_DISCONN_ACTIVE_TIMEOUT)
def host_down_with_lb_services(client, lb_port, host_down_count,
scale=2, lb_scale=2, globalf=False):
# Wait for hosts in "reconnecting" state to get to "active" state
check_hosts_state(client)
# Create environment with lb_service and 2 healthcheck enabled
# service targets
env, lb_service, service1, service2 = \
env_with_lb_service_with_health_check_enabled_targets(
client, lb_port, scale, lb_scale, globalf)
# Pick hosts (and collect instances that will fgo unhealthy) that need
# to be powered down
down_hosts = []
down_instances = []
for i in range(0, len(ha_host_list)):
host = ha_host_list[i]
instance_list = \
get_containers_on_host_for_service(client, host, lb_service)
if len(instance_list) > 0:
down_instances.extend(instance_list)
down_hosts.append(host)
if len(down_hosts) == host_down_count:
break
# Power Down hosts where lb service instances are running
for host in down_hosts:
host_name = host.hostname
print "power down- " + host_name
print ha_droplets[0]
action_on_digital_ocean_machine(ha_droplets[0][host_name], "power_off")
print "Waiting for the hosts to go to disconnected state"
for host in down_hosts:
wait_for_host_agent_state(client, host, "disconnected",
HOST_ACTIVE_DISCONN_TIMEOUT)
# Check for service reconcile
check_for_service_reconcile(
client, lb_service, down_instances,
instance_list, globalf)
validate_lb_service(client, lb_service,
lb_port, [service1, service2])
# Power on hosts that were powered off
for host in down_hosts:
host_name = host.hostname
print "power on- " + host_name
action_on_digital_ocean_machine(ha_droplets[0][host_name], "power_on")
print "Waiting for the hosts to go to active state"
for host in down_hosts:
wait_for_host_agent_state(client, host, "active",
HOST_DISCONN_ACTIVE_TIMEOUT)
# if service is global, validate that new instances of the service gets
# created on the host that gets powered on
if (globalf):
check_hosts_state(client)
wait_for_scale_to_adjust(client, service1, timeout=300)
wait_for_scale_to_adjust(client, service2, timeout=300)
wait_for_scale_to_adjust(client, lb_service, timeout=300)
validate_lb_service(client, lb_service,
lb_port, [service1, service2])
delete_all(client, [env])
