def global_service_with_reconn_disconn_host(client, state): # Pick one of the host and power down hosts host_down = ha_host_list[0] host_name = ha_host_list[0].hostname print "power down- " + host_name action_on_digital_ocean_machine(ha_droplets[0][host_name], "power_off") wait_for_host_agent_state(client, host_down, state, HOST_ACTIVE_DISCONN_TIMEOUT) # Create service launch_config = {"imageUuid": HEALTH_CHECK_IMAGE_UUID} launch_config["labels"] = {"io.rancher.scheduler.global": "true"} service, env = create_env_and_svc(client, launch_config) service = service.activate() service = client.wait_success(service, 300) assert service.state == "active" container_list = get_service_container_list(client, service) assert len(container_list) == get_service_instance_count(client, service) # Power on the host action_on_digital_ocean_machine(ha_droplets[0][host_name], "power_on") wait_for_host_agent_state(client, host_down, "active", HOST_DISCONN_ACTIVE_TIMEOUT) service = wait_success(client, service, 300) container_list = get_service_container_list(client, service) assert len(container_list) == get_service_instance_count(client, service) instance_list = get_containers_on_host_for_service( client, host_down, service) assert len(instance_list) == 1
def host_down_with_lb_services(super_client, client, lb_port, host_down_count, scale=2, lb_scale=2, globalf=False): # Wait for hosts in "reconnecting" state to get to "active" state check_hosts_state(client) # Create environment with lb_service and 2 healthcheck enabled # service targets env, lb_service, service1, service2 = \ env_with_lb_service_with_health_check_enabled_targets( super_client, client, lb_port, scale, lb_scale, globalf) # Pick hosts (and collect instances that will fgo unhealthy) that need # to be powered down down_hosts = [] down_instances = [] for i in range(0, len(ha_host_list)): host = ha_host_list[i] instance_list = \ get_containers_on_host_for_service(client, host, lb_service) if len(instance_list) > 0: down_instances.extend(instance_list) down_hosts.append(host) if len(down_hosts) == host_down_count: break # Power Down hosts where lb service instances are running for host in down_hosts: host_name = host.hostname print "power down- " + host_name print ha_droplets[0] action_on_digital_ocean_machine(ha_droplets[0][host_name], "power_off") # Check for service reconcile check_for_service_reconcile( super_client, client, lb_service, down_instances, instance_list) validate_lb_service(super_client, client, lb_service, lb_port, [service1, service2]) # Power on hosts that were powered off for host in down_hosts: host_name = host.hostname print "power on- " + host_name action_on_digital_ocean_machine(ha_droplets[0][host_name], "power_on") # if service is global, validate that new instances of the service gets # created on the host that gets powered on if (globalf): check_hosts_state(client) wait_for_scale_to_adjust(super_client, service1) wait_for_scale_to_adjust(super_client, service2) wait_for_scale_to_adjust(super_client, lb_service) validate_lb_service(super_client, client, lb_service, lb_port, [service1, service2]) delete_all(client, [env])
def host_down_with_healthcheck_services(super_client, client, host_down_count, retainIp=False): # Wait for hosts in "reconnecting" state to get to "active" state check_hosts_state(client) # Create service that is healthcheck enabled scale = 10 env, service = service_with_healthcheck_enabled( client, super_client, scale, retainIp=retainIp) # Pick hosts (and collect instances that will fgo unhealthy) that need # to be powered down down_hosts = [] down_instances = [] for i in range(0, len(ha_host_list)): host = ha_host_list[i] instance_list = \ get_containers_on_host_for_service(client, host, service) if len(instance_list) > 0: down_instances.extend(instance_list) down_hosts.append(host) if len(down_hosts) == host_down_count: break # Power Down hosts where service instances are running for host in down_hosts: host_name = host.hostname print "power down- " + host_name print ha_droplets[0] action_on_digital_ocean_machine(ha_droplets[0][host_name], "power_off") # Check for service reconcile check_for_service_reconcile( super_client, client, service, down_instances, instance_list) # If retainIp is turned on , make sure that ip address assigned to # reconciled instances are the same if (retainIp): for con in down_instances: container_name = con.name containers = super_client.list_container(name=container_name, removed_null=True) assert len(containers) == 1 container = containers[0] assert container.primaryIpAddress == con.primaryIpAddress assert container.externalId != con.externalId # Power on hosts that were powered off delete_all(client, [env]) for host in down_hosts: host_name = host.hostname print "power on- " + host_name action_on_digital_ocean_machine(ha_droplets[0][host_name], "power_on")
def check_host_state_power_on(client): print "Power on hosts that are in disconnected or reconnecting state" print ha_droplets inactive_hosts = client.list_host( kind='docker', removed_null=True, agentState="disconnected") print "Disconnected hosts:" print inactive_hosts reconn_hosts = client.list_host( kind='docker', removed_null=True, agentState="reconnecting") print "Reconnecting hosts:" print reconn_hosts inactive_hosts_dropletids = [] inactive_hosts_list = [] # Get droplet Id and hosts from disconnected hosts for host in inactive_hosts: host_name = host.hostname print host_name droplet_id = ha_droplets[0][host_name] inactive_hosts_dropletids.append(droplet_id) inactive_hosts_list.append(host) # Get droplet Id and hosts from reconnecting hosts # and append to the inactive host/droplet lists for host in reconn_hosts: host_name = host.hostname print host_name droplet_id = ha_droplets[0][host_name] inactive_hosts_dropletids.append(droplet_id) inactive_hosts_list.append(host) print "List of all disconnected/reconnecting hosts" print inactive_hosts_list print inactive_hosts_dropletids # Power on the droplets for dropletid in inactive_hosts_dropletids: print "Power on droplet " + str(droplet_id) action_on_digital_ocean_machine(dropletid, "power_on") # Wait for the host agent state to become active for host in inactive_hosts_list: print "In host wait method" wait_for_host_agent_state(client, host, "active", 600)
def host_down_with_services(client, host_down_count, globalf=False): # Wait for hosts in "reconnecting" state to get to "active" state check_hosts_state(client) # Create service launch_config = {"imageUuid": HEALTH_CHECK_IMAGE_UUID} if globalf: launch_config["labels"] = {"io.rancher.scheduler.global": "true"} scale = 0 else: scale = 10 service, env = create_env_and_svc(client, launch_config, scale) service = service.activate() service = client.wait_success(service, 300) assert service.state == "active" container_list = get_service_container_list(client, service) assert len(container_list) == get_service_instance_count(client, service) # Pick hosts (and collect instances that will go unhealthy) that need # to be powered down down_hosts = [] down_instances = [] for i in range(0, len(ha_host_list)): host = ha_host_list[i] instance_list = \ get_containers_on_host_for_service(client, host, service) if len(instance_list) > 0: down_instances.extend(instance_list) down_hosts.append(host) if len(down_hosts) == host_down_count: break # Power Down hosts where service instances are running for host in down_hosts: host_name = host.hostname print "power down- " + host_name print ha_droplets[0] action_on_digital_ocean_machine(ha_droplets[0][host_name], "power_off") print "Waiting for the hosts to go to disconnected state" for host in down_hosts: wait_for_host_agent_state(client, host, "disconnected", HOST_DISCONN_ACTIVE_TIMEOUT) # There will be no service reconcile since the instances will continue # to be "running" state in rancher-server for con in down_instances: assert con.state == "running" service = client.reload(service) assert service.state == "active" # Power on hosts that were powered off . # "stopped" state of the containers on the host will get synced and # service reconcile will trigger containers to be started. for host in down_hosts: host_name = host.hostname print "power on- " + host_name action_on_digital_ocean_machine(ha_droplets[0][host_name], "power_on") print "Waiting for the hosts to go to active state" for host in down_hosts: wait_for_host_agent_state(client, host, "active", HOST_DISCONN_ACTIVE_TIMEOUT) wait_for_condition( client, service, lambda x: x.state == 'active', lambda x: 'State is: ' + x.state) for con in down_instances: assert con.state == "running" delete_all(client, [env])
def host_down_with_healthcheck_services(client, host_down_count, retainIp=False): # Wait for hosts in "reconnecting" state to get to "active" state check_hosts_state(client) # Create service that is healthcheck enabled scale = 10 env, service = service_with_healthcheck_enabled( client, scale, retainIp=retainIp) # Pick hosts (and collect instances that will fgo unhealthy) that need # to be powered down down_hosts = [] down_instances = [] for i in range(0, len(ha_host_list)): host = ha_host_list[i] instance_list = \ get_containers_on_host_for_service(client, host, service) if len(instance_list) > 0: down_instances.extend(instance_list) down_hosts.append(host) if len(down_hosts) == host_down_count: break # Power Down hosts where service instances are running for host in down_hosts: host_name = host.hostname print "power down- " + host_name print ha_droplets[0] action_on_digital_ocean_machine(ha_droplets[0][host_name], "power_off") print "Waiting for the hosts to go to disconnected state" for host in down_hosts: wait_for_host_agent_state(client, host, "disconnected", HOST_ACTIVE_DISCONN_TIMEOUT) # Check for service reconcile check_for_service_reconcile( client, service, down_instances, instance_list) # If retainIp is turned on , make sure that ip address assigned to # reconciled instances are the same if (retainIp): for con in down_instances: container_name = con.name containers = client.list_container(name=container_name, removed_null=True) assert len(containers) == 1 container = containers[0] assert container.primaryIpAddress == con.primaryIpAddress assert container.externalId != con.externalId # Power on hosts that were powered off delete_all(client, [env]) for host in down_hosts: host_name = host.hostname print "power on- " + host_name action_on_digital_ocean_machine(ha_droplets[0][host_name], "power_on") print "Waiting for the hosts to go to active state" for host in down_hosts: wait_for_host_agent_state(client, host, "active", HOST_DISCONN_ACTIVE_TIMEOUT)
def host_down_with_lb_services(client, lb_port, host_down_count, scale=2, lb_scale=2, globalf=False): # Wait for hosts in "reconnecting" state to get to "active" state check_hosts_state(client) # Create environment with lb_service and 2 healthcheck enabled # service targets env, lb_service, service1, service2 = \ env_with_lb_service_with_health_check_enabled_targets( client, lb_port, scale, lb_scale, globalf) # Pick hosts (and collect instances that will fgo unhealthy) that need # to be powered down down_hosts = [] down_instances = [] for i in range(0, len(ha_host_list)): host = ha_host_list[i] instance_list = \ get_containers_on_host_for_service(client, host, lb_service) if len(instance_list) > 0: down_instances.extend(instance_list) down_hosts.append(host) if len(down_hosts) == host_down_count: break # Power Down hosts where lb service instances are running for host in down_hosts: host_name = host.hostname print "power down- " + host_name print ha_droplets[0] action_on_digital_ocean_machine(ha_droplets[0][host_name], "power_off") print "Waiting for the hosts to go to disconnected state" for host in down_hosts: wait_for_host_agent_state(client, host, "disconnected", HOST_ACTIVE_DISCONN_TIMEOUT) # Check for service reconcile check_for_service_reconcile( client, lb_service, down_instances, instance_list, globalf) validate_lb_service(client, lb_service, lb_port, [service1, service2]) # Power on hosts that were powered off for host in down_hosts: host_name = host.hostname print "power on- " + host_name action_on_digital_ocean_machine(ha_droplets[0][host_name], "power_on") print "Waiting for the hosts to go to active state" for host in down_hosts: wait_for_host_agent_state(client, host, "active", HOST_DISCONN_ACTIVE_TIMEOUT) # if service is global, validate that new instances of the service gets # created on the host that gets powered on if (globalf): check_hosts_state(client) wait_for_scale_to_adjust(client, service1, timeout=300) wait_for_scale_to_adjust(client, service2, timeout=300) wait_for_scale_to_adjust(client, lb_service, timeout=300) validate_lb_service(client, lb_service, lb_port, [service1, service2]) delete_all(client, [env])