def wait_for_failover_from(self, from_ip, timeout=30):
"""
Waits for the IP address of the `redis` service in Consul to change
from what we knew the IP address to be prior to failing over
"""
# verify "redis" address changes
while timeout > 0:
addresses = self.get_service_addresses_from_consul('redis')
if (len(addresses) > 0 and addresses[0] != from_ip):
new_master_ip = addresses[0]
break
time.sleep(1)
timeout -= 1
else:
raise WaitTimeoutError("Timed out waiting for redis service to be updated in Consul.")
# verify old master becoems a replica and new master is no longer a replica
while timeout > 0:
addresses = self.get_service_addresses_from_consul('redis-replica')
if from_ip in addresses and new_master_ip not in addresses:
break
time.sleep(1)
timeout -= 1
else:
raise WaitTimeoutError("Timed out waiting for redis service to be updated in Consul.")
def wait_for_containers(self, expected={}, timeout=30):
"""
Waits for all containers to be marked as 'Up' for all services.
`expected` should be a dict of {"service_name": count}.
TODO: lower this into the base class implementation.
"""
svc_regex = re.compile(r'^{}_(\w+)_\d+$'.format(self.project_name))
def get_service_name(container_name):
return svc_regex.match(container_name).group(1)
while timeout > 0:
containers = self.compose_ps()
found = defaultdict(int)
states = []
for container in containers:
service = get_service_name(container.name)
found[service] = found[service] + 1
states.append(container.state == 'Up')
if all(states):
if not expected or found == expected:
break
time.sleep(1)
timeout -= 1
else:
raise WaitTimeoutError(
"Timed out waiting for containers to start.")
def wait_for_replicated_value(self, replica, key, value, timeout=30):
"""
Waits for the given key/value pair to be written to the given replica
"""
while timeout > 0:
replicated_value = self.docker_exec(replica, 'redis-cli get test:repl').strip("\n")
if (replicated_value == value):
break
time.sleep(1)
timeout -= 1
else:
raise WaitTimeoutError("Timed out waiting for redis replica to receive replicated value.")
def settle(self, service, count, timeout=60):
"""
Wait for the service to appear healthy and correctly in Consul
"""
try:
nodes = self.instrument(self.wait_for_service,
service,
count,
timeout=timeout)
if len(nodes) < count:
raise WaitTimeoutError()
self.instrument(self.assert_consul_correctness)
except WaitTimeoutError:
self.fail('Failed to scale {} to {} instances'.format(
service, count))
def settle(self, count, timeout=30):
"""
Waits for all containers to marked as 'Up'
"""
while timeout > 0:
containers = self.compose_ps()
up_containers = [
container for container in containers
if container.state == 'Up'
]
if len(up_containers) == count:
break
time.sleep(1)
timeout -= 1
else:
raise WaitTimeoutError(
"Timed out waiting for containers to start.")
def settle(self, service, count, timeout=30):
"""
Wait for `count` instance of `service` to read as 'Up' and
then wait for them to show up in the Nginx virtualhost config.
"""
nodes = self.wait_for_service(service, count, timeout)
if len(nodes) < count:
self.fail('Failed to scale {} to {} instances'.format(
service, count))
timeout = timeout
while timeout > 0:
servers = self.get_upstream_blocks(service)
if len(servers) == count:
break
time.sleep(1)
timeout -= 1
else:
raise WaitTimeoutError('Timed out waiting for {} server '
'in nginx upstream'.format(service))