def wait_until_application_is_in_state(app, state, timeout=None,
poll_timeout=None):
"""Wait until an application is in a given state."""
if timeout is None:
timeout = 900
if poll_timeout is None:
poll_timeout = 10
end_time = time.time() + timeout
while True:
if time.time() > end_time:
break
poll_end_time = time.time() + poll_timeout
app = cache.get_application(app['id'], force_reload=True)
appstate = get_application_state(app)
if appstate == state:
return app
console.show_progress(appstate[0])
time.sleep(max(0, poll_end_time - time.time()))
error.raise_error("Application `{0}` did not reach state '{1}' within "
"{2} seconds.", app['name'], state, timeout)
return app
def wait_until_application_accepts_ssh(app, vms, timeout=None,
poll_timeout=None):
"""Wait until an application is reachable by ssh.
An application is reachable by SSH if all the VMs that have a public key in
their userdata are connect()able on port 22.
"""
if timeout is None:
timeout = 300
if poll_timeout is None:
poll_timeout = 5
waitaddrs = set((vm['dynamicMetadata']['externalIp']
for vm in app['vms'] if vm['name'] in vms))
aliveaddrs = set()
end_time = time.time() + timeout
# For the intricate details on non-blocking connect()'s, see Stevens,
# UNIX network programming, volume 1, chapter 16.3 and following.
while True:
if time.time() > end_time:
break
waitfds = {}
for addr in waitaddrs:
sock = socket.socket()
sock.setblocking(False)
try:
sock.connect((addr, 22))
except socket.error as e:
if e.errno not in nb_connect_errors:
console.debug('connect(): errno {.errno}'.format(e))
continue
waitfds[sock.fileno()] = (sock, addr)
poll_end_time = time.time() + poll_timeout
while True:
timeout = poll_end_time - time.time()
if timeout < 0:
for fd in waitfds:
sock, _ = waitfds[fd]
sock.close()
break
try:
wfds = list(waitfds)
_, wfds, _ = select.select([], wfds, [], timeout)
except select.error as e:
if e.args[0] == errno.EINTR:
continue
console.debug('select(): errno {.errno}'.format(e))
raise
for fd in wfds:
assert fd in waitfds
sock, addr = waitfds[fd]
try:
err = sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
except socket.error as e:
err = e.errno
sock.close()
if not err:
aliveaddrs.add(addr)
waitaddrs.remove(addr)
del waitfds[fd]
if not waitfds:
break
if not waitaddrs:
return
console.show_progress('C') # 'C' = Connecting
time.sleep(max(0, poll_end_time - time.time()))
unreachable = set((vm['name'] for vm in app['vms']
if vm['dynamicMetadata']['externalIp'] in waitaddrs))
noun = inflect.plural_noun('VM', len(unreachable))
vmnames = '`{0}`'.format('`, `'.join(sorted(unreachable)))
error.raise_error('{0} `{1}` did not become reachable within {2} seconds.',
noun, vmnames, timeout)
