def wait_till_all_up(attempts=90, interval=10, node=None, waitdown=True, contrail_role='all'):
''' Waits for given nodes to go down and then comeup within the given attempts.
Defaults: attempts = 90 retries
interval = 10 secs
node = env.roledefs['all']
'''
if contrail_role == 'compute':
nodes = node or env.roledefs['compute']
else:
nodes = node or env.roledefs['all']
nodes = [nodes] if type(nodes) is str else nodes
#Waits for node to shutdown
if waitdown != 'False':
for node in nodes:
nodeip = node.split('@')[1]
print 'Waiting for the node (%s) to go down...' %nodeip
common.wait_until_host_down(host=nodeip, wait=900)
print 'Given Nodes are down... Waiting for nodes to come back'
for node in nodes:
with hide('everything'):
with settings(host_string=node, timeout=int(interval),
connection_attempts=int(attempts),
linewise=True, warn_only=True,
password=env.passwords[node]):
connections.connect(env.host_string)
return 0
def wait_till_all_up(attempts=90, interval=10, node=None, waitdown=True, contrail_role='all'):
''' Waits for given nodes to go down and then comeup within the given attempts.
Defaults: attempts = 90 retries
interval = 10 secs
node = env.roledefs['all']
'''
if contrail_role == 'compute':
nodes = node or env.roledefs['compute']
else:
nodes = node or env.roledefs['all']
nodes = [nodes] if type(nodes) is str else nodes
#Waits for node to shutdown
if waitdown != 'False':
for node in nodes:
nodeip = node.split('@')[1]
print 'Waiting for the node (%s) to go down...' %nodeip
common.wait_until_host_down(host=nodeip, wait=900)
print 'Given Nodes are down... Waiting for nodes to come back'
for node in nodes:
user, hostip = node.split('@')
count = 0
while not verify_sshd(hostip,
user,
env.passwords[node]):
sys.stdout.write('.')
sleep(int(interval))
count+=1
if count <= int(attempts):
continue
else:
print 'Timed out waiting for node (%s) to come back up...' %node
sys.exit(1)
return 0
def reboot_node(*args):
for host_string in args:
user, hostip = host_string.split('@')
with settings(hide('running'), host_string=host_string, warn_only=True):
#Fabric hangs when reboot --force is issued, so adding timeout
#as workaround.
try:
sudo("/etc/contrail/contrail_reboot", timeout=5)
except CommandTimeout:
pass
print 'Reboot issued; Waiting for the node (%s) to go down...' % hostip
common.wait_until_host_down(wait=300, host=hostip)
print 'Node (%s) is down... Waiting for node to come back' % hostip
sys.stdout.write('.')
count = 0
while not verify_sshd(hostip,
user,
env.passwords[host_string]):
sys.stdout.write('.')
sleep(2)
count+=1
if count <=1000:
continue
else:
print 'Timed out waiting for node to come back up'
sys.exit(1)
def reboot_node(waitup, *args):
for host_string in args:
user, hostip = host_string.split('@')
with settings(hide('running'), host_string=host_string,
warn_only=True):
#Fabric hangs when reboot --force is issued, so adding timeout
#as workaround.
try:
sudo("/etc/contrail/contrail_reboot", timeout=5)
except CommandTimeout:
pass
print 'Reboot issued; Waiting for the node (%s) to go down...' % hostip
common.wait_until_host_down(wait=300, host=hostip)
if waitup == 'no':
return
print 'Node (%s) is down... Waiting for node to come back' % hostip
sys.stdout.write('.')
count = 0
while not verify_sshd(hostip, user, get_env_passwords(host_string)):
sys.stdout.write('.')
sleep(2)
count += 1
if count <= 1000:
continue
else:
print 'Timed out waiting for node to come back up'
sys.exit(1)
def wait_till_all_up(attempts=90, interval=10, node=None, waitdown=True, contrail_role='all'):
''' Waits for given nodes to go down and then comeup within the given attempts.
Defaults: attempts = 90 retries
interval = 10 secs
node = env.roledefs['all']
'''
if contrail_role == 'compute':
nodes = node or env.roledefs['compute']
else:
nodes = node or env.roledefs['all']
nodes = [nodes] if type(nodes) is str else nodes
#Waits for node to shutdown
if waitdown != 'False':
for node in nodes:
nodeip = node.split('@')[1]
print 'Waiting for the node (%s) to go down...' %nodeip
common.wait_until_host_down(host=nodeip, wait=900)
print 'Given Nodes are down... Waiting for nodes to come back'
for node in nodes:
with hide('everything'):
with settings(host_string=node, timeout=int(interval),
connection_attempts=int(attempts),
linewise=True, warn_only=True,
password=env.passwords[node]):
connections.connect(env.host_string)
return 0
def check_reimage_status():
user, hostip = env.host_string.split("@")
print "Reimage issued; Waiting for the node (%s) to go down..." % hostip
common.wait_until_host_down(wait=300, host=hostip)
print "Node (%s) is down... Waiting for node to come back up" % hostip
sys.stdout.write(".")
while not verify_sshd(hostip, user, env.passwords[env.host_string]):
sys.stdout.write(".")
sys.stdout.flush()
sleep(2)
continue
def check_reimage_status():
user, hostip = env.host_string.split('@')
print 'Reimage issued; Waiting for the node (%s) to go down...' % hostip
common.wait_until_host_down(wait=300, host=hostip)
print 'Node (%s) is down... Waiting for node to come back up' % hostip
sys.stdout.write('.')
while not verify_sshd(hostip, user, get_env_passwords(env.host_string)):
sys.stdout.write('.')
sys.stdout.flush()
sleep(2)
continue
def check_reimage_status():
user, hostip = env.host_string.split('@')
print 'Reimage issued; Waiting for the node (%s) to go down...' % hostip
common.wait_until_host_down(wait=300, host=hostip)
print 'Node (%s) is down... Waiting for node to come back up' % hostip
sys.stdout.write('.')
while not verify_sshd(hostip,
user,
get_env_passwords(env.host_string)):
sys.stdout.write('.')
sys.stdout.flush()
sleep(2)
continue
def wait_till_all_up(attempts=90,
interval=10,
node=None,
waitdown=True,
contrail_role='all',
reimaged=False):
''' Waits for given nodes to go down and then comeup within the given attempts.
Defaults: attempts = 90 retries
interval = 10 secs
node = env.roledefs['all']
'''
if node:
nodes = node
else:
nodes = env.roledefs[contrail_role][:]
if reimaged:
esxi_hosts = getattr(testbed, 'esxi_hosts', None)
if esxi_hosts:
for esxi in esxi_hosts:
hstr = esxi_hosts[esxi]['username'] + '@' + esxi_hosts[
esxi]['ip']
nodes.append(hstr)
env.passwords[hstr] = esxi_hosts[esxi]['password']
nodes.remove(esxi_hosts[esxi]['contrail_vm']['host'])
nodes = [nodes] if type(nodes) is str else nodes
#Waits for node to shutdown
if waitdown != 'False' and not (reimaged and esxi_hosts):
for node in nodes:
nodeip = node.split('@')[1]
print 'Waiting for the node (%s) to go down...' % nodeip
common.wait_until_host_down(host=nodeip, wait=900)
print 'Given Nodes are down... Waiting for nodes to come back'
for node in nodes:
user, hostip = node.split('@')
count = 0
while not verify_sshd(hostip, user, get_env_passwords(node)):
sys.stdout.write('.')
sleep(int(interval))
count += 1
if count <= int(attempts):
continue
else:
print 'Timed out waiting for node (%s) to come back up...' % node
sys.exit(1)
return 0
def wait_till_all_up(attempts=90, interval=10, node=None, waitdown=True, contrail_role='all', reimaged=False):
''' Waits for given nodes to go down and then comeup within the given attempts.
Defaults: attempts = 90 retries
interval = 10 secs
node = env.roledefs['all']
'''
if node:
nodes = node
else:
nodes = env.roledefs[contrail_role][:]
if reimaged:
esxi_hosts = getattr(testbed, 'esxi_hosts', None)
if esxi_hosts:
for esxi in esxi_hosts:
hstr = esxi_hosts[esxi]['username'] + '@' + esxi_hosts[esxi]['ip']
nodes.append(hstr)
env.passwords[hstr] = esxi_hosts[esxi]['password']
nodes.remove(esxi_hosts[esxi]['contrail_vm']['host'])
nodes = [nodes] if type(nodes) is str else nodes
#Waits for node to shutdown
if waitdown != 'False' and not (reimaged and esxi_hosts):
for node in nodes:
nodeip = node.split('@')[1]
print 'Waiting for the node (%s) to go down...' %nodeip
common.wait_until_host_down(host=nodeip, wait=900)
print 'Given Nodes are down... Waiting for nodes to come back'
for node in nodes:
user, hostip = node.split('@')
count = 0
while not verify_sshd(hostip,
user,
get_env_passwords(node)):
sys.stdout.write('.')
sleep(int(interval))
count+=1
if count <= int(attempts):
continue
else:
print 'Timed out waiting for node (%s) to come back up...' %node
sys.exit(1)
return 0
def wait_till_all_up(attempts=90,
interval=10,
node=None,
waitdown=True,
contrail_role='all'):
''' Waits for given nodes to go down and then comeup within the given attempts.
Defaults: attempts = 90 retries
interval = 10 secs
node = env.roledefs['all']
'''
if contrail_role == 'compute':
nodes = node or env.roledefs['compute']
else:
nodes = node or env.roledefs['all']
nodes = [nodes] if type(nodes) is str else nodes
#Waits for node to shutdown
if waitdown != 'False':
for node in nodes:
nodeip = node.split('@')[1]
print 'Waiting for the node (%s) to go down...' % nodeip
common.wait_until_host_down(host=nodeip, wait=900)
print 'Given Nodes are down... Waiting for nodes to come back'
for node in nodes:
user, hostip = node.split('@')
count = 0
while not verify_sshd(hostip, user, env.passwords[node]):
sys.stdout.write('.')
sleep(int(interval))
count += 1
if count <= int(attempts):
continue
else:
print 'Timed out waiting for node (%s) to come back up...' % node
sys.exit(1)
return 0