def wait_till_all_up(attempts=90, interval=10, node=None, waitdown=True, contrail_role='all'):
    ''' Waits for given nodes to go down and then comeup within the given attempts.
        Defaults: attempts = 90 retries
                  interval = 10 secs
                  node     = env.roledefs['all']
    '''
    if contrail_role == 'compute':
        nodes = node or env.roledefs['compute']
    else:
        nodes = node or env.roledefs['all']
    nodes = [nodes] if type(nodes) is str else nodes
    #Waits for node to shutdown
    if waitdown != 'False':
        for node in nodes:
            nodeip = node.split('@')[1]
            print 'Waiting for the node (%s) to go down...' %nodeip
            common.wait_until_host_down(host=nodeip, wait=900)

    print 'Given Nodes are down... Waiting for nodes to come back'
    for node in nodes:
        with hide('everything'):
            with settings(host_string=node, timeout=int(interval),
                          connection_attempts=int(attempts),
                          linewise=True, warn_only=True,
                          password=env.passwords[node]): 
                connections.connect(env.host_string)
    return 0
Example #2
0
def wait_till_all_up(attempts=90, interval=10, node=None, waitdown=True, contrail_role='all'):
    ''' Waits for given nodes to go down and then comeup within the given attempts.
        Defaults: attempts = 90 retries
                  interval = 10 secs
                  node     = env.roledefs['all']
    '''
    if contrail_role == 'compute':
        nodes = node or env.roledefs['compute']
    else:
        nodes = node or env.roledefs['all']
    nodes = [nodes] if type(nodes) is str else nodes
    #Waits for node to shutdown
    if waitdown != 'False':
        for node in nodes:
            nodeip = node.split('@')[1]
            print 'Waiting for the node (%s) to go down...' %nodeip
            common.wait_until_host_down(host=nodeip, wait=900)

    print 'Given Nodes are down... Waiting for nodes to come back'
    for node in nodes:
        user, hostip = node.split('@')
        count = 0
        while not verify_sshd(hostip,
                user,
                env.passwords[node]):
            sys.stdout.write('.')
            sleep(int(interval))
            count+=1
            if count <= int(attempts):
                continue
            else:
                print 'Timed out waiting for node (%s) to come back up...' %node
                sys.exit(1)
    return 0
def reboot_node(*args):
    for host_string in args:
        user, hostip = host_string.split('@')
        with settings(hide('running'), host_string=host_string, warn_only=True):
            #Fabric hangs when reboot --force is issued, so adding timeout
            #as workaround.
            try:
                sudo("/etc/contrail/contrail_reboot", timeout=5)
            except CommandTimeout:
                pass

        print 'Reboot issued; Waiting for the node (%s) to go down...' % hostip
        common.wait_until_host_down(wait=300, host=hostip)
        print 'Node (%s) is down... Waiting for node to come back' % hostip
        sys.stdout.write('.')
        count = 0
        while not verify_sshd(hostip,
                          user,
                          env.passwords[host_string]):
            sys.stdout.write('.')
            sleep(2)
            count+=1
            if count <=1000:
                continue
            else:
                print 'Timed out waiting for node to come back up'
                sys.exit(1)
Example #4
0
def reboot_node(waitup, *args):
    for host_string in args:
        user, hostip = host_string.split('@')
        with settings(hide('running'), host_string=host_string,
                      warn_only=True):
            #Fabric hangs when reboot --force is issued, so adding timeout
            #as workaround.
            try:
                sudo("/etc/contrail/contrail_reboot", timeout=5)
            except CommandTimeout:
                pass

        print 'Reboot issued; Waiting for the node (%s) to go down...' % hostip
        common.wait_until_host_down(wait=300, host=hostip)
        if waitup == 'no':
            return
        print 'Node (%s) is down... Waiting for node to come back' % hostip
        sys.stdout.write('.')
        count = 0
        while not verify_sshd(hostip, user, get_env_passwords(host_string)):
            sys.stdout.write('.')
            sleep(2)
            count += 1
            if count <= 1000:
                continue
            else:
                print 'Timed out waiting for node to come back up'
                sys.exit(1)
Example #5
0
def wait_till_all_up(attempts=90, interval=10, node=None, waitdown=True, contrail_role='all'):
    ''' Waits for given nodes to go down and then comeup within the given attempts.
        Defaults: attempts = 90 retries
                  interval = 10 secs
                  node     = env.roledefs['all']
    '''
    if contrail_role == 'compute':
        nodes = node or env.roledefs['compute']
    else:
        nodes = node or env.roledefs['all']
    nodes = [nodes] if type(nodes) is str else nodes
    #Waits for node to shutdown
    if waitdown != 'False':
        for node in nodes:
            nodeip = node.split('@')[1]
            print 'Waiting for the node (%s) to go down...' %nodeip
            common.wait_until_host_down(host=nodeip, wait=900)

    print 'Given Nodes are down... Waiting for nodes to come back'
    for node in nodes:
        with hide('everything'):
            with settings(host_string=node, timeout=int(interval),
                          connection_attempts=int(attempts),
                          linewise=True, warn_only=True,
                          password=env.passwords[node]): 
                connections.connect(env.host_string)
    return 0
def check_reimage_status():
    user, hostip = env.host_string.split("@")
    print "Reimage issued; Waiting for the node (%s) to go down..." % hostip
    common.wait_until_host_down(wait=300, host=hostip)
    print "Node (%s) is down... Waiting for node to come back up" % hostip
    sys.stdout.write(".")
    while not verify_sshd(hostip, user, env.passwords[env.host_string]):
        sys.stdout.write(".")
        sys.stdout.flush()
        sleep(2)
        continue
Example #7
0
def check_reimage_status():
    user, hostip = env.host_string.split('@')
    print 'Reimage issued; Waiting for the node (%s) to go down...' % hostip
    common.wait_until_host_down(wait=300, host=hostip)
    print 'Node (%s) is down... Waiting for node to come back up' % hostip
    sys.stdout.write('.')
    while not verify_sshd(hostip, user, get_env_passwords(env.host_string)):
        sys.stdout.write('.')
        sys.stdout.flush()
        sleep(2)
        continue
Example #8
0
def check_reimage_status():
    user, hostip = env.host_string.split('@')
    print 'Reimage issued; Waiting for the node (%s) to go down...' % hostip
    common.wait_until_host_down(wait=300, host=hostip)
    print 'Node (%s) is down... Waiting for node to come back up' % hostip
    sys.stdout.write('.')
    while not verify_sshd(hostip,
                          user,
                          get_env_passwords(env.host_string)):
        sys.stdout.write('.')
        sys.stdout.flush()
        sleep(2)
        continue
Example #9
0
def wait_till_all_up(attempts=90,
                     interval=10,
                     node=None,
                     waitdown=True,
                     contrail_role='all',
                     reimaged=False):
    ''' Waits for given nodes to go down and then comeup within the given attempts.
        Defaults: attempts = 90 retries
                  interval = 10 secs
                  node     = env.roledefs['all']
    '''
    if node:
        nodes = node
    else:
        nodes = env.roledefs[contrail_role][:]
        if reimaged:
            esxi_hosts = getattr(testbed, 'esxi_hosts', None)
            if esxi_hosts:
                for esxi in esxi_hosts:
                    hstr = esxi_hosts[esxi]['username'] + '@' + esxi_hosts[
                        esxi]['ip']
                    nodes.append(hstr)
                    env.passwords[hstr] = esxi_hosts[esxi]['password']
                    nodes.remove(esxi_hosts[esxi]['contrail_vm']['host'])

    nodes = [nodes] if type(nodes) is str else nodes
    #Waits for node to shutdown
    if waitdown != 'False' and not (reimaged and esxi_hosts):
        for node in nodes:
            nodeip = node.split('@')[1]
            print 'Waiting for the node (%s) to go down...' % nodeip
            common.wait_until_host_down(host=nodeip, wait=900)

    print 'Given Nodes are down... Waiting for nodes to come back'
    for node in nodes:
        user, hostip = node.split('@')
        count = 0
        while not verify_sshd(hostip, user, get_env_passwords(node)):
            sys.stdout.write('.')
            sleep(int(interval))
            count += 1
            if count <= int(attempts):
                continue
            else:
                print 'Timed out waiting for node (%s) to come back up...' % node
                sys.exit(1)
    return 0
Example #10
0
def wait_till_all_up(attempts=90, interval=10, node=None, waitdown=True, contrail_role='all', reimaged=False):
    ''' Waits for given nodes to go down and then comeup within the given attempts.
        Defaults: attempts = 90 retries
                  interval = 10 secs
                  node     = env.roledefs['all']
    '''
    if node:
        nodes = node
    else:
        nodes = env.roledefs[contrail_role][:]
        if reimaged:
            esxi_hosts = getattr(testbed, 'esxi_hosts', None)
            if esxi_hosts:
                for esxi in esxi_hosts:
                    hstr = esxi_hosts[esxi]['username'] + '@' + esxi_hosts[esxi]['ip']
                    nodes.append(hstr)
                    env.passwords[hstr] = esxi_hosts[esxi]['password']
                    nodes.remove(esxi_hosts[esxi]['contrail_vm']['host'])

    nodes = [nodes] if type(nodes) is str else nodes
    #Waits for node to shutdown
    if waitdown != 'False' and not (reimaged and esxi_hosts):
        for node in nodes:
            nodeip = node.split('@')[1]
            print 'Waiting for the node (%s) to go down...' %nodeip
            common.wait_until_host_down(host=nodeip, wait=900)

    print 'Given Nodes are down... Waiting for nodes to come back'
    for node in nodes:
        user, hostip = node.split('@')
        count = 0
        while not verify_sshd(hostip,
                user,
                get_env_passwords(node)):
            sys.stdout.write('.')
            sleep(int(interval))
            count+=1
            if count <= int(attempts):
                continue
            else:
                print 'Timed out waiting for node (%s) to come back up...' %node
                sys.exit(1)
    return 0
Example #11
0
def wait_till_all_up(attempts=90,
                     interval=10,
                     node=None,
                     waitdown=True,
                     contrail_role='all'):
    ''' Waits for given nodes to go down and then comeup within the given attempts.
        Defaults: attempts = 90 retries
                  interval = 10 secs
                  node     = env.roledefs['all']
    '''
    if contrail_role == 'compute':
        nodes = node or env.roledefs['compute']
    else:
        nodes = node or env.roledefs['all']
    nodes = [nodes] if type(nodes) is str else nodes
    #Waits for node to shutdown
    if waitdown != 'False':
        for node in nodes:
            nodeip = node.split('@')[1]
            print 'Waiting for the node (%s) to go down...' % nodeip
            common.wait_until_host_down(host=nodeip, wait=900)

    print 'Given Nodes are down... Waiting for nodes to come back'
    for node in nodes:
        user, hostip = node.split('@')
        count = 0
        while not verify_sshd(hostip, user, env.passwords[node]):
            sys.stdout.write('.')
            sleep(int(interval))
            count += 1
            if count <= int(attempts):
                continue
            else:
                print 'Timed out waiting for node (%s) to come back up...' % node
                sys.exit(1)
    return 0