def check_machine(jenkins, client, machine, error_counts):
    try:
        server = client.servers.get(machine.external_id)
    except:
        print "Unable to get server detail, will retry"
        traceback.print_exc()
        return

    if server.status == "ACTIVE":
        ip = utils.get_public_ip(server)
        if not ip and "os-floating-ips" in utils.get_extensions(client):
            utils.add_public_ip(server)
            ip = utils.get_public_ip(server)
        if not ip:
            raise Exception("Unable to find public ip of server")

        machine.ip = ip
        print "Machine %s is running, testing ssh" % machine.id
        if utils.ssh_connect(ip, "jenkins"):
            print "Adding machine %s to Jenkins" % machine.id
            create_jenkins_node(jenkins, machine)
            print "Machine %s is ready" % machine.id
            machine.state = vmdatabase.READY
            return
    elif not server.status.startswith("BUILD"):
        count = error_counts.get(machine.id, 0)
        count += 1
        error_counts[machine.id] = count
        print "Machine %s is in error %s (%s/5)" % (machine.id, server.status, count)
        if count >= 5:
            raise Exception("Too many errors querying machine %s" % machine.id)
    else:
        if time.time() - machine.state_time >= ABANDON_TIMEOUT:
            raise Exception("Waited too long for machine %s" % machine.id)
Example #2
0
def check_machine(jenkins, client, machine, error_counts):
    try:
        server = client.servers.get(machine.external_id)
    except:
        print "Unable to get server detail, will retry"
        traceback.print_exc()
        return

    if server.status == 'ACTIVE':
        ip = utils.get_public_ip(server)
        if not ip and 'os-floating-ips' in utils.get_extensions(client):
            utils.add_public_ip(server)
            ip = utils.get_public_ip(server)
        if not ip:
            raise Exception("Unable to find public ip of server")

        machine.ip = ip
        print "Machine %s is running, testing ssh" % machine.id
        if utils.ssh_connect(ip, 'jenkins'):
            print "Adding machine %s to Jenkins" % machine.id
            create_jenkins_node(jenkins, machine)
            print "Machine %s is ready" % machine.id
            machine.state = vmdatabase.READY
            return
    elif not server.status.startswith('BUILD'):
        count = error_counts.get(machine.id, 0)
        count += 1
        error_counts[machine.id] = count
        print "Machine %s is in error %s (%s/5)" % (machine.id, server.status,
                                                    count)
        if count >= 5:
            raise Exception("Too many errors querying machine %s" % machine.id)
    else:
        if time.time() - machine.state_time >= ABANDON_TIMEOUT:
            raise Exception("Waited too long for machine %s" % machine.id)
def bootstrap_server(provider, server, admin_pass, key):
    client = server.manager.api
    ip = utils.get_public_ip(server)
    if not ip and "os-floating-ips" in utils.get_extensions(client):
        utils.add_public_ip(server)
        ip = utils.get_public_ip(server)
    if not ip:
        raise Exception("Unable to find public ip of server")

    ssh_kwargs = {}
    if key:
        ssh_kwargs["pkey"] = key
    else:
        ssh_kwargs["password"] = admin_pass

    for username in ["root", "ubuntu"]:
        client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600)
        if client:
            break

    if not client:
        raise Exception("Unable to log in via SSH")

    # hpcloud can't reliably set the hostname
    gerrit_url = "https://review.openstack.org/p/openstack-infra/config.git"
    client.ssh("set hostname", "sudo hostname %s" % server.name)
    client.ssh(
        "get puppet repo deb",
        "sudo /usr/bin/wget "
        "http://apt.puppetlabs.com/puppetlabs-release-"
        "`lsb_release -c -s`.deb -O /root/puppet-repo.deb",
    )
    client.ssh("install puppet repo deb", "sudo dpkg -i /root/puppet-repo.deb")
    client.ssh("update apt cache", "sudo apt-get update")
    client.ssh(
        "upgrading system packages",
        "sudo DEBIAN_FRONTEND=noninteractive apt-get "
        '--option "Dpkg::Options::=--force-confold"'
        " --assume-yes dist-upgrade",
    )
    client.ssh(
        "install git and puppet",
        "sudo DEBIAN_FRONTEND=noninteractive apt-get "
        '--option "Dpkg::Options::=--force-confold"'
        " --assume-yes install git puppet",
    )
    client.ssh("clone puppret repo", "sudo git clone %s /root/config" % gerrit_url)
    client.ssh("install puppet modules", "sudo /bin/bash /root/config/install_modules.sh")
    client.ssh(
        "run puppet",
        "sudo puppet apply --modulepath=/root/config/modules:" "/etc/puppet/modules " '-e "%s"' % PUPPET_CLASS,
    )
def bootstrap_server(provider, server, admin_pass, key):
    client = server.manager.api
    ip = utils.get_public_ip(server)
    if not ip and 'os-floating-ips' in utils.get_extensions(client):
        utils.add_public_ip(server)
        ip = utils.get_public_ip(server)
    if not ip:
        raise Exception("Unable to find public ip of server")

    ssh_kwargs = {}
    if key:
        ssh_kwargs['pkey'] = key
    else:
        ssh_kwargs['password'] = admin_pass

    for username in ['root', 'ubuntu']:
        client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600)
        if client:
            break

    if not client:
        raise Exception("Unable to log in via SSH")

    # hpcloud can't reliably set the hostname
    gerrit_url = 'https://review.openstack.org/p/openstack-infra/config.git'
    client.ssh("set hostname", "sudo hostname %s" % server.name)
    client.ssh(
        "get puppet repo deb", "sudo /usr/bin/wget "
        "http://apt.puppetlabs.com/puppetlabs-release-"
        "`lsb_release -c -s`.deb -O /root/puppet-repo.deb")
    client.ssh("install puppet repo deb", "sudo dpkg -i /root/puppet-repo.deb")
    client.ssh("update apt cache", "sudo apt-get update")
    client.ssh(
        "upgrading system packages",
        'sudo DEBIAN_FRONTEND=noninteractive apt-get '
        '--option "Dpkg::Options::=--force-confold"'
        ' --assume-yes dist-upgrade')
    client.ssh(
        "install git and puppet",
        'sudo DEBIAN_FRONTEND=noninteractive apt-get '
        '--option "Dpkg::Options::=--force-confold"'
        ' --assume-yes install git puppet')
    client.ssh("clone puppret repo",
               "sudo git clone %s /root/config" % gerrit_url)
    client.ssh("install puppet modules",
               "sudo /bin/bash /root/config/install_modules.sh")
    client.ssh(
        "run puppet", "sudo puppet apply --modulepath=/root/config/modules:"
        "/etc/puppet/modules "
        '-e "%s"' % PUPPET_CLASS)
def check_machine(jenkins, client, machine, error_counts):
    try:
        server = client.servers.get(machine.external_id)
    except:
        print "Unable to get server detail, will retry"
        traceback.print_exc()
        return

    if server.status == 'ACTIVE':
        ip = utils.get_public_ip(server)
        if not ip and 'os-floating-ips' in utils.get_extensions(client):
            utils.add_public_ip(server)
            ip = utils.get_public_ip(server)
        if not ip:
            raise Exception("Unable to find public ip of server")

        machine.ip = ip
        print "Machine %s is running, testing ssh" % machine.id
        if utils.ssh_connect(ip, 'jenkins'):
            if statsd:
                dt = int((time.time() - machine.state_time) * 1000)
                key = 'devstack.launch.%s' % machine.base_image.provider.name
                statsd.timing(key, dt)
                statsd.incr(key)
            print "Adding machine %s to Jenkins" % machine.id
            create_jenkins_node(jenkins, machine)
            print "Machine %s is ready" % machine.id
            machine.state = vmdatabase.READY
            return
    elif not server.status.startswith('BUILD'):
        count = error_counts.get(machine.id, 0)
        count += 1
        error_counts[machine.id] = count
        print "Machine %s is in error %s (%s/5)" % (machine.id,
                                                    server.status,
                                                    count)
        if count >= 5:
            if statsd:
                statsd.incr('devstack.error.%s' %
                            machine.base_image.provider.name)
            raise Exception("Too many errors querying machine %s" % machine.id)
    else:
        if time.time() - machine.state_time >= ABANDON_TIMEOUT:
            if statsd:
                statsd.incr('devstack.timeout.%s' %
                            machine.base_image.provider.name)
            raise Exception("Waited too long for machine %s" % machine.id)
def check_machine(jenkins, client, machine, error_counts, credentials_id):
    try:
        server = client.servers.get(machine.external_id)
    except:
        print "Unable to get server detail, will retry"
        traceback.print_exc()
        return

    if server.status == 'ACTIVE':
        ip = utils.get_public_ip(server)
        if not ip and 'os-floating-ips' in utils.get_extensions(client):
            utils.add_public_ip(server)
            ip = utils.get_public_ip(server)
        if not ip:
            raise Exception("Unable to find public ip of server")

        machine.ip = ip
        print "Machine %s is running, testing ssh" % machine.id
        if utils.ssh_connect(ip, 'jenkins'):
            if statsd:
                dt = int((time.time() - machine.state_time) * 1000)
                key = 'devstack.launch.%s' % machine.base_image.provider.name
                statsd.timing(key, dt)
                statsd.incr(key)
            print "Adding machine %s to Jenkins" % machine.id
            create_jenkins_node(jenkins, machine, credentials_id)
            print "Machine %s is ready" % machine.id
            machine.state = vmdatabase.READY
            utils.log.debug("Online ID: %s" % machine.id)
            return
    elif not server.status.startswith('BUILD'):
        count = error_counts.get(machine.id, 0)
        count += 1
        error_counts[machine.id] = count
        print "Machine %s is in error %s (%s/5)" % (machine.id, server.status,
                                                    count)
        if count >= 5:
            if statsd:
                statsd.incr('devstack.error.%s' %
                            machine.base_image.provider.name)
            raise Exception("Too many errors querying machine %s" % machine.id)
    else:
        if time.time() - machine.state_time >= ABANDON_TIMEOUT:
            if statsd:
                statsd.incr('devstack.timeout.%s' %
                            machine.base_image.provider.name)
            raise Exception("Waited too long for machine %s" % machine.id)
Example #7
0
def bootstrap_server(provider, server, admin_pass, key):
    client = server.manager.api
    ip = utils.get_public_ip(server)
    if not ip and 'os-floating-ips' in utils.get_extensions(client):
        utils.add_public_ip(server)
        ip = utils.get_public_ip(server)
    if not ip:
        raise Exception("Unable to find public ip of server")

    ssh_kwargs = {}
    if key:
        ssh_kwargs['pkey'] = key
    else:
        ssh_kwargs['password'] = admin_pass

    for username in ['root', 'ubuntu']:
        client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600)
        if client:
            break

    if not client:
        raise Exception("Unable to log in via SSH")

    # hpcloud can't reliably set the hostname
    gerrit_url = 'https://review.openstack.org/p/openstack/' \
                 'openstack-ci-puppet.git'
    client.ssh("set hostname", "sudo hostname %s" % server.name)
    client.ssh("update apt cache", "sudo apt-get update")
    client.ssh(
        "upgrading system packages",
        'sudo DEBIAN_FRONTEND=noninteractive apt-get '
        '--option "Dpkg::Options::=--force-confold"'
        ' --assume-yes upgrade')
    client.ssh(
        "install git and puppet",
        'sudo DEBIAN_FRONTEND=noninteractive apt-get '
        '--option "Dpkg::Options::=--force-confold"'
        ' --assume-yes install git puppet')
    client.ssh("clone puppret repo",
               "sudo git clone %s /root/openstack-ci-puppet" % gerrit_url)
    client.ssh(
        "run puppet", "sudo puppet apply "
        "--modulepath=/root/openstack-ci-puppet/modules"
        "/root/openstack-ci-puppet/manifests/site.pp")
def bootstrap_server(provider, server, admin_pass, key):
    client = server.manager.api
    ip = utils.get_public_ip(server)
    if not ip and 'os-floating-ips' in utils.get_extensions(client):
        utils.add_public_ip(server)
        ip = utils.get_public_ip(server)
    if not ip:
        raise Exception("Unable to find public ip of server")

    ssh_kwargs = {}
    if key:
        ssh_kwargs['pkey'] = key
    else:
        ssh_kwargs['password'] = admin_pass

    for username in ['root', 'ubuntu']:
        client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600)
        if client:
            break

    if not client:
        raise Exception("Unable to log in via SSH")

    # hpcloud can't reliably set the hostname
    gerrit_url = 'https://review.openstack.org/p/openstack/' \
                 'openstack-ci-puppet.git'
    client.ssh("set hostname", "sudo hostname %s" % server.name)
    client.ssh("update apt cache", "sudo apt-get update")
    client.ssh("upgrading system packages",
               'sudo DEBIAN_FRONTEND=noninteractive apt-get '
               '--option "Dpkg::Options::=--force-confold"'
               ' --assume-yes upgrade')
    client.ssh("install git and puppet",
               'sudo DEBIAN_FRONTEND=noninteractive apt-get '
               '--option "Dpkg::Options::=--force-confold"'
               ' --assume-yes install git puppet')
    client.ssh("clone puppret repo",
               "sudo git clone %s /root/openstack-ci-puppet" % gerrit_url)
    client.ssh("run puppet",
               "sudo puppet apply "
               "--modulepath=/root/openstack-ci-puppet/modules"
               "/root/openstack-ci-puppet/manifests/site.pp")