def check_machine(jenkins, client, machine, error_counts): try: server = client.servers.get(machine.external_id) except: print "Unable to get server detail, will retry" traceback.print_exc() return if server.status == "ACTIVE": ip = utils.get_public_ip(server) if not ip and "os-floating-ips" in utils.get_extensions(client): utils.add_public_ip(server) ip = utils.get_public_ip(server) if not ip: raise Exception("Unable to find public ip of server") machine.ip = ip print "Machine %s is running, testing ssh" % machine.id if utils.ssh_connect(ip, "jenkins"): print "Adding machine %s to Jenkins" % machine.id create_jenkins_node(jenkins, machine) print "Machine %s is ready" % machine.id machine.state = vmdatabase.READY return elif not server.status.startswith("BUILD"): count = error_counts.get(machine.id, 0) count += 1 error_counts[machine.id] = count print "Machine %s is in error %s (%s/5)" % (machine.id, server.status, count) if count >= 5: raise Exception("Too many errors querying machine %s" % machine.id) else: if time.time() - machine.state_time >= ABANDON_TIMEOUT: raise Exception("Waited too long for machine %s" % machine.id)
def check_machine(jenkins, client, machine, error_counts): try: server = client.servers.get(machine.external_id) except: print "Unable to get server detail, will retry" traceback.print_exc() return if server.status == 'ACTIVE': ip = utils.get_public_ip(server) if not ip and 'os-floating-ips' in utils.get_extensions(client): utils.add_public_ip(server) ip = utils.get_public_ip(server) if not ip: raise Exception("Unable to find public ip of server") machine.ip = ip print "Machine %s is running, testing ssh" % machine.id if utils.ssh_connect(ip, 'jenkins'): print "Adding machine %s to Jenkins" % machine.id create_jenkins_node(jenkins, machine) print "Machine %s is ready" % machine.id machine.state = vmdatabase.READY return elif not server.status.startswith('BUILD'): count = error_counts.get(machine.id, 0) count += 1 error_counts[machine.id] = count print "Machine %s is in error %s (%s/5)" % (machine.id, server.status, count) if count >= 5: raise Exception("Too many errors querying machine %s" % machine.id) else: if time.time() - machine.state_time >= ABANDON_TIMEOUT: raise Exception("Waited too long for machine %s" % machine.id)
def bootstrap_server(provider, server, admin_pass, key): client = server.manager.api ip = utils.get_public_ip(server) if not ip and "os-floating-ips" in utils.get_extensions(client): utils.add_public_ip(server) ip = utils.get_public_ip(server) if not ip: raise Exception("Unable to find public ip of server") ssh_kwargs = {} if key: ssh_kwargs["pkey"] = key else: ssh_kwargs["password"] = admin_pass for username in ["root", "ubuntu"]: client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) if client: break if not client: raise Exception("Unable to log in via SSH") # hpcloud can't reliably set the hostname gerrit_url = "https://review.openstack.org/p/openstack-infra/config.git" client.ssh("set hostname", "sudo hostname %s" % server.name) client.ssh( "get puppet repo deb", "sudo /usr/bin/wget " "http://apt.puppetlabs.com/puppetlabs-release-" "`lsb_release -c -s`.deb -O /root/puppet-repo.deb", ) client.ssh("install puppet repo deb", "sudo dpkg -i /root/puppet-repo.deb") client.ssh("update apt cache", "sudo apt-get update") client.ssh( "upgrading system packages", "sudo DEBIAN_FRONTEND=noninteractive apt-get " '--option "Dpkg::Options::=--force-confold"' " --assume-yes dist-upgrade", ) client.ssh( "install git and puppet", "sudo DEBIAN_FRONTEND=noninteractive apt-get " '--option "Dpkg::Options::=--force-confold"' " --assume-yes install git puppet", ) client.ssh("clone puppret repo", "sudo git clone %s /root/config" % gerrit_url) client.ssh("install puppet modules", "sudo /bin/bash /root/config/install_modules.sh") client.ssh( "run puppet", "sudo puppet apply --modulepath=/root/config/modules:" "/etc/puppet/modules " '-e "%s"' % PUPPET_CLASS, )
def bootstrap_server(provider, server, admin_pass, key): client = server.manager.api ip = utils.get_public_ip(server) if not ip and 'os-floating-ips' in utils.get_extensions(client): utils.add_public_ip(server) ip = utils.get_public_ip(server) if not ip: raise Exception("Unable to find public ip of server") ssh_kwargs = {} if key: ssh_kwargs['pkey'] = key else: ssh_kwargs['password'] = admin_pass for username in ['root', 'ubuntu']: client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) if client: break if not client: raise Exception("Unable to log in via SSH") # hpcloud can't reliably set the hostname gerrit_url = 'https://review.openstack.org/p/openstack-infra/config.git' client.ssh("set hostname", "sudo hostname %s" % server.name) client.ssh( "get puppet repo deb", "sudo /usr/bin/wget " "http://apt.puppetlabs.com/puppetlabs-release-" "`lsb_release -c -s`.deb -O /root/puppet-repo.deb") client.ssh("install puppet repo deb", "sudo dpkg -i /root/puppet-repo.deb") client.ssh("update apt cache", "sudo apt-get update") client.ssh( "upgrading system packages", 'sudo DEBIAN_FRONTEND=noninteractive apt-get ' '--option "Dpkg::Options::=--force-confold"' ' --assume-yes dist-upgrade') client.ssh( "install git and puppet", 'sudo DEBIAN_FRONTEND=noninteractive apt-get ' '--option "Dpkg::Options::=--force-confold"' ' --assume-yes install git puppet') client.ssh("clone puppret repo", "sudo git clone %s /root/config" % gerrit_url) client.ssh("install puppet modules", "sudo /bin/bash /root/config/install_modules.sh") client.ssh( "run puppet", "sudo puppet apply --modulepath=/root/config/modules:" "/etc/puppet/modules " '-e "%s"' % PUPPET_CLASS)
def check_machine(jenkins, client, machine, error_counts): try: server = client.servers.get(machine.external_id) except: print "Unable to get server detail, will retry" traceback.print_exc() return if server.status == 'ACTIVE': ip = utils.get_public_ip(server) if not ip and 'os-floating-ips' in utils.get_extensions(client): utils.add_public_ip(server) ip = utils.get_public_ip(server) if not ip: raise Exception("Unable to find public ip of server") machine.ip = ip print "Machine %s is running, testing ssh" % machine.id if utils.ssh_connect(ip, 'jenkins'): if statsd: dt = int((time.time() - machine.state_time) * 1000) key = 'devstack.launch.%s' % machine.base_image.provider.name statsd.timing(key, dt) statsd.incr(key) print "Adding machine %s to Jenkins" % machine.id create_jenkins_node(jenkins, machine) print "Machine %s is ready" % machine.id machine.state = vmdatabase.READY return elif not server.status.startswith('BUILD'): count = error_counts.get(machine.id, 0) count += 1 error_counts[machine.id] = count print "Machine %s is in error %s (%s/5)" % (machine.id, server.status, count) if count >= 5: if statsd: statsd.incr('devstack.error.%s' % machine.base_image.provider.name) raise Exception("Too many errors querying machine %s" % machine.id) else: if time.time() - machine.state_time >= ABANDON_TIMEOUT: if statsd: statsd.incr('devstack.timeout.%s' % machine.base_image.provider.name) raise Exception("Waited too long for machine %s" % machine.id)
def check_machine(jenkins, client, machine, error_counts, credentials_id): try: server = client.servers.get(machine.external_id) except: print "Unable to get server detail, will retry" traceback.print_exc() return if server.status == 'ACTIVE': ip = utils.get_public_ip(server) if not ip and 'os-floating-ips' in utils.get_extensions(client): utils.add_public_ip(server) ip = utils.get_public_ip(server) if not ip: raise Exception("Unable to find public ip of server") machine.ip = ip print "Machine %s is running, testing ssh" % machine.id if utils.ssh_connect(ip, 'jenkins'): if statsd: dt = int((time.time() - machine.state_time) * 1000) key = 'devstack.launch.%s' % machine.base_image.provider.name statsd.timing(key, dt) statsd.incr(key) print "Adding machine %s to Jenkins" % machine.id create_jenkins_node(jenkins, machine, credentials_id) print "Machine %s is ready" % machine.id machine.state = vmdatabase.READY utils.log.debug("Online ID: %s" % machine.id) return elif not server.status.startswith('BUILD'): count = error_counts.get(machine.id, 0) count += 1 error_counts[machine.id] = count print "Machine %s is in error %s (%s/5)" % (machine.id, server.status, count) if count >= 5: if statsd: statsd.incr('devstack.error.%s' % machine.base_image.provider.name) raise Exception("Too many errors querying machine %s" % machine.id) else: if time.time() - machine.state_time >= ABANDON_TIMEOUT: if statsd: statsd.incr('devstack.timeout.%s' % machine.base_image.provider.name) raise Exception("Waited too long for machine %s" % machine.id)
def bootstrap_server(provider, server, admin_pass, key): client = server.manager.api ip = utils.get_public_ip(server) if not ip and 'os-floating-ips' in utils.get_extensions(client): utils.add_public_ip(server) ip = utils.get_public_ip(server) if not ip: raise Exception("Unable to find public ip of server") ssh_kwargs = {} if key: ssh_kwargs['pkey'] = key else: ssh_kwargs['password'] = admin_pass for username in ['root', 'ubuntu']: client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) if client: break if not client: raise Exception("Unable to log in via SSH") # hpcloud can't reliably set the hostname gerrit_url = 'https://review.openstack.org/p/openstack/' \ 'openstack-ci-puppet.git' client.ssh("set hostname", "sudo hostname %s" % server.name) client.ssh("update apt cache", "sudo apt-get update") client.ssh( "upgrading system packages", 'sudo DEBIAN_FRONTEND=noninteractive apt-get ' '--option "Dpkg::Options::=--force-confold"' ' --assume-yes upgrade') client.ssh( "install git and puppet", 'sudo DEBIAN_FRONTEND=noninteractive apt-get ' '--option "Dpkg::Options::=--force-confold"' ' --assume-yes install git puppet') client.ssh("clone puppret repo", "sudo git clone %s /root/openstack-ci-puppet" % gerrit_url) client.ssh( "run puppet", "sudo puppet apply " "--modulepath=/root/openstack-ci-puppet/modules" "/root/openstack-ci-puppet/manifests/site.pp")
def bootstrap_server(provider, server, admin_pass, key): client = server.manager.api ip = utils.get_public_ip(server) if not ip and 'os-floating-ips' in utils.get_extensions(client): utils.add_public_ip(server) ip = utils.get_public_ip(server) if not ip: raise Exception("Unable to find public ip of server") ssh_kwargs = {} if key: ssh_kwargs['pkey'] = key else: ssh_kwargs['password'] = admin_pass for username in ['root', 'ubuntu']: client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) if client: break if not client: raise Exception("Unable to log in via SSH") # hpcloud can't reliably set the hostname gerrit_url = 'https://review.openstack.org/p/openstack/' \ 'openstack-ci-puppet.git' client.ssh("set hostname", "sudo hostname %s" % server.name) client.ssh("update apt cache", "sudo apt-get update") client.ssh("upgrading system packages", 'sudo DEBIAN_FRONTEND=noninteractive apt-get ' '--option "Dpkg::Options::=--force-confold"' ' --assume-yes upgrade') client.ssh("install git and puppet", 'sudo DEBIAN_FRONTEND=noninteractive apt-get ' '--option "Dpkg::Options::=--force-confold"' ' --assume-yes install git puppet') client.ssh("clone puppret repo", "sudo git clone %s /root/openstack-ci-puppet" % gerrit_url) client.ssh("run puppet", "sudo puppet apply " "--modulepath=/root/openstack-ci-puppet/modules" "/root/openstack-ci-puppet/manifests/site.pp")