def bootstrap_server(name, server, admin_pass, key, environment): client = server.manager.api ip = utils.get_public_ip(server) if not ip: raise Exception("Unable to find public ip of server") ssh_kwargs = {} if not key: ssh_kwargs['password'] = admin_pass for username in ['root', 'ubuntu']: ssh_client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) if ssh_client: break if not ssh_client: raise Exception("Unable to log in via SSH") ssh_client.ssh('sudo hostname `curl http://169.254.169.254/2009-04-04/meta-data/hostname`' % name) if username != 'root': ssh_client.ssh("sudo cp ~/.ssh/authorized_keys" " ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chmod 644 ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chown root.root ~root/.ssh/authorized_keys") ssh_client = utils.ssh_connect(ip, 'root', ssh_kwargs, timeout=600) ssh_client.ssh('git clone https://github.com/emonty/config') ssh_client.ssh('sudo bash -x config/install_modules.sh') ssh_client.ssh('sudo puppet apply' ' --modulepath=`pwd`/config/modules:/etc/puppet/modules' ' config/manifests/site.pp')
def bootstrap_server(server, admin_pass, key, cert): client = server.manager.api ip = utils.get_public_ip(server) if not ip: raise Exception("Unable to find public ip of server") ssh_kwargs = {} if key: ssh_kwargs['pkey'] = key else: ssh_kwargs['password'] = admin_pass for username in ['root', 'ubuntu']: ssh_client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) if ssh_client: break if not ssh_client: raise Exception("Unable to log in via SSH") if username != 'root': ssh_client.ssh("sudo cp ~/.ssh/authorized_keys" " ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chmod 644 ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chown root.root ~root/.ssh/authorized_keys") ssh_client = utils.ssh_connect(ip, 'root', ssh_kwargs, timeout=600) ssh_client.ssh("apt-get update") ssh_client.ssh("DEBIAN_FRONTEND=noninteractive apt-get --option" " 'Dpkg::Options::=--force-confold'" " --assume-yes upgrade") ssh_client.ssh("apt-get install -y --force-yes puppet") certname = cert[:0-len('.pem')] ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/certs") ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/private_keys") ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/public_keys") ssh_client.ssh("chown -R puppet:root /var/lib/puppet/ssl") ssh_client.ssh("chmod 0771 /var/lib/puppet/ssl") ssh_client.ssh("chmod 0755 /var/lib/puppet/ssl/certs") ssh_client.ssh("chmod 0750 /var/lib/puppet/ssl/private_keys") ssh_client.ssh("chmod 0755 /var/lib/puppet/ssl/public_keys") for ssldir in ['/var/lib/puppet/ssl/certs/', '/var/lib/puppet/ssl/private_keys/', '/var/lib/puppet/ssl/public_keys/']: ssh_client.scp(os.path.join(ssldir, cert), os.path.join(ssldir, cert)) ssh_client.scp("/var/lib/puppet/ssl/crl.pem", "/var/lib/puppet/ssl/crl.pem") ssh_client.scp("/var/lib/puppet/ssl/certs/ca.pem", "/var/lib/puppet/ssl/certs/ca.pem") ssh_client.ssh("puppet agent " "--server ci-puppetmaster.openstack.org " "--no-daemonize --verbose --onetime " "--certname %s" % certname)
def bootstrap_server(server, admin_pass, key, cert): client = server.manager.api ip = utils.get_public_ip(server) if not ip: raise Exception("Unable to find public ip of server") ssh_kwargs = {} if key: ssh_kwargs['pkey'] = key else: ssh_kwargs['password'] = admin_pass for username in ['root', 'ubuntu']: ssh_client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) if ssh_client: break if not ssh_client: raise Exception("Unable to log in via SSH") if username != 'root': ssh_client.ssh("sudo cp ~/.ssh/authorized_keys" " ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chmod 644 ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chown root.root ~root/.ssh/authorized_keys") ssh_client = utils.ssh_connect(ip, 'root', ssh_kwargs, timeout=600) ssh_client.ssh("apt-get update") ssh_client.ssh("DEBIAN_FRONTEND=noninteractive apt-get --option" " 'Dpkg::Options::=--force-confold'" " --assume-yes upgrade") ssh_client.ssh("apt-get install -y --force-yes puppet") certname = cert[:0 - len('.pem')] ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/certs") ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/private_keys") ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/public_keys") ssh_client.ssh("chown -R puppet:root /var/lib/puppet/ssl") ssh_client.ssh("chmod 0771 /var/lib/puppet/ssl") ssh_client.ssh("chmod 0755 /var/lib/puppet/ssl/certs") ssh_client.ssh("chmod 0750 /var/lib/puppet/ssl/private_keys") ssh_client.ssh("chmod 0755 /var/lib/puppet/ssl/public_keys") for ssldir in [ '/var/lib/puppet/ssl/certs/', '/var/lib/puppet/ssl/private_keys/', '/var/lib/puppet/ssl/public_keys/' ]: ssh_client.scp(os.path.join(ssldir, cert), os.path.join(ssldir, cert)) ssh_client.scp("/var/lib/puppet/ssl/crl.pem", "/var/lib/puppet/ssl/crl.pem") ssh_client.scp("/var/lib/puppet/ssl/certs/ca.pem", "/var/lib/puppet/ssl/certs/ca.pem") ssh_client.ssh("puppet agent " "--server ci-puppetmaster.openstack.org " "--no-daemonize --verbose --onetime " "--certname %s" % certname)
def bootstrap_server(server, admin_pass, key, cert, environment): client = server.manager.api ip = utils.get_public_ip(server) if not ip: raise Exception("Unable to find public ip of server") ssh_kwargs = {} if key: ssh_kwargs['pkey'] = key else: ssh_kwargs['password'] = admin_pass for username in ['root', 'ubuntu']: ssh_client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) if ssh_client: break if not ssh_client: raise Exception("Unable to log in via SSH") if username != 'root': ssh_client.ssh("sudo cp ~/.ssh/authorized_keys" " ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chmod 644 ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chown root.root ~root/.ssh/authorized_keys") ssh_client = utils.ssh_connect(ip, 'root', ssh_kwargs, timeout=600) ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'install_puppet.sh'), 'install_puppet.sh') ssh_client.ssh('bash -x install_puppet.sh') certname = cert[:0-len('.pem')] ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/certs") ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/private_keys") ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/public_keys") ssh_client.ssh("chown -R puppet:root /var/lib/puppet/ssl") ssh_client.ssh("chmod 0771 /var/lib/puppet/ssl") ssh_client.ssh("chmod 0755 /var/lib/puppet/ssl/certs") ssh_client.ssh("chmod 0750 /var/lib/puppet/ssl/private_keys") ssh_client.ssh("chmod 0755 /var/lib/puppet/ssl/public_keys") for ssldir in ['/var/lib/puppet/ssl/certs/', '/var/lib/puppet/ssl/private_keys/', '/var/lib/puppet/ssl/public_keys/']: ssh_client.scp(os.path.join(ssldir, cert), os.path.join(ssldir, cert)) ssh_client.scp("/var/lib/puppet/ssl/crl.pem", "/var/lib/puppet/ssl/crl.pem") ssh_client.scp("/var/lib/puppet/ssl/certs/ca.pem", "/var/lib/puppet/ssl/certs/ca.pem") ssh_client.ssh("puppet agent " "--environment %s " "--server ci-puppetmaster.openstack.org " "--no-daemonize --verbose --onetime --pluginsync true " "--certname %s" % (environment, certname))
def check_machine(jenkins, client, machine, error_counts): try: server = client.servers.get(machine.external_id) except: print "Unable to get server detail, will retry" traceback.print_exc() return if server.status == "ACTIVE": ip = utils.get_public_ip(server) if not ip and "os-floating-ips" in utils.get_extensions(client): utils.add_public_ip(server) ip = utils.get_public_ip(server) if not ip: raise Exception("Unable to find public ip of server") machine.ip = ip print "Machine %s is running, testing ssh" % machine.id if utils.ssh_connect(ip, "jenkins"): print "Adding machine %s to Jenkins" % machine.id create_jenkins_node(jenkins, machine) print "Machine %s is ready" % machine.id machine.state = vmdatabase.READY return elif not server.status.startswith("BUILD"): count = error_counts.get(machine.id, 0) count += 1 error_counts[machine.id] = count print "Machine %s is in error %s (%s/5)" % (machine.id, server.status, count) if count >= 5: raise Exception("Too many errors querying machine %s" % machine.id) else: if time.time() - machine.state_time >= ABANDON_TIMEOUT: raise Exception("Waited too long for machine %s" % machine.id)
def check_machine(jenkins, client, machine, error_counts): try: server = client.servers.get(machine.external_id) except: print "Unable to get server detail, will retry" traceback.print_exc() return if server.status == 'ACTIVE': ip = utils.get_public_ip(server) if not ip and 'os-floating-ips' in utils.get_extensions(client): utils.add_public_ip(server) ip = utils.get_public_ip(server) if not ip: raise Exception("Unable to find public ip of server") machine.ip = ip print "Machine %s is running, testing ssh" % machine.id if utils.ssh_connect(ip, 'jenkins'): print "Adding machine %s to Jenkins" % machine.id create_jenkins_node(jenkins, machine) print "Machine %s is ready" % machine.id machine.state = vmdatabase.READY return elif not server.status.startswith('BUILD'): count = error_counts.get(machine.id, 0) count += 1 error_counts[machine.id] = count print "Machine %s is in error %s (%s/5)" % (machine.id, server.status, count) if count >= 5: raise Exception("Too many errors querying machine %s" % machine.id) else: if time.time() - machine.state_time >= ABANDON_TIMEOUT: raise Exception("Waited too long for machine %s" % machine.id)
def bootstrap_server(provider, server, admin_pass, key): client = server.manager.api ip = utils.get_public_ip(server) if not ip and "os-floating-ips" in utils.get_extensions(client): utils.add_public_ip(server) ip = utils.get_public_ip(server) if not ip: raise Exception("Unable to find public ip of server") ssh_kwargs = {} if key: ssh_kwargs["pkey"] = key else: ssh_kwargs["password"] = admin_pass for username in ["root", "ubuntu"]: client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) if client: break if not client: raise Exception("Unable to log in via SSH") # hpcloud can't reliably set the hostname gerrit_url = "https://review.openstack.org/p/openstack-infra/config.git" client.ssh("set hostname", "sudo hostname %s" % server.name) client.ssh( "get puppet repo deb", "sudo /usr/bin/wget " "http://apt.puppetlabs.com/puppetlabs-release-" "`lsb_release -c -s`.deb -O /root/puppet-repo.deb", ) client.ssh("install puppet repo deb", "sudo dpkg -i /root/puppet-repo.deb") client.ssh("update apt cache", "sudo apt-get update") client.ssh( "upgrading system packages", "sudo DEBIAN_FRONTEND=noninteractive apt-get " '--option "Dpkg::Options::=--force-confold"' " --assume-yes dist-upgrade", ) client.ssh( "install git and puppet", "sudo DEBIAN_FRONTEND=noninteractive apt-get " '--option "Dpkg::Options::=--force-confold"' " --assume-yes install git puppet", ) client.ssh("clone puppret repo", "sudo git clone %s /root/config" % gerrit_url) client.ssh("install puppet modules", "sudo /bin/bash /root/config/install_modules.sh") client.ssh( "run puppet", "sudo puppet apply --modulepath=/root/config/modules:" "/etc/puppet/modules " '-e "%s"' % PUPPET_CLASS, )
def bootstrap_server(provider, server, admin_pass, key): client = server.manager.api ip = utils.get_public_ip(server) if not ip and 'os-floating-ips' in utils.get_extensions(client): utils.add_public_ip(server) ip = utils.get_public_ip(server) if not ip: raise Exception("Unable to find public ip of server") ssh_kwargs = {} if key: ssh_kwargs['pkey'] = key else: ssh_kwargs['password'] = admin_pass for username in ['root', 'ubuntu']: client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) if client: break if not client: raise Exception("Unable to log in via SSH") # hpcloud can't reliably set the hostname gerrit_url = 'https://review.openstack.org/p/openstack-infra/config.git' client.ssh("set hostname", "sudo hostname %s" % server.name) client.ssh( "get puppet repo deb", "sudo /usr/bin/wget " "http://apt.puppetlabs.com/puppetlabs-release-" "`lsb_release -c -s`.deb -O /root/puppet-repo.deb") client.ssh("install puppet repo deb", "sudo dpkg -i /root/puppet-repo.deb") client.ssh("update apt cache", "sudo apt-get update") client.ssh( "upgrading system packages", 'sudo DEBIAN_FRONTEND=noninteractive apt-get ' '--option "Dpkg::Options::=--force-confold"' ' --assume-yes dist-upgrade') client.ssh( "install git and puppet", 'sudo DEBIAN_FRONTEND=noninteractive apt-get ' '--option "Dpkg::Options::=--force-confold"' ' --assume-yes install git puppet') client.ssh("clone puppret repo", "sudo git clone %s /root/config" % gerrit_url) client.ssh("install puppet modules", "sudo /bin/bash /root/config/install_modules.sh") client.ssh( "run puppet", "sudo puppet apply --modulepath=/root/config/modules:" "/etc/puppet/modules " '-e "%s"' % PUPPET_CLASS)
def check_machine(jenkins, client, machine, error_counts, credentials_id): try: server = client.servers.get(machine.external_id) except: print "Unable to get server detail, will retry" traceback.print_exc() return if server.status == 'ACTIVE': ip = utils.get_public_ip(server) if not ip and 'os-floating-ips' in utils.get_extensions(client): utils.add_public_ip(server) ip = utils.get_public_ip(server) if not ip: raise Exception("Unable to find public ip of server") machine.ip = ip print "Machine %s is running, testing ssh" % machine.id if utils.ssh_connect(ip, 'jenkins'): if statsd: dt = int((time.time() - machine.state_time) * 1000) key = 'devstack.launch.%s' % machine.base_image.provider.name statsd.timing(key, dt) statsd.incr(key) print "Adding machine %s to Jenkins" % machine.id create_jenkins_node(jenkins, machine, credentials_id) print "Machine %s is ready" % machine.id machine.state = vmdatabase.READY utils.log.debug("Online ID: %s" % machine.id) return elif not server.status.startswith('BUILD'): count = error_counts.get(machine.id, 0) count += 1 error_counts[machine.id] = count print "Machine %s is in error %s (%s/5)" % (machine.id, server.status, count) if count >= 5: if statsd: statsd.incr('devstack.error.%s' % machine.base_image.provider.name) raise Exception("Too many errors querying machine %s" % machine.id) else: if time.time() - machine.state_time >= ABANDON_TIMEOUT: if statsd: statsd.incr('devstack.timeout.%s' % machine.base_image.provider.name) raise Exception("Waited too long for machine %s" % machine.id)
def check_machine(jenkins, client, machine, error_counts): try: server = client.servers.get(machine.external_id) except: print "Unable to get server detail, will retry" traceback.print_exc() return if server.status == 'ACTIVE': ip = utils.get_public_ip(server) if not ip and 'os-floating-ips' in utils.get_extensions(client): utils.add_public_ip(server) ip = utils.get_public_ip(server) if not ip: raise Exception("Unable to find public ip of server") machine.ip = ip print "Machine %s is running, testing ssh" % machine.id if utils.ssh_connect(ip, 'jenkins'): if statsd: dt = int((time.time() - machine.state_time) * 1000) key = 'devstack.launch.%s' % machine.base_image.provider.name statsd.timing(key, dt) statsd.incr(key) print "Adding machine %s to Jenkins" % machine.id create_jenkins_node(jenkins, machine) print "Machine %s is ready" % machine.id machine.state = vmdatabase.READY return elif not server.status.startswith('BUILD'): count = error_counts.get(machine.id, 0) count += 1 error_counts[machine.id] = count print "Machine %s is in error %s (%s/5)" % (machine.id, server.status, count) if count >= 5: if statsd: statsd.incr('devstack.error.%s' % machine.base_image.provider.name) raise Exception("Too many errors querying machine %s" % machine.id) else: if time.time() - machine.state_time >= ABANDON_TIMEOUT: if statsd: statsd.incr('devstack.timeout.%s' % machine.base_image.provider.name) raise Exception("Waited too long for machine %s" % machine.id)
def check_machine(jenkins, machine): utils.log.debug("Check ID: %s" % machine.id) try: if utils.ssh_connect(machine.ip, 'jenkins'): return except: utils.log.exception("Check failed ID: %s" % machine.id) utils.log.debug("Set deleted ID: %s old state: %s" % ( machine.id, machine.state)) machine.state = vmdatabase.DELETE if jenkins: if machine.jenkins_name: if jenkins.node_exists(machine.jenkins_name): utils.log.debug("Delete jenkins node ID: %s" % machine.id) jenkins.delete_node(machine.jenkins_name) machine.delete()
def check_machine(jenkins, machine): utils.log.debug("Check ID: %s" % machine.id) try: if utils.ssh_connect(machine.ip, 'jenkins'): return except: utils.log.exception("Check failed ID: %s" % machine.id) utils.log.debug("Set deleted ID: %s old state: %s" % (machine.id, machine.state)) machine.state = vmdatabase.DELETE if jenkins: if machine.jenkins_name: if jenkins.node_exists(machine.jenkins_name): utils.log.debug("Delete jenkins node ID: %s" % machine.id) jenkins.delete_node(machine.jenkins_name) machine.delete()
def bootstrap_server(provider, server, admin_pass, key): client = server.manager.api ip = utils.get_public_ip(server) if not ip and 'os-floating-ips' in utils.get_extensions(client): utils.add_public_ip(server) ip = utils.get_public_ip(server) if not ip: raise Exception("Unable to find public ip of server") ssh_kwargs = {} if key: ssh_kwargs['pkey'] = key else: ssh_kwargs['password'] = admin_pass for username in ['root', 'ubuntu']: client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) if client: break if not client: raise Exception("Unable to log in via SSH") # hpcloud can't reliably set the hostname gerrit_url = 'https://review.openstack.org/p/openstack/' \ 'openstack-ci-puppet.git' client.ssh("set hostname", "sudo hostname %s" % server.name) client.ssh("update apt cache", "sudo apt-get update") client.ssh( "upgrading system packages", 'sudo DEBIAN_FRONTEND=noninteractive apt-get ' '--option "Dpkg::Options::=--force-confold"' ' --assume-yes upgrade') client.ssh( "install git and puppet", 'sudo DEBIAN_FRONTEND=noninteractive apt-get ' '--option "Dpkg::Options::=--force-confold"' ' --assume-yes install git puppet') client.ssh("clone puppret repo", "sudo git clone %s /root/openstack-ci-puppet" % gerrit_url) client.ssh( "run puppet", "sudo puppet apply " "--modulepath=/root/openstack-ci-puppet/modules" "/root/openstack-ci-puppet/manifests/site.pp")
def bootstrap_server(provider, server, admin_pass, key): client = server.manager.api ip = utils.get_public_ip(server) if not ip and 'os-floating-ips' in utils.get_extensions(client): utils.add_public_ip(server) ip = utils.get_public_ip(server) if not ip: raise Exception("Unable to find public ip of server") ssh_kwargs = {} if key: ssh_kwargs['pkey'] = key else: ssh_kwargs['password'] = admin_pass for username in ['root', 'ubuntu']: client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) if client: break if not client: raise Exception("Unable to log in via SSH") # hpcloud can't reliably set the hostname gerrit_url = 'https://review.openstack.org/p/openstack/' \ 'openstack-ci-puppet.git' client.ssh("set hostname", "sudo hostname %s" % server.name) client.ssh("update apt cache", "sudo apt-get update") client.ssh("upgrading system packages", 'sudo DEBIAN_FRONTEND=noninteractive apt-get ' '--option "Dpkg::Options::=--force-confold"' ' --assume-yes upgrade') client.ssh("install git and puppet", 'sudo DEBIAN_FRONTEND=noninteractive apt-get ' '--option "Dpkg::Options::=--force-confold"' ' --assume-yes install git puppet') client.ssh("clone puppret repo", "sudo git clone %s /root/openstack-ci-puppet" % gerrit_url) client.ssh("run puppet", "sudo puppet apply " "--modulepath=/root/openstack-ci-puppet/modules" "/root/openstack-ci-puppet/manifests/site.pp")
def bootstrap_server(server, key, name, volume_device, keep, mount_path, fs_label): ip = server.public_v4 ssh_kwargs = dict(pkey=key) print 'Public IP', ip for username in ['root', 'ubuntu', 'centos', 'admin']: ssh_client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) if ssh_client: break if not ssh_client: raise Exception("Unable to log in via SSH") # cloud-init puts the "please log in as user foo" message and # subsequent exit() in root's authorized_keys -- overwrite it with # a normal version to get root login working again. if username != 'root': ssh_client.ssh("sudo cp ~/.ssh/authorized_keys" " ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chmod 644 ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chown root.root ~root/.ssh/authorized_keys") ssh_client = utils.ssh_connect(ip, 'root', ssh_kwargs, timeout=600) if server.public_v6: ssh_client.ssh('ping6 -c5 -Q 0x10 review.openstack.org ' '|| ping6 -c5 -Q 0x10 wiki.openstack.org') ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'make_swap.sh'), 'make_swap.sh') ssh_client.ssh('bash -x make_swap.sh') if volume_device: ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'mount_volume.sh'), 'mount_volume.sh') ssh_client.ssh('bash -x mount_volume.sh %s %s %s' % (volume_device, mount_path, fs_label)) # This next chunk should really exist as a playbook, but whatev ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'install_puppet.sh'), 'install_puppet.sh') ssh_client.ssh('bash -x install_puppet.sh') # Write out the private SSH key we generated key_file = tempfile.NamedTemporaryFile(delete=not keep) key.write_private_key(key_file) key_file.flush() # Write out inventory inventory_file = tempfile.NamedTemporaryFile(delete=not keep) inventory_file.write("{host} ansible_host={ip} ansible_user=root".format( host=name, ip=server.interface_ip)) inventory_file.flush() ansible_cmd = [ 'ansible-playbook', '-i', inventory_file.name, '-l', name, '--private-key={key}'.format(key=key_file.name), "--ssh-common-args='-o StrictHostKeyChecking=no'", '-e', 'target={name}'.format(name=name), ] # Run the remote puppet apply playbook limited to just this server # we just created try: for playbook in [ 'set_hostnames.yml', 'remote_puppet_adhoc.yaml']: print subprocess.check_output( ansible_cmd + [ os.path.join( SCRIPT_DIR, '..', 'playbooks', playbook)], stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: print "Subprocess failed" print e.output raise try: ssh_client.ssh("reboot") except Exception as e: # Some init system kill the connection too fast after reboot. # Deal with it by ignoring ssh errors when rebooting. if e.rc == -1: pass else: raise
def bootstrap_server(server, admin_pass, key, cert, environment, name, puppetmaster, volume, floating_ip_pool): ip = utils.get_public_ip(server, floating_ip_pool=floating_ip_pool) if not ip: raise Exception("Unable to find public ip of server") ssh_kwargs = {} if key: ssh_kwargs['pkey'] = key else: ssh_kwargs['password'] = admin_pass for username in ['root', 'ubuntu', 'centos']: ssh_client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) if ssh_client: break if not ssh_client: raise Exception("Unable to log in via SSH") # cloud-init puts the "please log in as user foo" message and # subsequent exit() in root's authorized_keys -- overwrite it with # a normal version to get root login working again. if username != 'root': ssh_client.ssh("sudo cp ~/.ssh/authorized_keys" " ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chmod 644 ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chown root.root ~root/.ssh/authorized_keys") ssh_client = utils.ssh_connect(ip, 'root', ssh_kwargs, timeout=600) if IPV6: ssh_client.ssh('ping6 -c5 -Q 0x10 review.openstack.org ' '|| ping6 -c5 -Q 0x10 wiki.openstack.org') ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'make_swap.sh'), 'make_swap.sh') ssh_client.ssh('bash -x make_swap.sh') if volume: ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'mount_volume.sh'), 'mount_volume.sh') ssh_client.ssh('bash -x mount_volume.sh') ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'install_puppet.sh'), 'install_puppet.sh') ssh_client.ssh('bash -x install_puppet.sh') certname = cert[:(0 - len('.pem'))] ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/certs") ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/private_keys") ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/public_keys") ssh_client.ssh("chown -R puppet:root /var/lib/puppet/ssl") ssh_client.ssh("chown -R puppet:puppet /var/lib/puppet/ssl/private_keys") ssh_client.ssh("chmod 0771 /var/lib/puppet/ssl") ssh_client.ssh("chmod 0755 /var/lib/puppet/ssl/certs") ssh_client.ssh("chmod 0750 /var/lib/puppet/ssl/private_keys") ssh_client.ssh("chmod 0755 /var/lib/puppet/ssl/public_keys") for ssldir in [ '/var/lib/puppet/ssl/certs/', '/var/lib/puppet/ssl/private_keys/', '/var/lib/puppet/ssl/public_keys/' ]: ssh_client.scp(os.path.join(ssldir, cert), os.path.join(ssldir, cert)) ssh_client.scp("/var/lib/puppet/ssl/crl.pem", "/var/lib/puppet/ssl/crl.pem") ssh_client.scp("/var/lib/puppet/ssl/certs/ca.pem", "/var/lib/puppet/ssl/certs/ca.pem") (rc, output) = ssh_client.ssh( "puppet agent " "--environment %s " "--server %s " "--detailed-exitcodes " "--no-daemonize --verbose --onetime --pluginsync true " "--certname %s" % (environment, puppetmaster, certname), error_ok=True) utils.interpret_puppet_exitcodes(rc, output) ssh_client.ssh("reboot")
def bootstrap_server(server, key, cert, environment, name, puppetmaster, volume, floating_ip_pool): ip = server.public_v4 ssh_kwargs = dict(pkey=key) print 'Public IP', ip for username in ['root', 'ubuntu', 'centos', 'admin']: ssh_client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) if ssh_client: break if not ssh_client: raise Exception("Unable to log in via SSH") # cloud-init puts the "please log in as user foo" message and # subsequent exit() in root's authorized_keys -- overwrite it with # a normal version to get root login working again. if username != 'root': ssh_client.ssh("sudo cp ~/.ssh/authorized_keys" " ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chmod 644 ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chown root.root ~root/.ssh/authorized_keys") ssh_client = utils.ssh_connect(ip, 'root', ssh_kwargs, timeout=600) if server.public_v6: ssh_client.ssh('ping6 -c5 -Q 0x10 review.openstack.org ' '|| ping6 -c5 -Q 0x10 wiki.openstack.org') ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'make_swap.sh'), 'make_swap.sh') ssh_client.ssh('bash -x make_swap.sh') if volume: ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'mount_volume.sh'), 'mount_volume.sh') ssh_client.ssh('bash -x mount_volume.sh') ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'install_puppet.sh'), 'install_puppet.sh') ssh_client.ssh('bash -x install_puppet.sh') certname = cert[:(0 - len('.pem'))] shortname = name.split('.')[0] with ssh_client.open('/etc/hosts', 'w') as f: f.write('127.0.0.1 localhost\n') f.write('127.0.1.1 %s %s\n' % (name, shortname)) with ssh_client.open('/etc/hostname', 'w') as f: f.write('%s\n' % (shortname,)) ssh_client.ssh("hostname %s" % (name,)) ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/certs") ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/private_keys") ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/public_keys") ssh_client.ssh("chown -R puppet:root /var/lib/puppet/ssl") ssh_client.ssh("chown -R puppet:puppet /var/lib/puppet/ssl/private_keys") ssh_client.ssh("chmod 0771 /var/lib/puppet/ssl") ssh_client.ssh("chmod 0755 /var/lib/puppet/ssl/certs") ssh_client.ssh("chmod 0750 /var/lib/puppet/ssl/private_keys") ssh_client.ssh("chmod 0755 /var/lib/puppet/ssl/public_keys") for ssldir in ['/var/lib/puppet/ssl/certs/', '/var/lib/puppet/ssl/private_keys/', '/var/lib/puppet/ssl/public_keys/']: ssh_client.scp(os.path.join(ssldir, cert), os.path.join(ssldir, cert)) ssh_client.scp("/var/lib/puppet/ssl/crl.pem", "/var/lib/puppet/ssl/crl.pem") ssh_client.scp("/var/lib/puppet/ssl/certs/ca.pem", "/var/lib/puppet/ssl/certs/ca.pem") (rc, output) = ssh_client.ssh( "puppet agent " "--environment %s " "--server %s " "--detailed-exitcodes " "--no-daemonize --verbose --onetime --pluginsync true " "--certname %s" % (environment, puppetmaster, certname), error_ok=True) utils.interpret_puppet_exitcodes(rc, output) try: ssh_client.ssh("reboot") except Exception as e: # Some init system kill the connection too fast after reboot. # Deal with it by ignoring ssh errors when rebooting. if e.rc == -1: pass else: raise
def bootstrap_server(server, admin_pass, key, cert, environment, name, puppetmaster, volume): ip = utils.get_public_ip(server) if not ip: raise Exception("Unable to find public ip of server") ssh_kwargs = {} if key: ssh_kwargs['pkey'] = key else: ssh_kwargs['password'] = admin_pass for username in ['root', 'ubuntu']: ssh_client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) if ssh_client: break if not ssh_client: raise Exception("Unable to log in via SSH") if username != 'root': ssh_client.ssh("sudo cp ~/.ssh/authorized_keys" " ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chmod 644 ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chown root.root ~root/.ssh/authorized_keys") ssh_client = utils.ssh_connect(ip, 'root', ssh_kwargs, timeout=600) if IPV6: ssh_client.ssh('ping6 -c5 -Q 0x10 review.openstack.org ' '|| ping6 -c5 -Q 0x10 wiki.openstack.org') ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'make_swap.sh'), 'make_swap.sh') ssh_client.ssh('bash -x make_swap.sh') if volume: ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'mount_volume.sh'), 'mount_volume.sh') ssh_client.ssh('bash -x mount_volume.sh') # BH: Life seems better with this faked FQDN. It's not real. ssh_client.ssh('echo "10.242.0.208 ci-puppetmaster.openstacklocal" \ >> /etc/hosts') ssh_client.ssh('echo "%s" > /etc/hostname && service hostname restart' % name) # BH: this should be in the puppet: ssh_client.ssh('mkdir /var/www') ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'install_puppet.sh'), 'install_puppet.sh') # install correct puppet ssh_client.ssh('export PUPPET_VERSION=2.7 && bash -x install_puppet.sh') certname = cert[:(0 - len('.pem'))] ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/certs") ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/private_keys") ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/public_keys") ssh_client.ssh("chown -R puppet:root /var/lib/puppet/ssl") ssh_client.ssh("chown -R puppet:puppet /var/lib/puppet/ssl/private_keys") ssh_client.ssh("chmod 0771 /var/lib/puppet/ssl") ssh_client.ssh("chmod 0755 /var/lib/puppet/ssl/certs") ssh_client.ssh("chmod 0750 /var/lib/puppet/ssl/private_keys") ssh_client.ssh("chmod 0755 /var/lib/puppet/ssl/public_keys") for ssldir in ['/var/lib/puppet/ssl/certs/', '/var/lib/puppet/ssl/private_keys/', '/var/lib/puppet/ssl/public_keys/']: ssh_client.scp(os.path.join(ssldir, cert), os.path.join(ssldir, cert)) ssh_client.scp("/var/lib/puppet/ssl/crl.pem", "/var/lib/puppet/ssl/crl.pem") ssh_client.scp("/var/lib/puppet/ssl/certs/ca.pem", "/var/lib/puppet/ssl/certs/ca.pem") ssh_client.ssh("puppet agent " "--environment %s " "--server %s " "--no-daemonize --verbose --onetime --pluginsync true " "--certname %s" % (environment, puppetmaster, certname)) ssh_client.ssh("reboot")
def bootstrap_server(server, admin_pass, key, cert, environment, name, salt_priv, salt_pub): client = server.manager.api ip = utils.get_public_ip(server) if not ip: raise Exception("Unable to find public ip of server") ssh_kwargs = {} if key: ssh_kwargs['pkey'] = key else: ssh_kwargs['password'] = admin_pass for username in ['root', 'ubuntu']: ssh_client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) if ssh_client: break if not ssh_client: raise Exception("Unable to log in via SSH") if username != 'root': ssh_client.ssh("sudo cp ~/.ssh/authorized_keys" " ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chmod 644 ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chown root.root ~root/.ssh/authorized_keys") ssh_client = utils.ssh_connect(ip, 'root', ssh_kwargs, timeout=600) ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'install_puppet.sh'), 'install_puppet.sh') ssh_client.ssh('bash -x install_puppet.sh') certname = cert[:0 - len('.pem')] ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/certs") ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/private_keys") ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/public_keys") ssh_client.ssh("chown -R puppet:root /var/lib/puppet/ssl") ssh_client.ssh("chmod 0771 /var/lib/puppet/ssl") ssh_client.ssh("chmod 0755 /var/lib/puppet/ssl/certs") ssh_client.ssh("chmod 0750 /var/lib/puppet/ssl/private_keys") ssh_client.ssh("chmod 0755 /var/lib/puppet/ssl/public_keys") # Assuming salt-master is running on the puppetmaster shutil.copyfile(salt_pub, os.path.join(SALT_MASTER_PKI, 'minions', name)) ssh_client.ssh('mkdir -p {0}'.format(SALT_MINION_PKI)) ssh_client.scp(salt_pub, os.path.join(SALT_MINION_PKI, 'minion.pub')) ssh_client.scp(salt_priv, os.path.join(SALT_MINION_PKI, 'minion.pem')) for ssldir in [ '/var/lib/puppet/ssl/certs/', '/var/lib/puppet/ssl/private_keys/', '/var/lib/puppet/ssl/public_keys/' ]: ssh_client.scp(os.path.join(ssldir, cert), os.path.join(ssldir, cert)) ssh_client.scp("/var/lib/puppet/ssl/crl.pem", "/var/lib/puppet/ssl/crl.pem") ssh_client.scp("/var/lib/puppet/ssl/certs/ca.pem", "/var/lib/puppet/ssl/certs/ca.pem") ssh_client.ssh("puppet agent " "--environment %s " "--server ci-puppetmaster.openstack.org " "--no-daemonize --verbose --onetime --pluginsync true " "--certname %s" % (environment, certname))
def bootstrap_server(server, admin_pass, key, cert, environment, name, salt_priv, salt_pub, puppetmaster): client = server.manager.api ip = utils.get_public_ip(server) if not ip: raise Exception("Unable to find public ip of server") ssh_kwargs = {} if key: ssh_kwargs["pkey"] = key else: ssh_kwargs["password"] = admin_pass for username in ["root", "ubuntu"]: ssh_client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) if ssh_client: break if not ssh_client: raise Exception("Unable to log in via SSH") if username != "root": ssh_client.ssh("sudo cp ~/.ssh/authorized_keys" " ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chmod 644 ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chown root.root ~root/.ssh/authorized_keys") ssh_client = utils.ssh_connect(ip, "root", ssh_kwargs, timeout=600) if IPV6: ssh_client.ssh("ping6 -c5 -Q 0x10 review.openstack.org " "|| ping6 -c5 -Q 0x10 wiki.openstack.org") ssh_client.scp(os.path.join(SCRIPT_DIR, "..", "install_puppet.sh"), "install_puppet.sh") ssh_client.ssh("bash -x install_puppet.sh") certname = cert[: 0 - len(".pem")] ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/certs") ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/private_keys") ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/public_keys") ssh_client.ssh("chown -R puppet:root /var/lib/puppet/ssl") ssh_client.ssh("chmod 0771 /var/lib/puppet/ssl") ssh_client.ssh("chmod 0755 /var/lib/puppet/ssl/certs") ssh_client.ssh("chmod 0750 /var/lib/puppet/ssl/private_keys") ssh_client.ssh("chmod 0755 /var/lib/puppet/ssl/public_keys") if salt_pub and salt_priv: # Assuming salt-master is running on the puppetmaster shutil.copyfile(salt_pub, os.path.join(SALT_MASTER_PKI, "minions", name)) ssh_client.ssh("mkdir -p {0}".format(SALT_MINION_PKI)) ssh_client.scp(salt_pub, os.path.join(SALT_MINION_PKI, "minion.pub")) ssh_client.scp(salt_priv, os.path.join(SALT_MINION_PKI, "minion.pem")) for ssldir in [ "/var/lib/puppet/ssl/certs/", "/var/lib/puppet/ssl/private_keys/", "/var/lib/puppet/ssl/public_keys/", ]: ssh_client.scp(os.path.join(ssldir, cert), os.path.join(ssldir, cert)) ssh_client.scp("/var/lib/puppet/ssl/crl.pem", "/var/lib/puppet/ssl/crl.pem") ssh_client.scp("/var/lib/puppet/ssl/certs/ca.pem", "/var/lib/puppet/ssl/certs/ca.pem") ssh_client.ssh( "puppet agent " "--environment %s " "--server %s " "--no-daemonize --verbose --onetime --pluginsync true " "--certname %s" % (environment, puppetmaster, certname) ) ssh_client.ssh("reboot")
def bootstrap_server(server, admin_pass, key, cert, environment, name, salt_priv, salt_pub, puppetmaster): ip = utils.get_public_ip(server) if not ip: raise Exception("Unable to find public ip of server") ssh_kwargs = {} if key: ssh_kwargs['pkey'] = key else: ssh_kwargs['password'] = admin_pass for username in ['root', 'ubuntu']: ssh_client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) if ssh_client: break if not ssh_client: raise Exception("Unable to log in via SSH") if username != 'root': ssh_client.ssh("sudo cp ~/.ssh/authorized_keys" " ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chmod 644 ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chown root.root ~root/.ssh/authorized_keys") ssh_client = utils.ssh_connect(ip, 'root', ssh_kwargs, timeout=600) if IPV6: ssh_client.ssh('ping6 -c5 -Q 0x10 review.opencontrail.org ' '|| ping6 -c5 -Q 0x10 wiki.opencontrail.org') ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'install_puppet.sh'), 'install_puppet.sh') ssh_client.ssh('bash -x install_puppet.sh') certname = cert[:(0 - len('.pem'))] ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/certs") ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/private_keys") ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/public_keys") ssh_client.ssh("chown -R puppet:root /var/lib/puppet/ssl") ssh_client.ssh("chmod 0771 /var/lib/puppet/ssl") ssh_client.ssh("chmod 0755 /var/lib/puppet/ssl/certs") ssh_client.ssh("chmod 0750 /var/lib/puppet/ssl/private_keys") ssh_client.ssh("chmod 0755 /var/lib/puppet/ssl/public_keys") if salt_pub and salt_priv: # Assuming salt-master is running on the puppetmaster shutil.copyfile(salt_pub, os.path.join(SALT_MASTER_PKI, 'minions', name)) ssh_client.ssh('mkdir -p {0}'.format(SALT_MINION_PKI)) ssh_client.scp(salt_pub, os.path.join(SALT_MINION_PKI, 'minion.pub')) ssh_client.scp(salt_priv, os.path.join(SALT_MINION_PKI, 'minion.pem')) for ssldir in ['/var/lib/puppet/ssl/certs/', '/var/lib/puppet/ssl/private_keys/', '/var/lib/puppet/ssl/public_keys/']: ssh_client.scp(os.path.join(ssldir, cert), os.path.join(ssldir, cert)) ssh_client.scp("/var/lib/puppet/ssl/crl.pem", "/var/lib/puppet/ssl/crl.pem") ssh_client.scp("/var/lib/puppet/ssl/certs/ca.pem", "/var/lib/puppet/ssl/certs/ca.pem") ssh_client.ssh("puppet agent " "--environment %s " "--server %s " "--no-daemonize --verbose --onetime --pluginsync true " "--certname %s" % (environment, puppetmaster, certname)) ssh_client.ssh("reboot")
def bootstrap_server(server, key, name, volume_device, keep, mount_path, fs_label, environment): ip = server.public_v4 ssh_kwargs = dict(pkey=key) print("--- Running initial configuration on host %s ---" % ip) for username in ['root', 'ubuntu', 'centos', 'admin']: ssh_client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) if ssh_client: break if not ssh_client: raise Exception("Unable to log in via SSH") # cloud-init puts the "please log in as user foo" message and # subsequent exit() in root's authorized_keys -- overwrite it with # a normal version to get root login working again. if username != 'root': ssh_client.ssh("sudo cp ~/.ssh/authorized_keys" " ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chmod 644 ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chown root.root ~root/.ssh/authorized_keys") ssh_client = utils.ssh_connect(ip, 'root', ssh_kwargs, timeout=600) # Something up with RAX images that they have the ipv6 interface in # /etc/network/interfaces but eth0 hasn't noticed yet; reload it ssh_client.ssh('ifdown eth0 && ifup eth0') if server.public_v6: ssh_client.ssh('ping6 -c5 -Q 0x10 review.openstack.org ' '|| ping6 -c5 -Q 0x10 wiki.openstack.org') ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'make_swap.sh'), 'make_swap.sh') ssh_client.ssh('bash -x make_swap.sh') if volume_device: ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'mount_volume.sh'), 'mount_volume.sh') ssh_client.ssh('bash -x mount_volume.sh %s %s %s' % (volume_device, mount_path, fs_label)) # This next chunk should really exist as a playbook, but whatev ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'install_puppet.sh'), 'install_puppet.sh') ssh_client.ssh('bash -x install_puppet.sh') # Zero the ansible inventory cache so that next run finds the new server inventory_cache = '/var/cache/ansible-inventory/ansible-inventory.cache' if os.path.exists(inventory_cache): with open(inventory_cache, 'w'): pass with JobDir(keep) as jobdir: # Update the generated-groups file globally and incorporate it # into our inventory # Remove cloud and region from the environment to work # around a bug in occ expand_env = os.environ.copy() for env_key in expand_env.keys(): if env_key.startswith('OS_'): expand_env.pop(env_key, None) expand_env['ANSIBLE_LOG_PATH'] = jobdir.ansible_log # Regenerate inventory cache, throwing an error if there is an issue # so that we don't generate a bogus groups file try: run(['/etc/ansible/hosts/openstack', '--list'], env=expand_env) except subprocess.CalledProcessError as e: print "Inventory regeneration failed" print e.output raise run('/usr/local/bin/expand-groups.sh', env=expand_env, stderr=subprocess.STDOUT) # Write out the private SSH key we generated with open(jobdir.key, 'w') as key_file: key.write_private_key(key_file) os.chmod(jobdir.key, 0o600) # Write out inventory with open(jobdir.hosts, 'w') as inventory_file: inventory_file.write( "{host} ansible_host={ip} ansible_user=root".format( host=name, ip=server.interface_ip)) os.symlink('/etc/ansible/hosts/generated-groups', jobdir.groups) t = threading.Thread(target=stream_syslog, args=(ssh_client,)) t.daemon = True t.start() ansible_cmd = [ 'ansible-playbook', '-i', jobdir.inventory_root, '-l', name, '--private-key={key}'.format(key=jobdir.key), "--ssh-common-args='-o StrictHostKeyChecking=no'", '-e', 'target={name}'.format(name=name), ] if environment is not None: ansible_cmd += [ '-e', 'puppet_environment={env}'.format(env=environment)] # Run the remote puppet apply playbook limited to just this server # we just created for playbook in [ 'set_hostnames.yml', 'remote_puppet_adhoc.yaml']: run(ansible_cmd + [ os.path.join(SCRIPT_DIR, '..', 'playbooks', playbook)], env=jobdir.env) try: ssh_client.ssh("reboot") except Exception as e: # Some init system kill the connection too fast after reboot. # Deal with it by ignoring ssh errors when rebooting. if e.rc == -1: pass else: raise
def bootstrap_server(server, key, name, volume_device, keep, mount_path, fs_label, environment, timeout): ip = server.public_v4 ssh_kwargs = dict(pkey=key) print("--- Running initial configuration on host %s ---" % ip) for username in ['root', 'ubuntu', 'centos', 'admin']: ssh_client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=timeout) if ssh_client: break if not ssh_client: raise Exception("Unable to log in via SSH") # cloud-init puts the "please log in as user foo" message and # subsequent exit() in root's authorized_keys -- overwrite it with # a normal version to get root login working again. if username != 'root': ssh_client.ssh("sudo cp ~/.ssh/authorized_keys" " ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chmod 644 ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chown root.root ~root/.ssh/authorized_keys") ssh_client = utils.ssh_connect(ip, 'root', ssh_kwargs, timeout=timeout) # Something up with RAX images that they have the ipv6 interface in # /etc/network/interfaces but eth0 hasn't noticed yet; reload it ssh_client.ssh('(ifdown eth0 && ifup eth0) || true') if server.public_v6: ssh_client.ssh('ping6 -c5 -Q 0x10 review.openstack.org ' '|| ping6 -c5 -Q 0x10 wiki.openstack.org') ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'make_swap.sh'), 'make_swap.sh') ssh_client.ssh('bash -x make_swap.sh') if volume_device: ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'mount_volume.sh'), 'mount_volume.sh') ssh_client.ssh('bash -x mount_volume.sh %s %s %s' % (volume_device, mount_path, fs_label)) # Zero the ansible inventory cache so that next run finds the new server inventory_cache_dir = '/var/cache/ansible/inventory' for inventory_cache in os.listdir(inventory_cache_dir): os.unlink(os.path.join(inventory_cache_dir, inventory_cache)) with JobDir(keep) as jobdir: # Update the generated-groups file globally and incorporate it # into our inventory # Remove cloud and region from the environment to work # around a bug in occ expand_env = os.environ.copy() for env_key in list(expand_env.keys()): if env_key.startswith('OS_'): expand_env.pop(env_key, None) expand_env['ANSIBLE_LOG_PATH'] = jobdir.ansible_log # Write out the private SSH key we generated with open(jobdir.key, 'w') as key_file: key.write_private_key(key_file) os.chmod(jobdir.key, 0o600) # Write out inventory with open(jobdir.hosts, 'w') as inventory_file: inventory_file.write( "{host} ansible_host={ip} ansible_user=root {python}".format( host=name, ip=server.interface_ip, python='ansible_python_interpreter=/usr/bin/python3')) t = threading.Thread(target=stream_syslog, args=(ssh_client,)) t.daemon = True t.start() inventory_list = ','.join([ jobdir.inventory_root, '/opt/system-config/inventory/openstack.yaml', '/opt/system-config/inventory/groups.yaml', '/opt/system-config/inventory/emergency.yaml', ]) ansible_cmd = [ 'ansible-playbook', '--flush-cache', '-i', inventory_list, '-l', name, '--private-key={key}'.format(key=jobdir.key), "--ssh-common-args='-o StrictHostKeyChecking=no'", '-e', 'target={name}'.format(name=name), ] # Run the base playbook limited to just this server we just created for playbook in [ 'set-hostnames.yaml', 'base.yaml', 'apply-package-updates.yaml', ]: run(ansible_cmd + [ os.path.join(SCRIPT_DIR, '..', 'playbooks', playbook)], env=jobdir.env) try: ssh_client.ssh("reboot") except Exception as e: # Some init system kill the connection too fast after reboot. # Deal with it by ignoring ssh errors when rebooting. if e.rc == -1: pass else: raise
def bootstrap_server(server, key, name, volume, keep): ip = server.public_v4 ssh_kwargs = dict(pkey=key) print 'Public IP', ip for username in ['root', 'ubuntu', 'centos', 'admin']: ssh_client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) if ssh_client: break if not ssh_client: raise Exception("Unable to log in via SSH") # cloud-init puts the "please log in as user foo" message and # subsequent exit() in root's authorized_keys -- overwrite it with # a normal version to get root login working again. if username != 'root': ssh_client.ssh("sudo cp ~/.ssh/authorized_keys" " ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chmod 644 ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chown root.root ~root/.ssh/authorized_keys") ssh_client = utils.ssh_connect(ip, 'root', ssh_kwargs, timeout=600) if server.public_v6: ssh_client.ssh('ping6 -c5 -Q 0x10 review.openstack.org ' '|| ping6 -c5 -Q 0x10 wiki.openstack.org') ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'make_swap.sh'), 'make_swap.sh') ssh_client.ssh('bash -x make_swap.sh') if volume: ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'mount_volume.sh'), 'mount_volume.sh') ssh_client.ssh('bash -x mount_volume.sh') # This next chunk should really exist as a playbook, but whatev ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'install_puppet.sh'), 'install_puppet.sh') ssh_client.ssh('bash -x install_puppet.sh') # Write out the private SSH key we generated key_file = tempfile.NamedTemporaryFile(delete=not keep) key.write_private_key(key_file) key_file.flush() # Write out inventory inventory_file = tempfile.NamedTemporaryFile(delete=not keep) inventory_file.write("{host} ansible_host={ip} ansible_user=root".format( host=name, ip=server.interface_ip)) inventory_file.flush() ansible_cmd = [ 'ansible-playbook', '-i', inventory_file.name, '-l', name, '--private-key={key}'.format(key=key_file.name), "--ssh-common-args='-o StrictHostKeyChecking=no'", '-e', 'target={name}'.format(name=name), ] # Run the remote puppet apply playbook limited to just this server # we just created try: for playbook in [ 'set_hostnames.yml', 'remote_puppet_adhoc.yaml']: print subprocess.check_output( ansible_cmd + [ os.path.join( SCRIPT_DIR, '..', 'playbooks', playbook)], stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: print "Subprocess failed" print e.output raise try: ssh_client.ssh("reboot") except Exception as e: # Some init system kill the connection too fast after reboot. # Deal with it by ignoring ssh errors when rebooting. if e.rc == -1: pass else: raise
def bootstrap_server(server, key, name, volume_device, keep, mount_path, fs_label): ip = server.public_v4 ssh_kwargs = dict(pkey=key) print 'Public IP', ip for username in ['root', 'ubuntu', 'centos', 'admin']: ssh_client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) if ssh_client: break if not ssh_client: raise Exception("Unable to log in via SSH") # cloud-init puts the "please log in as user foo" message and # subsequent exit() in root's authorized_keys -- overwrite it with # a normal version to get root login working again. if username != 'root': ssh_client.ssh("sudo cp ~/.ssh/authorized_keys" " ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chmod 644 ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chown root.root ~root/.ssh/authorized_keys") ssh_client = utils.ssh_connect(ip, 'root', ssh_kwargs, timeout=600) if server.public_v6: ssh_client.ssh('ping6 -c5 -Q 0x10 review.openstack.org ' '|| ping6 -c5 -Q 0x10 wiki.openstack.org') ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'make_swap.sh'), 'make_swap.sh') ssh_client.ssh('bash -x make_swap.sh') if volume_device: ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'mount_volume.sh'), 'mount_volume.sh') ssh_client.ssh('bash -x mount_volume.sh %s %s %s' % (volume_device, mount_path, fs_label)) # This next chunk should really exist as a playbook, but whatev ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'install_puppet.sh'), 'install_puppet.sh') ssh_client.ssh('bash -x install_puppet.sh') # Zero the ansible inventory cache so that next run finds the new server inventory_cache = '/var/cache/ansible-inventory/ansible-inventory.cache' if os.path.exists(inventory_cache): with open(inventory_cache, 'w'): pass # Update the generated-groups file globally and incorporate it # into our inventory # Remove cloud and region from the environment to work around a bug in occ expand_env = os.environ.copy() for env_key in expand_env.keys(): if env_key.startswith('OS_'): expand_env.pop(env_key, None) # Regenerate inventory cache, throwing an error if there is an issue # so that we don't generate a bogus groups file try: subprocess.check_output(['/etc/ansible/hosts/openstack', '--list'], env=expand_env) except subprocess.CalledProcessError as e: print "Inventory regeneration failed" print e.output raise print subprocess.check_output('/usr/local/bin/expand-groups.sh', env=expand_env, stderr=subprocess.STDOUT) with JobDir(keep) as jobdir: # Write out the private SSH key we generated with open(jobdir.key, 'w') as key_file: key.write_private_key(key_file) os.chmod(jobdir.key, 0o600) # Write out inventory with open(jobdir.hosts, 'w') as inventory_file: inventory_file.write( "{host} ansible_host={ip} ansible_user=root".format( host=name, ip=server.interface_ip)) os.symlink('/etc/ansible/hosts/generated-groups', jobdir.groups) ansible_cmd = [ 'ansible-playbook', '-i', jobdir.inventory_root, '-l', name, '--private-key={key}'.format(key=jobdir.key), "--ssh-common-args='-o StrictHostKeyChecking=no'", '-e', 'target={name}'.format(name=name), ] # Run the remote puppet apply playbook limited to just this server # we just created try: for playbook in ['set_hostnames.yml', 'remote_puppet_adhoc.yaml']: print subprocess.check_output( ansible_cmd + [os.path.join(SCRIPT_DIR, '..', 'playbooks', playbook)], stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: print "Subprocess failed" print e.output raise try: ssh_client.ssh("reboot") except Exception as e: # Some init system kill the connection too fast after reboot. # Deal with it by ignoring ssh errors when rebooting. if e.rc == -1: pass else: raise
def bootstrap_server(server, admin_pass, key, cert, environment, name, puppetmaster, volume): ip = utils.get_public_ip(server) if not ip: raise Exception("Unable to find public ip of server") ssh_kwargs = {} if key: ssh_kwargs['pkey'] = key else: ssh_kwargs['password'] = admin_pass for username in ['root', 'ubuntu']: ssh_client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) if ssh_client: break if not ssh_client: raise Exception("Unable to log in via SSH") if username != 'root': ssh_client.ssh("sudo cp ~/.ssh/authorized_keys" " ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chmod 644 ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chown root.root ~root/.ssh/authorized_keys") ssh_client = utils.ssh_connect(ip, 'root', ssh_kwargs, timeout=600) if IPV6: ssh_client.ssh('ping6 -c5 -Q 0x10 review.openstack.org ' '|| ping6 -c5 -Q 0x10 wiki.openstack.org') ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'make_swap.sh'), 'make_swap.sh') ssh_client.ssh('bash -x make_swap.sh') if volume: ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'mount_volume.sh'), 'mount_volume.sh') ssh_client.ssh('bash -x mount_volume.sh') ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'install_puppet.sh'), 'install_puppet.sh') ssh_client.ssh('bash -x install_puppet.sh') certname = cert[:(0 - len('.pem'))] ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/certs") ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/private_keys") ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/public_keys") ssh_client.ssh("chown -R puppet:root /var/lib/puppet/ssl") ssh_client.ssh("chown -R puppet:puppet /var/lib/puppet/ssl/private_keys") ssh_client.ssh("chmod 0771 /var/lib/puppet/ssl") ssh_client.ssh("chmod 0755 /var/lib/puppet/ssl/certs") ssh_client.ssh("chmod 0750 /var/lib/puppet/ssl/private_keys") ssh_client.ssh("chmod 0755 /var/lib/puppet/ssl/public_keys") for ssldir in [ '/var/lib/puppet/ssl/certs/', '/var/lib/puppet/ssl/private_keys/', '/var/lib/puppet/ssl/public_keys/' ]: ssh_client.scp(os.path.join(ssldir, cert), os.path.join(ssldir, cert)) ssh_client.scp("/var/lib/puppet/ssl/crl.pem", "/var/lib/puppet/ssl/crl.pem") ssh_client.scp("/var/lib/puppet/ssl/certs/ca.pem", "/var/lib/puppet/ssl/certs/ca.pem") ssh_client.ssh("puppet agent " "--environment %s " "--server %s " "--no-daemonize --verbose --onetime --pluginsync true " "--certname %s" % (environment, puppetmaster, certname)) ssh_client.ssh("reboot")
def bootstrap_server(server, admin_pass, key, cert, environment, name, puppetmaster, volume, floating_ip_pool): ip = utils.get_public_ip(server, floating_ip_pool=floating_ip_pool) if not ip: raise Exception("Unable to find public ip of server") ssh_kwargs = {} if key: ssh_kwargs['pkey'] = key else: ssh_kwargs['password'] = admin_pass for username in ['root', 'ubuntu', 'centos']: ssh_client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) if ssh_client: break if not ssh_client: raise Exception("Unable to log in via SSH") # cloud-init puts the "please log in as user foo" message and # subsequent exit() in root's authorized_keys -- overwrite it with # a normal version to get root login working again. if username != 'root': ssh_client.ssh("sudo cp ~/.ssh/authorized_keys" " ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chmod 644 ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chown root.root ~root/.ssh/authorized_keys") ssh_client = utils.ssh_connect(ip, 'root', ssh_kwargs, timeout=600) if IPV6: ssh_client.ssh('ping6 -c5 -Q 0x10 review.openstack.org ' '|| ping6 -c5 -Q 0x10 wiki.openstack.org') ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'make_swap.sh'), 'make_swap.sh') ssh_client.ssh('bash -x make_swap.sh') if volume: ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'mount_volume.sh'), 'mount_volume.sh') ssh_client.ssh('bash -x mount_volume.sh') ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'install_puppet.sh'), 'install_puppet.sh') ssh_client.ssh('bash -x install_puppet.sh') certname = cert[:(0 - len('.pem'))] ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/certs") ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/private_keys") ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/public_keys") ssh_client.ssh("chown -R puppet:root /var/lib/puppet/ssl") ssh_client.ssh("chown -R puppet:puppet /var/lib/puppet/ssl/private_keys") ssh_client.ssh("chmod 0771 /var/lib/puppet/ssl") ssh_client.ssh("chmod 0755 /var/lib/puppet/ssl/certs") ssh_client.ssh("chmod 0750 /var/lib/puppet/ssl/private_keys") ssh_client.ssh("chmod 0755 /var/lib/puppet/ssl/public_keys") for ssldir in ['/var/lib/puppet/ssl/certs/', '/var/lib/puppet/ssl/private_keys/', '/var/lib/puppet/ssl/public_keys/']: ssh_client.scp(os.path.join(ssldir, cert), os.path.join(ssldir, cert)) ssh_client.scp("/var/lib/puppet/ssl/crl.pem", "/var/lib/puppet/ssl/crl.pem") ssh_client.scp("/var/lib/puppet/ssl/certs/ca.pem", "/var/lib/puppet/ssl/certs/ca.pem") (rc, output) = ssh_client.ssh( "puppet agent " "--environment %s " "--server %s " "--detailed-exitcodes " "--no-daemonize --verbose --onetime --pluginsync true " "--certname %s" % (environment, puppetmaster, certname), error_ok=True) utils.interpret_puppet_exitcodes(rc, output) ssh_client.ssh("reboot")
def bootstrap_server(server, key, name, volume_device, keep, mount_path, fs_label, environment, timeout): ip = server.public_v4 ssh_kwargs = dict(pkey=key) print("--- Running initial configuration on host %s ---" % ip) for username in ['root', 'ubuntu', 'centos', 'admin']: ssh_client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=timeout) if ssh_client: break if not ssh_client: raise Exception("Unable to log in via SSH") # cloud-init puts the "please log in as user foo" message and # subsequent exit() in root's authorized_keys -- overwrite it with # a normal version to get root login working again. if username != 'root': ssh_client.ssh("sudo cp ~/.ssh/authorized_keys" " ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chmod 644 ~root/.ssh/authorized_keys") ssh_client.ssh("sudo chown root.root ~root/.ssh/authorized_keys") ssh_client = utils.ssh_connect(ip, 'root', ssh_kwargs, timeout=timeout) # Something up with RAX images that they have the ipv6 interface in # /etc/network/interfaces but eth0 hasn't noticed yet; reload it ssh_client.ssh('(ifdown eth0 && ifup eth0) || true') if server.public_v6: ssh_client.ssh('ping6 -c5 -Q 0x10 review.openstack.org ' '|| ping6 -c5 -Q 0x10 wiki.openstack.org') ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'make_swap.sh'), 'make_swap.sh') ssh_client.ssh('bash -x make_swap.sh') if volume_device: ssh_client.scp(os.path.join(SCRIPT_DIR, '..', 'mount_volume.sh'), 'mount_volume.sh') ssh_client.ssh('bash -x mount_volume.sh %s %s %s' % (volume_device, mount_path, fs_label)) # Zero the ansible inventory cache so that next run finds the new server inventory_cache_dir = '/var/cache/ansible/inventory' for inventory_cache in os.listdir(inventory_cache_dir): os.unlink(os.path.join(inventory_cache_dir, inventory_cache)) with JobDir(keep) as jobdir: # Update the generated-groups file globally and incorporate it # into our inventory # Remove cloud and region from the environment to work # around a bug in occ expand_env = os.environ.copy() for env_key in list(expand_env.keys()): if env_key.startswith('OS_'): expand_env.pop(env_key, None) expand_env['ANSIBLE_LOG_PATH'] = jobdir.ansible_log # Write out the private SSH key we generated with open(jobdir.key, 'w') as key_file: key.write_private_key(key_file) os.chmod(jobdir.key, 0o600) # Write out inventory with open(jobdir.hosts, 'w') as inventory_file: inventory_file.write( "{host} ansible_host={ip} ansible_user=root {python}".format( host=name, ip=server.interface_ip, python='ansible_python_interpreter=/usr/bin/python3')) t = threading.Thread(target=stream_syslog, args=(ssh_client, )) t.daemon = True t.start() ansible_cmd = [ 'ansible-playbook', '-i', jobdir.inventory_root, '-l', name, '--private-key={key}'.format(key=jobdir.key), "--ssh-common-args='-o StrictHostKeyChecking=no'", '-e', 'target={name}'.format(name=name), ] # Run the base playbook limited to just this server we just created for playbook in [ 'set-hostnames.yaml', 'base.yaml', ]: run(ansible_cmd + [os.path.join(SCRIPT_DIR, '..', 'playbooks', playbook)], env=jobdir.env) try: ssh_client.ssh("reboot") except Exception as e: # Some init system kill the connection too fast after reboot. # Deal with it by ignoring ssh errors when rebooting. if e.rc == -1: pass else: raise