def install_packages(self): """ Install required packages. """ log_green('install_packages') apt_install(packages=self.required_packages()) self.install_docker()
def start_github_handler_instance(self): """ Start the Github PR Handler service. """ log_green('start_github_handler_instance') repo = 'github-pr-handler' git_clone('https://github.com/ClusterHQ/github-pr-handler', repo) with cd(repo): self.build_docker_image(os.path.curdir, repo) secrets = self.secrets()['env']['default']['github_pr_handler'] cmd = ('docker run ' '-d ' '-p {port}:{port} ' '-e GITHUB_SECRET={github_secret} ' '-e JENKINS_USERNAME={jenkins_username} ' '-e JENKINS_API_TOKEN={jenkins_api_token} ' '{image} ' '-p {port} ' '-u {jenkins_server}' ).format( port=secrets['port'], image=repo, github_secret=secrets['github_secret'], jenkins_username=secrets['username'], jenkins_api_token=secrets['api_token'], jenkins_server=secrets['jenkins_url'] ) sudo(cmd)
def add_user_to_docker_group(self): """ Make sure the ubuntu user is part of the docker group. """ log_green('adding the ubuntu user to the docker group') data = load_state_from_disk() with settings(hide('warnings', 'running', 'stdout', 'stderr'), warn_only=True, capture=True): user_ensure('ubuntu', home='/home/ubuntu', shell='/bin/bash') group_ensure('docker', gid=55) group_user_ensure('docker', 'ubuntu')
def build_docker_image(self, dockerfile, image_name): """ Build a Dockerfile with a given tag. :param dockerfile: The path of the Dockerfile to build. :param image_name: The name to give to the created image. """ log_green('build_docker_image') cmd = ('docker build -t {image_name} {dockerfile}').format( image_name=image_name, dockerfile=dockerfile ) sudo(cmd)
def create_etc_slave_config(): """ creates /etc/slave_config directory on master /etc/slave_config is used by jenkins slave_plugin. it allows files to be copied from the master to the slave. These files are copied to /etc/slave_config on the slave. """ # TODO: fix these permissions, likely ubuntu/centos/jenkins users # need read/write permissions. log_green('create /etc/slave_config') with settings(hide('warnings', 'running', 'stdout', 'stderr'), warn_only=True, capture=True): dir_ensure('/etc/slave_config', mode="777", use_sudo=True)
def add_user_to_docker_group(distro): """ make sure the user running jenkins is part of the docker group """ log_green('adding the user running jenkins into the docker group') with settings(hide('warnings', 'running', 'stdout', 'stderr'), warn_only=True, capture=True): if 'centos' in distro.value: user_ensure('centos', home='/home/centos', shell='/bin/bash') group_ensure('docker', gid=55) group_user_ensure('docker', 'centos') if 'ubuntu' in distro.value: user_ensure('ubuntu', home='/home/ubuntu', shell='/bin/bash') group_ensure('docker', gid=55) group_user_ensure('docker', 'ubuntu')
def acceptance_tests_common_tests_for_flocker(distribution): """ Runs checks that are common to all platforms related to Flocker :param string distribution: which OS to use 'centos7', 'ubuntu1404' """ with settings(): env.platform_family = detect.detect() # Jenkins should call the correct interpreter based on the shebang # However, # We noticed that our Ubuntu /bin/bash calls were being executed # as /bin/sh. # So we as part of the slave image build process symlinked # /bin/sh -> /bin/bash. # https://clusterhq.atlassian.net/browse/FLOC-2986 log_green('check that /bin/sh is symlinked to bash') assert file.is_link("/bin/sh") assert 'bash' in run('ls -l /bin/sh') # umask needs to be set to 022, so that the packages we build # through the flocker tests have the correct permissions. # otherwise rpmlint fails with permssion errors. log_green('check that our umask matches 022') assert '022' in run('umask') # we need to keep the PATH so that we can run virtualenv with sudo log_green('check that the environment is not reset on sudo') assert sudo("sudo grep " "'Defaults:\%wheel\ \!env_reset\,\!secure_path'" " /etc/sudoers") # the run acceptance tests fail if we don't have a known_hosts file # so we make sure it exists log_green('check that /root/.ssh/known_hosts exists') # known_hosts needs to have 600 permissions assert sudo("ls /root/.ssh/known_hosts") assert "600" in sudo("stat -c %a /root/.ssh/known_hosts") # fpm is used for building RPMs/DEBs log_green('check that fpm is installed') assert 'fpm' in sudo('gem list') # A lot of Flocker tests use different docker images, # we don't want to have to download those images every time we # spin up a new slave node. So we make sure they are cached # locally when we bake the image. log_green('check that images have been downloaded locally') for image in local_docker_images(): log_green(' checking %s' % image) if ':' in image: parts = image.split(':') expression = parts[0] + '.*' + parts[1] assert re.search(expression, sudo('docker images')) else: assert image in sudo('docker images') # CentOS 7 provides us with a fairly old git version, we install # a recent version in /usr/local/bin log_green('check that git is installed locally') assert file.exists("/usr/local/bin/git") # and then update the PATH so that our new git comes first log_green('check that /usr/local/bin is in path') assert '/usr/local/bin/git' in run('which git') # update pip # We have a devpi cache in AWS which we will consume instead of # going upstream to the PyPi servers. # We specify that devpi caching server using -i \$PIP_INDEX_URL # which requires as to include --trusted_host as we are not (yet) # using SSL on our caching box. # The --trusted-host option is only available with pip 7 log_green('check that pip is the latest version') assert '7.' in run('pip --version') # The /tmp/acceptance.yaml file is deployed to the jenkins slave # during bootstrapping. These are copied from the Jenkins Master # /etc/slave_config directory. # We just need to make sure that directory exists. log_green('check that /etc/slave_config exists') assert file.dir_exists("/etc/slave_config") assert file.mode_is("/etc/slave_config", "777") # pypy will be used in the acceptance tests log_green('check that pypy is available') assert '2.6.1' in run('pypy --version') # the client acceptance tests run on docker instances log_green('check that docker is running') assert sudo('docker --version | grep "1.10."') assert process.is_up("docker")
def acceptance_tests_on_ubuntu14_img_for_flocker(distribution): """ checks that the Ubuntu 14 image is suitable for running the Flocker acceptance tests :param string distribution: which OS to use 'centos7', 'ubuntu1404' """ with settings(): env.platform_family = detect.detect() # Jenkins should call the correct interpreter based on the shebang # However, # We noticed that our Ubuntu /bin/bash call were being executed # as /bin/sh. # So we as part of the slave image build process symlinked # /bin/sh -> /bin/bash. # https://clusterhq.atlassian.net/browse/FLOC-2986 log_green('check that /bin/sh is symlinked to bash') assert file.is_link("/bin/sh") assert 'bash' in run('ls -l /bin/sh') # the client acceptance tests run on docker instances log_green('check that docker is enabled') assert 'docker' in run('ls -l /etc/init') # make sure we installed all the packages we need log_green('assert that required deb packages are installed') for pkg in ubuntu14_required_packages(): log_green('... checking package: %s' % pkg) assert package.installed(pkg) # Our tests require us to run docker as ubuntu. # So we add the user ubuntu to the docker group. # During bootstrapping of the node, jenkins will update the init # file so that docker is running with the correct group. # TODO: move that jenkins code here log_green('check that ubuntu is part of group docker') assert user.exists("ubuntu") assert group.is_exists("docker") assert user.is_belonging_group("ubuntu", "docker") # the acceptance tests look for a package in a yum repository, # we provide one by starting a webserver and pointing the tests # to look over there. # for that we need 'nginx' installed and running log_green('check that nginx is running') assert package.installed('nginx') assert port.is_listening(80, "tcp") assert process.is_up("nginx") assert 'nginx' in run('ls -l /etc/init.d/')
def acceptance_tests_on_centos7_img_for_flocker(distribution): """ checks that the CentOS 7 image is suitable for running the Flocker acceptance tests :param string distribution: which OS to use 'centos7', 'ubuntu1404' """ with settings(): env.platform_family = detect.detect() # disable requiretty # http://tinyurl.com/peoffwk log_green("check that tty are not required when sudo'ing") assert sudo('grep "^\#Defaults.*requiretty" /etc/sudoers') # the epel-release repository is required for a bunch of packages log_green('assert that EPEL is installed') assert package.installed('epel-release') # make sure we installed all the packages we need log_green('assert that required rpm packages are installed') for pkg in centos7_required_packages(): # we can't check meta-packages if '@' not in pkg: log_green('... checking %s' % pkg) assert package.installed(pkg) # ZFS will be required for the ZFS acceptance tests log_green('check that the zfs repository is installed') assert package.installed('zfs-release') log_green('check that zfs from testing repository is installed') assert run( 'grep "SPL_DKMS_DISABLE_STRIP=y" /etc/sysconfig/spl') assert run( 'grep "ZFS_DKMS_DISABLE_STRIP=y" /etc/sysconfig/zfs') assert package.installed("zfs") assert run('lsmod |grep zfs') # We now need SELinux enabled log_green('check that SElinux is enforcing') assert sudo('getenforce | grep -i "enforcing"') # And Firewalld should be running too log_green('check that firewalld is enabled') assert sudo("systemctl is-enabled firewalld") # EL, won't allow us to run docker as non-root # http://tinyurl.com/qfuyxjm # but our tests require us to, so we add the 'centos' user to the # docker group. # and the jenkins bootstrapping of the node will change the # docker sysconfig file to run as 'docker' group. # TODO: move that jenkins code here # https://clusterhq.atlassian.net/browse/FLOC-2995 log_green('check that centos is part of group docker') assert user.exists("centos") assert group.is_exists("docker") assert user.is_belonging_group("centos", "docker") # the acceptance tests look for a package in a yum repository, # we provide one by starting a webserver and pointing the tests # to look over there. # for that we need 'nginx' installed and running log_green('check that nginx is running') assert package.installed('nginx') assert port.is_listening(80, "tcp") assert process.is_up("nginx") assert sudo("systemctl is-enabled nginx") # the client acceptance tests run on docker instances log_green('check that docker is running') assert sudo('rpm -q docker-engine | grep "1.10."') assert process.is_up("docker") assert sudo("systemctl is-enabled docker")
def install_docker(self): log_green('install_docker') install_docker() log_green('docker installed') create_docker_group() self.add_user_to_docker_group()
def acceptance_tests(self, distribution): """ checks that the mesos cluster is configured correctly :param string distribution: which OS to use 'ubuntu1404' """ # required for our acceptance tests using envassert env.platform_family = detect.detect() if 'ubuntu' in distribution: # make sure we installed all the packages we need log_green('assert that required deb packages are installed') for pkg in self.ubuntu14_required_packages(): log_green('... checking package: %s' % pkg) assert package.installed(pkg) # the acceptance tests look for a package in a yum repository, # we provide one by starting a webserver and pointing the tests # to look over there. # for that we need 'nginx' installed and running log_green('check that nginx is running') assert package.installed('nginx') assert port.is_listening(80, "tcp") assert process.is_up("nginx") with hide('running', 'stdout'): assert 'nginx' in run('ls -l /etc/init.d/') # check that mesos is installed log_green('check that zookeeper is running') assert package.installed('zookeeper') assert port.is_listening('2181', "tcp") with hide('running', 'stdout'): assert 'zookeeper' in run('ls -l /etc/init/') log_green('check that mesos-master is running') assert package.installed('mesos') assert port.is_listening('5050', "tcp") with hide('running', 'stdout'): assert 'mesos-master' in run('ls -l /etc/init/') log_green('check that mesos-slave is running') assert package.installed('mesos') assert port.is_listening('5051', "tcp") with hide('running', 'stdout'): assert 'mesos-slave' in run('ls -l /etc/init/') log_green('check that marathon is running') assert package.installed('marathon') assert port.is_listening('8080', "tcp") with hide('running', 'stdout'): assert 'marathon' in run('ls -l /etc/init/') log_green('check that virtualbox is installed ') assert package.installed('virtualbox-5.0') with hide('running', 'stdout'): assert 'vboxdrv' in sudo('lsmod') log_green('check that vagrant is installed ') assert package.installed('vagrant') log_green('check that vagrant plugins are installed ') with hide('running', 'stdout'): assert 'vagrant-reload' in sudo('vagrant plugin list') # update pip # We have a devpi cache in AWS which we will consume instead of # going upstream to the PyPi servers. # We specify that devpi caching server using -i \$PIP_INDEX_URL # which requires as to include --trusted_host as we are not (yet) # using SSL on our caching box. # The --trusted-host option is only available with pip 7 log_green('check that pip is the latest version') with hide('running', 'stdout'): assert '7.' in run('pip --version')
def bootstrap_mesos_on_ubuntu14(self): log_green('enabling APT repositories ...') enable_apt_repositories('deb', 'http://archive.ubuntu.com/ubuntu', '$(lsb_release -sc)', 'main universe restricted multiverse') log_green('installing OS updates...') install_os_updates(distribution='ubuntu14.04') log_green('installing required packages...') apt_install(packages=self.ubuntu14_required_packages()) log_green('installing mesos on a single node...') install_mesos_single_box_mode(distribution='ubuntu14.04') # to use wheels, we want the latest pip log_green('updating pip...') update_system_pip_to_latest_pip() # nginx is used during the acceptance tests, the VM built by # flocker provision will connect to the jenkins slave on p 80 # and retrieve the just generated rpm/deb file log_green('installing virtualbox 5...') install_virtualbox(distribution='ubuntu14.04') log_green('installing vagrant 1.7.4...') install_vagrant('ubuntu1404', '1.7.4') install_vagrant_plugin('vagrant-reload')