def setUp(self): super(TestInstallation, self).setUp() self.pa_installer = PrestoadminInstaller(self) self.setup_cluster(NoHadoopBareImageProvider(), self.BARE_CLUSTER) dist_dir = self.pa_installer._build_dist_if_necessary(self.cluster) self.pa_installer._copy_dist_to_host(self.cluster, dist_dir, self.cluster.master)
def test_uninstall_lost_host(self): self.setup_cluster(self.PA_ONLY_CLUSTER) pa_installer = PrestoadminInstaller(self) pa_installer.install() topology = { "coordinator": self.cluster.internal_slaves[0], "workers": [ self.cluster.internal_master, self.cluster.internal_slaves[1], self.cluster.internal_slaves[2] ] } self.upload_topology(topology) self.installer.install() start_output = self.run_prestoadmin('server start') process_per_host = self.get_process_per_host(start_output.splitlines()) self.assert_started(process_per_host) down_node = self.cluster.internal_slaves[0] self.cluster.stop_host(self.cluster.slaves[0]) expected = self.down_node_connection_error( self.cluster.internal_slaves[0]) cmd_output = self.run_prestoadmin('server uninstall') self.assertRegexpMatches(cmd_output, expected) process_per_active_host = [] for host, pid in process_per_host: if host not in down_node: process_per_active_host.append((host, pid)) self.assert_stopped(process_per_active_host) for container in [ self.cluster.internal_master, self.cluster.internal_slaves[1], self.cluster.internal_slaves[2] ]: self.assert_uninstalled_dirs_removed(container)
def setUp(self): super(TestInstaller, self).setUp() self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_BARE_CLUSTER) self.centos_container = \ self.__create_and_start_single_centos_container() self.pa_installer = PrestoadminInstaller(self)
def test_uninstall_lost_host(self): self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER) pa_installer = PrestoadminInstaller(self) pa_installer.install() topology = {"coordinator": self.cluster.internal_slaves[0], "workers": [self.cluster.internal_master, self.cluster.internal_slaves[1], self.cluster.internal_slaves[2]]} self.upload_topology(topology) self.installer.install() start_output = self.run_prestoadmin('server start') process_per_host = self.get_process_per_host(start_output.splitlines()) self.assert_started(process_per_host) down_node = self.cluster.internal_slaves[0] self.cluster.stop_host( self.cluster.slaves[0]) expected = self.down_node_connection_error( self.cluster.internal_slaves[0]) cmd_output = self.run_prestoadmin('server uninstall', raise_error=False) self.assertRegexpMatches(cmd_output, expected) process_per_active_host = [] for host, pid in process_per_host: if host not in down_node: process_per_active_host.append((host, pid)) self.assert_stopped(process_per_active_host) for container in [self.cluster.internal_master, self.cluster.internal_slaves[1], self.cluster.internal_slaves[2]]: self.assert_uninstalled_dirs_removed(container)
def setUp(self): super(TestInstallation, self).setUp() self.pa_installer = PrestoadminInstaller(self) self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_BARE_CLUSTER) dist_dir = self.pa_installer._build_dist_if_necessary(self.cluster) self.pa_installer._copy_dist_to_host(self.cluster, dist_dir, self.cluster.master)
def setUp(self): super(TestInstaller, self).setUp() self.centos_container = \ self.__create_and_start_single_centos_container() self.pa_installer = PrestoadminInstaller(self)
class TestInstallation(BaseProductTestCase): def setUp(self): super(TestInstallation, self).setUp() self.pa_installer = PrestoadminInstaller(self) self.setup_cluster(NoHadoopBareImageProvider(), self.BARE_CLUSTER) dist_dir = self.pa_installer._build_dist_if_necessary(self.cluster) self.pa_installer._copy_dist_to_host(self.cluster, dist_dir, self.cluster.master) @attr('smoketest') @docker_only def test_install_non_root(self): install_dir = '/home/app-admin' script = """ set -e cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir} chown app-admin {install_dir}/prestoadmin-*.tar.bz2 cd {install_dir} sudo -u app-admin tar jxf prestoadmin-*.tar.bz2 cd prestoadmin sudo -u app-admin ./install-prestoadmin.sh """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir) self.assertRaisesRegexp( OSError, 'mkdir: cannot create directory ' '`/var/log/prestoadmin\': Permission denied', self.cluster.run_script_on_host, script, self.cluster.master) @attr('smoketest') def test_install_from_different_dir(self): install_dir = '/opt' script = """ set -e cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir} cd {install_dir} tar jxf prestoadmin-*.tar.bz2 ./prestoadmin/install-prestoadmin.sh """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir) self.assertRaisesRegexp( OSError, r'IOError: \[Errno 2\] No such file or directory: ' r'\'/opt/prestoadmin-1.2-py2-none-any.whl\'', self.cluster.run_script_on_host, script, self.cluster.master) @attr('smoketest') @docker_only def test_install_on_wrong_os_offline_installer(self): image = 'ubuntu' tag = '14.04' host = image + '-master' ubuntu_container = DockerCluster(host, [], DEFAULT_LOCAL_MOUNT_POINT, DEFAULT_DOCKER_MOUNT_POINT) try: ubuntu_container.fetch_image_if_not_present(image, tag) ubuntu_container.start_containers( image + ':' + tag, cmd='tail -f /var/log/bootstrap.log') self.retry(lambda: ubuntu_container.run_script_on_host( install_py26_script, ubuntu_container.master)) ubuntu_container.exec_cmd_on_host(ubuntu_container.master, 'sudo apt-get -y install wget') self.assertRaisesRegexp( OSError, r'ERROR\n' r'Paramiko could not be imported. This usually means that', self.pa_installer.install, cluster=ubuntu_container) finally: ubuntu_container.tear_down() @attr('smoketest') def test_cert_arg_to_installation_nonexistent_file(self): install_dir = '/opt' script = """ set -e cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir} cd {install_dir} tar jxf prestoadmin-*.tar.bz2 cd prestoadmin ./install-prestoadmin.sh dummy_cert.cert """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir) output = self.cluster.run_script_on_host(script, self.cluster.master) self.assertRegexpMatches( output, r'Adding pypi.python.org as ' 'trusted\-host. Cannot find certificate ' 'file: dummy_cert.cert') @attr('smoketest') def test_cert_arg_to_installation_real_cert(self): self.cluster.copy_to_host(certifi.where(), self.cluster.master) install_dir = '/opt' cert_file = os.path.basename(certifi.where()) script = """ set -e cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir} cd {install_dir} tar jxf prestoadmin-*.tar.bz2 cd prestoadmin ./install-prestoadmin.sh {mount_dir}/{cacert} """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir, cacert=cert_file) output = self.cluster.run_script_on_host(script, self.cluster.master) self.assertTrue( 'Adding pypi.python.org as trusted-host. Cannot find' ' certificate file: %s' % cert_file not in output, 'Unable to find cert file; output: %s' % output) def test_additional_dirs_created(self): install_dir = '/opt' script = """ set -e cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir} cd {install_dir} tar jxf prestoadmin-*.tar.bz2 cd prestoadmin ./install-prestoadmin.sh """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir) self.cluster.run_script_on_host(script, self.cluster.master) pa_etc_dir = '/etc/opt/prestoadmin' connectors_dir = pa_etc_dir + '/connectors' self.assert_path_exists(self.cluster.master, connectors_dir) coordinator_dir = pa_etc_dir + '/coordinator' self.assert_path_exists(self.cluster.master, coordinator_dir) workers_dir = pa_etc_dir + '/workers' self.assert_path_exists(self.cluster.master, workers_dir)
class TestInstallation(BaseProductTestCase): def setUp(self): super(TestInstallation, self).setUp() self.pa_installer = PrestoadminInstaller(self) self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_BARE_CLUSTER) dist_dir = self.pa_installer._build_dist_if_necessary(self.cluster) self.pa_installer._copy_dist_to_host(self.cluster, dist_dir, self.cluster.master) @attr('smoketest') @docker_only def test_install_non_root(self): install_dir = '/home/app-admin' script = """ set -e cp {mount_dir}/prestoadmin-*.tar.gz {install_dir} chown app-admin {install_dir}/prestoadmin-*.tar.gz cd {install_dir} sudo -u app-admin tar zxf prestoadmin-*.tar.gz cd prestoadmin sudo -u app-admin ./install-prestoadmin.sh """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir) self.cluster.run_script_on_host(script, self.cluster.master) pa_config_dir = '/home/app-admin/.prestoadmin' catalog_dir = os.path.join(pa_config_dir, 'catalog') self.assert_path_exists(self.cluster.master, catalog_dir) coordinator_dir = os.path.join(pa_config_dir, 'coordinator') self.assert_path_exists(self.cluster.master, coordinator_dir) workers_dir = os.path.join(pa_config_dir, 'workers') self.assert_path_exists(self.cluster.master, workers_dir) @attr('smoketest') def test_cert_arg_to_installation_nonexistent_file(self): install_dir = '~' script = """ set -e cp {mount_dir}/prestoadmin-*.tar.gz {install_dir} cd {install_dir} tar zxf prestoadmin-*.tar.gz cd prestoadmin ./install-prestoadmin.sh dummy_cert.cert """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir) output = self.cluster.run_script_on_host(script, self.cluster.master) self.assertRegexpMatches(output, r'Adding pypi.python.org as ' 'trusted\-host. Cannot find certificate ' 'file: dummy_cert.cert') @attr('smoketest') def test_cert_arg_to_installation_real_cert(self): self.cluster.copy_to_host(certifi.where(), self.cluster.master) install_dir = '~' cert_file = os.path.basename(certifi.where()) script = """ set -e cp {mount_dir}/prestoadmin-*.tar.gz {install_dir} cd {install_dir} tar zxf prestoadmin-*.tar.gz cd prestoadmin ./install-prestoadmin.sh {mount_dir}/{cacert} """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir, cacert=cert_file) output = self.cluster.run_script_on_host(script, self.cluster.master) self.assertTrue('Adding pypi.python.org as trusted-host. Cannot find' ' certificate file: %s' % cert_file not in output, 'Unable to find cert file; output: %s' % output) def test_additional_dirs_created(self): install_dir = '~' script = """ set -e cp {mount_dir}/prestoadmin-*.tar.gz {install_dir} cd {install_dir} tar zxf prestoadmin-*.tar.gz cd prestoadmin ./install-prestoadmin.sh """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir) self.cluster.run_script_on_host(script, self.cluster.master) self.assert_path_exists(self.cluster.master, get_catalog_directory()) self.assert_path_exists(self.cluster.master, get_coordinator_directory()) self.assert_path_exists(self.cluster.master, get_workers_directory())
class TestInstallation(BaseProductTestCase): def setUp(self): super(TestInstallation, self).setUp() self.pa_installer = PrestoadminInstaller(self) self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_BARE_CLUSTER) dist_dir = self.pa_installer._build_dist_if_necessary(self.cluster) self.pa_installer._copy_dist_to_host(self.cluster, dist_dir, self.cluster.master) @attr('smoketest') @docker_only def test_install_non_root(self): install_dir = '/home/app-admin' script = """ set -e cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir} chown app-admin {install_dir}/prestoadmin-*.tar.bz2 cd {install_dir} sudo -u app-admin tar jxf prestoadmin-*.tar.bz2 cd prestoadmin sudo -u app-admin ./install-prestoadmin.sh """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir) self.assertRaisesRegexp(OSError, 'mkdir: cannot create directory ' '`/var/log/prestoadmin\': Permission denied', self.cluster.run_script_on_host, script, self.cluster.master) @attr('smoketest') def test_install_from_different_dir(self): install_dir = '/opt' script = """ set -e cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir} cd {install_dir} tar jxf prestoadmin-*.tar.bz2 ./prestoadmin/install-prestoadmin.sh """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir) self.assertRaisesRegexp( OSError, r'IOError: \[Errno 2\] No such file or directory: ' r'\'/opt/prestoadmin-.*-py2-none-any.whl\'', self.cluster.run_script_on_host, script, self.cluster.master ) @attr('smoketest', 'offline_installer') @docker_only def test_install_on_wrong_os_offline_installer(self): image = 'teradatalabs/ubuntu-trusty-python2.6' tag = BASE_IMAGES_TAG host = 'wrong-os-master' ubuntu_container = DockerCluster(host, [], DEFAULT_LOCAL_MOUNT_POINT, DEFAULT_DOCKER_MOUNT_POINT) if not DockerCluster._check_for_images(image, image, tag): raise RuntimeError("Docker images have not been created") try: ubuntu_container.start_containers( image + ':' + tag, cmd='tail -f /var/log/bootstrap.log') self.assertRaisesRegexp( OSError, r'ERROR\n' r'Paramiko could not be imported. This usually means that', self.pa_installer.install, cluster=ubuntu_container ) finally: ubuntu_container.tear_down() @attr('smoketest') def test_cert_arg_to_installation_nonexistent_file(self): install_dir = '/opt' script = """ set -e cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir} cd {install_dir} tar jxf prestoadmin-*.tar.bz2 cd prestoadmin ./install-prestoadmin.sh dummy_cert.cert """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir) output = self.cluster.run_script_on_host(script, self.cluster.master) self.assertRegexpMatches(output, r'Adding pypi.python.org as ' 'trusted\-host. Cannot find certificate ' 'file: dummy_cert.cert') @attr('smoketest') def test_cert_arg_to_installation_real_cert(self): self.cluster.copy_to_host(certifi.where(), self.cluster.master) install_dir = '/opt' cert_file = os.path.basename(certifi.where()) script = """ set -e cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir} cd {install_dir} tar jxf prestoadmin-*.tar.bz2 cd prestoadmin ./install-prestoadmin.sh {mount_dir}/{cacert} """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir, cacert=cert_file) output = self.cluster.run_script_on_host(script, self.cluster.master) self.assertTrue('Adding pypi.python.org as trusted-host. Cannot find' ' certificate file: %s' % cert_file not in output, 'Unable to find cert file; output: %s' % output) def test_additional_dirs_created(self): install_dir = '/opt' script = """ set -e cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir} cd {install_dir} tar jxf prestoadmin-*.tar.bz2 cd prestoadmin ./install-prestoadmin.sh """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir) self.cluster.run_script_on_host(script, self.cluster.master) pa_etc_dir = '/etc/opt/prestoadmin' connectors_dir = pa_etc_dir + '/connectors' self.assert_path_exists(self.cluster.master, connectors_dir) coordinator_dir = pa_etc_dir + '/coordinator' self.assert_path_exists(self.cluster.master, coordinator_dir) workers_dir = pa_etc_dir + '/workers' self.assert_path_exists(self.cluster.master, workers_dir)
class TestInstallation(BaseProductTestCase): def setUp(self): super(TestInstallation, self).setUp() self.pa_installer = PrestoadminInstaller(self) self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_BARE_CLUSTER) dist_dir = self.pa_installer._build_dist_if_necessary(self.cluster) self.pa_installer._copy_dist_to_host(self.cluster, dist_dir, self.cluster.master) @attr('smoketest') @docker_only def test_install_non_root(self): install_dir = '/home/app-admin' script = """ set -e cp {mount_dir}/prestoadmin-*.tar.gz {install_dir} chown app-admin {install_dir}/prestoadmin-*.tar.gz cd {install_dir} sudo -u app-admin tar zxf prestoadmin-*.tar.gz cd prestoadmin sudo -u app-admin ./install-prestoadmin.sh """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir) self.cluster.run_script_on_host(script, self.cluster.master) pa_config_dir = '/home/app-admin/.prestoadmin' catalog_dir = os.path.join(pa_config_dir, 'catalog') self.assert_path_exists(self.cluster.master, catalog_dir) coordinator_dir = os.path.join(pa_config_dir, 'coordinator') self.assert_path_exists(self.cluster.master, coordinator_dir) workers_dir = os.path.join(pa_config_dir, 'workers') self.assert_path_exists(self.cluster.master, workers_dir) @attr('smoketest', 'offline_installer') @docker_only def test_install_on_wrong_os_offline_installer(self): image = 'teradatalabs/ubuntu-trusty-python2.6' tag = BASE_IMAGES_TAG host = 'wrong-os-master' ubuntu_container = DockerCluster(host, [], DEFAULT_LOCAL_MOUNT_POINT, DEFAULT_DOCKER_MOUNT_POINT) if not DockerCluster._check_for_images(image, image, tag): raise RuntimeError("Docker images have not been created") try: ubuntu_container.start_containers( image + ':' + tag, cmd='tail -f /var/log/bootstrap.log') self.assertRaisesRegexp( OSError, r'ERROR\n' r'Paramiko could not be imported. This usually means that', self.pa_installer.install, cluster=ubuntu_container) finally: ubuntu_container.tear_down() @attr('smoketest') def test_cert_arg_to_installation_nonexistent_file(self): install_dir = '~' script = """ set -e cp {mount_dir}/prestoadmin-*.tar.gz {install_dir} cd {install_dir} tar zxf prestoadmin-*.tar.gz cd prestoadmin ./install-prestoadmin.sh dummy_cert.cert """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir) output = self.cluster.run_script_on_host(script, self.cluster.master) self.assertRegexpMatches( output, r'Adding pypi.python.org as ' 'trusted\-host. Cannot find certificate ' 'file: dummy_cert.cert') @attr('smoketest') def test_cert_arg_to_installation_real_cert(self): self.cluster.copy_to_host(certifi.where(), self.cluster.master) install_dir = '~' cert_file = os.path.basename(certifi.where()) script = """ set -e cp {mount_dir}/prestoadmin-*.tar.gz {install_dir} cd {install_dir} tar zxf prestoadmin-*.tar.gz cd prestoadmin ./install-prestoadmin.sh {mount_dir}/{cacert} """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir, cacert=cert_file) output = self.cluster.run_script_on_host(script, self.cluster.master) self.assertTrue( 'Adding pypi.python.org as trusted-host. Cannot find' ' certificate file: %s' % cert_file not in output, 'Unable to find cert file; output: %s' % output) def test_additional_dirs_created(self): install_dir = '~' script = """ set -e cp {mount_dir}/prestoadmin-*.tar.gz {install_dir} cd {install_dir} tar zxf prestoadmin-*.tar.gz cd prestoadmin ./install-prestoadmin.sh """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir) self.cluster.run_script_on_host(script, self.cluster.master) self.assert_path_exists(self.cluster.master, get_catalog_directory()) self.assert_path_exists(self.cluster.master, get_coordinator_directory()) self.assert_path_exists(self.cluster.master, get_workers_directory())
class TestInstaller(BaseProductTestCase): def setUp(self): super(TestInstaller, self).setUp() self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_BARE_CLUSTER) self.centos_container = \ self.__create_and_start_single_centos_container() self.pa_installer = PrestoadminInstaller(self) def tearDown(self): super(TestInstaller, self).tearDown() self.centos_container.tear_down() @attr('smoketest') def test_online_installer(self): self.pa_installer._build_installer_in_docker(self.centos_container, online_installer=True, unique=True) self.__verify_third_party_dir(False) self.pa_installer.install(dist_dir=self.centos_container.get_dist_dir( unique=True)) self.run_prestoadmin('--help', raise_error=True) @attr('smoketest', 'offline_installer') @docker_only def test_offline_installer(self): self.pa_installer._build_installer_in_docker(self.centos_container, online_installer=False, unique=True) self.__verify_third_party_dir(True) self.centos_container.exec_cmd_on_host( # IMPORTANT: ifdown eth0 fails silently without taking the # interface down if the NET_ADMIN capability isn't set for the # container. ifconfig eth0 down accomplishes the same thing, but # results in a failure if it fails. self.centos_container.master, 'ifconfig eth0 down') self.pa_installer.install(dist_dir=self.centos_container.get_dist_dir( unique=True)) self.run_prestoadmin('--help', raise_error=True) def __create_and_start_single_centos_container(self): cluster_type = 'installer_tester' bare_image_provider = NoHadoopBareImageProvider(BASE_IMAGES_TAG) centos_container, bare_cluster = DockerCluster.start_cluster( bare_image_provider, cluster_type, 'master', [], cap_add=['NET_ADMIN']) if bare_cluster: centos_container.commit_images(bare_image_provider, cluster_type) return centos_container def __verify_third_party_dir(self, is_third_party_present): matches = fnmatch.filter( os.listdir(self.centos_container.get_dist_dir(unique=True)), 'prestoadmin-*.tar.gz') if len(matches) > 1: raise RuntimeError( 'More than one archive found in the dist directory ' + ' '.join(matches)) cmd_to_run = [ 'tar', '-tf', os.path.join(self.centos_container.get_dist_dir(unique=True), matches[0]) ] popen_obj = subprocess.Popen(cmd_to_run, cwd=main_dir, stdout=subprocess.PIPE) retcode = popen_obj.returncode if retcode: raise RuntimeError('Non zero return code when executing ' + ' '.join(cmd_to_run)) stdout = popen_obj.communicate()[0] match = re.search('/third-party/', stdout) if is_third_party_present and match is None: raise RuntimeError('Expected to have an offline installer with ' 'a third-party directory. Found no ' 'third-party directory in the installer ' 'archive.') elif not is_third_party_present and match: raise RuntimeError('Expected to have an online installer with no ' 'third-party directory. Found a third-party ' 'directory in the installer archive.')
def setUp(self): super(TestInstallation, self).setUp() self.pa_installer = PrestoadminInstaller(self) self.setup_cluster(self.BARE_CLUSTER) dist_dir = self.pa_installer._build_dist_if_necessary(self.cluster) self.pa_installer._copy_dist_to_host(self.cluster, dist_dir, self.cluster.master)
class TestInstaller(BaseProductTestCase): def setUp(self): super(TestInstaller, self).setUp() self.centos_container = \ self.__create_and_start_single_centos_container() self.pa_installer = PrestoadminInstaller(self) def tearDown(self): super(TestInstaller, self).tearDown() self.centos_container.tear_down() @attr('smoketest') @docker_only def test_online_installer(self): self.pa_installer._build_installer_in_docker(self.centos_container, online_installer=True, unique=True) self.__verify_third_party_dir(False) self.pa_installer.install( dist_dir=self.centos_container.get_dist_dir(unique=True)) self.run_prestoadmin('--help', raise_error=True, cluster=self.centos_container) @attr('smoketest') @docker_only def test_offline_installer(self): self.pa_installer._build_installer_in_docker( self.centos_container, online_installer=False, unique=True) self.__verify_third_party_dir(True) self.centos_container.exec_cmd_on_host( self.centos_container.master, 'ifdown eth0') self.pa_installer.install( dist_dir=self.centos_container.get_dist_dir(unique=True)) self.run_prestoadmin('--help', raise_error=True, cluster=self.centos_container) def __create_and_start_single_centos_container(self): centos_container = DockerCluster( 'master', [], DEFAULT_LOCAL_MOUNT_POINT, DEFAULT_DOCKER_MOUNT_POINT) # we can't assume that another test has created the image centos_container.create_image( BASE_TD_DOCKERFILE_DIR, BASE_TD_IMAGE_NAME, BASE_IMAGE_NAME ) centos_container.start_containers( BASE_TD_IMAGE_NAME, cap_add=['NET_ADMIN'] ) return centos_container def __verify_third_party_dir(self, is_third_party_present): matches = fnmatch.filter( os.listdir(self.centos_container.get_dist_dir(unique=True)), 'prestoadmin-*.tar.bz2') if len(matches) > 1: raise RuntimeError( 'More than one archive found in the dist directory ' + ' '.join(matches) ) cmd_to_run = ['tar', '-tf', os.path.join( self.centos_container.get_dist_dir(unique=True), matches[0]) ] popen_obj = subprocess.Popen(cmd_to_run, cwd=main_dir, stdout=subprocess.PIPE) retcode = popen_obj.returncode if retcode: raise RuntimeError('Non zero return code when executing ' + ' '.join(cmd_to_run)) stdout = popen_obj.communicate()[0] match = re.search('/third-party/', stdout) if is_third_party_present and match is None: raise RuntimeError('Expected to have an offline installer with ' 'a third-party directory. Found no ' 'third-party directory in the installer ' 'archive.') elif not is_third_party_present and match: raise RuntimeError('Expected to have an online installer with no ' 'third-party directory. Found a third-party ' 'directory in the installer archive.')
class TestInstaller(BaseProductTestCase): def setUp(self): super(TestInstaller, self).setUp() self.centos_container = \ self.__create_and_start_single_centos_container() self.pa_installer = PrestoadminInstaller(self) def tearDown(self): super(TestInstaller, self).tearDown() self.centos_container.tear_down() @attr('smoketest') @docker_only def test_online_installer(self): self.pa_installer._build_installer_in_docker(self.centos_container, online_installer=True, unique=True) self.__verify_third_party_dir(False) self.pa_installer.install(dist_dir=self.centos_container.get_dist_dir( unique=True)) self.run_prestoadmin('--help', raise_error=True, cluster=self.centos_container) @attr('smoketest') @docker_only def test_offline_installer(self): self.pa_installer._build_installer_in_docker(self.centos_container, online_installer=False, unique=True) self.__verify_third_party_dir(True) self.centos_container.exec_cmd_on_host(self.centos_container.master, 'ifdown eth0') self.pa_installer.install(dist_dir=self.centos_container.get_dist_dir( unique=True)) self.run_prestoadmin('--help', raise_error=True, cluster=self.centos_container) def __create_and_start_single_centos_container(self): centos_container = DockerCluster('master', [], DEFAULT_LOCAL_MOUNT_POINT, DEFAULT_DOCKER_MOUNT_POINT) # we can't assume that another test has created the image centos_container.create_image(BASE_TD_DOCKERFILE_DIR, BASE_TD_IMAGE_NAME, BASE_IMAGE_NAME) centos_container.start_containers(BASE_TD_IMAGE_NAME, cap_add=['NET_ADMIN']) return centos_container def __verify_third_party_dir(self, is_third_party_present): matches = fnmatch.filter( os.listdir(self.centos_container.get_dist_dir(unique=True)), 'prestoadmin-*.tar.bz2') if len(matches) > 1: raise RuntimeError( 'More than one archive found in the dist directory ' + ' '.join(matches)) cmd_to_run = [ 'tar', '-tf', os.path.join(self.centos_container.get_dist_dir(unique=True), matches[0]) ] popen_obj = subprocess.Popen(cmd_to_run, cwd=main_dir, stdout=subprocess.PIPE) retcode = popen_obj.returncode if retcode: raise RuntimeError('Non zero return code when executing ' + ' '.join(cmd_to_run)) stdout = popen_obj.communicate()[0] match = re.search('/third-party/', stdout) if is_third_party_present and match is None: raise RuntimeError('Expected to have an offline installer with ' 'a third-party directory. Found no ' 'third-party directory in the installer ' 'archive.') elif not is_third_party_present and match: raise RuntimeError('Expected to have an online installer with no ' 'third-party directory. Found a third-party ' 'directory in the installer archive.')
class TestInstallation(BaseProductTestCase): def setUp(self): super(TestInstallation, self).setUp() self.pa_installer = PrestoadminInstaller(self) self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_BARE_CLUSTER) dist_dir = self.pa_installer._build_dist_if_necessary(self.cluster) self.pa_installer._copy_dist_to_host(self.cluster, dist_dir, self.cluster.master) @attr('smoketest') @docker_only def test_install_non_root(self): install_dir = '/home/app-admin' script = """ set -e cp {mount_dir}/prestoadmin-*.tar.gz {install_dir} chown app-admin {install_dir}/prestoadmin-*.tar.gz cd {install_dir} sudo -u app-admin tar zxf prestoadmin-*.tar.gz cd prestoadmin sudo -u app-admin ./install-prestoadmin.sh """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir) self.cluster.run_script_on_host(script, self.cluster.master) pa_config_dir = '/home/app-admin/.prestoadmin' catalog_dir = os.path.join(pa_config_dir, 'catalog') self.assert_path_exists(self.cluster.master, catalog_dir) coordinator_dir = os.path.join(pa_config_dir, 'coordinator') self.assert_path_exists(self.cluster.master, coordinator_dir) workers_dir = os.path.join(pa_config_dir, 'workers') self.assert_path_exists(self.cluster.master, workers_dir) @attr('smoketest') def test_cert_arg_to_installation_nonexistent_file(self): install_dir = '~' script = """ set -e cp {mount_dir}/prestoadmin-*.tar.gz {install_dir} cd {install_dir} tar zxf prestoadmin-*.tar.gz cd prestoadmin ./install-prestoadmin.sh dummy_cert.cert """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir) output = self.cluster.run_script_on_host(script, self.cluster.master) self.assertRegexpMatches( output, r'Adding pypi.python.org as ' 'trusted\-host. Cannot find certificate ' 'file: dummy_cert.cert') @attr('smoketest') def test_cert_arg_to_installation_real_cert(self): self.cluster.copy_to_host(certifi.where(), self.cluster.master) install_dir = '~' cert_file = os.path.basename(certifi.where()) script = """ set -e cp {mount_dir}/prestoadmin-*.tar.gz {install_dir} cd {install_dir} tar zxf prestoadmin-*.tar.gz cd prestoadmin ./install-prestoadmin.sh {mount_dir}/{cacert} """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir, cacert=cert_file) output = self.cluster.run_script_on_host(script, self.cluster.master) self.assertTrue( 'Adding pypi.python.org as trusted-host. Cannot find' ' certificate file: %s' % cert_file not in output, 'Unable to find cert file; output: %s' % output) def test_additional_dirs_created(self): install_dir = '~' script = """ set -e cp {mount_dir}/prestoadmin-*.tar.gz {install_dir} cd {install_dir} tar zxf prestoadmin-*.tar.gz cd prestoadmin ./install-prestoadmin.sh """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir) self.cluster.run_script_on_host(script, self.cluster.master) self.assert_path_exists(self.cluster.master, get_catalog_directory()) self.assert_path_exists(self.cluster.master, get_coordinator_directory()) self.assert_path_exists(self.cluster.master, get_workers_directory())
def setUp(self): super(TestInstaller, self).setUp() self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_BARE_CLUSTER) self.centos_container = \ self.__create_and_start_single_centos_container() self.pa_installer = PrestoadminInstaller(self)
class TestInstaller(BaseProductTestCase): def setUp(self): super(TestInstaller, self).setUp() self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_BARE_CLUSTER) self.centos_container = \ self.__create_and_start_single_centos_container() self.pa_installer = PrestoadminInstaller(self) def tearDown(self): super(TestInstaller, self).tearDown() self.centos_container.tear_down() @attr('smoketest') def test_online_installer(self): self.pa_installer._build_installer_in_docker(self.centos_container, online_installer=True, unique=True) self.__verify_third_party_dir(False) self.pa_installer.install( dist_dir=self.centos_container.get_dist_dir(unique=True)) self.run_prestoadmin('--help', raise_error=True) @attr('smoketest', 'offline_installer') @docker_only def test_offline_installer(self): self.pa_installer._build_installer_in_docker( self.centos_container, online_installer=False, unique=True) self.__verify_third_party_dir(True) self.centos_container.exec_cmd_on_host( # IMPORTANT: ifdown eth0 fails silently without taking the # interface down if the NET_ADMIN capability isn't set for the # container. ifconfig eth0 down accomplishes the same thing, but # results in a failure if it fails. self.centos_container.master, 'ifconfig eth0 down') self.pa_installer.install( dist_dir=self.centos_container.get_dist_dir(unique=True)) self.run_prestoadmin('--help', raise_error=True) def __create_and_start_single_centos_container(self): cluster_type = 'installer_tester' bare_image_provider = NoHadoopBareImageProvider(BASE_IMAGES_TAG) centos_container, bare_cluster = DockerCluster.start_cluster( bare_image_provider, cluster_type, 'master', [], cap_add=['NET_ADMIN']) if bare_cluster: centos_container.commit_images(bare_image_provider, cluster_type) return centos_container def __verify_third_party_dir(self, is_third_party_present): matches = fnmatch.filter( os.listdir(self.centos_container.get_dist_dir(unique=True)), 'prestoadmin-*.tar.bz2') if len(matches) > 1: raise RuntimeError( 'More than one archive found in the dist directory ' + ' '.join(matches) ) cmd_to_run = ['tar', '-tf', os.path.join( self.centos_container.get_dist_dir(unique=True), matches[0]) ] popen_obj = subprocess.Popen(cmd_to_run, cwd=main_dir, stdout=subprocess.PIPE) retcode = popen_obj.returncode if retcode: raise RuntimeError('Non zero return code when executing ' + ' '.join(cmd_to_run)) stdout = popen_obj.communicate()[0] match = re.search('/third-party/', stdout) if is_third_party_present and match is None: raise RuntimeError('Expected to have an offline installer with ' 'a third-party directory. Found no ' 'third-party directory in the installer ' 'archive.') elif not is_third_party_present and match: raise RuntimeError('Expected to have an online installer with no ' 'third-party directory. Found a third-party ' 'directory in the installer archive.')
class TestInstallation(BaseProductTestCase): def setUp(self): super(TestInstallation, self).setUp() self.pa_installer = PrestoadminInstaller(self) self.setup_cluster(NoHadoopBareImageProvider(), self.BARE_CLUSTER) dist_dir = self.pa_installer._build_dist_if_necessary(self.cluster) self.pa_installer._copy_dist_to_host(self.cluster, dist_dir, self.cluster.master) @attr('smoketest') @docker_only def test_install_non_root(self): install_dir = '/home/app-admin' script = """ set -e cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir} chown app-admin {install_dir}/prestoadmin-*.tar.bz2 cd {install_dir} sudo -u app-admin tar jxf prestoadmin-*.tar.bz2 cd prestoadmin sudo -u app-admin ./install-prestoadmin.sh """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir) self.assertRaisesRegexp(OSError, 'mkdir: cannot create directory ' '`/var/log/prestoadmin\': Permission denied', self.cluster.run_script_on_host, script, self.cluster.master) @attr('smoketest') def test_install_from_different_dir(self): install_dir = '/opt' script = """ set -e cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir} cd {install_dir} tar jxf prestoadmin-*.tar.bz2 ./prestoadmin/install-prestoadmin.sh """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir) self.assertRaisesRegexp( OSError, r'IOError: \[Errno 2\] No such file or directory: ' r'\'/opt/prestoadmin-1.2-py2-none-any.whl\'', self.cluster.run_script_on_host, script, self.cluster.master ) @attr('smoketest') @docker_only def test_install_on_wrong_os_offline_installer(self): image = 'ubuntu' tag = '14.04' host = image + '-master' ubuntu_container = DockerCluster(host, [], DEFAULT_LOCAL_MOUNT_POINT, DEFAULT_DOCKER_MOUNT_POINT) try: ubuntu_container.fetch_image_if_not_present(image, tag) ubuntu_container.start_containers( image + ':' + tag, cmd='tail -f /var/log/bootstrap.log') ubuntu_container.run_script_on_host(install_py26_script, ubuntu_container.master) ubuntu_container.exec_cmd_on_host( ubuntu_container.master, 'sudo apt-get -y install wget') self.assertRaisesRegexp( OSError, r'ERROR\n' r'Paramiko could not be imported. This usually means that', self.pa_installer.install, cluster=ubuntu_container ) finally: ubuntu_container.tear_down() @attr('smoketest') def test_cert_arg_to_installation_nonexistent_file(self): install_dir = '/opt' script = """ set -e cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir} cd {install_dir} tar jxf prestoadmin-*.tar.bz2 cd prestoadmin ./install-prestoadmin.sh dummy_cert.cert """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir) output = self.cluster.run_script_on_host(script, self.cluster.master) self.assertRegexpMatches(output, r'Adding pypi.python.org as ' 'trusted\-host. Cannot find certificate ' 'file: dummy_cert.cert') @attr('smoketest') def test_cert_arg_to_installation_real_cert(self): self.cluster.copy_to_host(certifi.where(), self.cluster.master) install_dir = '/opt' cert_file = os.path.basename(certifi.where()) script = """ set -e cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir} cd {install_dir} tar jxf prestoadmin-*.tar.bz2 cd prestoadmin ./install-prestoadmin.sh {mount_dir}/{cacert} """.format(mount_dir=self.cluster.mount_dir, install_dir=install_dir, cacert=cert_file) output = self.cluster.run_script_on_host(script, self.cluster.master) self.assertTrue('Adding pypi.python.org as trusted-host. Cannot find' ' certificate file: %s' % cert_file not in output, 'Unable to find cert file; output: %s' % output)
def setUp(self): super(TestInstaller, self).setUp() self.centos_container = \ self.__create_and_start_single_centos_container() self.pa_installer = PrestoadminInstaller(self)
class TestInstallation(BaseProductTestCase): def setUp(self): super(TestInstallation, self).setUp() self.pa_installer = PrestoadminInstaller(self) self.setup_cluster(self.BARE_CLUSTER) dist_dir = self.pa_installer._build_dist_if_necessary(self.cluster) self.pa_installer._copy_dist_to_host(self.cluster, dist_dir, self.cluster.master) @attr("smoketest") @docker_only def test_install_non_root(self): install_dir = "/home/app-admin" script = """ set -e cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir} chown app-admin {install_dir}/prestoadmin-*.tar.bz2 cd {install_dir} sudo -u app-admin tar jxf prestoadmin-*.tar.bz2 cd prestoadmin sudo -u app-admin ./install-prestoadmin.sh """.format( mount_dir=self.cluster.mount_dir, install_dir=install_dir ) self.assertRaisesRegexp( OSError, "mkdir: cannot create directory " "`/var/log/prestoadmin': Permission denied", self.cluster.run_script_on_host, script, self.cluster.master, ) @attr("smoketest") def test_install_from_different_dir(self): install_dir = "/opt" script = """ set -e cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir} cd {install_dir} tar jxf prestoadmin-*.tar.bz2 ./prestoadmin/install-prestoadmin.sh """.format( mount_dir=self.cluster.mount_dir, install_dir=install_dir ) self.assertRaisesRegexp( OSError, r"IOError: \[Errno 2\] No such file or directory: " r"\'/opt/prestoadmin-1.1-py2-none-any.whl\'", self.cluster.run_script_on_host, script, self.cluster.master, ) @attr("smoketest") @docker_only def test_install_on_wrong_os_offline_installer(self): image = "ubuntu" tag = "14.04" host = image + "-master" ubuntu_container = DockerCluster(host, [], DEFAULT_LOCAL_MOUNT_POINT, DEFAULT_DOCKER_MOUNT_POINT) try: ubuntu_container.fetch_image_if_not_present(image, tag) ubuntu_container.start_containers(image + ":" + tag, cmd="tail -f /var/log/bootstrap.log") ubuntu_container.run_script_on_host(install_py26_script, ubuntu_container.master) ubuntu_container.exec_cmd_on_host(ubuntu_container.master, "sudo apt-get -y install wget") self.assertRaisesRegexp( OSError, r"ERROR\n" r"Paramiko could not be imported. This usually means that", self.pa_installer.install, cluster=ubuntu_container, ) finally: ubuntu_container.tear_down() @attr("smoketest") def test_cert_arg_to_installation_nonexistent_file(self): install_dir = "/opt" script = """ set -e cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir} cd {install_dir} tar jxf prestoadmin-*.tar.bz2 cd prestoadmin ./install-prestoadmin.sh dummy_cert.cert """.format( mount_dir=self.cluster.mount_dir, install_dir=install_dir ) output = self.cluster.run_script_on_host(script, self.cluster.master) self.assertRegexpMatches( output, r"Adding pypi.python.org as " "trusted\-host. Cannot find certificate " "file: dummy_cert.cert" ) @attr("smoketest") def test_cert_arg_to_installation_real_cert(self): self.cluster.copy_to_host(certifi.where(), self.cluster.master) install_dir = "/opt" cert_file = os.path.basename(certifi.where()) script = """ set -e cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir} cd {install_dir} tar jxf prestoadmin-*.tar.bz2 cd prestoadmin ./install-prestoadmin.sh {mount_dir}/{cacert} """.format( mount_dir=self.cluster.mount_dir, install_dir=install_dir, cacert=cert_file ) output = self.cluster.run_script_on_host(script, self.cluster.master) self.assertTrue( "Adding pypi.python.org as trusted-host. Cannot find" " certificate file: %s" % cert_file not in output, "Unable to find cert file; output: %s" % output, )