def setUp(self):
super(TestInstallation, self).setUp()
self.pa_installer = PrestoadminInstaller(self)
self.setup_cluster(NoHadoopBareImageProvider(), self.BARE_CLUSTER)
dist_dir = self.pa_installer._build_dist_if_necessary(self.cluster)
self.pa_installer._copy_dist_to_host(self.cluster, dist_dir,
self.cluster.master)
def test_uninstall_lost_host(self):
self.setup_cluster(self.PA_ONLY_CLUSTER)
pa_installer = PrestoadminInstaller(self)
pa_installer.install()
topology = {
"coordinator":
self.cluster.internal_slaves[0],
"workers": [
self.cluster.internal_master, self.cluster.internal_slaves[1],
self.cluster.internal_slaves[2]
]
}
self.upload_topology(topology)
self.installer.install()
start_output = self.run_prestoadmin('server start')
process_per_host = self.get_process_per_host(start_output.splitlines())
self.assert_started(process_per_host)
down_node = self.cluster.internal_slaves[0]
self.cluster.stop_host(self.cluster.slaves[0])
expected = self.down_node_connection_error(
self.cluster.internal_slaves[0])
cmd_output = self.run_prestoadmin('server uninstall')
self.assertRegexpMatches(cmd_output, expected)
process_per_active_host = []
for host, pid in process_per_host:
if host not in down_node:
process_per_active_host.append((host, pid))
self.assert_stopped(process_per_active_host)
for container in [
self.cluster.internal_master, self.cluster.internal_slaves[1],
self.cluster.internal_slaves[2]
]:
self.assert_uninstalled_dirs_removed(container)
def setUp(self):
super(TestInstaller, self).setUp()
self.setup_cluster(NoHadoopBareImageProvider(),
STANDALONE_BARE_CLUSTER)
self.centos_container = \
self.__create_and_start_single_centos_container()
self.pa_installer = PrestoadminInstaller(self)
def test_uninstall_lost_host(self):
self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER)
pa_installer = PrestoadminInstaller(self)
pa_installer.install()
topology = {"coordinator": self.cluster.internal_slaves[0],
"workers": [self.cluster.internal_master,
self.cluster.internal_slaves[1],
self.cluster.internal_slaves[2]]}
self.upload_topology(topology)
self.installer.install()
start_output = self.run_prestoadmin('server start')
process_per_host = self.get_process_per_host(start_output.splitlines())
self.assert_started(process_per_host)
down_node = self.cluster.internal_slaves[0]
self.cluster.stop_host(
self.cluster.slaves[0])
expected = self.down_node_connection_error(
self.cluster.internal_slaves[0])
cmd_output = self.run_prestoadmin('server uninstall',
raise_error=False)
self.assertRegexpMatches(cmd_output, expected)
process_per_active_host = []
for host, pid in process_per_host:
if host not in down_node:
process_per_active_host.append((host, pid))
self.assert_stopped(process_per_active_host)
for container in [self.cluster.internal_master,
self.cluster.internal_slaves[1],
self.cluster.internal_slaves[2]]:
self.assert_uninstalled_dirs_removed(container)
def setUp(self):
super(TestInstallation, self).setUp()
self.pa_installer = PrestoadminInstaller(self)
self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_BARE_CLUSTER)
dist_dir = self.pa_installer._build_dist_if_necessary(self.cluster)
self.pa_installer._copy_dist_to_host(self.cluster, dist_dir,
self.cluster.master)
def setUp(self):
super(TestInstaller, self).setUp()
self.centos_container = \
self.__create_and_start_single_centos_container()
self.pa_installer = PrestoadminInstaller(self)
class TestInstallation(BaseProductTestCase):
def setUp(self):
super(TestInstallation, self).setUp()
self.pa_installer = PrestoadminInstaller(self)
self.setup_cluster(NoHadoopBareImageProvider(), self.BARE_CLUSTER)
dist_dir = self.pa_installer._build_dist_if_necessary(self.cluster)
self.pa_installer._copy_dist_to_host(self.cluster, dist_dir,
self.cluster.master)
@attr('smoketest')
@docker_only
def test_install_non_root(self):
install_dir = '/home/app-admin'
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir}
chown app-admin {install_dir}/prestoadmin-*.tar.bz2
cd {install_dir}
sudo -u app-admin tar jxf prestoadmin-*.tar.bz2
cd prestoadmin
sudo -u app-admin ./install-prestoadmin.sh
""".format(mount_dir=self.cluster.mount_dir, install_dir=install_dir)
self.assertRaisesRegexp(
OSError, 'mkdir: cannot create directory '
'`/var/log/prestoadmin\': Permission denied',
self.cluster.run_script_on_host, script, self.cluster.master)
@attr('smoketest')
def test_install_from_different_dir(self):
install_dir = '/opt'
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir}
cd {install_dir}
tar jxf prestoadmin-*.tar.bz2
./prestoadmin/install-prestoadmin.sh
""".format(mount_dir=self.cluster.mount_dir, install_dir=install_dir)
self.assertRaisesRegexp(
OSError, r'IOError: \[Errno 2\] No such file or directory: '
r'\'/opt/prestoadmin-1.2-py2-none-any.whl\'',
self.cluster.run_script_on_host, script, self.cluster.master)
@attr('smoketest')
@docker_only
def test_install_on_wrong_os_offline_installer(self):
image = 'ubuntu'
tag = '14.04'
host = image + '-master'
ubuntu_container = DockerCluster(host, [], DEFAULT_LOCAL_MOUNT_POINT,
DEFAULT_DOCKER_MOUNT_POINT)
try:
ubuntu_container.fetch_image_if_not_present(image, tag)
ubuntu_container.start_containers(
image + ':' + tag, cmd='tail -f /var/log/bootstrap.log')
self.retry(lambda: ubuntu_container.run_script_on_host(
install_py26_script, ubuntu_container.master))
ubuntu_container.exec_cmd_on_host(ubuntu_container.master,
'sudo apt-get -y install wget')
self.assertRaisesRegexp(
OSError, r'ERROR\n'
r'Paramiko could not be imported. This usually means that',
self.pa_installer.install,
cluster=ubuntu_container)
finally:
ubuntu_container.tear_down()
@attr('smoketest')
def test_cert_arg_to_installation_nonexistent_file(self):
install_dir = '/opt'
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir}
cd {install_dir}
tar jxf prestoadmin-*.tar.bz2
cd prestoadmin
./install-prestoadmin.sh dummy_cert.cert
""".format(mount_dir=self.cluster.mount_dir, install_dir=install_dir)
output = self.cluster.run_script_on_host(script, self.cluster.master)
self.assertRegexpMatches(
output, r'Adding pypi.python.org as '
'trusted\-host. Cannot find certificate '
'file: dummy_cert.cert')
@attr('smoketest')
def test_cert_arg_to_installation_real_cert(self):
self.cluster.copy_to_host(certifi.where(), self.cluster.master)
install_dir = '/opt'
cert_file = os.path.basename(certifi.where())
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir}
cd {install_dir}
tar jxf prestoadmin-*.tar.bz2
cd prestoadmin
./install-prestoadmin.sh {mount_dir}/{cacert}
""".format(mount_dir=self.cluster.mount_dir,
install_dir=install_dir,
cacert=cert_file)
output = self.cluster.run_script_on_host(script, self.cluster.master)
self.assertTrue(
'Adding pypi.python.org as trusted-host. Cannot find'
' certificate file: %s' % cert_file not in output,
'Unable to find cert file; output: %s' % output)
def test_additional_dirs_created(self):
install_dir = '/opt'
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir}
cd {install_dir}
tar jxf prestoadmin-*.tar.bz2
cd prestoadmin
./install-prestoadmin.sh
""".format(mount_dir=self.cluster.mount_dir, install_dir=install_dir)
self.cluster.run_script_on_host(script, self.cluster.master)
pa_etc_dir = '/etc/opt/prestoadmin'
connectors_dir = pa_etc_dir + '/connectors'
self.assert_path_exists(self.cluster.master, connectors_dir)
coordinator_dir = pa_etc_dir + '/coordinator'
self.assert_path_exists(self.cluster.master, coordinator_dir)
workers_dir = pa_etc_dir + '/workers'
self.assert_path_exists(self.cluster.master, workers_dir)
class TestInstallation(BaseProductTestCase):
def setUp(self):
super(TestInstallation, self).setUp()
self.pa_installer = PrestoadminInstaller(self)
self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_BARE_CLUSTER)
dist_dir = self.pa_installer._build_dist_if_necessary(self.cluster)
self.pa_installer._copy_dist_to_host(self.cluster, dist_dir, self.cluster.master)
@attr('smoketest')
@docker_only
def test_install_non_root(self):
install_dir = '/home/app-admin'
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.gz {install_dir}
chown app-admin {install_dir}/prestoadmin-*.tar.gz
cd {install_dir}
sudo -u app-admin tar zxf prestoadmin-*.tar.gz
cd prestoadmin
sudo -u app-admin ./install-prestoadmin.sh
""".format(mount_dir=self.cluster.mount_dir, install_dir=install_dir)
self.cluster.run_script_on_host(script, self.cluster.master)
pa_config_dir = '/home/app-admin/.prestoadmin'
catalog_dir = os.path.join(pa_config_dir, 'catalog')
self.assert_path_exists(self.cluster.master, catalog_dir)
coordinator_dir = os.path.join(pa_config_dir, 'coordinator')
self.assert_path_exists(self.cluster.master, coordinator_dir)
workers_dir = os.path.join(pa_config_dir, 'workers')
self.assert_path_exists(self.cluster.master, workers_dir)
@attr('smoketest')
def test_cert_arg_to_installation_nonexistent_file(self):
install_dir = '~'
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.gz {install_dir}
cd {install_dir}
tar zxf prestoadmin-*.tar.gz
cd prestoadmin
./install-prestoadmin.sh dummy_cert.cert
""".format(mount_dir=self.cluster.mount_dir,
install_dir=install_dir)
output = self.cluster.run_script_on_host(script, self.cluster.master)
self.assertRegexpMatches(output, r'Adding pypi.python.org as '
'trusted\-host. Cannot find certificate '
'file: dummy_cert.cert')
@attr('smoketest')
def test_cert_arg_to_installation_real_cert(self):
self.cluster.copy_to_host(certifi.where(), self.cluster.master)
install_dir = '~'
cert_file = os.path.basename(certifi.where())
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.gz {install_dir}
cd {install_dir}
tar zxf prestoadmin-*.tar.gz
cd prestoadmin
./install-prestoadmin.sh {mount_dir}/{cacert}
""".format(mount_dir=self.cluster.mount_dir,
install_dir=install_dir,
cacert=cert_file)
output = self.cluster.run_script_on_host(script, self.cluster.master)
self.assertTrue('Adding pypi.python.org as trusted-host. Cannot find'
' certificate file: %s' % cert_file not in output,
'Unable to find cert file; output: %s' % output)
def test_additional_dirs_created(self):
install_dir = '~'
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.gz {install_dir}
cd {install_dir}
tar zxf prestoadmin-*.tar.gz
cd prestoadmin
./install-prestoadmin.sh
""".format(mount_dir=self.cluster.mount_dir,
install_dir=install_dir)
self.cluster.run_script_on_host(script, self.cluster.master)
self.assert_path_exists(self.cluster.master, get_catalog_directory())
self.assert_path_exists(self.cluster.master, get_coordinator_directory())
self.assert_path_exists(self.cluster.master, get_workers_directory())
class TestInstallation(BaseProductTestCase):
def setUp(self):
super(TestInstallation, self).setUp()
self.pa_installer = PrestoadminInstaller(self)
self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_BARE_CLUSTER)
dist_dir = self.pa_installer._build_dist_if_necessary(self.cluster)
self.pa_installer._copy_dist_to_host(self.cluster, dist_dir,
self.cluster.master)
@attr('smoketest')
@docker_only
def test_install_non_root(self):
install_dir = '/home/app-admin'
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir}
chown app-admin {install_dir}/prestoadmin-*.tar.bz2
cd {install_dir}
sudo -u app-admin tar jxf prestoadmin-*.tar.bz2
cd prestoadmin
sudo -u app-admin ./install-prestoadmin.sh
""".format(mount_dir=self.cluster.mount_dir,
install_dir=install_dir)
self.assertRaisesRegexp(OSError, 'mkdir: cannot create directory '
'`/var/log/prestoadmin\': Permission denied',
self.cluster.run_script_on_host, script,
self.cluster.master)
@attr('smoketest')
def test_install_from_different_dir(self):
install_dir = '/opt'
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir}
cd {install_dir}
tar jxf prestoadmin-*.tar.bz2
./prestoadmin/install-prestoadmin.sh
""".format(mount_dir=self.cluster.mount_dir,
install_dir=install_dir)
self.assertRaisesRegexp(
OSError,
r'IOError: \[Errno 2\] No such file or directory: '
r'\'/opt/prestoadmin-.*-py2-none-any.whl\'',
self.cluster.run_script_on_host,
script,
self.cluster.master
)
@attr('smoketest', 'offline_installer')
@docker_only
def test_install_on_wrong_os_offline_installer(self):
image = 'teradatalabs/ubuntu-trusty-python2.6'
tag = BASE_IMAGES_TAG
host = 'wrong-os-master'
ubuntu_container = DockerCluster(host, [], DEFAULT_LOCAL_MOUNT_POINT,
DEFAULT_DOCKER_MOUNT_POINT)
if not DockerCluster._check_for_images(image, image, tag):
raise RuntimeError("Docker images have not been created")
try:
ubuntu_container.start_containers(
image + ':' + tag, cmd='tail -f /var/log/bootstrap.log')
self.assertRaisesRegexp(
OSError,
r'ERROR\n'
r'Paramiko could not be imported. This usually means that',
self.pa_installer.install,
cluster=ubuntu_container
)
finally:
ubuntu_container.tear_down()
@attr('smoketest')
def test_cert_arg_to_installation_nonexistent_file(self):
install_dir = '/opt'
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir}
cd {install_dir}
tar jxf prestoadmin-*.tar.bz2
cd prestoadmin
./install-prestoadmin.sh dummy_cert.cert
""".format(mount_dir=self.cluster.mount_dir,
install_dir=install_dir)
output = self.cluster.run_script_on_host(script, self.cluster.master)
self.assertRegexpMatches(output, r'Adding pypi.python.org as '
'trusted\-host. Cannot find certificate '
'file: dummy_cert.cert')
@attr('smoketest')
def test_cert_arg_to_installation_real_cert(self):
self.cluster.copy_to_host(certifi.where(), self.cluster.master)
install_dir = '/opt'
cert_file = os.path.basename(certifi.where())
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir}
cd {install_dir}
tar jxf prestoadmin-*.tar.bz2
cd prestoadmin
./install-prestoadmin.sh {mount_dir}/{cacert}
""".format(mount_dir=self.cluster.mount_dir,
install_dir=install_dir,
cacert=cert_file)
output = self.cluster.run_script_on_host(script, self.cluster.master)
self.assertTrue('Adding pypi.python.org as trusted-host. Cannot find'
' certificate file: %s' % cert_file not in output,
'Unable to find cert file; output: %s' % output)
def test_additional_dirs_created(self):
install_dir = '/opt'
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir}
cd {install_dir}
tar jxf prestoadmin-*.tar.bz2
cd prestoadmin
./install-prestoadmin.sh
""".format(mount_dir=self.cluster.mount_dir,
install_dir=install_dir)
self.cluster.run_script_on_host(script, self.cluster.master)
pa_etc_dir = '/etc/opt/prestoadmin'
connectors_dir = pa_etc_dir + '/connectors'
self.assert_path_exists(self.cluster.master, connectors_dir)
coordinator_dir = pa_etc_dir + '/coordinator'
self.assert_path_exists(self.cluster.master, coordinator_dir)
workers_dir = pa_etc_dir + '/workers'
self.assert_path_exists(self.cluster.master, workers_dir)
class TestInstallation(BaseProductTestCase):
def setUp(self):
super(TestInstallation, self).setUp()
self.pa_installer = PrestoadminInstaller(self)
self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_BARE_CLUSTER)
dist_dir = self.pa_installer._build_dist_if_necessary(self.cluster)
self.pa_installer._copy_dist_to_host(self.cluster, dist_dir,
self.cluster.master)
@attr('smoketest')
@docker_only
def test_install_non_root(self):
install_dir = '/home/app-admin'
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.gz {install_dir}
chown app-admin {install_dir}/prestoadmin-*.tar.gz
cd {install_dir}
sudo -u app-admin tar zxf prestoadmin-*.tar.gz
cd prestoadmin
sudo -u app-admin ./install-prestoadmin.sh
""".format(mount_dir=self.cluster.mount_dir, install_dir=install_dir)
self.cluster.run_script_on_host(script, self.cluster.master)
pa_config_dir = '/home/app-admin/.prestoadmin'
catalog_dir = os.path.join(pa_config_dir, 'catalog')
self.assert_path_exists(self.cluster.master, catalog_dir)
coordinator_dir = os.path.join(pa_config_dir, 'coordinator')
self.assert_path_exists(self.cluster.master, coordinator_dir)
workers_dir = os.path.join(pa_config_dir, 'workers')
self.assert_path_exists(self.cluster.master, workers_dir)
@attr('smoketest', 'offline_installer')
@docker_only
def test_install_on_wrong_os_offline_installer(self):
image = 'teradatalabs/ubuntu-trusty-python2.6'
tag = BASE_IMAGES_TAG
host = 'wrong-os-master'
ubuntu_container = DockerCluster(host, [], DEFAULT_LOCAL_MOUNT_POINT,
DEFAULT_DOCKER_MOUNT_POINT)
if not DockerCluster._check_for_images(image, image, tag):
raise RuntimeError("Docker images have not been created")
try:
ubuntu_container.start_containers(
image + ':' + tag, cmd='tail -f /var/log/bootstrap.log')
self.assertRaisesRegexp(
OSError, r'ERROR\n'
r'Paramiko could not be imported. This usually means that',
self.pa_installer.install,
cluster=ubuntu_container)
finally:
ubuntu_container.tear_down()
@attr('smoketest')
def test_cert_arg_to_installation_nonexistent_file(self):
install_dir = '~'
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.gz {install_dir}
cd {install_dir}
tar zxf prestoadmin-*.tar.gz
cd prestoadmin
./install-prestoadmin.sh dummy_cert.cert
""".format(mount_dir=self.cluster.mount_dir, install_dir=install_dir)
output = self.cluster.run_script_on_host(script, self.cluster.master)
self.assertRegexpMatches(
output, r'Adding pypi.python.org as '
'trusted\-host. Cannot find certificate '
'file: dummy_cert.cert')
@attr('smoketest')
def test_cert_arg_to_installation_real_cert(self):
self.cluster.copy_to_host(certifi.where(), self.cluster.master)
install_dir = '~'
cert_file = os.path.basename(certifi.where())
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.gz {install_dir}
cd {install_dir}
tar zxf prestoadmin-*.tar.gz
cd prestoadmin
./install-prestoadmin.sh {mount_dir}/{cacert}
""".format(mount_dir=self.cluster.mount_dir,
install_dir=install_dir,
cacert=cert_file)
output = self.cluster.run_script_on_host(script, self.cluster.master)
self.assertTrue(
'Adding pypi.python.org as trusted-host. Cannot find'
' certificate file: %s' % cert_file not in output,
'Unable to find cert file; output: %s' % output)
def test_additional_dirs_created(self):
install_dir = '~'
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.gz {install_dir}
cd {install_dir}
tar zxf prestoadmin-*.tar.gz
cd prestoadmin
./install-prestoadmin.sh
""".format(mount_dir=self.cluster.mount_dir, install_dir=install_dir)
self.cluster.run_script_on_host(script, self.cluster.master)
self.assert_path_exists(self.cluster.master, get_catalog_directory())
self.assert_path_exists(self.cluster.master,
get_coordinator_directory())
self.assert_path_exists(self.cluster.master, get_workers_directory())
class TestInstaller(BaseProductTestCase):
def setUp(self):
super(TestInstaller, self).setUp()
self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_BARE_CLUSTER)
self.centos_container = \
self.__create_and_start_single_centos_container()
self.pa_installer = PrestoadminInstaller(self)
def tearDown(self):
super(TestInstaller, self).tearDown()
self.centos_container.tear_down()
@attr('smoketest')
def test_online_installer(self):
self.pa_installer._build_installer_in_docker(self.centos_container,
online_installer=True,
unique=True)
self.__verify_third_party_dir(False)
self.pa_installer.install(dist_dir=self.centos_container.get_dist_dir(
unique=True))
self.run_prestoadmin('--help', raise_error=True)
@attr('smoketest', 'offline_installer')
@docker_only
def test_offline_installer(self):
self.pa_installer._build_installer_in_docker(self.centos_container,
online_installer=False,
unique=True)
self.__verify_third_party_dir(True)
self.centos_container.exec_cmd_on_host(
# IMPORTANT: ifdown eth0 fails silently without taking the
# interface down if the NET_ADMIN capability isn't set for the
# container. ifconfig eth0 down accomplishes the same thing, but
# results in a failure if it fails.
self.centos_container.master,
'ifconfig eth0 down')
self.pa_installer.install(dist_dir=self.centos_container.get_dist_dir(
unique=True))
self.run_prestoadmin('--help', raise_error=True)
def __create_and_start_single_centos_container(self):
cluster_type = 'installer_tester'
bare_image_provider = NoHadoopBareImageProvider(BASE_IMAGES_TAG)
centos_container, bare_cluster = DockerCluster.start_cluster(
bare_image_provider,
cluster_type,
'master', [],
cap_add=['NET_ADMIN'])
if bare_cluster:
centos_container.commit_images(bare_image_provider, cluster_type)
return centos_container
def __verify_third_party_dir(self, is_third_party_present):
matches = fnmatch.filter(
os.listdir(self.centos_container.get_dist_dir(unique=True)),
'prestoadmin-*.tar.gz')
if len(matches) > 1:
raise RuntimeError(
'More than one archive found in the dist directory ' +
' '.join(matches))
cmd_to_run = [
'tar', '-tf',
os.path.join(self.centos_container.get_dist_dir(unique=True),
matches[0])
]
popen_obj = subprocess.Popen(cmd_to_run,
cwd=main_dir,
stdout=subprocess.PIPE)
retcode = popen_obj.returncode
if retcode:
raise RuntimeError('Non zero return code when executing ' +
' '.join(cmd_to_run))
stdout = popen_obj.communicate()[0]
match = re.search('/third-party/', stdout)
if is_third_party_present and match is None:
raise RuntimeError('Expected to have an offline installer with '
'a third-party directory. Found no '
'third-party directory in the installer '
'archive.')
elif not is_third_party_present and match:
raise RuntimeError('Expected to have an online installer with no '
'third-party directory. Found a third-party '
'directory in the installer archive.')
def setUp(self):
super(TestInstallation, self).setUp()
self.pa_installer = PrestoadminInstaller(self)
self.setup_cluster(self.BARE_CLUSTER)
dist_dir = self.pa_installer._build_dist_if_necessary(self.cluster)
self.pa_installer._copy_dist_to_host(self.cluster, dist_dir, self.cluster.master)
class TestInstaller(BaseProductTestCase):
def setUp(self):
super(TestInstaller, self).setUp()
self.centos_container = \
self.__create_and_start_single_centos_container()
self.pa_installer = PrestoadminInstaller(self)
def tearDown(self):
super(TestInstaller, self).tearDown()
self.centos_container.tear_down()
@attr('smoketest')
@docker_only
def test_online_installer(self):
self.pa_installer._build_installer_in_docker(self.centos_container,
online_installer=True,
unique=True)
self.__verify_third_party_dir(False)
self.pa_installer.install(
dist_dir=self.centos_container.get_dist_dir(unique=True))
self.run_prestoadmin('--help', raise_error=True,
cluster=self.centos_container)
@attr('smoketest')
@docker_only
def test_offline_installer(self):
self.pa_installer._build_installer_in_docker(
self.centos_container, online_installer=False, unique=True)
self.__verify_third_party_dir(True)
self.centos_container.exec_cmd_on_host(
self.centos_container.master, 'ifdown eth0')
self.pa_installer.install(
dist_dir=self.centos_container.get_dist_dir(unique=True))
self.run_prestoadmin('--help', raise_error=True,
cluster=self.centos_container)
def __create_and_start_single_centos_container(self):
centos_container = DockerCluster(
'master', [], DEFAULT_LOCAL_MOUNT_POINT,
DEFAULT_DOCKER_MOUNT_POINT)
# we can't assume that another test has created the image
centos_container.create_image(
BASE_TD_DOCKERFILE_DIR,
BASE_TD_IMAGE_NAME,
BASE_IMAGE_NAME
)
centos_container.start_containers(
BASE_TD_IMAGE_NAME,
cap_add=['NET_ADMIN']
)
return centos_container
def __verify_third_party_dir(self, is_third_party_present):
matches = fnmatch.filter(
os.listdir(self.centos_container.get_dist_dir(unique=True)),
'prestoadmin-*.tar.bz2')
if len(matches) > 1:
raise RuntimeError(
'More than one archive found in the dist directory ' +
' '.join(matches)
)
cmd_to_run = ['tar', '-tf',
os.path.join(
self.centos_container.get_dist_dir(unique=True),
matches[0])
]
popen_obj = subprocess.Popen(cmd_to_run,
cwd=main_dir, stdout=subprocess.PIPE)
retcode = popen_obj.returncode
if retcode:
raise RuntimeError('Non zero return code when executing ' +
' '.join(cmd_to_run))
stdout = popen_obj.communicate()[0]
match = re.search('/third-party/', stdout)
if is_third_party_present and match is None:
raise RuntimeError('Expected to have an offline installer with '
'a third-party directory. Found no '
'third-party directory in the installer '
'archive.')
elif not is_third_party_present and match:
raise RuntimeError('Expected to have an online installer with no '
'third-party directory. Found a third-party '
'directory in the installer archive.')
class TestInstaller(BaseProductTestCase):
def setUp(self):
super(TestInstaller, self).setUp()
self.centos_container = \
self.__create_and_start_single_centos_container()
self.pa_installer = PrestoadminInstaller(self)
def tearDown(self):
super(TestInstaller, self).tearDown()
self.centos_container.tear_down()
@attr('smoketest')
@docker_only
def test_online_installer(self):
self.pa_installer._build_installer_in_docker(self.centos_container,
online_installer=True,
unique=True)
self.__verify_third_party_dir(False)
self.pa_installer.install(dist_dir=self.centos_container.get_dist_dir(
unique=True))
self.run_prestoadmin('--help',
raise_error=True,
cluster=self.centos_container)
@attr('smoketest')
@docker_only
def test_offline_installer(self):
self.pa_installer._build_installer_in_docker(self.centos_container,
online_installer=False,
unique=True)
self.__verify_third_party_dir(True)
self.centos_container.exec_cmd_on_host(self.centos_container.master,
'ifdown eth0')
self.pa_installer.install(dist_dir=self.centos_container.get_dist_dir(
unique=True))
self.run_prestoadmin('--help',
raise_error=True,
cluster=self.centos_container)
def __create_and_start_single_centos_container(self):
centos_container = DockerCluster('master', [],
DEFAULT_LOCAL_MOUNT_POINT,
DEFAULT_DOCKER_MOUNT_POINT)
# we can't assume that another test has created the image
centos_container.create_image(BASE_TD_DOCKERFILE_DIR,
BASE_TD_IMAGE_NAME, BASE_IMAGE_NAME)
centos_container.start_containers(BASE_TD_IMAGE_NAME,
cap_add=['NET_ADMIN'])
return centos_container
def __verify_third_party_dir(self, is_third_party_present):
matches = fnmatch.filter(
os.listdir(self.centos_container.get_dist_dir(unique=True)),
'prestoadmin-*.tar.bz2')
if len(matches) > 1:
raise RuntimeError(
'More than one archive found in the dist directory ' +
' '.join(matches))
cmd_to_run = [
'tar', '-tf',
os.path.join(self.centos_container.get_dist_dir(unique=True),
matches[0])
]
popen_obj = subprocess.Popen(cmd_to_run,
cwd=main_dir,
stdout=subprocess.PIPE)
retcode = popen_obj.returncode
if retcode:
raise RuntimeError('Non zero return code when executing ' +
' '.join(cmd_to_run))
stdout = popen_obj.communicate()[0]
match = re.search('/third-party/', stdout)
if is_third_party_present and match is None:
raise RuntimeError('Expected to have an offline installer with '
'a third-party directory. Found no '
'third-party directory in the installer '
'archive.')
elif not is_third_party_present and match:
raise RuntimeError('Expected to have an online installer with no '
'third-party directory. Found a third-party '
'directory in the installer archive.')
class TestInstallation(BaseProductTestCase):
def setUp(self):
super(TestInstallation, self).setUp()
self.pa_installer = PrestoadminInstaller(self)
self.setup_cluster(NoHadoopBareImageProvider(),
STANDALONE_BARE_CLUSTER)
dist_dir = self.pa_installer._build_dist_if_necessary(self.cluster)
self.pa_installer._copy_dist_to_host(self.cluster, dist_dir,
self.cluster.master)
@attr('smoketest')
@docker_only
def test_install_non_root(self):
install_dir = '/home/app-admin'
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.gz {install_dir}
chown app-admin {install_dir}/prestoadmin-*.tar.gz
cd {install_dir}
sudo -u app-admin tar zxf prestoadmin-*.tar.gz
cd prestoadmin
sudo -u app-admin ./install-prestoadmin.sh
""".format(mount_dir=self.cluster.mount_dir, install_dir=install_dir)
self.cluster.run_script_on_host(script, self.cluster.master)
pa_config_dir = '/home/app-admin/.prestoadmin'
catalog_dir = os.path.join(pa_config_dir, 'catalog')
self.assert_path_exists(self.cluster.master, catalog_dir)
coordinator_dir = os.path.join(pa_config_dir, 'coordinator')
self.assert_path_exists(self.cluster.master, coordinator_dir)
workers_dir = os.path.join(pa_config_dir, 'workers')
self.assert_path_exists(self.cluster.master, workers_dir)
@attr('smoketest')
def test_cert_arg_to_installation_nonexistent_file(self):
install_dir = '~'
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.gz {install_dir}
cd {install_dir}
tar zxf prestoadmin-*.tar.gz
cd prestoadmin
./install-prestoadmin.sh dummy_cert.cert
""".format(mount_dir=self.cluster.mount_dir, install_dir=install_dir)
output = self.cluster.run_script_on_host(script, self.cluster.master)
self.assertRegexpMatches(
output, r'Adding pypi.python.org as '
'trusted\-host. Cannot find certificate '
'file: dummy_cert.cert')
@attr('smoketest')
def test_cert_arg_to_installation_real_cert(self):
self.cluster.copy_to_host(certifi.where(), self.cluster.master)
install_dir = '~'
cert_file = os.path.basename(certifi.where())
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.gz {install_dir}
cd {install_dir}
tar zxf prestoadmin-*.tar.gz
cd prestoadmin
./install-prestoadmin.sh {mount_dir}/{cacert}
""".format(mount_dir=self.cluster.mount_dir,
install_dir=install_dir,
cacert=cert_file)
output = self.cluster.run_script_on_host(script, self.cluster.master)
self.assertTrue(
'Adding pypi.python.org as trusted-host. Cannot find'
' certificate file: %s' % cert_file not in output,
'Unable to find cert file; output: %s' % output)
def test_additional_dirs_created(self):
install_dir = '~'
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.gz {install_dir}
cd {install_dir}
tar zxf prestoadmin-*.tar.gz
cd prestoadmin
./install-prestoadmin.sh
""".format(mount_dir=self.cluster.mount_dir, install_dir=install_dir)
self.cluster.run_script_on_host(script, self.cluster.master)
self.assert_path_exists(self.cluster.master, get_catalog_directory())
self.assert_path_exists(self.cluster.master,
get_coordinator_directory())
self.assert_path_exists(self.cluster.master, get_workers_directory())
def setUp(self):
super(TestInstaller, self).setUp()
self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_BARE_CLUSTER)
self.centos_container = \
self.__create_and_start_single_centos_container()
self.pa_installer = PrestoadminInstaller(self)
class TestInstaller(BaseProductTestCase):
def setUp(self):
super(TestInstaller, self).setUp()
self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_BARE_CLUSTER)
self.centos_container = \
self.__create_and_start_single_centos_container()
self.pa_installer = PrestoadminInstaller(self)
def tearDown(self):
super(TestInstaller, self).tearDown()
self.centos_container.tear_down()
@attr('smoketest')
def test_online_installer(self):
self.pa_installer._build_installer_in_docker(self.centos_container,
online_installer=True,
unique=True)
self.__verify_third_party_dir(False)
self.pa_installer.install(
dist_dir=self.centos_container.get_dist_dir(unique=True))
self.run_prestoadmin('--help', raise_error=True)
@attr('smoketest', 'offline_installer')
@docker_only
def test_offline_installer(self):
self.pa_installer._build_installer_in_docker(
self.centos_container, online_installer=False, unique=True)
self.__verify_third_party_dir(True)
self.centos_container.exec_cmd_on_host(
# IMPORTANT: ifdown eth0 fails silently without taking the
# interface down if the NET_ADMIN capability isn't set for the
# container. ifconfig eth0 down accomplishes the same thing, but
# results in a failure if it fails.
self.centos_container.master, 'ifconfig eth0 down')
self.pa_installer.install(
dist_dir=self.centos_container.get_dist_dir(unique=True))
self.run_prestoadmin('--help', raise_error=True)
def __create_and_start_single_centos_container(self):
cluster_type = 'installer_tester'
bare_image_provider = NoHadoopBareImageProvider(BASE_IMAGES_TAG)
centos_container, bare_cluster = DockerCluster.start_cluster(
bare_image_provider, cluster_type, 'master', [],
cap_add=['NET_ADMIN'])
if bare_cluster:
centos_container.commit_images(bare_image_provider, cluster_type)
return centos_container
def __verify_third_party_dir(self, is_third_party_present):
matches = fnmatch.filter(
os.listdir(self.centos_container.get_dist_dir(unique=True)),
'prestoadmin-*.tar.bz2')
if len(matches) > 1:
raise RuntimeError(
'More than one archive found in the dist directory ' +
' '.join(matches)
)
cmd_to_run = ['tar', '-tf',
os.path.join(
self.centos_container.get_dist_dir(unique=True),
matches[0])
]
popen_obj = subprocess.Popen(cmd_to_run,
cwd=main_dir, stdout=subprocess.PIPE)
retcode = popen_obj.returncode
if retcode:
raise RuntimeError('Non zero return code when executing ' +
' '.join(cmd_to_run))
stdout = popen_obj.communicate()[0]
match = re.search('/third-party/', stdout)
if is_third_party_present and match is None:
raise RuntimeError('Expected to have an offline installer with '
'a third-party directory. Found no '
'third-party directory in the installer '
'archive.')
elif not is_third_party_present and match:
raise RuntimeError('Expected to have an online installer with no '
'third-party directory. Found a third-party '
'directory in the installer archive.')
class TestInstallation(BaseProductTestCase):
def setUp(self):
super(TestInstallation, self).setUp()
self.pa_installer = PrestoadminInstaller(self)
self.setup_cluster(NoHadoopBareImageProvider(), self.BARE_CLUSTER)
dist_dir = self.pa_installer._build_dist_if_necessary(self.cluster)
self.pa_installer._copy_dist_to_host(self.cluster, dist_dir,
self.cluster.master)
@attr('smoketest')
@docker_only
def test_install_non_root(self):
install_dir = '/home/app-admin'
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir}
chown app-admin {install_dir}/prestoadmin-*.tar.bz2
cd {install_dir}
sudo -u app-admin tar jxf prestoadmin-*.tar.bz2
cd prestoadmin
sudo -u app-admin ./install-prestoadmin.sh
""".format(mount_dir=self.cluster.mount_dir,
install_dir=install_dir)
self.assertRaisesRegexp(OSError, 'mkdir: cannot create directory '
'`/var/log/prestoadmin\': Permission denied',
self.cluster.run_script_on_host, script,
self.cluster.master)
@attr('smoketest')
def test_install_from_different_dir(self):
install_dir = '/opt'
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir}
cd {install_dir}
tar jxf prestoadmin-*.tar.bz2
./prestoadmin/install-prestoadmin.sh
""".format(mount_dir=self.cluster.mount_dir,
install_dir=install_dir)
self.assertRaisesRegexp(
OSError,
r'IOError: \[Errno 2\] No such file or directory: '
r'\'/opt/prestoadmin-1.2-py2-none-any.whl\'',
self.cluster.run_script_on_host,
script,
self.cluster.master
)
@attr('smoketest')
@docker_only
def test_install_on_wrong_os_offline_installer(self):
image = 'ubuntu'
tag = '14.04'
host = image + '-master'
ubuntu_container = DockerCluster(host, [], DEFAULT_LOCAL_MOUNT_POINT,
DEFAULT_DOCKER_MOUNT_POINT)
try:
ubuntu_container.fetch_image_if_not_present(image, tag)
ubuntu_container.start_containers(
image + ':' + tag, cmd='tail -f /var/log/bootstrap.log')
ubuntu_container.run_script_on_host(install_py26_script,
ubuntu_container.master)
ubuntu_container.exec_cmd_on_host(
ubuntu_container.master, 'sudo apt-get -y install wget')
self.assertRaisesRegexp(
OSError,
r'ERROR\n'
r'Paramiko could not be imported. This usually means that',
self.pa_installer.install,
cluster=ubuntu_container
)
finally:
ubuntu_container.tear_down()
@attr('smoketest')
def test_cert_arg_to_installation_nonexistent_file(self):
install_dir = '/opt'
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir}
cd {install_dir}
tar jxf prestoadmin-*.tar.bz2
cd prestoadmin
./install-prestoadmin.sh dummy_cert.cert
""".format(mount_dir=self.cluster.mount_dir,
install_dir=install_dir)
output = self.cluster.run_script_on_host(script, self.cluster.master)
self.assertRegexpMatches(output, r'Adding pypi.python.org as '
'trusted\-host. Cannot find certificate '
'file: dummy_cert.cert')
@attr('smoketest')
def test_cert_arg_to_installation_real_cert(self):
self.cluster.copy_to_host(certifi.where(), self.cluster.master)
install_dir = '/opt'
cert_file = os.path.basename(certifi.where())
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir}
cd {install_dir}
tar jxf prestoadmin-*.tar.bz2
cd prestoadmin
./install-prestoadmin.sh {mount_dir}/{cacert}
""".format(mount_dir=self.cluster.mount_dir,
install_dir=install_dir,
cacert=cert_file)
output = self.cluster.run_script_on_host(script, self.cluster.master)
self.assertTrue('Adding pypi.python.org as trusted-host. Cannot find'
' certificate file: %s' % cert_file not in output,
'Unable to find cert file; output: %s' % output)
def setUp(self):
super(TestInstaller, self).setUp()
self.centos_container = \
self.__create_and_start_single_centos_container()
self.pa_installer = PrestoadminInstaller(self)
class TestInstallation(BaseProductTestCase):
def setUp(self):
super(TestInstallation, self).setUp()
self.pa_installer = PrestoadminInstaller(self)
self.setup_cluster(self.BARE_CLUSTER)
dist_dir = self.pa_installer._build_dist_if_necessary(self.cluster)
self.pa_installer._copy_dist_to_host(self.cluster, dist_dir, self.cluster.master)
@attr("smoketest")
@docker_only
def test_install_non_root(self):
install_dir = "/home/app-admin"
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir}
chown app-admin {install_dir}/prestoadmin-*.tar.bz2
cd {install_dir}
sudo -u app-admin tar jxf prestoadmin-*.tar.bz2
cd prestoadmin
sudo -u app-admin ./install-prestoadmin.sh
""".format(
mount_dir=self.cluster.mount_dir, install_dir=install_dir
)
self.assertRaisesRegexp(
OSError,
"mkdir: cannot create directory " "`/var/log/prestoadmin': Permission denied",
self.cluster.run_script_on_host,
script,
self.cluster.master,
)
@attr("smoketest")
def test_install_from_different_dir(self):
install_dir = "/opt"
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir}
cd {install_dir}
tar jxf prestoadmin-*.tar.bz2
./prestoadmin/install-prestoadmin.sh
""".format(
mount_dir=self.cluster.mount_dir, install_dir=install_dir
)
self.assertRaisesRegexp(
OSError,
r"IOError: \[Errno 2\] No such file or directory: " r"\'/opt/prestoadmin-1.1-py2-none-any.whl\'",
self.cluster.run_script_on_host,
script,
self.cluster.master,
)
@attr("smoketest")
@docker_only
def test_install_on_wrong_os_offline_installer(self):
image = "ubuntu"
tag = "14.04"
host = image + "-master"
ubuntu_container = DockerCluster(host, [], DEFAULT_LOCAL_MOUNT_POINT, DEFAULT_DOCKER_MOUNT_POINT)
try:
ubuntu_container.fetch_image_if_not_present(image, tag)
ubuntu_container.start_containers(image + ":" + tag, cmd="tail -f /var/log/bootstrap.log")
ubuntu_container.run_script_on_host(install_py26_script, ubuntu_container.master)
ubuntu_container.exec_cmd_on_host(ubuntu_container.master, "sudo apt-get -y install wget")
self.assertRaisesRegexp(
OSError,
r"ERROR\n" r"Paramiko could not be imported. This usually means that",
self.pa_installer.install,
cluster=ubuntu_container,
)
finally:
ubuntu_container.tear_down()
@attr("smoketest")
def test_cert_arg_to_installation_nonexistent_file(self):
install_dir = "/opt"
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir}
cd {install_dir}
tar jxf prestoadmin-*.tar.bz2
cd prestoadmin
./install-prestoadmin.sh dummy_cert.cert
""".format(
mount_dir=self.cluster.mount_dir, install_dir=install_dir
)
output = self.cluster.run_script_on_host(script, self.cluster.master)
self.assertRegexpMatches(
output, r"Adding pypi.python.org as " "trusted\-host. Cannot find certificate " "file: dummy_cert.cert"
)
@attr("smoketest")
def test_cert_arg_to_installation_real_cert(self):
self.cluster.copy_to_host(certifi.where(), self.cluster.master)
install_dir = "/opt"
cert_file = os.path.basename(certifi.where())
script = """
set -e
cp {mount_dir}/prestoadmin-*.tar.bz2 {install_dir}
cd {install_dir}
tar jxf prestoadmin-*.tar.bz2
cd prestoadmin
./install-prestoadmin.sh {mount_dir}/{cacert}
""".format(
mount_dir=self.cluster.mount_dir, install_dir=install_dir, cacert=cert_file
)
output = self.cluster.run_script_on_host(script, self.cluster.master)
self.assertTrue(
"Adding pypi.python.org as trusted-host. Cannot find" " certificate file: %s" % cert_file not in output,
"Unable to find cert file; output: %s" % output,
)
