def run_script_from_prestoadmin_dir(self, script_contents, host='', raise_error=True, **kwargs): if not host: host = self.cluster.master script_contents = self.replace_keywords(script_contents, **kwargs) temp_script = os.path.join(get_install_directory(), 'tmp.sh') self.cluster.write_content_to_host( '#!/bin/bash\ncd %s\n%s' % (get_install_directory(), script_contents), temp_script, host) self.cluster.exec_cmd_on_host( host, 'chmod +x %s' % temp_script) return self.cluster.exec_cmd_on_host( host, temp_script, raise_error=raise_error)
def test_collect_logs_nonstandard_location(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) version = self.cluster.exec_cmd_on_host( self.cluster.master, 'rpm -q --qf \"%{VERSION}\\n\" ' + PRESTO_RPM_NAME) if '127t' not in version: print 'test_collect_logs_nonstandard_location only valid for 127t' return new_log_location = '/var/presto' self._add_custom_log_location(new_log_location) self.run_prestoadmin('server start') self._collect_logs_and_unzip() collected_logs_dir = os.path.join(get_install_directory(), 'logs') self.assert_path_exists( self.cluster.master, os.path.join(collected_logs_dir, ' presto-admin.log')) for host in self.cluster.all_internal_hosts(): host_directory = os.path.join(collected_logs_dir, host) self.assert_path_exists(self.cluster.master, os.path.join(host_directory, 'server.log')) self.assert_path_exists( self.cluster.master, os.path.join(host_directory, 'launcher.log'))
def _assert_no_logs_downloaded(self): self._collect_logs_and_unzip() collected_logs_dir = os.path.join(get_install_directory(), 'logs') self.assert_path_exists(self.cluster.master, os.path.join(collected_logs_dir, 'presto-admin.log')) for host in self.cluster.all_internal_hosts(): host_directory = os.path.join(collected_logs_dir, host) self.assert_path_exists(self.cluster.master, host_directory) self.assert_path_removed(self.cluster.master, os.path.join(host_directory, '*'))
def run_prestoadmin_expect(self, command, expect_statements): temp_script = os.path.join(get_install_directory(), 'tmp.expect') script_content = '#!/usr/bin/expect\n' + \ 'spawn %s %s\n%s' % \ (get_presto_admin_path(), command, expect_statements) self.cluster.write_content_to_host(script_content, temp_script, self.cluster.master) self.cluster.exec_cmd_on_host(self.cluster.master, 'chmod +x %s' % temp_script) return self.cluster.exec_cmd_on_host(self.cluster.master, temp_script)
def test_upgrade_works_with_symlink(self): self.run_prestoadmin('configuration deploy') for container in self.cluster.all_hosts(): self.real_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) path_on_cluster = self.copy_upgrade_rpm_to_cluster() symlink = os.path.join(get_install_directory(), 'link.rpm') self.cluster.exec_cmd_on_host(self.cluster.master, 'ln -s %s %s' % (path_on_cluster, symlink)) self.upgrade_and_assert_success(symlink)
def test_upgrade_works_with_symlink(self): self.run_prestoadmin('configuration deploy') for container in self.cluster.all_hosts(): self.real_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) path_on_cluster = self.copy_upgrade_rpm_to_cluster() symlink = os.path.join(get_install_directory(), 'link.rpm') self.cluster.exec_cmd_on_host( self.cluster.master, 'ln -s %s %s' % (path_on_cluster, symlink)) self.upgrade_and_assert_success(symlink)
def run_prestoadmin_expect(self, command, expect_statements): temp_script = os.path.join(get_install_directory(), 'tmp.expect') script_content = '#!/usr/bin/expect\n' + \ 'spawn %s %s\n%s' % \ (get_presto_admin_path(), command, expect_statements) self.cluster.write_content_to_host(script_content, temp_script, self.cluster.master) self.cluster.exec_cmd_on_host( self.cluster.master, 'chmod +x %s' % temp_script) return self.cluster.exec_cmd_on_host( self.cluster.master, temp_script)
def test_configuration_preserved_on_upgrade(self): book_content = 'Call me Ishmael ... FINIS' book_path = '/etc/presto/moby_dick_abridged' self.run_prestoadmin('configuration deploy') big_files = {} for container in self.cluster.all_hosts(): self.real_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) big_file = self.cluster.exec_cmd_on_host( container, "find /usr -size +2M -ls | " "sort -nk7 | " "tail -1 | " "awk '{print $NF}'").strip() self.cluster.exec_cmd_on_host(container, "cp %s /etc/presto" % (big_file, ), invoke_sudo=True) big_files[container] = os.path.join("/etc/presto", os.path.basename(big_file)) self.cluster.write_content_to_host(book_content, book_path, host=container) self.cluster.exec_cmd_on_host(container, "chown presto:games %s" % (book_path, ), invoke_sudo=True) self.cluster.exec_cmd_on_host(container, "chmod 272 %s" % (book_path, ), invoke_sudo=True) self.assert_file_content(container, book_path, book_content) self.assert_file_perm_owner(container, book_path, '--w-rwx-w-', 'presto', 'games') self.assert_path_exists(container, big_files[container]) self.add_dummy_properties_to_host(self.cluster.slaves[1]) path_on_cluster = self.copy_upgrade_rpm_to_cluster() symlink = os.path.join(get_install_directory(), 'link.rpm') self.cluster.exec_cmd_on_host( self.cluster.master, 'ln -s %s %s' % (path_on_cluster, symlink)) self.run_prestoadmin('server upgrade ' + path_on_cluster) self.assert_dummy_properties(self.cluster.slaves[1]) for container in self.cluster.all_hosts(): self.assert_file_content(container, book_path, book_content) self.assert_file_perm_owner(container, book_path, '--w-rwx-w-', 'presto', 'games') self.assert_path_exists(container, big_files[container])
def test_configuration_preserved_on_upgrade(self): book_content = 'Call me Ishmael ... FINIS' book_path = '/etc/presto/moby_dick_abridged' self.run_prestoadmin('configuration deploy') big_files = {} for container in self.cluster.all_hosts(): self.real_installer.assert_installed(self, container) self.assert_has_default_config(container) self.assert_has_default_catalog(container) big_file = self.cluster.exec_cmd_on_host( container, "find /usr -size +2M -ls | " "sort -nk7 | " "tail -1 | " "awk '{print $NF}'").strip() self.cluster.exec_cmd_on_host( container, "cp %s /etc/presto" % (big_file,), invoke_sudo=True) big_files[container] = os.path.join("/etc/presto", os.path.basename(big_file)) self.cluster.write_content_to_host(book_content, book_path, host=container) self.cluster.exec_cmd_on_host(container, "chown presto:games %s" % (book_path,), invoke_sudo=True) self.cluster.exec_cmd_on_host(container, "chmod 272 %s" % (book_path,), invoke_sudo=True) self.assert_file_content(container, book_path, book_content) self.assert_file_perm_owner(container, book_path, '--w-rwx-w-', 'presto', 'games') self.assert_path_exists(container, big_files[container]) self.add_dummy_properties_to_host(self.cluster.slaves[1]) path_on_cluster = self.copy_upgrade_rpm_to_cluster() symlink = os.path.join(get_install_directory(), 'link.rpm') self.cluster.exec_cmd_on_host(self.cluster.master, 'ln -s %s %s' % (path_on_cluster, symlink)) self.run_prestoadmin('server upgrade ' + path_on_cluster) self.assert_dummy_properties(self.cluster.slaves[1]) for container in self.cluster.all_hosts(): self.assert_file_content(container, book_path, book_content) self.assert_file_perm_owner(container, book_path, '--w-rwx-w-', 'presto', 'games') self.assert_path_exists(container, big_files[container])
def test_collect_logs_nonstandard_location(self): self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER) version = self.cluster.exec_cmd_on_host(self.cluster.master, 'rpm -q --qf \"%{VERSION}\\n\" presto-server-rpm') if '127t' not in version: print 'test_collect_logs_nonstandard_location only valid for 127t' return new_log_location = '/var/presto' self._add_custom_log_location(new_log_location) self.run_prestoadmin('server start') self._collect_logs_and_unzip() collected_logs_dir = os.path.join(get_install_directory(), 'logs') self.assert_path_exists(self.cluster.master, os.path.join(collected_logs_dir, ' presto-admin.log')) for host in self.cluster.all_internal_hosts(): host_directory = os.path.join(collected_logs_dir, host) self.assert_path_exists(self.cluster.master, os.path.join(host_directory, 'server.log')) self.assert_path_exists(self.cluster.master, os.path.join(host_directory, 'launcher.log'))
def tear_down(self): for host in self.all_hosts(): # Remove the rm -rf /var/log/presto when the following issue # is resolved https://github.com/prestodb/presto-admin/issues/226 script = """ sudo service presto stop sudo rpm -e presto-server-rpm rm -rf {install_dir} rm -rf ~/prestoadmin*.tar.gz rm -rf {config_dir} sudo rm -rf /etc/presto/ sudo rm -rf /usr/lib/presto/ sudo rm -rf /tmp/presto-debug sudo rm -rf /tmp/presto-debug-remote sudo rm -rf /var/log/presto rm -rf {mount_dir} """.format(install_dir=get_install_directory(), config_dir=get_config_directory(), mount_dir=self.mount_dir) self.run_script_on_host(script, host)
# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ product tests for presto-admin plugin commands """ import os from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase from tests.product.cluster_types import STANDALONE_PA_CLUSTER from tests.product.config_dir_utils import get_install_directory TMP_JAR_PATH = os.path.join(get_install_directory(), 'pretend.jar') STD_REMOTE_PATH = '/usr/lib/presto/plugin/hive-cdh5/pretend.jar' class TestPlugin(BaseProductTestCase): def setUp(self): super(TestPlugin, self).setUp() self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) def deploy_jar_to_master(self): self.cluster.write_content_to_host('A PRETEND JAR', TMP_JAR_PATH, self.cluster.master) def test_basic_add_jars(self): self.upload_topology() self.deploy_jar_to_master()
def test_run_script(self): script_path = os.path.join(get_install_directory(), 'script.sh') # basic run script self.cluster.write_content_to_host('#!/bin/bash\necho hello', script_path, self.cluster.master) output = self.run_prestoadmin('file run %s' % script_path) self.assertEqualIgnoringOrder(output, """[slave2] out: hello [slave2] out: [slave1] out: hello [slave1] out: [master] out: hello [master] out: [slave3] out: hello [slave3] out: """) # specify remote directory self.cluster.write_content_to_host('#!/bin/bash\necho hello', script_path, self.cluster.master) output = self.run_prestoadmin('file run %s' % script_path) self.assertEqualIgnoringOrder(output, """[slave2] out: hello [slave2] out: [slave1] out: hello [slave1] out: [master] out: hello [master] out: [slave3] out: hello [slave3] out: """) # remote and local are the same self.cluster.write_content_to_host('#!/bin/bash\necho hello', '/tmp/script.sh', self.cluster.master) output = self.run_prestoadmin('file run %s' % script_path) self.assertEqualIgnoringOrder(output, """[slave2] out: hello [slave2] out: [slave1] out: hello [slave1] out: [master] out: hello [master] out: [slave3] out: hello [slave3] out: """) # invalid script self.cluster.write_content_to_host('not a valid script', script_path, self.cluster.master) output = self.run_prestoadmin('file run %s' % script_path, raise_error=False) self.assertEqualIgnoringOrder(output, """ Fatal error: [slave2] sudo() received nonzero return code 127 while executing! Requested: /tmp/script.sh Executed: sudo -S -p 'sudo password:'******'sudo password:'******'sudo password:'******'sudo password:' /bin/bash -l -c "/tmp/script.sh" Aborting. [slave1] out: /tmp/script.sh: line 1: not: command not found [slave1] out: """)
def test_run_script(self): script_path = os.path.join(get_install_directory(), 'script.sh') # basic run script self.cluster.write_content_to_host('#!/bin/bash\necho hello', script_path, self.cluster.master) output = self.run_prestoadmin('file run %s' % script_path) self.assertEqualIgnoringOrder( output, """[slave2] out: hello [slave2] out: [slave1] out: hello [slave1] out: [master] out: hello [master] out: [slave3] out: hello [slave3] out: """) # specify remote directory self.cluster.write_content_to_host('#!/bin/bash\necho hello', script_path, self.cluster.master) output = self.run_prestoadmin('file run %s' % script_path) self.assertEqualIgnoringOrder( output, """[slave2] out: hello [slave2] out: [slave1] out: hello [slave1] out: [master] out: hello [master] out: [slave3] out: hello [slave3] out: """) # remote and local are the same self.cluster.write_content_to_host('#!/bin/bash\necho hello', '/tmp/script.sh', self.cluster.master) output = self.run_prestoadmin('file run %s' % script_path) self.assertEqualIgnoringOrder( output, """[slave2] out: hello [slave2] out: [slave1] out: hello [slave1] out: [master] out: hello [master] out: [slave3] out: hello [slave3] out: """) # invalid script self.cluster.write_content_to_host('not a valid script', script_path, self.cluster.master) output = self.run_prestoadmin('file run %s' % script_path, raise_error=False) self.assertEqualIgnoringOrder( output, """ Fatal error: [slave2] sudo() received nonzero return code 127 while executing! Requested: /tmp/script.sh Executed: sudo -S -p 'sudo password:'******'sudo password:'******'sudo password:'******'sudo password:' /bin/bash -l -c "/tmp/script.sh" Aborting. [slave1] out: /tmp/script.sh: line 1: not: command not found [slave1] out: """)
# distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ product tests for presto-admin plugin commands """ import os from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider from tests.product.base_product_case import BaseProductTestCase from tests.product.cluster_types import STANDALONE_PA_CLUSTER from tests.product.config_dir_utils import get_install_directory TMP_JAR_PATH = os.path.join(get_install_directory(), 'pretend.jar') STD_REMOTE_PATH = '/usr/lib/presto/lib/plugin/hive-cdh5/pretend.jar' class TestPlugin(BaseProductTestCase): def setUp(self): super(TestPlugin, self).setUp() self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER) def deploy_jar_to_master(self): self.cluster.write_content_to_host('A PRETEND JAR', TMP_JAR_PATH, self.cluster.master) def test_basic_add_jars(self): self.upload_topology() self.deploy_jar_to_master()
def assert_installed(testcase, msg=None): cluster = testcase.cluster cluster.exec_cmd_on_host(cluster.master, 'test -x %s' % get_install_directory())