def run_script_from_prestoadmin_dir(self, script_contents, host='',
raise_error=True, **kwargs):
if not host:
host = self.cluster.master
script_contents = self.replace_keywords(script_contents,
**kwargs)
temp_script = os.path.join(get_install_directory(), 'tmp.sh')
self.cluster.write_content_to_host(
'#!/bin/bash\ncd %s\n%s' % (get_install_directory(), script_contents),
temp_script, host)
self.cluster.exec_cmd_on_host(
host, 'chmod +x %s' % temp_script)
return self.cluster.exec_cmd_on_host(
host, temp_script, raise_error=raise_error)
def run_script_from_prestoadmin_dir(self, script_contents, host='',
raise_error=True, **kwargs):
if not host:
host = self.cluster.master
script_contents = self.replace_keywords(script_contents,
**kwargs)
temp_script = os.path.join(get_install_directory(), 'tmp.sh')
self.cluster.write_content_to_host(
'#!/bin/bash\ncd %s\n%s' % (get_install_directory(), script_contents),
temp_script, host)
self.cluster.exec_cmd_on_host(
host, 'chmod +x %s' % temp_script)
return self.cluster.exec_cmd_on_host(
host, temp_script, raise_error=raise_error)
def test_collect_logs_nonstandard_location(self):
self.setup_cluster(NoHadoopBareImageProvider(),
STANDALONE_PRESTO_CLUSTER)
version = self.cluster.exec_cmd_on_host(
self.cluster.master,
'rpm -q --qf \"%{VERSION}\\n\" ' + PRESTO_RPM_NAME)
if '127t' not in version:
print 'test_collect_logs_nonstandard_location only valid for 127t'
return
new_log_location = '/var/presto'
self._add_custom_log_location(new_log_location)
self.run_prestoadmin('server start')
self._collect_logs_and_unzip()
collected_logs_dir = os.path.join(get_install_directory(), 'logs')
self.assert_path_exists(
self.cluster.master,
os.path.join(collected_logs_dir, ' presto-admin.log'))
for host in self.cluster.all_internal_hosts():
host_directory = os.path.join(collected_logs_dir, host)
self.assert_path_exists(self.cluster.master,
os.path.join(host_directory, 'server.log'))
self.assert_path_exists(
self.cluster.master,
os.path.join(host_directory, 'launcher.log'))
def _assert_no_logs_downloaded(self):
self._collect_logs_and_unzip()
collected_logs_dir = os.path.join(get_install_directory(), 'logs')
self.assert_path_exists(self.cluster.master, os.path.join(collected_logs_dir, 'presto-admin.log'))
for host in self.cluster.all_internal_hosts():
host_directory = os.path.join(collected_logs_dir, host)
self.assert_path_exists(self.cluster.master, host_directory)
self.assert_path_removed(self.cluster.master, os.path.join(host_directory, '*'))
def _assert_no_logs_downloaded(self):
self._collect_logs_and_unzip()
collected_logs_dir = os.path.join(get_install_directory(), 'logs')
self.assert_path_exists(self.cluster.master, os.path.join(collected_logs_dir, 'presto-admin.log'))
for host in self.cluster.all_internal_hosts():
host_directory = os.path.join(collected_logs_dir, host)
self.assert_path_exists(self.cluster.master, host_directory)
self.assert_path_removed(self.cluster.master, os.path.join(host_directory, '*'))
def run_prestoadmin_expect(self, command, expect_statements):
temp_script = os.path.join(get_install_directory(), 'tmp.expect')
script_content = '#!/usr/bin/expect\n' + \
'spawn %s %s\n%s' % \
(get_presto_admin_path(), command, expect_statements)
self.cluster.write_content_to_host(script_content, temp_script,
self.cluster.master)
self.cluster.exec_cmd_on_host(self.cluster.master,
'chmod +x %s' % temp_script)
return self.cluster.exec_cmd_on_host(self.cluster.master, temp_script)
def test_upgrade_works_with_symlink(self):
self.run_prestoadmin('configuration deploy')
for container in self.cluster.all_hosts():
self.real_installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_catalog(container)
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
symlink = os.path.join(get_install_directory(), 'link.rpm')
self.cluster.exec_cmd_on_host(self.cluster.master, 'ln -s %s %s'
% (path_on_cluster, symlink))
self.upgrade_and_assert_success(symlink)
def test_upgrade_works_with_symlink(self):
self.run_prestoadmin('configuration deploy')
for container in self.cluster.all_hosts():
self.real_installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_catalog(container)
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
symlink = os.path.join(get_install_directory(), 'link.rpm')
self.cluster.exec_cmd_on_host(
self.cluster.master, 'ln -s %s %s' % (path_on_cluster, symlink))
self.upgrade_and_assert_success(symlink)
def run_prestoadmin_expect(self, command, expect_statements):
temp_script = os.path.join(get_install_directory(), 'tmp.expect')
script_content = '#!/usr/bin/expect\n' + \
'spawn %s %s\n%s' % \
(get_presto_admin_path(), command, expect_statements)
self.cluster.write_content_to_host(script_content, temp_script,
self.cluster.master)
self.cluster.exec_cmd_on_host(
self.cluster.master, 'chmod +x %s' % temp_script)
return self.cluster.exec_cmd_on_host(
self.cluster.master, temp_script)
def test_configuration_preserved_on_upgrade(self):
book_content = 'Call me Ishmael ... FINIS'
book_path = '/etc/presto/moby_dick_abridged'
self.run_prestoadmin('configuration deploy')
big_files = {}
for container in self.cluster.all_hosts():
self.real_installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_catalog(container)
big_file = self.cluster.exec_cmd_on_host(
container, "find /usr -size +2M -ls | "
"sort -nk7 | "
"tail -1 | "
"awk '{print $NF}'").strip()
self.cluster.exec_cmd_on_host(container,
"cp %s /etc/presto" % (big_file, ),
invoke_sudo=True)
big_files[container] = os.path.join("/etc/presto",
os.path.basename(big_file))
self.cluster.write_content_to_host(book_content,
book_path,
host=container)
self.cluster.exec_cmd_on_host(container,
"chown presto:games %s" %
(book_path, ),
invoke_sudo=True)
self.cluster.exec_cmd_on_host(container,
"chmod 272 %s" % (book_path, ),
invoke_sudo=True)
self.assert_file_content(container, book_path, book_content)
self.assert_file_perm_owner(container, book_path, '--w-rwx-w-',
'presto', 'games')
self.assert_path_exists(container, big_files[container])
self.add_dummy_properties_to_host(self.cluster.slaves[1])
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
symlink = os.path.join(get_install_directory(), 'link.rpm')
self.cluster.exec_cmd_on_host(
self.cluster.master, 'ln -s %s %s' % (path_on_cluster, symlink))
self.run_prestoadmin('server upgrade ' + path_on_cluster)
self.assert_dummy_properties(self.cluster.slaves[1])
for container in self.cluster.all_hosts():
self.assert_file_content(container, book_path, book_content)
self.assert_file_perm_owner(container, book_path, '--w-rwx-w-',
'presto', 'games')
self.assert_path_exists(container, big_files[container])
def test_configuration_preserved_on_upgrade(self):
book_content = 'Call me Ishmael ... FINIS'
book_path = '/etc/presto/moby_dick_abridged'
self.run_prestoadmin('configuration deploy')
big_files = {}
for container in self.cluster.all_hosts():
self.real_installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_catalog(container)
big_file = self.cluster.exec_cmd_on_host(
container,
"find /usr -size +2M -ls | "
"sort -nk7 | "
"tail -1 | "
"awk '{print $NF}'").strip()
self.cluster.exec_cmd_on_host(
container, "cp %s /etc/presto" % (big_file,), invoke_sudo=True)
big_files[container] = os.path.join("/etc/presto", os.path.basename(big_file))
self.cluster.write_content_to_host(book_content, book_path, host=container)
self.cluster.exec_cmd_on_host(container, "chown presto:games %s" % (book_path,), invoke_sudo=True)
self.cluster.exec_cmd_on_host(container, "chmod 272 %s" % (book_path,), invoke_sudo=True)
self.assert_file_content(container, book_path, book_content)
self.assert_file_perm_owner(container, book_path, '--w-rwx-w-', 'presto', 'games')
self.assert_path_exists(container, big_files[container])
self.add_dummy_properties_to_host(self.cluster.slaves[1])
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
symlink = os.path.join(get_install_directory(), 'link.rpm')
self.cluster.exec_cmd_on_host(self.cluster.master, 'ln -s %s %s'
% (path_on_cluster, symlink))
self.run_prestoadmin('server upgrade ' + path_on_cluster)
self.assert_dummy_properties(self.cluster.slaves[1])
for container in self.cluster.all_hosts():
self.assert_file_content(container, book_path, book_content)
self.assert_file_perm_owner(container, book_path, '--w-rwx-w-', 'presto', 'games')
self.assert_path_exists(container, big_files[container])
def test_collect_logs_nonstandard_location(self):
self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER)
version = self.cluster.exec_cmd_on_host(self.cluster.master, 'rpm -q --qf \"%{VERSION}\\n\" presto-server-rpm')
if '127t' not in version:
print 'test_collect_logs_nonstandard_location only valid for 127t'
return
new_log_location = '/var/presto'
self._add_custom_log_location(new_log_location)
self.run_prestoadmin('server start')
self._collect_logs_and_unzip()
collected_logs_dir = os.path.join(get_install_directory(), 'logs')
self.assert_path_exists(self.cluster.master, os.path.join(collected_logs_dir, ' presto-admin.log'))
for host in self.cluster.all_internal_hosts():
host_directory = os.path.join(collected_logs_dir, host)
self.assert_path_exists(self.cluster.master, os.path.join(host_directory, 'server.log'))
self.assert_path_exists(self.cluster.master, os.path.join(host_directory, 'launcher.log'))
def tear_down(self):
for host in self.all_hosts():
# Remove the rm -rf /var/log/presto when the following issue
# is resolved https://github.com/prestodb/presto-admin/issues/226
script = """
sudo service presto stop
sudo rpm -e presto-server-rpm
rm -rf {install_dir}
rm -rf ~/prestoadmin*.tar.gz
rm -rf {config_dir}
sudo rm -rf /etc/presto/
sudo rm -rf /usr/lib/presto/
sudo rm -rf /tmp/presto-debug
sudo rm -rf /tmp/presto-debug-remote
sudo rm -rf /var/log/presto
rm -rf {mount_dir}
""".format(install_dir=get_install_directory(),
config_dir=get_config_directory(),
mount_dir=self.mount_dir)
self.run_script_on_host(script, host)
def tear_down(self):
for host in self.all_hosts():
# Remove the rm -rf /var/log/presto when the following issue
# is resolved https://github.com/prestodb/presto-admin/issues/226
script = """
sudo service presto stop
sudo rpm -e presto-server-rpm
rm -rf {install_dir}
rm -rf ~/prestoadmin*.tar.gz
rm -rf {config_dir}
sudo rm -rf /etc/presto/
sudo rm -rf /usr/lib/presto/
sudo rm -rf /tmp/presto-debug
sudo rm -rf /tmp/presto-debug-remote
sudo rm -rf /var/log/presto
rm -rf {mount_dir}
""".format(install_dir=get_install_directory(),
config_dir=get_config_directory(),
mount_dir=self.mount_dir)
self.run_script_on_host(script, host)
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
product tests for presto-admin plugin commands
"""
import os
from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider
from tests.product.base_product_case import BaseProductTestCase
from tests.product.cluster_types import STANDALONE_PA_CLUSTER
from tests.product.config_dir_utils import get_install_directory
TMP_JAR_PATH = os.path.join(get_install_directory(), 'pretend.jar')
STD_REMOTE_PATH = '/usr/lib/presto/plugin/hive-cdh5/pretend.jar'
class TestPlugin(BaseProductTestCase):
def setUp(self):
super(TestPlugin, self).setUp()
self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER)
def deploy_jar_to_master(self):
self.cluster.write_content_to_host('A PRETEND JAR', TMP_JAR_PATH,
self.cluster.master)
def test_basic_add_jars(self):
self.upload_topology()
self.deploy_jar_to_master()
def test_run_script(self):
script_path = os.path.join(get_install_directory(), 'script.sh')
# basic run script
self.cluster.write_content_to_host('#!/bin/bash\necho hello',
script_path,
self.cluster.master)
output = self.run_prestoadmin('file run %s' % script_path)
self.assertEqualIgnoringOrder(output, """[slave2] out: hello
[slave2] out:
[slave1] out: hello
[slave1] out:
[master] out: hello
[master] out:
[slave3] out: hello
[slave3] out:
""")
# specify remote directory
self.cluster.write_content_to_host('#!/bin/bash\necho hello',
script_path,
self.cluster.master)
output = self.run_prestoadmin('file run %s' % script_path)
self.assertEqualIgnoringOrder(output, """[slave2] out: hello
[slave2] out:
[slave1] out: hello
[slave1] out:
[master] out: hello
[master] out:
[slave3] out: hello
[slave3] out:
""")
# remote and local are the same
self.cluster.write_content_to_host('#!/bin/bash\necho hello',
'/tmp/script.sh',
self.cluster.master)
output = self.run_prestoadmin('file run %s' % script_path)
self.assertEqualIgnoringOrder(output, """[slave2] out: hello
[slave2] out:
[slave1] out: hello
[slave1] out:
[master] out: hello
[master] out:
[slave3] out: hello
[slave3] out:
""")
# invalid script
self.cluster.write_content_to_host('not a valid script',
script_path,
self.cluster.master)
output = self.run_prestoadmin('file run %s' % script_path,
raise_error=False)
self.assertEqualIgnoringOrder(output, """
Fatal error: [slave2] sudo() received nonzero return code 127 while executing!
Requested: /tmp/script.sh
Executed: sudo -S -p 'sudo password:'******'sudo password:'******'sudo password:'******'sudo password:' /bin/bash -l -c "/tmp/script.sh"
Aborting.
[slave1] out: /tmp/script.sh: line 1: not: command not found
[slave1] out:
""")
def test_run_script(self):
script_path = os.path.join(get_install_directory(), 'script.sh')
# basic run script
self.cluster.write_content_to_host('#!/bin/bash\necho hello',
script_path, self.cluster.master)
output = self.run_prestoadmin('file run %s' % script_path)
self.assertEqualIgnoringOrder(
output, """[slave2] out: hello
[slave2] out:
[slave1] out: hello
[slave1] out:
[master] out: hello
[master] out:
[slave3] out: hello
[slave3] out:
""")
# specify remote directory
self.cluster.write_content_to_host('#!/bin/bash\necho hello',
script_path, self.cluster.master)
output = self.run_prestoadmin('file run %s' % script_path)
self.assertEqualIgnoringOrder(
output, """[slave2] out: hello
[slave2] out:
[slave1] out: hello
[slave1] out:
[master] out: hello
[master] out:
[slave3] out: hello
[slave3] out:
""")
# remote and local are the same
self.cluster.write_content_to_host('#!/bin/bash\necho hello',
'/tmp/script.sh',
self.cluster.master)
output = self.run_prestoadmin('file run %s' % script_path)
self.assertEqualIgnoringOrder(
output, """[slave2] out: hello
[slave2] out:
[slave1] out: hello
[slave1] out:
[master] out: hello
[master] out:
[slave3] out: hello
[slave3] out:
""")
# invalid script
self.cluster.write_content_to_host('not a valid script', script_path,
self.cluster.master)
output = self.run_prestoadmin('file run %s' % script_path,
raise_error=False)
self.assertEqualIgnoringOrder(
output, """
Fatal error: [slave2] sudo() received nonzero return code 127 while executing!
Requested: /tmp/script.sh
Executed: sudo -S -p 'sudo password:'******'sudo password:'******'sudo password:'******'sudo password:' /bin/bash -l -c "/tmp/script.sh"
Aborting.
[slave1] out: /tmp/script.sh: line 1: not: command not found
[slave1] out:
""")
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
product tests for presto-admin plugin commands
"""
import os
from tests.no_hadoop_bare_image_provider import NoHadoopBareImageProvider
from tests.product.base_product_case import BaseProductTestCase
from tests.product.cluster_types import STANDALONE_PA_CLUSTER
from tests.product.config_dir_utils import get_install_directory
TMP_JAR_PATH = os.path.join(get_install_directory(), 'pretend.jar')
STD_REMOTE_PATH = '/usr/lib/presto/lib/plugin/hive-cdh5/pretend.jar'
class TestPlugin(BaseProductTestCase):
def setUp(self):
super(TestPlugin, self).setUp()
self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER)
def deploy_jar_to_master(self):
self.cluster.write_content_to_host('A PRETEND JAR', TMP_JAR_PATH,
self.cluster.master)
def test_basic_add_jars(self):
self.upload_topology()
self.deploy_jar_to_master()
def assert_installed(testcase, msg=None):
cluster = testcase.cluster
cluster.exec_cmd_on_host(cluster.master,
'test -x %s' % get_install_directory())
def assert_installed(testcase, msg=None):
cluster = testcase.cluster
cluster.exec_cmd_on_host(cluster.master, 'test -x %s' % get_install_directory())
