def test_install_interactive(self):
installer = StandalonePrestoInstaller(self)
self.cluster.write_content_to_host(
'connector.name=jmx',
os.path.join(get_catalog_directory(), 'jmx.properties'),
self.cluster.master)
rpm_name = installer.copy_presto_rpm_to_master()
self.write_test_configs(self.cluster)
additional_keywords = {
'user': self.cluster.user,
'rpm_dir': self.cluster.rpm_cache_dir,
'rpm': rpm_name
}
cmd_output = self.run_script_from_prestoadmin_dir(
'echo -e "%(user)s\n22\n%(master)s\n%(slave1)s\n" | '
'./presto-admin server install %(rpm_dir)s/%(rpm)s ',
**additional_keywords)
actual = cmd_output.splitlines()
expected = [
r'Enter user name for SSH connection to all nodes: '
r'\[root\] '
r'Enter port number for SSH connections to all nodes: '
r'\[22\] '
r'Enter host name or IP address for coordinator node. '
r'Enter an external host name or ip address if this is a '
r'multi-node cluster: \[localhost\] '
r'Enter host names or IP addresses for worker nodes '
r'separated by spaces: '
r'\[localhost\] Using rpm_specifier as a local path',
r'Package deployed successfully on: ' +
self.cluster.internal_master,
r'Package installed successfully on: ' +
self.cluster.internal_master,
r'Package deployed successfully on: ' +
self.cluster.internal_slaves[0],
r'Package installed successfully on: ' +
self.cluster.internal_slaves[0],
r'Deploying configuration on: ' + self.cluster.internal_master,
r'Deploying jmx.properties, tpch.properties catalog '
r'configurations on: ' + self.cluster.internal_master,
r'Deploying configuration on: ' + self.cluster.internal_slaves[0],
r'Deploying jmx.properties, tpch.properties catalog '
r'configurations on: ' + self.cluster.internal_slaves[0],
r'Deploying rpm on .*\.\.\.', r'Deploying rpm on .*\.\.\.',
r'Fetching local presto rpm at path: .*',
r'Found existing rpm at: .*'
]
self.assertRegexpMatchesLineByLine(actual, expected)
for container in [self.cluster.master, self.cluster.slaves[0]]:
installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_catalog(container)
self.assert_has_jmx_catalog(container)
def test_install_interactive_with_hostnames(self):
installer = StandalonePrestoInstaller(self)
self.cluster.write_content_to_host(
'connector.name=jmx',
os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'),
self.cluster.master
)
rpm_name = installer.copy_presto_rpm_to_master()
self.write_test_configs(self.cluster)
cmd_output = self.run_script_from_prestoadmin_dir(
'echo -e "root\n22\n%(master)s\n%(slave1)s\n" | '
'./presto-admin server install /mnt/presto-admin/%(rpm)s ',
rpm=rpm_name)
actual = cmd_output.splitlines()
expected = [r'Enter user name for SSH connection to all nodes: '
r'\[root\] '
r'Enter port number for SSH connections to all nodes: '
r'\[22\] '
r'Enter host name or IP address for coordinator node. '
r'Enter an external host name or ip address if this is a '
r'multi-node cluster: \[localhost\] '
r'Enter host names or IP addresses for worker nodes '
r'separated by spaces: '
r'\[localhost\] Using rpm_specifier as a local path',
r'Package deployed successfully on: ' +
self.cluster.internal_master,
r'Package installed successfully on: ' +
self.cluster.internal_master,
r'Package deployed successfully on: ' +
self.cluster.internal_slaves[0],
r'Package installed successfully on: ' +
self.cluster.internal_slaves[0],
r'Deploying configuration on: ' +
self.cluster.internal_master,
r'Deploying jmx.properties, tpch.properties connector '
r'configurations on: ' +
self.cluster.internal_master,
r'Deploying configuration on: ' +
self.cluster.internal_slaves[0],
r'Deploying jmx.properties, tpch.properties connector '
r'configurations on: ' +
self.cluster.internal_slaves[0],
r'Deploying rpm on .*\.\.\.',
r'Deploying rpm on .*\.\.\.',
r'Fetching local presto rpm at path: .*',
r'Found existing rpm at: .*'
]
self.assertRegexpMatchesLineByLine(actual, expected)
for container in [self.cluster.master,
self.cluster.slaves[0]]:
installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
self.assert_has_jmx_connector(container)
def test_install_interactive_with_ips(self):
installer = StandalonePrestoInstaller(self)
ips = self.cluster.get_ip_address_dict()
rpm_name = installer.copy_presto_rpm_to_master()
self.write_test_configs(self.cluster)
additional_keywords = {
'rpm': rpm_name,
'master_ip': ips[self.cluster.master],
'slave1_ip': ips[self.cluster.slaves[0]]
}
cmd_output = self.run_script_from_prestoadmin_dir(
'echo -e "root\n22\n%(master_ip)s\n%(slave1_ip)s\n" | '
'./presto-admin server install /mnt/presto-admin/%(rpm)s ',
**additional_keywords).splitlines()
expected = [r'Enter user name for SSH connection to all nodes: '
r'\[root\] '
r'Enter port number for SSH connections to all nodes: '
r'\[22\] '
r'Enter host name or IP address for coordinator node. '
r'Enter an external host name or ip address if this is a '
r'multi-node cluster: \[localhost\] '
r'Enter host names or IP addresses for worker nodes '
r'separated by spaces: '
r'\[localhost\] Using rpm_specifier as a local path',
r'Package deployed successfully on: ' +
ips[self.cluster.master],
r'Package installed successfully on: ' +
ips[self.cluster.master],
r'Package deployed successfully on: ' +
ips[self.cluster.slaves[0]],
r'Package installed successfully on: ' +
ips[self.cluster.slaves[0]],
r'Deploying configuration on: ' +
ips[self.cluster.master],
r'Deploying tpch.properties connector '
r'configurations on: ' +
ips[self.cluster.master] + r' ',
r'Deploying configuration on: ' +
ips[self.cluster.slaves[0]],
r'Deploying tpch.properties connector '
r'configurations on: ' +
ips[self.cluster.slaves[0]] + r' ',
r'Deploying rpm on .*\.\.\.',
r'Deploying rpm on .*\.\.\.',
r'Fetching local presto rpm at path: .*',
r'Found existing rpm at: .*']
cmd_output.sort()
expected.sort()
for expected_regexp, actual_line in zip(expected, cmd_output):
self.assertRegexpMatches(actual_line, expected_regexp)
self.assert_installed_with_regex_configs(
self.cluster.master,
[self.cluster.slaves[0]])
def test_install_with_wrong_topology(self):
installer = StandalonePrestoInstaller(self)
rpm_name = installer.copy_presto_rpm_to_master()
topology = {'coordinator': 'dummy_master', 'workers': ['slave1']}
self.upload_topology(topology)
expected = 'u\'dummy_master\' is not a valid ip address or' \
' host name.' \
' More detailed information can be found in ' \
'/var/log/prestoadmin/presto-admin.log\n'
self.assertRaisesRegexp(OSError,
expected,
self.run_prestoadmin,
'server install /mnt/presto-admin/%(rpm)s ',
rpm=rpm_name)
def test_install_with_malformed_topology(self):
installer = StandalonePrestoInstaller(self)
rpm_name = installer.copy_presto_rpm_to_master()
topology = {'coordinator': 'master', 'workers': 'slave1' 'slave2'}
self.upload_topology(topology)
expected = 'Workers must be of type list. Found <type \'unicode\'>.' \
' More detailed information can be found in ' \
'/var/log/prestoadmin/presto-admin.log'
self.assertRaisesRegexp(OSError,
expected,
self.run_prestoadmin,
'server install /mnt/presto-admin/%(rpm)s ',
rpm=rpm_name)
def test_install_with_wrong_topology(self):
installer = StandalonePrestoInstaller(self)
rpm_name = installer.copy_presto_rpm_to_master()
topology = {'coordinator': 'dummy_master', 'workers': ['slave1']}
self.upload_topology(topology)
expected = 'u\'dummy_master\' is not a valid ip address or' \
' host name.' \
' More detailed information can be found in ' \
'/var/log/prestoadmin/presto-admin.log\n'
self.assertRaisesRegexp(OSError,
expected,
self.run_prestoadmin,
'server install /mnt/presto-admin/%(rpm)s ',
rpm=rpm_name)
def test_install_interactive_with_ips(self):
installer = StandalonePrestoInstaller(self)
ips = self.cluster.get_ip_address_dict()
rpm_name = installer.copy_presto_rpm_to_master()
self.write_test_configs(self.cluster)
additional_keywords = {
'rpm': rpm_name,
'master_ip': ips[self.cluster.master],
'slave1_ip': ips[self.cluster.slaves[0]]
}
cmd_output = self.run_script_from_prestoadmin_dir(
'echo -e "root\n22\n%(master_ip)s\n%(slave1_ip)s\n" | '
'./presto-admin server install /mnt/presto-admin/%(rpm)s ',
**additional_keywords).splitlines()
expected = [
r'Enter user name for SSH connection to all nodes: '
r'\[root\] '
r'Enter port number for SSH connections to all nodes: '
r'\[22\] '
r'Enter host name or IP address for coordinator node. '
r'Enter an external host name or ip address if this is a '
r'multi-node cluster: \[localhost\] '
r'Enter host names or IP addresses for worker nodes '
r'separated by spaces: '
r'\[localhost\] Using rpm_specifier as a local path',
r'Package deployed successfully on: ' + ips[self.cluster.master],
r'Package installed successfully on: ' + ips[self.cluster.master],
r'Package deployed successfully on: ' +
ips[self.cluster.slaves[0]],
r'Package installed successfully on: ' +
ips[self.cluster.slaves[0]], r'Deploying configuration on: ' +
ips[self.cluster.master], r'Deploying tpch.properties connector '
r'configurations on: ' + ips[self.cluster.master] + r' ',
r'Deploying configuration on: ' + ips[self.cluster.slaves[0]],
r'Deploying tpch.properties connector '
r'configurations on: ' + ips[self.cluster.slaves[0]] + r' ',
r'Deploying rpm on .*\.\.\.', r'Deploying rpm on .*\.\.\.',
r'Fetching local presto rpm at path: .*',
r'Found existing rpm at: .*'
]
cmd_output.sort()
expected.sort()
for expected_regexp, actual_line in zip(expected, cmd_output):
self.assertRegexpMatches(actual_line, expected_regexp)
self.assert_installed_with_regex_configs(self.cluster.master,
[self.cluster.slaves[0]])
def test_install_interactive_with_hostnames(self):
installer = StandalonePrestoInstaller(self)
self.cluster.write_content_to_host(
'connector.name=jmx',
os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'),
self.cluster.master)
rpm_name = installer.copy_presto_rpm_to_master()
self.write_test_configs(self.cluster)
cmd_output = self.run_script_from_prestoadmin_dir(
'echo -e "root\n22\n%(master)s\n%(slave1)s\n" | '
'./presto-admin server install /mnt/presto-admin/%(rpm)s ',
rpm=rpm_name)
actual = cmd_output.splitlines()
expected = [
r'Enter user name for SSH connection to all nodes: '
r'\[root\] '
r'Enter port number for SSH connections to all nodes: '
r'\[22\] '
r'Enter host name or IP address for coordinator node. '
r'Enter an external host name or ip address if this is a '
r'multi-node cluster: \[localhost\] '
r'Enter host names or IP addresses for worker nodes '
r'separated by spaces: '
r'\[localhost\] Deploying rpm on .*\.\.\.',
r'Package deployed successfully on: ' +
self.cluster.internal_master,
r'Package installed successfully on: ' +
self.cluster.internal_master,
r'Package deployed successfully on: ' +
self.cluster.internal_slaves[0],
r'Package installed successfully on: ' +
self.cluster.internal_slaves[0],
r'Deploying configuration on: ' + self.cluster.internal_master,
r'Deploying jmx.properties, tpch.properties connector '
r'configurations on: ' + self.cluster.internal_master,
r'Deploying configuration on: ' + self.cluster.internal_slaves[0],
r'Deploying jmx.properties, tpch.properties connector '
r'configurations on: ' + self.cluster.internal_slaves[0],
r'Deploying rpm on .*\.\.\.'
]
self.assertRegexpMatchesLineByLine(actual, expected)
for container in [self.cluster.master, self.cluster.slaves[0]]:
installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
self.assert_has_jmx_connector(container)
def test_install_with_malformed_topology(self):
installer = StandalonePrestoInstaller(self)
rpm_name = installer.copy_presto_rpm_to_master()
topology = {'coordinator': 'master',
'workers': 'slave1' 'slave2'}
self.upload_topology(topology)
expected = 'Workers must be of type list. Found <type \'unicode\'>.' \
' More detailed information can be found in ' \
'/var/log/prestoadmin/presto-admin.log'
self.assertRaisesRegexp(OSError,
expected,
self.run_prestoadmin,
'server install /mnt/presto-admin/%(rpm)s ',
rpm=rpm_name)
def test_install_with_no_perm_to_local_path(self):
installer = StandalonePrestoInstaller(self)
rpm_name = installer.copy_presto_rpm_to_master()
self.upload_topology()
self.run_prestoadmin("configuration deploy")
script = 'chmod 600 /mnt/presto-admin/%(rpm)s; su app-admin -c ' \
'"./presto-admin server install /mnt/presto-admin/%(rpm)s "'
error_msg = '\nFatal error: [%(host)s] error: ' \
'/mnt/presto-admin/%(rpm)s: ' \
'open failed: Permission denied\n\nAborting.\n'
expected = ''
for host in self.cluster.all_internal_hosts():
expected += error_msg % {'host': host, 'rpm': rpm_name}
actual = self.run_script_from_prestoadmin_dir(script, rpm=rpm_name,
raise_error=False)
self.assertEqualIgnoringOrder(actual, expected)
def test_install_non_root_user(self):
installer = StandalonePrestoInstaller(self)
self.upload_topology({
"coordinator": "master",
"workers": ["slave1", "slave2", "slave3"],
"username": "******"
})
rpm_name = installer.copy_presto_rpm_to_master(cluster=self.cluster)
self.write_test_configs(self.cluster)
self.run_prestoadmin(
'server install {rpm_dir}/{name} -p password'.format(
rpm_dir=self.cluster.rpm_cache_dir, name=rpm_name))
for container in self.cluster.all_hosts():
installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_catalog(container)
def test_install_non_root_user(self):
installer = StandalonePrestoInstaller(self)
self.upload_topology({
"coordinator": "master",
"workers": ["slave1", "slave2", "slave3"],
"username": "******"
})
rpm_name = installer.copy_presto_rpm_to_master(cluster=self.cluster)
self.write_test_configs(self.cluster)
self.run_prestoadmin('server install ' +
os.path.join(self.cluster.mount_dir, rpm_name) +
' -p password')
for container in self.cluster.all_hosts():
installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
def test_install_non_root_user(self):
installer = StandalonePrestoInstaller(self)
self.upload_topology(
{"coordinator": "master",
"workers": ["slave1", "slave2", "slave3"],
"username": "******"}
)
rpm_name = installer.copy_presto_rpm_to_master(cluster=self.cluster)
self.write_test_configs(self.cluster)
self.run_prestoadmin(
'server install ' + os.path.join(self.cluster.mount_dir, rpm_name) + ' -p password'
)
for container in self.cluster.all_hosts():
installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
def test_install_interactive(self):
installer = StandalonePrestoInstaller(self)
self.cluster.write_content_to_host(
'connector.name=jmx',
os.path.join(get_catalog_directory(), 'jmx.properties'),
self.cluster.master)
ips = self.cluster.get_ip_address_dict()
self.upload_topology({
"coordinator":
ips[self.cluster.internal_master],
"workers": [
ips[self.cluster.internal_slaves[0]],
ips[self.cluster.internal_slaves[1]],
ips[self.cluster.internal_slaves[2]]
],
"username":
"******"
})
rpm_name = installer.copy_presto_rpm_to_master()
self.write_test_configs(self.cluster,
coordinator=ips[self.cluster.internal_master])
additional_keywords = {
'user': "******",
'rpm_dir': self.cluster.rpm_cache_dir,
'rpm': rpm_name
}
self.run_script_from_prestoadmin_dir(
'echo -e "%(user)s\n22\n%(master)s\n%(slave1)s\n" | '
'./presto-admin server install %(rpm_dir)s/%(rpm)s -p password',
**additional_keywords)
self.assert_installed_with_regex_configs(self.cluster.master, [
self.cluster.slaves[0], self.cluster.slaves[1],
self.cluster.slaves[2]
])
for container in self.cluster.all_hosts():
installer.assert_installed(self, container)
self.assert_has_default_catalog(container)
self.assert_has_jmx_catalog(container)
class TestPackageInstall(BaseProductTestCase):
def setUp(self):
super(TestPackageInstall, self).setUp()
self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER)
self.upload_topology()
self.installer = StandalonePrestoInstaller(self)
def tearDown(self):
self._assert_uninstall()
super(TestPackageInstall, self).tearDown()
def _assert_uninstall(self):
output = self.run_prestoadmin(
'package uninstall presto-server-rpm --force')
for container in self.cluster.all_hosts():
self.installer.assert_uninstalled(container, msg=output)
@attr('smoketest')
def test_package_installer(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
# install
output = self.run_prestoadmin('package install %(rpm)s',
rpm=os.path.join(
self.cluster.rpm_cache_dir,
rpm_name))
for container in self.cluster.all_hosts():
self.installer.assert_installed(self, container, msg=output)
# uninstall
output = self.run_prestoadmin('package uninstall presto-server-rpm')
for container in self.cluster.all_hosts():
self.installer.assert_uninstalled(container, msg=output)
def test_install_using_dash_h(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
# install onto master and slave2
output = self.run_prestoadmin(
'package install %(rpm)s -H %(master)s,%(slave2)s',
rpm=os.path.join(self.cluster.rpm_cache_dir, rpm_name))
self.installer.assert_installed(self, self.cluster.master, msg=output)
self.installer.assert_installed(self,
self.cluster.slaves[1],
msg=output)
self.installer.assert_uninstalled(self.cluster.slaves[0], msg=output)
self.installer.assert_uninstalled(self.cluster.slaves[2], msg=output)
# uninstall on slave2
output = self.run_prestoadmin(
'package uninstall presto-server-rpm -H %(slave2)s')
self.installer.assert_installed(self, self.cluster.master, msg=output)
for container in self.cluster.slaves:
self.installer.assert_uninstalled(container, msg=output)
# uninstall on rest
output = self.run_prestoadmin(
'package uninstall presto-server-rpm --force')
for container in self.cluster.all_hosts():
self.installer.assert_uninstalled(container, msg=output)
def test_install_exclude_nodes(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
output = self.run_prestoadmin(
'package install %(rpm)s -x %(master)s,%(slave2)s',
rpm=os.path.join(self.cluster.rpm_cache_dir, rpm_name))
# install
self.installer.assert_uninstalled(self.cluster.master, msg=output)
self.installer.assert_uninstalled(self.cluster.slaves[1], msg=output)
self.installer.assert_installed(self,
self.cluster.slaves[0],
msg=output)
self.installer.assert_installed(self,
self.cluster.slaves[2],
msg=output)
# uninstall
output = self.run_prestoadmin(
'package uninstall presto-server-rpm -x %(master)s,%(slave2)s')
for container in self.cluster.all_hosts():
self.installer.assert_uninstalled(container, msg=output)
# skip this tests as it depends on OS package names
@attr('quarantine')
@docker_only
def test_install_rpm_missing_dependency(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
self.cluster.exec_cmd_on_host(self.cluster.master,
'rpm -e --nodeps python-2.6.6')
self.assertRaisesRegexp(OSError,
'package python-2.6.6 is not installed',
self.cluster.exec_cmd_on_host,
self.cluster.master, 'rpm -q python-2.6.6')
cmd_output = self.run_prestoadmin(
'package install %(rpm)s -H %(master)s',
rpm=os.path.join(self.cluster.rpm_cache_dir, rpm_name),
raise_error=False)
expected = self.replace_keywords(
"""
Fatal error: [%(master)s] sudo() received nonzero return code 1 while \
executing!
Requested: rpm -i /opt/prestoadmin/packages/%(rpm)s
Executed: sudo -S -p 'sudo password:'******'quarantine')
@docker_only
def test_install_rpm_with_nodeps(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
self.cluster.exec_cmd_on_host(self.cluster.master,
'rpm -e --nodeps python-2.6.6')
self.assertRaisesRegexp(OSError,
'package python-2.6.6 is not installed',
self.cluster.exec_cmd_on_host,
self.cluster.master, 'rpm -q python-2.6.6')
cmd_output = self.run_prestoadmin(
'package install %(rpm)s -H %(master)s --nodeps',
rpm=os.path.join(self.cluster.rpm_cache_dir, rpm_name))
expected = 'Deploying rpm on %(host)s...\n' \
'Package deployed successfully on: %(host)s\n' \
'Package installed successfully on: %(host)s' \
% {'host': self.cluster.internal_master}
self.assertEqualIgnoringOrder(expected, cmd_output)
class TestPackageInstall(BaseProductTestCase):
def setUp(self):
super(TestPackageInstall, self).setUp()
self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER)
self.upload_topology()
self.installer = StandalonePrestoInstaller(self)
def tearDown(self):
self._assert_uninstall()
super(TestPackageInstall, self).tearDown()
def _assert_uninstall(self):
output = self.run_prestoadmin('package uninstall presto-server-rpm --force')
for container in self.cluster.all_hosts():
self.installer.assert_uninstalled(container, msg=output)
@attr('smoketest')
def test_package_installer(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
# install
output = self.run_prestoadmin('package install %(rpm)s',
rpm=os.path.join(self.cluster.rpm_cache_dir, rpm_name))
for container in self.cluster.all_hosts():
self.installer.assert_installed(self, container, msg=output)
# uninstall
output = self.run_prestoadmin('package uninstall presto-server-rpm')
for container in self.cluster.all_hosts():
self.installer.assert_uninstalled(container, msg=output)
def test_install_using_dash_h(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
# install onto master and slave2
output = self.run_prestoadmin('package install %(rpm)s -H %(master)s,%(slave2)s',
rpm=os.path.join(self.cluster.rpm_cache_dir, rpm_name))
self.installer.assert_installed(self, self.cluster.master, msg=output)
self.installer.assert_installed(self, self.cluster.slaves[1], msg=output)
self.installer.assert_uninstalled(self.cluster.slaves[0], msg=output)
self.installer.assert_uninstalled(self.cluster.slaves[2], msg=output)
# uninstall on slave2
output = self.run_prestoadmin('package uninstall presto-server-rpm -H %(slave2)s')
self.installer.assert_installed(self, self.cluster.master, msg=output)
for container in self.cluster.slaves:
self.installer.assert_uninstalled(container, msg=output)
# uninstall on rest
output = self.run_prestoadmin('package uninstall presto-server-rpm --force')
for container in self.cluster.all_hosts():
self.installer.assert_uninstalled(container, msg=output)
def test_install_exclude_nodes(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
output = self.run_prestoadmin('package install %(rpm)s -x %(master)s,%(slave2)s',
rpm=os.path.join(self.cluster.rpm_cache_dir, rpm_name))
# install
self.installer.assert_uninstalled(self.cluster.master, msg=output)
self.installer.assert_uninstalled(self.cluster.slaves[1], msg=output)
self.installer.assert_installed(self, self.cluster.slaves[0], msg=output)
self.installer.assert_installed(self, self.cluster.slaves[2], msg=output)
# uninstall
output = self.run_prestoadmin('package uninstall presto-server-rpm -x %(master)s,%(slave2)s')
for container in self.cluster.all_hosts():
self.installer.assert_uninstalled(container, msg=output)
# skip this tests as it depends on OS package names
@attr('quarantine')
@docker_only
def test_install_rpm_missing_dependency(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
self.cluster.exec_cmd_on_host(
self.cluster.master, 'rpm -e --nodeps python-2.6.6')
self.assertRaisesRegexp(OSError,
'package python-2.6.6 is not installed',
self.cluster.exec_cmd_on_host,
self.cluster.master,
'rpm -q python-2.6.6')
cmd_output = self.run_prestoadmin(
'package install %(rpm)s -H %(master)s',
rpm=os.path.join(self.cluster.rpm_cache_dir, rpm_name),
raise_error=False)
expected = self.replace_keywords("""
Fatal error: [%(master)s] sudo() received nonzero return code 1 while \
executing!
Requested: rpm -i /opt/prestoadmin/packages/%(rpm)s
Executed: sudo -S -p 'sudo password:'******'quarantine')
@docker_only
def test_install_rpm_with_nodeps(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
self.cluster.exec_cmd_on_host(
self.cluster.master, 'rpm -e --nodeps python-2.6.6')
self.assertRaisesRegexp(OSError,
'package python-2.6.6 is not installed',
self.cluster.exec_cmd_on_host,
self.cluster.master,
'rpm -q python-2.6.6')
cmd_output = self.run_prestoadmin(
'package install %(rpm)s -H %(master)s --nodeps',
rpm=os.path.join(self.cluster.rpm_cache_dir, rpm_name)
)
expected = 'Deploying rpm on %(host)s...\n' \
'Package deployed successfully on: %(host)s\n' \
'Package installed successfully on: %(host)s' \
% {'host': self.cluster.internal_master}
self.assertEqualIgnoringOrder(expected, cmd_output)
class TestPackageInstall(BaseProductTestCase):
def setUp(self):
super(TestPackageInstall, self).setUp()
self.setup_cluster(self.PA_ONLY_CLUSTER)
self.upload_topology()
self.installer = StandalonePrestoInstaller(self)
@attr("smoketest")
def test_package_install(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
output = self.run_prestoadmin("package install " "/mnt/presto-admin/%(rpm)s", rpm=rpm_name)
for container in self.cluster.all_hosts():
self.installer.assert_installed(self, container, msg=output)
def test_install_coord_using_dash_h(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
output = self.run_prestoadmin("package install /mnt/presto-admin/%(rpm)s -H %(master)s", rpm=rpm_name)
self.installer.assert_installed(self, self.cluster.master)
for slave in self.cluster.slaves:
self.installer.assert_uninstalled(slave, msg=output)
def test_install_worker_using_dash_h(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
output = self.run_prestoadmin("package install /mnt/presto-admin/%(rpm)s -H %(slave1)s", rpm=rpm_name)
self.installer.assert_installed(self, self.cluster.slaves[0], msg=output)
self.installer.assert_uninstalled(self.cluster.master, msg=output)
self.installer.assert_uninstalled(self.cluster.slaves[1], msg=output)
self.installer.assert_uninstalled(self.cluster.slaves[2], msg=output)
def test_install_workers_using_dash_h(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
output = self.run_prestoadmin(
"package install /mnt/presto-admin/" "%(rpm)s -H %(slave1)s,%(slave2)s", rpm=rpm_name
)
self.installer.assert_installed(self, self.cluster.slaves[0], msg=output)
self.installer.assert_installed(self, self.cluster.slaves[1], msg=output)
self.installer.assert_uninstalled(self.cluster.master, msg=output)
self.installer.assert_uninstalled(self.cluster.slaves[2], msg=output)
def test_install_exclude_coord(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
output = self.run_prestoadmin("package install /mnt/presto-admin/" "%(rpm)s -x %(master)s", rpm=rpm_name)
self.installer.assert_uninstalled(self.cluster.master, msg=output)
for slave in self.cluster.slaves:
self.installer.assert_installed(self, slave, msg=output)
def test_install_exclude_worker(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
output = self.run_prestoadmin("package install /mnt/presto-admin/" "%(rpm)s -x %(slave1)s", rpm=rpm_name)
self.installer.assert_uninstalled(self.cluster.slaves[0], msg=output)
self.installer.assert_installed(self, self.cluster.slaves[1], msg=output)
self.installer.assert_installed(self, self.cluster.master, msg=output)
self.installer.assert_installed(self, self.cluster.slaves[2], msg=output)
def test_install_exclude_workers(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
output = self.run_prestoadmin(
"package install /mnt/presto-admin/" "%(rpm)s -x %(slave1)s,%(slave2)s", rpm=rpm_name
)
self.installer.assert_uninstalled(self.cluster.slaves[0], msg=output)
self.installer.assert_uninstalled(self.cluster.slaves[1], msg=output)
self.installer.assert_installed(self, self.cluster.master, msg=output)
self.installer.assert_installed(self, self.cluster.slaves[2], msg=output)
def test_install_invalid_path(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
cmd_output = self.run_prestoadmin("package install /mnt/presto-admin" "/invalid-path/presto.rpm", rpm=rpm_name)
error = (
"\nFatal error: [%s] error: "
"/mnt/presto-admin/invalid-path/presto.rpm: open failed: "
"No such file or directory\n\nAborting.\n"
)
expected = ""
for host in self.cluster.all_internal_hosts():
expected += error % host
self.assertEqualIgnoringOrder(cmd_output, expected)
def test_install_no_path_arg(self):
self.installer.copy_presto_rpm_to_master()
output = self.run_prestoadmin("package install", raise_error=False)
self.assertEqual(
output,
"Incorrect number of arguments to task.\n\n"
"Displaying detailed information for task "
"'package install':\n\n"
" Install the rpm package on the cluster\n"
" \n Args:\n"
" local_path: Absolute path to the rpm"
" to be installed\n "
"--nodeps (optional): Flag to indicate if "
"rpm install\n"
" should ignore c"
"hecking package dependencies. Equivalent\n"
" to adding --nodeps flag to rpm "
"-i.\n\n",
)
def test_install_already_installed(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
self.run_prestoadmin("package install /mnt/presto-admin/%(rpm)s -H " "%(master)s", rpm=rpm_name)
self.installer.assert_installed(self, self.cluster.master)
cmd_output = self.run_prestoadmin("package install /mnt/presto-admin/%(rpm)s -H %(master)s", rpm=rpm_name)
expected = self.escape_for_regex(
self.replace_keywords(
"""
Fatal error: [%(master)s] sudo() received nonzero return code 1 while \
executing!
Requested: rpm -i /opt/prestoadmin/packages/%(rpm)s
Executed: sudo -S -p 'sudo password:'******'sudo password:' /bin/bash -l -c "rpm -i \
/opt/prestoadmin/packages/%(rpm)s"
Aborting.
Deploying rpm on %(master)s...
Package deployed successfully on: %(master)s
[%(master)s] out: error: Failed dependencies:
[%(master)s] out: python >= 2.4 is needed by %(rpm_basename)s
[%(master)s] out: """,
**self.installer.get_keywords(rpm_name)
)
self.assertRegexpMatchesLineByLine(cmd_output.splitlines(), self.escape_for_regex(expected).splitlines())
@docker_only
def test_install_rpm_with_nodeps(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
self.cluster.exec_cmd_on_host(self.cluster.master, "rpm -e --nodeps python-2.6.6")
self.assertRaisesRegexp(
OSError,
"package python-2.6.6 is not installed",
self.cluster.exec_cmd_on_host,
self.cluster.master,
"rpm -q python-2.6.6",
)
cmd_output = self.run_prestoadmin(
"package install /mnt/presto-admin/%(rpm)s -H %(master)s --nodeps", rpm=rpm_name
)
expected = (
"Deploying rpm on %(host)s...\n"
"Package deployed successfully on: %(host)s\n"
"Package installed successfully on: %(host)s" % {"host": self.cluster.internal_master}
)
self.assertEqualIgnoringOrder(expected, cmd_output)
class TestPackageInstall(BaseProductTestCase):
def setUp(self):
super(TestPackageInstall, self).setUp()
self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER)
self.upload_topology()
self.installer = StandalonePrestoInstaller(self)
@attr('smoketest')
def test_package_install(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
output = self.run_prestoadmin('package install '
'/mnt/presto-admin/%(rpm)s',
rpm=rpm_name)
for container in self.cluster.all_hosts():
self.installer.assert_installed(self, container,
msg=output)
def test_install_coord_using_dash_h(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
output = self.run_prestoadmin(
'package install /mnt/presto-admin/%(rpm)s -H %(master)s',
rpm=rpm_name)
self.installer.assert_installed(self, self.cluster.master)
for slave in self.cluster.slaves:
self.installer.assert_uninstalled(slave, msg=output)
def test_install_worker_using_dash_h(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
output = self.run_prestoadmin(
'package install /mnt/presto-admin/%(rpm)s -H %(slave1)s',
rpm=rpm_name)
self.installer.assert_installed(self, self.cluster.slaves[0],
msg=output)
self.installer.assert_uninstalled(self.cluster.master, msg=output)
self.installer.assert_uninstalled(self.cluster.slaves[1], msg=output)
self.installer.assert_uninstalled(self.cluster.slaves[2], msg=output)
def test_install_workers_using_dash_h(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
output = self.run_prestoadmin('package install /mnt/presto-admin/'
'%(rpm)s -H %(slave1)s,%(slave2)s',
rpm=rpm_name)
self.installer.assert_installed(self, self.cluster.slaves[0],
msg=output)
self.installer.assert_installed(self, self.cluster.slaves[1],
msg=output)
self.installer.assert_uninstalled(self.cluster.master, msg=output)
self.installer.assert_uninstalled(self.cluster.slaves[2], msg=output)
def test_install_exclude_coord(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
output = self.run_prestoadmin('package install /mnt/presto-admin/'
'%(rpm)s -x %(master)s', rpm=rpm_name)
self.installer.assert_uninstalled(self.cluster.master, msg=output)
for slave in self.cluster.slaves:
self.installer.assert_installed(self, slave, msg=output)
def test_install_exclude_worker(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
output = self.run_prestoadmin('package install /mnt/presto-admin/'
'%(rpm)s -x %(slave1)s', rpm=rpm_name)
self.installer.assert_uninstalled(self.cluster.slaves[0], msg=output)
self.installer.assert_installed(self, self.cluster.slaves[1],
msg=output)
self.installer.assert_installed(self, self.cluster.master, msg=output)
self.installer.assert_installed(self, self.cluster.slaves[2],
msg=output)
def test_install_exclude_workers(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
output = self.run_prestoadmin('package install /mnt/presto-admin/'
'%(rpm)s -x %(slave1)s,%(slave2)s',
rpm=rpm_name)
self.installer.assert_uninstalled(self.cluster.slaves[0], msg=output)
self.installer.assert_uninstalled(self.cluster.slaves[1], msg=output)
self.installer.assert_installed(self, self.cluster.master, msg=output)
self.installer.assert_installed(self, self.cluster.slaves[2],
msg=output)
def test_install_invalid_path(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
cmd_output = self.run_prestoadmin('package install /mnt/presto-admin'
'/invalid-path/presto.rpm',
rpm=rpm_name, raise_error=False)
error = '\nFatal error: [%s] error: ' \
'/mnt/presto-admin/invalid-path/presto.rpm: open failed: ' \
'No such file or directory\n\nAborting.\n'
expected = ''
for host in self.cluster.all_internal_hosts():
expected += error % host
self.assertEqualIgnoringOrder(cmd_output, expected)
def test_install_no_path_arg(self):
self.installer.copy_presto_rpm_to_master()
output = self.run_prestoadmin('package install', raise_error=False)
self.assertEqual(output, 'Incorrect number of arguments to task.\n\n'
'Displaying detailed information for task '
'\'package install\':\n\n'
' Install the rpm package on the cluster\n'
' \n Args:\n'
' local_path: Absolute path to the rpm'
' to be installed\n '
'--nodeps (optional): Flag to indicate if '
'rpm install\n'
' should ignore c'
'hecking package dependencies. Equivalent\n'
' to adding --nodeps flag to rpm '
'-i.\n\n')
def test_install_already_installed(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
self.run_prestoadmin('package install /mnt/presto-admin/%(rpm)s -H '
'%(master)s', rpm=rpm_name)
self.installer.assert_installed(self, self.cluster.master)
cmd_output = self.run_prestoadmin(
'package install /mnt/presto-admin/%(rpm)s -H %(master)s',
rpm=rpm_name, raise_error=False)
expected = self.escape_for_regex(self.replace_keywords("""
Fatal error: [%(master)s] sudo() received nonzero return code 1 while \
executing!
Requested: rpm -i /opt/prestoadmin/packages/%(rpm)s
Executed: sudo -S -p 'sudo password:'******'package install '
'/etc/opt/prestoadmin/config.json',
raise_error=False)
error = """
Fatal error: \[%s\] error: (/etc/opt/prestoadmin/config.json: )?not an rpm \
package
Aborting.
"""
expected = ''
for host in self.cluster.all_internal_hosts():
expected += error % host
self.assertRegexpMatchesLineByLine(cmd_output.splitlines(),
expected.splitlines())
@docker_only
def test_install_rpm_missing_dependency(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
self.cluster.exec_cmd_on_host(
self.cluster.master, 'rpm -e --nodeps python-2.6.6')
self.assertRaisesRegexp(OSError,
'package python-2.6.6 is not installed',
self.cluster.exec_cmd_on_host,
self.cluster.master,
'rpm -q python-2.6.6')
cmd_output = self.run_prestoadmin(
'package install /mnt/presto-admin/%(rpm)s -H %(master)s',
rpm=rpm_name, raise_error=False)
expected = self.replace_keywords("""
Fatal error: [%(master)s] sudo() received nonzero return code 1 while \
executing!
Requested: rpm -i /opt/prestoadmin/packages/%(rpm)s
Executed: sudo -S -p 'sudo password:'******'rpm -e --nodeps python-2.6.6')
self.assertRaisesRegexp(OSError,
'package python-2.6.6 is not installed',
self.cluster.exec_cmd_on_host,
self.cluster.master,
'rpm -q python-2.6.6')
cmd_output = self.run_prestoadmin(
'package install /mnt/presto-admin/%(rpm)s -H %(master)s --nodeps',
rpm=rpm_name
)
expected = 'Deploying rpm on %(host)s...\n' \
'Package deployed successfully on: %(host)s\n' \
'Package installed successfully on: %(host)s' \
% {'host': self.cluster.internal_master}
self.assertEqualIgnoringOrder(expected, cmd_output)
class TestServerInstall(BaseProductTestCase):
default_workers_config_with_slave1_ = """coordinator=false
discovery.uri=http://slave1:8080
http-server.http.port=8080
query.max-memory-per-node=512MB
query.max-memory=50GB\n"""
default_coord_config_with_slave1_ = """coordinator=true
discovery-server.enabled=true
discovery.uri=http://slave1:8080
http-server.http.port=8080
node-scheduler.include-coordinator=false
query.max-memory-per-node=512MB
query.max-memory=50GB\n"""
default_workers_config_regex_ = """coordinator=false
discovery.uri=http:.*:8080
http-server.http.port=8080
query.max-memory-per-node=512MB
query.max-memory=50GB\n"""
default_coord_config_regex_ = """coordinator=true
discovery-server.enabled=true
discovery.uri=http:.*:8080
http-server.http.port=8080
node-scheduler.include-coordinator=false
query.max-memory-per-node=512MB
query.max-memory=50GB\n"""
def setUp(self):
super(TestServerInstall, self).setUp()
self.setup_cluster(self.PA_ONLY_CLUSTER)
self.installer = StandalonePrestoInstaller(self)
def assert_common_configs(self, container):
self.installer.assert_installed(self, container)
self.assert_file_content(container, '/etc/presto/jvm.config',
self.default_jvm_config_)
self.assert_node_config(container, self.default_node_properties_)
self.assert_has_default_connector(container)
def assert_installed_with_configs(self, master, slaves):
self.assert_common_configs(master)
self.assert_file_content(master,
'/etc/presto/config.properties',
self.default_coord_config_with_slave1_)
for container in slaves:
self.assert_common_configs(container)
self.assert_file_content(container,
'/etc/presto/config.properties',
self.default_workers_config_with_slave1_)
def assert_installed_with_regex_configs(self, master, slaves):
self.assert_common_configs(master)
self.assert_file_content_regex(master,
'/etc/presto/config.properties',
self.default_coord_config_regex_)
for container in slaves:
self.assert_common_configs(container)
self.assert_file_content_regex(container,
'/etc/presto/config.properties',
self.default_workers_config_regex_)
@attr('smoketest')
def test_install(self, dummy=False):
self.upload_topology()
cmd_output = self.installer.install(dummy)
expected = installed_all_hosts_output
actual = cmd_output.splitlines()
self.assertEqual(sorted(expected), sorted(actual))
for container in self.cluster.all_hosts():
self.installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
def test_install_worker_is_pa_master(self):
topology = {"coordinator": "slave1",
"workers": ["master", "slave2", "slave3"]}
self.upload_topology(topology)
cmd_output = self.installer.install(dummy=True)
expected = install_with_worker_pa_master_out
actual = cmd_output.splitlines()
self.assertEqual(sorted(expected), sorted(actual))
self.assert_installed_with_configs(
self.cluster.slaves[0],
[self.cluster.slaves[1],
self.cluster.slaves[2],
self.cluster.master])
def test_install_ext_host_is_pa_master(self):
topology = {"coordinator": "slave1",
"workers": ["slave2", "slave3"]}
self.upload_topology(topology)
cmd_output = self.installer.install(dummy=True)
expected = install_with_ext_host_pa_master_out
actual = cmd_output.splitlines()
self.assertEqual(sorted(expected), sorted(actual))
self.assert_installed_with_configs(
self.cluster.slaves[0],
[self.cluster.slaves[1],
self.cluster.slaves[2]])
def test_install_when_connector_json_exists(self):
topology = {"coordinator": "master",
"workers": ["slave1"]}
self.upload_topology(topology)
self.cluster.write_content_to_host(
'connector.name=jmx',
os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'),
self.cluster.master
)
cmd_output = self.installer.install(dummy=True)
expected = ['Deploying rpm on master...',
'Deploying rpm on slave1...',
'Package deployed successfully on: slave1',
'Package installed successfully on: slave1',
'Package deployed successfully on: master',
'Package installed successfully on: master',
'Deploying configuration on: master',
'Deploying jmx.properties, tpch.properties '
'connector configurations on: master ',
'Deploying configuration on: slave1',
'Deploying jmx.properties, tpch.properties '
'connector configurations on: slave1 ']
actual = cmd_output.splitlines()
self.assertEqual(sorted(expected), sorted(actual))
for container in [self.cluster.master,
self.cluster.slaves[0]]:
self.installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
self.assert_has_jmx_connector(container)
def test_install_when_topology_has_ips(self):
ips = self.cluster.get_ip_address_dict()
topology = {"coordinator": ips[self.cluster.master],
"workers": [ips[self.cluster.slaves[0]]]}
self.upload_topology(topology)
self.cluster.write_content_to_host(
'connector.name=jmx',
os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'),
self.cluster.master
)
cmd_output = self.installer.install(dummy=True).splitlines()
expected = [
r'Deploying rpm on %s...' % ips[self.cluster.master],
r'Deploying rpm on %s...' % ips[self.cluster.slaves[0]],
r'Package deployed successfully on: ' + ips[
self.cluster.master],
r'Package installed successfully on: ' + ips[
self.cluster.master],
r'Package deployed successfully on: '
+ ips[self.cluster.slaves[0]],
r'Package installed successfully on: ' +
ips[self.cluster.slaves[0]],
r'Deploying configuration on: ' +
ips[self.cluster.master],
r'Deploying jmx.properties, tpch.properties '
r'connector configurations on: ' +
ips[self.cluster.master],
r'Deploying configuration on: ' +
ips[self.cluster.slaves[0]],
r'Deploying jmx.properties, tpch.properties '
r'connector configurations on: ' +
ips[self.cluster.slaves[0]]]
cmd_output.sort()
expected.sort()
self.assertRegexpMatchesLineByLine(expected, cmd_output)
self.assert_installed_with_regex_configs(
self.cluster.master,
[self.cluster.slaves[0]])
for container in [self.cluster.master,
self.cluster.slaves[0]]:
self.assert_has_jmx_connector(container)
def test_install_interactive_with_hostnames(self):
self.cluster.write_content_to_host(
'connector.name=jmx',
os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'),
self.cluster.master
)
rpm_name = self.installer.copy_presto_rpm_to_master()
self.write_test_configs(self.cluster)
cmd_output = self.run_script_from_prestoadmin_dir(
'echo -e "root\n22\n%(master)s\n%(slave1)s\n" | '
'./presto-admin server install /mnt/presto-admin/%(rpm)s ',
rpm=rpm_name)
actual = cmd_output.splitlines()
expected = [r'Enter user name for SSH connection to all nodes: '
r'\[root\] '
r'Enter port number for SSH connections to all nodes: '
r'\[22\] '
r'Enter host name or IP address for coordinator node. '
r'Enter an external host name or ip address if this is a '
r'multi-node cluster: \[localhost\] '
r'Enter host names or IP addresses for worker nodes '
r'separated by spaces: '
r'\[localhost\] Deploying rpm on .*\.\.\.',
r'Package deployed successfully on: ' +
self.cluster.internal_master,
r'Package installed successfully on: ' +
self.cluster.internal_master,
r'Package deployed successfully on: ' +
self.cluster.internal_slaves[0],
r'Package installed successfully on: ' +
self.cluster.internal_slaves[0],
r'Deploying configuration on: ' +
self.cluster.internal_master,
r'Deploying jmx.properties, tpch.properties connector '
r'configurations on: ' +
self.cluster.internal_master,
r'Deploying configuration on: ' +
self.cluster.internal_slaves[0],
r'Deploying jmx.properties, tpch.properties connector '
r'configurations on: ' +
self.cluster.internal_slaves[0],
r'Deploying rpm on .*\.\.\.']
self.assertRegexpMatchesLineByLine(actual, expected)
for container in [self.cluster.master,
self.cluster.slaves[0]]:
self.installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
self.assert_has_jmx_connector(container)
def test_install_interactive_with_ips(self):
ips = self.cluster.get_ip_address_dict()
rpm_name = self.installer.copy_presto_rpm_to_master()
self.write_test_configs(self.cluster)
additional_keywords = {
'rpm': rpm_name,
'master_ip': ips[self.cluster.master],
'slave1_ip': ips[self.cluster.slaves[0]]
}
cmd_output = self.run_script_from_prestoadmin_dir(
'echo -e "root\n22\n%(master_ip)s\n%(slave1_ip)s\n" | '
'./presto-admin server install /mnt/presto-admin/%(rpm)s ',
**additional_keywords).splitlines()
expected = [r'Enter user name for SSH connection to all nodes: '
r'\[root\] '
r'Enter port number for SSH connections to all nodes: '
r'\[22\] '
r'Enter host name or IP address for coordinator node. '
r'Enter an external host name or ip address if this is a '
r'multi-node cluster: \[localhost\] '
r'Enter host names or IP addresses for worker nodes '
r'separated by spaces: '
r'\[localhost\] Deploying rpm on .*\.\.\.',
r'Package deployed successfully on: ' +
ips[self.cluster.master],
r'Package installed successfully on: ' +
ips[self.cluster.master],
r'Package deployed successfully on: '
+ ips[self.cluster.slaves[0]],
r'Package installed successfully on: '
+ ips[self.cluster.slaves[0]],
r'Deploying configuration on: ' +
ips[self.cluster.master],
r'Deploying tpch.properties connector '
r'configurations on: ' +
ips[self.cluster.master],
r'Deploying configuration on: ' +
ips[self.cluster.slaves[0]],
r'Deploying tpch.properties connector '
r'configurations on: ' +
ips[self.cluster.slaves[0]],
r'Deploying rpm on .*\.\.\.']
cmd_output.sort()
expected.sort()
for expected_regexp, actual_line in zip(expected, cmd_output):
self.assertRegexpMatches(actual_line, expected_regexp)
self.assert_installed_with_regex_configs(
self.cluster.master,
[self.cluster.slaves[0]])
def test_install_with_wrong_topology(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
topology = {'coordinator': 'dummy_master', 'workers': ['slave1']}
self.upload_topology(topology)
expected = 'u\'dummy_master\' is not a valid ip address or' \
' host name.' \
' More detailed information can be found in ' \
'/var/log/prestoadmin/presto-admin.log\n'
self.assertRaisesRegexp(OSError,
expected,
self.run_prestoadmin,
'server install /mnt/presto-admin/%(rpm)s ',
rpm=rpm_name)
def test_install_with_malformed_topology(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
topology = {'coordinator': 'master',
'workers': 'slave1' 'slave2'}
self.upload_topology(topology)
expected = 'Workers must be of type list. Found <type \'unicode\'>.' \
' More detailed information can be found in ' \
'/var/log/prestoadmin/presto-admin.log'
self.assertRaisesRegexp(OSError,
expected,
self.run_prestoadmin,
'server install /mnt/presto-admin/%(rpm)s ',
rpm=rpm_name)
def test_install_with_malformed_connector(self):
self.upload_topology()
self.cluster.write_content_to_host(
'connectr.typo:invalid',
os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'),
self.cluster.master
)
actual_out = self.installer.install(dummy=True)
expected = 'Underlying exception:\n Catalog configuration ' \
'jmx.properties does not contain connector.name'
self.assertRegexpMatches(actual_out, expected)
for container in self.cluster.all_hosts():
self.installer.assert_installed(self, container)
self.assert_has_default_config(container)
def test_connection_to_coord_lost(self):
down_node = self.cluster.internal_slaves[0]
topology = {"coordinator": down_node,
"workers": [self.cluster.internal_master,
self.cluster.internal_slaves[1],
self.cluster.internal_slaves[2]]}
self.upload_topology(topology=topology)
self.cluster.stop_host(
self.cluster.slaves[0])
actual_out = self.installer.install(dummy=True)
self.assertRegexpMatches(
actual_out,
self.down_node_connection_error(down_node)
)
for container in [self.cluster.master,
self.cluster.slaves[1],
self.cluster.slaves[2]]:
self.assert_common_configs(container)
self.assert_file_content(
container,
'/etc/presto/config.properties',
self.default_workers_config_with_slave1_.replace(
down_node, self.cluster.get_down_hostname(down_node)
)
)
@docker_only
def test_install_with_no_perm_to_local_path(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
self.upload_topology()
self.run_prestoadmin("configuration deploy")
script = 'chmod 600 /mnt/presto-admin/%(rpm)s; su app-admin -c ' \
'"./presto-admin server install /mnt/presto-admin/%(rpm)s "'
error_msg = '\nFatal error: [%(host)s] error: ' \
'/mnt/presto-admin/%(rpm)s: ' \
'open failed: Permission denied\n\nAborting.\n'
expected = ''
for host in self.cluster.all_internal_hosts():
expected += error_msg % {'host': host, 'rpm': rpm_name}
actual = self.run_script_from_prestoadmin_dir(script, rpm=rpm_name)
self.assertEqualIgnoringOrder(actual, expected)
def test_install_twice(self):
self.test_install(dummy=True)
output = self.installer.install(dummy=True)
with open(os.path.join(LOCAL_RESOURCES_DIR, 'install_twice.txt'), 'r') \
as f:
expected = f.read()
expected = self.escape_for_regex(
self.replace_keywords(expected))
self.assertRegexpMatchesLineByLine(output.splitlines(),
expected.splitlines())
for container in self.cluster.all_hosts():
self.installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
class TestServerUpgrade(BaseProductTestCase):
def setUp(self):
super(TestServerUpgrade, self).setUp()
self.setup_cluster(NoHadoopBareImageProvider(),
STANDALONE_PRESTO_CLUSTER)
self.dummy_installer = StandalonePrestoInstaller(
self, (os.path.join(prestoadmin.main_dir, 'tests', 'product',
'resources'), 'dummy-rpm.rpm'))
self.real_installer = StandalonePrestoInstaller(self)
def start_and_assert_started(self):
cmd_output = self.run_prestoadmin('server start')
process_per_host = self.get_process_per_host(cmd_output.splitlines())
self.assert_started(process_per_host)
#
# The dummy RPM is not guaranteed to have any functionality beyond not
# including any real payload and adding the random README file. It's a
# hacky one-off that satisfies the requirement of having *something* to
# upgrade to without downloading another copy of the real RPM. This is NOT
# the place to test functionality that the presto-server-rpm normally
# provides, because the dummy rpm probably doesn't provide it, or worse,
# provides an old and/or broken version of it.
#
def assert_upgraded_to_dummy_rpm(self, hosts):
for container in hosts:
# Still should have the same configs
self.dummy_installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
# However, dummy_rpm.rpm removes /usr/lib/presto/lib and
# /usr/lib/presto/lib/plugin
self.assert_path_removed(container, '/usr/lib/presto/lib')
self.assert_path_removed(container, '/usr/lib/presto/lib/plugin')
# And adds /usr/lib/presto/README.txt
self.assert_path_exists(container, '/usr/lib/presto/README.txt')
# And modifies the text of the readme in
# /usr/shared/doc/presto/README.txt
self.assert_file_content_regex(
container, '/usr/shared/doc/presto/README.txt',
r'.*New line of text here.$')
@attr('smoketest')
def test_upgrade(self):
self.start_and_assert_started()
self.run_prestoadmin('configuration deploy')
for container in self.cluster.all_hosts():
self.real_installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
self.upgrade_and_assert_success(path_on_cluster)
def upgrade_and_assert_success(self, path_on_cluster, extra_arguments=''):
self.run_prestoadmin('server upgrade ' + path_on_cluster +
extra_arguments)
self.assert_upgraded_to_dummy_rpm(self.cluster.all_hosts())
@docker_only
def test_rolling_upgrade(self):
# Test that if a node is down, and then you upgrade again, it works
self.run_prestoadmin('configuration deploy')
self.cluster.stop_host(self.cluster.slaves[0])
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
self.run_prestoadmin('server upgrade ' + path_on_cluster,
raise_error=False)
running_hosts = self.cluster.all_hosts()[:]
running_hosts.remove(self.cluster.slaves[0])
self.assert_upgraded_to_dummy_rpm(running_hosts)
self.cluster.start_host(self.cluster.slaves[0])
self.retry(lambda: self.upgrade_and_assert_success(path_on_cluster))
def copy_upgrade_rpm_to_cluster(self):
rpm_name = self.dummy_installer.copy_presto_rpm_to_master()
return os.path.join(self.cluster.mount_dir, rpm_name)
def test_upgrade_fails_given_directory(self):
dir_on_cluster = '/opt/prestoadmin'
self.assertRaisesRegexp(OSError,
'RPM file not found at %s.' % dir_on_cluster,
self.run_prestoadmin,
'server upgrade ' + dir_on_cluster)
def test_upgrade_works_with_symlink(self):
self.run_prestoadmin('configuration deploy')
for container in self.cluster.all_hosts():
self.real_installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
symlink = '/opt/prestoadmin/link.rpm'
self.cluster.exec_cmd_on_host(
self.cluster.master, 'ln -s %s %s' % (path_on_cluster, symlink))
self.upgrade_and_assert_success(symlink)
def test_configuration_preserved_on_upgrade(self):
self.run_prestoadmin('configuration deploy')
for container in self.cluster.all_hosts():
self.real_installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
self.add_dummy_properties_to_host(self.cluster.slaves[1])
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
symlink = '/opt/prestoadmin/link.rpm'
self.cluster.exec_cmd_on_host(
self.cluster.master, 'ln -s %s %s' % (path_on_cluster, symlink))
self.run_prestoadmin('server upgrade ' + path_on_cluster)
self.assert_dummy_properties(self.cluster.slaves[1])
def test_upgrade_non_root_user(self):
self.upload_topology({
"coordinator": "master",
"workers": ["slave1", "slave2", "slave3"],
"username": "******"
})
self.run_prestoadmin('configuration deploy -p password')
for container in self.cluster.all_hosts():
self.real_installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
self.upgrade_and_assert_success(path_on_cluster,
extra_arguments=' -p password')
def add_dummy_properties_to_host(self, host):
self.cluster.write_content_to_host('com.facebook.presto=INFO',
'/etc/presto/log.properties', host)
self.cluster.write_content_to_host('dummy config file',
'/etc/presto/jvm.config', host)
def assert_dummy_properties(self, host):
# assert log properties file is there
self.assert_file_content(host, '/etc/presto/log.properties',
'com.facebook.presto=INFO')
# assert dummy jvm config is there too
self.assert_file_content(host, '/etc/presto/jvm.config',
'dummy config file')
class TestServerUpgrade(BaseProductTestCase):
def setUp(self):
super(TestServerUpgrade, self).setUp()
self.setup_cluster(NoHadoopBareImageProvider(),
STANDALONE_PRESTO_CLUSTER)
self.dummy_installer = StandalonePrestoInstaller(
self, (os.path.join(prestoadmin.main_dir, 'tests', 'product',
'resources'), 'dummy-rpm.rpm'))
self.real_installer = StandalonePrestoInstaller(self)
def start_and_assert_started(self):
cmd_output = self.run_prestoadmin('server start')
process_per_host = self.get_process_per_host(cmd_output.splitlines())
self.assert_started(process_per_host)
#
# The dummy RPM is not guaranteed to have any functionality beyond not
# including any real payload and adding the random README file. It's a
# hacky one-off that satisfies the requirement of having *something* to
# upgrade to without downloading another copy of the real RPM. This is NOT
# the place to test functionality that the presto-server-rpm normally
# provides, because the dummy rpm probably doesn't provide it, or worse,
# provides an old and/or broken version of it.
#
def assert_upgraded_to_dummy_rpm(self, hosts):
for container in hosts:
# Still should have the same configs
self.dummy_installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_catalog(container)
# However, dummy_rpm.rpm removes /usr/lib/presto/lib and
# /usr/lib/presto/lib/plugin
self.assert_path_removed(container, '/usr/lib/presto/lib')
self.assert_path_removed(container, '/usr/lib/presto/lib/plugin')
# And modifies the text of the readme in
# /usr/shared/doc/presto/README.txt
self.assert_file_content_regex(
container, '/usr/shared/doc/presto/README.txt',
r'.*New line of text here.$')
@attr('smoketest')
def test_upgrade(self):
self.start_and_assert_started()
self.run_prestoadmin('configuration deploy')
for container in self.cluster.all_hosts():
self.real_installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_catalog(container)
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
self.upgrade_and_assert_success(path_on_cluster)
def upgrade_and_assert_success(self, path_on_cluster, extra_arguments=''):
self.run_prestoadmin('server upgrade ' + path_on_cluster +
extra_arguments)
self.assert_upgraded_to_dummy_rpm(self.cluster.all_hosts())
def copy_upgrade_rpm_to_cluster(self):
rpm_name = self.dummy_installer.copy_presto_rpm_to_master()
return os.path.join(self.cluster.rpm_cache_dir, rpm_name)
def test_upgrade_fails_given_directory(self):
dir_on_cluster = '/opt/prestoadmin'
self.assertRaisesRegexp(OSError,
'RPM file not found at %s.' % dir_on_cluster,
self.run_prestoadmin,
'server upgrade ' + dir_on_cluster)
def test_upgrade_works_with_symlink(self):
self.run_prestoadmin('configuration deploy')
for container in self.cluster.all_hosts():
self.real_installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_catalog(container)
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
symlink = os.path.join(get_install_directory(), 'link.rpm')
self.cluster.exec_cmd_on_host(
self.cluster.master, 'ln -s %s %s' % (path_on_cluster, symlink))
self.upgrade_and_assert_success(symlink)
def test_configuration_preserved_on_upgrade(self):
book_content = 'Call me Ishmael ... FINIS'
book_path = '/etc/presto/moby_dick_abridged'
self.run_prestoadmin('configuration deploy')
big_files = {}
for container in self.cluster.all_hosts():
self.real_installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_catalog(container)
big_file = self.cluster.exec_cmd_on_host(
container, "find /usr -size +2M -ls | "
"sort -nk7 | "
"tail -1 | "
"awk '{print $NF}'").strip()
self.cluster.exec_cmd_on_host(container,
"cp %s /etc/presto" % (big_file, ),
invoke_sudo=True)
big_files[container] = os.path.join("/etc/presto",
os.path.basename(big_file))
self.cluster.write_content_to_host(book_content,
book_path,
host=container)
self.cluster.exec_cmd_on_host(container,
"chown presto:games %s" %
(book_path, ),
invoke_sudo=True)
self.cluster.exec_cmd_on_host(container,
"chmod 272 %s" % (book_path, ),
invoke_sudo=True)
self.assert_file_content(container, book_path, book_content)
self.assert_file_perm_owner(container, book_path, '--w-rwx-w-',
'presto', 'games')
self.assert_path_exists(container, big_files[container])
self.add_dummy_properties_to_host(self.cluster.slaves[1])
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
symlink = os.path.join(get_install_directory(), 'link.rpm')
self.cluster.exec_cmd_on_host(
self.cluster.master, 'ln -s %s %s' % (path_on_cluster, symlink))
self.run_prestoadmin('server upgrade ' + path_on_cluster)
self.assert_dummy_properties(self.cluster.slaves[1])
for container in self.cluster.all_hosts():
self.assert_file_content(container, book_path, book_content)
self.assert_file_perm_owner(container, book_path, '--w-rwx-w-',
'presto', 'games')
self.assert_path_exists(container, big_files[container])
def test_upgrade_non_root_user(self):
self.upload_topology({
"coordinator": "master",
"workers": ["slave1", "slave2", "slave3"],
"username": "******"
})
self.run_prestoadmin('configuration deploy -p password')
for container in self.cluster.all_hosts():
self.real_installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_catalog(container)
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
self.upgrade_and_assert_success(path_on_cluster,
extra_arguments=' -p password')
def add_dummy_properties_to_host(self, host):
self.cluster.write_content_to_host('io.prestosql=INFO',
'/etc/presto/log.properties', host)
self.cluster.write_content_to_host('dummy config file',
'/etc/presto/jvm.config', host)
def assert_dummy_properties(self, host):
# assert log properties file is there
self.assert_file_content(host, '/etc/presto/log.properties',
'io.prestosql=INFO')
# assert dummy jvm config is there too
self.assert_file_content(host, '/etc/presto/jvm.config',
'dummy config file')
class TestServerUpgrade(BaseProductTestCase):
def setUp(self):
super(TestServerUpgrade, self).setUp()
self.setup_cluster(NoHadoopBareImageProvider(),
self.STANDALONE_PRESTO_CLUSTER)
self.installer = StandalonePrestoInstaller(self)
def start_and_assert_started(self):
cmd_output = self.run_prestoadmin('server start')
process_per_host = self.get_process_per_host(cmd_output.splitlines())
self.assert_started(process_per_host)
def assert_upgraded_to_dummy_rpm(self, hosts):
for container in hosts:
# Still should have the same configs
self.installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
# However, dummy_rpm.rpm removes /usr/lib/presto/lib and
# /usr/lib/presto/lib/plugin
self.assert_path_removed(container, '/usr/lib/presto/lib')
self.assert_path_removed(container, '/usr/lib/presto/lib/plugin')
# And adds /usr/lib/presto/README.txt
self.assert_path_exists(container, '/usr/lib/presto/README.txt')
# And modifies the text of the readme in
# /usr/shared/doc/presto/README.txt
self.assert_file_content_regex(
container,
'/usr/shared/doc/presto/README.txt',
r'.*New line of text here.$'
)
@attr('smoketest')
def test_upgrade(self):
self.start_and_assert_started()
self.run_prestoadmin('configuration deploy')
for container in self.cluster.all_hosts():
self.installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
self.upgrade_and_assert_success(path_on_cluster)
def upgrade_and_assert_success(self, path_on_cluster):
self.run_prestoadmin('server upgrade ' + path_on_cluster)
self.assert_upgraded_to_dummy_rpm(self.cluster.all_hosts())
@docker_only
def test_rolling_upgrade(self):
# Test that if a node is down, and then you upgrade again, it works
self.run_prestoadmin('configuration deploy')
self.cluster.stop_host(self.cluster.slaves[0])
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
self.run_prestoadmin('server upgrade ' + path_on_cluster,
raise_error=False)
running_hosts = self.cluster.all_hosts()[:]
running_hosts.remove(self.cluster.slaves[0])
self.assert_upgraded_to_dummy_rpm(running_hosts)
self.cluster.start_host(self.cluster.slaves[0])
self.retry(lambda: self.upgrade_and_assert_success(path_on_cluster))
def copy_upgrade_rpm_to_cluster(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
return os.path.join(self.cluster.mount_dir, rpm_name)
class TestServerUpgrade(BaseProductTestCase):
def setUp(self):
super(TestServerUpgrade, self).setUp()
self.setup_cluster(self.STANDALONE_PRESTO_CLUSTER)
self.installer = StandalonePrestoInstaller(self)
def start_and_assert_started(self):
cmd_output = self.run_prestoadmin('server start')
process_per_host = self.get_process_per_host(cmd_output.splitlines())
self.assert_started(process_per_host)
def assert_upgraded_to_dummy_rpm(self, hosts):
for container in hosts:
# Still should have the same configs
self.installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
# However, dummy_rpm.rpm removes /usr/lib/presto/lib and
# /usr/lib/presto/lib/plugin
self.assert_path_removed(container, '/usr/lib/presto/lib')
self.assert_path_removed(container, '/usr/lib/presto/lib/plugin')
# And adds /usr/lib/presto/README.txt
self.assert_path_exists(container, '/usr/lib/presto/README.txt')
# And modifies the text of the readme in
# /usr/shared/doc/presto/README.txt
self.assert_file_content_regex(
container, '/usr/shared/doc/presto/README.txt',
r'.*New line of text here.$')
@attr('smoketest')
def test_upgrade(self):
self.start_and_assert_started()
self.run_prestoadmin('configuration deploy')
for container in self.cluster.all_hosts():
self.installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
self.upgrade_and_assert_success(path_on_cluster)
def upgrade_and_assert_success(self, path_on_cluster):
self.run_prestoadmin('server upgrade ' + path_on_cluster)
self.assert_upgraded_to_dummy_rpm(self.cluster.all_hosts())
@docker_only
def test_rolling_upgrade(self):
# Test that if a node is down, and then you upgrade again, it works
self.run_prestoadmin('configuration deploy')
self.cluster.stop_host(self.cluster.slaves[0])
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
self.run_prestoadmin('server upgrade ' + path_on_cluster)
running_hosts = self.cluster.all_hosts()[:]
running_hosts.remove(self.cluster.slaves[0])
self.assert_upgraded_to_dummy_rpm(running_hosts)
self.cluster.start_host(self.cluster.slaves[0])
self.retry(lambda: self.upgrade_and_assert_success(path_on_cluster))
def copy_upgrade_rpm_to_cluster(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
return os.path.join(self.cluster.mount_dir, rpm_name)
class TestServerUpgrade(BaseProductTestCase):
def setUp(self):
super(TestServerUpgrade, self).setUp()
self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER)
self.dummy_installer = StandalonePrestoInstaller(
self, (os.path.join(prestoadmin.main_dir, 'tests', 'product',
'resources'), 'dummy-rpm.rpm'))
self.real_installer = StandalonePrestoInstaller(self)
def start_and_assert_started(self):
cmd_output = self.run_prestoadmin('server start')
process_per_host = self.get_process_per_host(cmd_output.splitlines())
self.assert_started(process_per_host)
#
# The dummy RPM is not guaranteed to have any functionality beyond not
# including any real payload and adding the random README file. It's a
# hacky one-off that satisfies the requirement of having *something* to
# upgrade to without downloading another copy of the real RPM. This is NOT
# the place to test functionality that the presto-server-rpm normally
# provides, because the dummy rpm probably doesn't provide it, or worse,
# provides an old and/or broken version of it.
#
def assert_upgraded_to_dummy_rpm(self, hosts):
for container in hosts:
# Still should have the same configs
self.dummy_installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_catalog(container)
# However, dummy_rpm.rpm removes /usr/lib/presto/lib and
# /usr/lib/presto/lib/plugin
self.assert_path_removed(container, '/usr/lib/presto/lib')
self.assert_path_removed(container, '/usr/lib/presto/lib/plugin')
# And adds /usr/lib/presto/README.txt
self.assert_path_exists(container, '/usr/lib/presto/README.txt')
# And modifies the text of the readme in
# /usr/shared/doc/presto/README.txt
self.assert_file_content_regex(
container,
'/usr/shared/doc/presto/README.txt',
r'.*New line of text here.$'
)
@attr('smoketest')
def test_upgrade(self):
self.start_and_assert_started()
self.run_prestoadmin('configuration deploy')
for container in self.cluster.all_hosts():
self.real_installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_catalog(container)
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
self.upgrade_and_assert_success(path_on_cluster)
def upgrade_and_assert_success(self, path_on_cluster, extra_arguments=''):
self.run_prestoadmin('server upgrade ' + path_on_cluster + extra_arguments)
self.assert_upgraded_to_dummy_rpm(self.cluster.all_hosts())
def copy_upgrade_rpm_to_cluster(self):
rpm_name = self.dummy_installer.copy_presto_rpm_to_master()
return os.path.join(self.cluster.rpm_cache_dir, rpm_name)
def test_upgrade_fails_given_directory(self):
dir_on_cluster = '/opt/prestoadmin'
self.assertRaisesRegexp(
OSError,
'RPM file not found at %s.' % dir_on_cluster,
self.run_prestoadmin,
'server upgrade ' + dir_on_cluster
)
def test_upgrade_works_with_symlink(self):
self.run_prestoadmin('configuration deploy')
for container in self.cluster.all_hosts():
self.real_installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_catalog(container)
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
symlink = os.path.join(get_install_directory(), 'link.rpm')
self.cluster.exec_cmd_on_host(self.cluster.master, 'ln -s %s %s'
% (path_on_cluster, symlink))
self.upgrade_and_assert_success(symlink)
def test_configuration_preserved_on_upgrade(self):
book_content = 'Call me Ishmael ... FINIS'
book_path = '/etc/presto/moby_dick_abridged'
self.run_prestoadmin('configuration deploy')
big_files = {}
for container in self.cluster.all_hosts():
self.real_installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_catalog(container)
big_file = self.cluster.exec_cmd_on_host(
container,
"find /usr -size +2M -ls | "
"sort -nk7 | "
"tail -1 | "
"awk '{print $NF}'").strip()
self.cluster.exec_cmd_on_host(
container, "cp %s /etc/presto" % (big_file,), invoke_sudo=True)
big_files[container] = os.path.join("/etc/presto", os.path.basename(big_file))
self.cluster.write_content_to_host(book_content, book_path, host=container)
self.cluster.exec_cmd_on_host(container, "chown presto:games %s" % (book_path,), invoke_sudo=True)
self.cluster.exec_cmd_on_host(container, "chmod 272 %s" % (book_path,), invoke_sudo=True)
self.assert_file_content(container, book_path, book_content)
self.assert_file_perm_owner(container, book_path, '--w-rwx-w-', 'presto', 'games')
self.assert_path_exists(container, big_files[container])
self.add_dummy_properties_to_host(self.cluster.slaves[1])
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
symlink = os.path.join(get_install_directory(), 'link.rpm')
self.cluster.exec_cmd_on_host(self.cluster.master, 'ln -s %s %s'
% (path_on_cluster, symlink))
self.run_prestoadmin('server upgrade ' + path_on_cluster)
self.assert_dummy_properties(self.cluster.slaves[1])
for container in self.cluster.all_hosts():
self.assert_file_content(container, book_path, book_content)
self.assert_file_perm_owner(container, book_path, '--w-rwx-w-', 'presto', 'games')
self.assert_path_exists(container, big_files[container])
def test_upgrade_non_root_user(self):
self.upload_topology(
{"coordinator": "master",
"workers": ["slave1", "slave2", "slave3"],
"username": "******"}
)
self.run_prestoadmin('configuration deploy -p password')
for container in self.cluster.all_hosts():
self.real_installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_catalog(container)
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
self.upgrade_and_assert_success(path_on_cluster, extra_arguments=' -p password')
def add_dummy_properties_to_host(self, host):
self.cluster.write_content_to_host(
'com.facebook.presto=INFO',
'/etc/presto/log.properties',
host
)
self.cluster.write_content_to_host(
'dummy config file',
'/etc/presto/jvm.config',
host
)
def assert_dummy_properties(self, host):
# assert log properties file is there
self.assert_file_content(
host,
'/etc/presto/log.properties',
'com.facebook.presto=INFO'
)
# assert dummy jvm config is there too
self.assert_file_content(
host,
'/etc/presto/jvm.config',
'dummy config file'
)
class TestServerInstall(BaseProductTestCase):
default_workers_config_with_slave1_ = """coordinator=false
discovery.uri=http://slave1:8080
http-server.http.port=8080
query.max-memory-per-node=512MB
query.max-memory=50GB\n"""
default_coord_config_with_slave1_ = """coordinator=true
discovery-server.enabled=true
discovery.uri=http://slave1:8080
http-server.http.port=8080
node-scheduler.include-coordinator=false
query.max-memory-per-node=512MB
query.max-memory=50GB\n"""
default_workers_config_regex_ = """coordinator=false
discovery.uri=http:.*:8080
http-server.http.port=8080
query.max-memory-per-node=512MB
query.max-memory=50GB\n"""
default_coord_config_regex_ = """coordinator=true
discovery-server.enabled=true
discovery.uri=http:.*:8080
http-server.http.port=8080
node-scheduler.include-coordinator=false
query.max-memory-per-node=512MB
query.max-memory=50GB\n"""
def setUp(self):
super(TestServerInstall, self).setUp()
self.setup_cluster(self.PA_ONLY_CLUSTER)
self.installer = StandalonePrestoInstaller(self)
def assert_common_configs(self, container):
self.installer.assert_installed(self, container)
self.assert_file_content(container, '/etc/presto/jvm.config',
self.default_jvm_config_)
self.assert_node_config(container, self.default_node_properties_)
self.assert_has_default_connector(container)
def assert_installed_with_configs(self, master, slaves):
self.assert_common_configs(master)
self.assert_file_content(master, '/etc/presto/config.properties',
self.default_coord_config_with_slave1_)
for container in slaves:
self.assert_common_configs(container)
self.assert_file_content(container,
'/etc/presto/config.properties',
self.default_workers_config_with_slave1_)
def assert_installed_with_regex_configs(self, master, slaves):
self.assert_common_configs(master)
self.assert_file_content_regex(master, '/etc/presto/config.properties',
self.default_coord_config_regex_)
for container in slaves:
self.assert_common_configs(container)
self.assert_file_content_regex(container,
'/etc/presto/config.properties',
self.default_workers_config_regex_)
@attr('smoketest')
def test_install(self, dummy=False):
self.upload_topology()
cmd_output = self.installer.install(dummy)
expected = installed_all_hosts_output
actual = cmd_output.splitlines()
self.assertEqual(sorted(expected), sorted(actual))
for container in self.cluster.all_hosts():
self.installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
def test_install_worker_is_pa_master(self):
topology = {
"coordinator": "slave1",
"workers": ["master", "slave2", "slave3"]
}
self.upload_topology(topology)
cmd_output = self.installer.install(dummy=True)
expected = install_with_worker_pa_master_out
actual = cmd_output.splitlines()
self.assertEqual(sorted(expected), sorted(actual))
self.assert_installed_with_configs(self.cluster.slaves[0], [
self.cluster.slaves[1], self.cluster.slaves[2], self.cluster.master
])
def test_install_ext_host_is_pa_master(self):
topology = {"coordinator": "slave1", "workers": ["slave2", "slave3"]}
self.upload_topology(topology)
cmd_output = self.installer.install(dummy=True)
expected = install_with_ext_host_pa_master_out
actual = cmd_output.splitlines()
self.assertEqual(sorted(expected), sorted(actual))
self.assert_installed_with_configs(
self.cluster.slaves[0],
[self.cluster.slaves[1], self.cluster.slaves[2]])
def test_install_when_connector_json_exists(self):
topology = {"coordinator": "master", "workers": ["slave1"]}
self.upload_topology(topology)
self.cluster.write_content_to_host(
'connector.name=jmx',
os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'),
self.cluster.master)
cmd_output = self.installer.install(dummy=True)
expected = [
'Deploying rpm on master...', 'Deploying rpm on slave1...',
'Package deployed successfully on: slave1',
'Package installed successfully on: slave1',
'Package deployed successfully on: master',
'Package installed successfully on: master',
'Deploying configuration on: master',
'Deploying jmx.properties, tpch.properties '
'connector configurations on: master ',
'Deploying configuration on: slave1',
'Deploying jmx.properties, tpch.properties '
'connector configurations on: slave1 '
]
actual = cmd_output.splitlines()
self.assertEqual(sorted(expected), sorted(actual))
for container in [self.cluster.master, self.cluster.slaves[0]]:
self.installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
self.assert_has_jmx_connector(container)
def test_install_when_topology_has_ips(self):
ips = self.cluster.get_ip_address_dict()
topology = {
"coordinator": ips[self.cluster.master],
"workers": [ips[self.cluster.slaves[0]]]
}
self.upload_topology(topology)
self.cluster.write_content_to_host(
'connector.name=jmx',
os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'),
self.cluster.master)
cmd_output = self.installer.install(dummy=True).splitlines()
expected = [
r'Deploying rpm on %s...' % ips[self.cluster.master],
r'Deploying rpm on %s...' % ips[self.cluster.slaves[0]],
r'Package deployed successfully on: ' + ips[self.cluster.master],
r'Package installed successfully on: ' + ips[self.cluster.master],
r'Package deployed successfully on: ' +
ips[self.cluster.slaves[0]],
r'Package installed successfully on: ' +
ips[self.cluster.slaves[0]],
r'Deploying configuration on: ' + ips[self.cluster.master],
r'Deploying jmx.properties, tpch.properties '
r'connector configurations on: ' + ips[self.cluster.master],
r'Deploying configuration on: ' + ips[self.cluster.slaves[0]],
r'Deploying jmx.properties, tpch.properties '
r'connector configurations on: ' + ips[self.cluster.slaves[0]]
]
cmd_output.sort()
expected.sort()
self.assertRegexpMatchesLineByLine(expected, cmd_output)
self.assert_installed_with_regex_configs(self.cluster.master,
[self.cluster.slaves[0]])
for container in [self.cluster.master, self.cluster.slaves[0]]:
self.assert_has_jmx_connector(container)
def test_install_interactive_with_hostnames(self):
self.cluster.write_content_to_host(
'connector.name=jmx',
os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'),
self.cluster.master)
rpm_name = self.installer.copy_presto_rpm_to_master()
self.write_test_configs(self.cluster)
cmd_output = self.run_script_from_prestoadmin_dir(
'echo -e "root\n22\n%(master)s\n%(slave1)s\n" | '
'./presto-admin server install /mnt/presto-admin/%(rpm)s ',
rpm=rpm_name)
actual = cmd_output.splitlines()
expected = [
r'Enter user name for SSH connection to all nodes: '
r'\[root\] '
r'Enter port number for SSH connections to all nodes: '
r'\[22\] '
r'Enter host name or IP address for coordinator node. '
r'Enter an external host name or ip address if this is a '
r'multi-node cluster: \[localhost\] '
r'Enter host names or IP addresses for worker nodes '
r'separated by spaces: '
r'\[localhost\] Deploying rpm on .*\.\.\.',
r'Package deployed successfully on: ' +
self.cluster.internal_master,
r'Package installed successfully on: ' +
self.cluster.internal_master,
r'Package deployed successfully on: ' +
self.cluster.internal_slaves[0],
r'Package installed successfully on: ' +
self.cluster.internal_slaves[0],
r'Deploying configuration on: ' + self.cluster.internal_master,
r'Deploying jmx.properties, tpch.properties connector '
r'configurations on: ' + self.cluster.internal_master,
r'Deploying configuration on: ' + self.cluster.internal_slaves[0],
r'Deploying jmx.properties, tpch.properties connector '
r'configurations on: ' + self.cluster.internal_slaves[0],
r'Deploying rpm on .*\.\.\.'
]
self.assertRegexpMatchesLineByLine(actual, expected)
for container in [self.cluster.master, self.cluster.slaves[0]]:
self.installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
self.assert_has_jmx_connector(container)
def test_install_interactive_with_ips(self):
ips = self.cluster.get_ip_address_dict()
rpm_name = self.installer.copy_presto_rpm_to_master()
self.write_test_configs(self.cluster)
additional_keywords = {
'rpm': rpm_name,
'master_ip': ips[self.cluster.master],
'slave1_ip': ips[self.cluster.slaves[0]]
}
cmd_output = self.run_script_from_prestoadmin_dir(
'echo -e "root\n22\n%(master_ip)s\n%(slave1_ip)s\n" | '
'./presto-admin server install /mnt/presto-admin/%(rpm)s ',
**additional_keywords).splitlines()
expected = [
r'Enter user name for SSH connection to all nodes: '
r'\[root\] '
r'Enter port number for SSH connections to all nodes: '
r'\[22\] '
r'Enter host name or IP address for coordinator node. '
r'Enter an external host name or ip address if this is a '
r'multi-node cluster: \[localhost\] '
r'Enter host names or IP addresses for worker nodes '
r'separated by spaces: '
r'\[localhost\] Deploying rpm on .*\.\.\.',
r'Package deployed successfully on: ' + ips[self.cluster.master],
r'Package installed successfully on: ' + ips[self.cluster.master],
r'Package deployed successfully on: ' +
ips[self.cluster.slaves[0]],
r'Package installed successfully on: ' +
ips[self.cluster.slaves[0]], r'Deploying configuration on: ' +
ips[self.cluster.master], r'Deploying tpch.properties connector '
r'configurations on: ' + ips[self.cluster.master],
r'Deploying configuration on: ' + ips[self.cluster.slaves[0]],
r'Deploying tpch.properties connector '
r'configurations on: ' + ips[self.cluster.slaves[0]],
r'Deploying rpm on .*\.\.\.'
]
cmd_output.sort()
expected.sort()
for expected_regexp, actual_line in zip(expected, cmd_output):
self.assertRegexpMatches(actual_line, expected_regexp)
self.assert_installed_with_regex_configs(self.cluster.master,
[self.cluster.slaves[0]])
def test_install_with_wrong_topology(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
topology = {'coordinator': 'dummy_master', 'workers': ['slave1']}
self.upload_topology(topology)
expected = 'u\'dummy_master\' is not a valid ip address or' \
' host name.' \
' More detailed information can be found in ' \
'/var/log/prestoadmin/presto-admin.log\n'
self.assertRaisesRegexp(OSError,
expected,
self.run_prestoadmin,
'server install /mnt/presto-admin/%(rpm)s ',
rpm=rpm_name)
def test_install_with_malformed_topology(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
topology = {'coordinator': 'master', 'workers': 'slave1' 'slave2'}
self.upload_topology(topology)
expected = 'Workers must be of type list. Found <type \'unicode\'>.' \
' More detailed information can be found in ' \
'/var/log/prestoadmin/presto-admin.log'
self.assertRaisesRegexp(OSError,
expected,
self.run_prestoadmin,
'server install /mnt/presto-admin/%(rpm)s ',
rpm=rpm_name)
def test_install_with_malformed_connector(self):
self.upload_topology()
self.cluster.write_content_to_host(
'connectr.typo:invalid',
os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'),
self.cluster.master)
actual_out = self.installer.install(dummy=True)
expected = 'Underlying exception:\n Catalog configuration ' \
'jmx.properties does not contain connector.name'
self.assertRegexpMatches(actual_out, expected)
for container in self.cluster.all_hosts():
self.installer.assert_installed(self, container)
self.assert_has_default_config(container)
def test_connection_to_coord_lost(self):
down_node = self.cluster.internal_slaves[0]
topology = {
"coordinator":
down_node,
"workers": [
self.cluster.internal_master, self.cluster.internal_slaves[1],
self.cluster.internal_slaves[2]
]
}
self.upload_topology(topology=topology)
self.cluster.stop_host(self.cluster.slaves[0])
actual_out = self.installer.install(dummy=True)
self.assertRegexpMatches(actual_out,
self.down_node_connection_error(down_node))
for container in [
self.cluster.master, self.cluster.slaves[1],
self.cluster.slaves[2]
]:
self.assert_common_configs(container)
self.assert_file_content(
container, '/etc/presto/config.properties',
self.default_workers_config_with_slave1_.replace(
down_node, self.cluster.get_down_hostname(down_node)))
@docker_only
def test_install_with_no_perm_to_local_path(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
self.upload_topology()
self.run_prestoadmin("configuration deploy")
script = 'chmod 600 /mnt/presto-admin/%(rpm)s; su app-admin -c ' \
'"./presto-admin server install /mnt/presto-admin/%(rpm)s "'
error_msg = '\nFatal error: [%(host)s] error: ' \
'/mnt/presto-admin/%(rpm)s: ' \
'open failed: Permission denied\n\nAborting.\n'
expected = ''
for host in self.cluster.all_internal_hosts():
expected += error_msg % {'host': host, 'rpm': rpm_name}
actual = self.run_script_from_prestoadmin_dir(script, rpm=rpm_name)
self.assertEqualIgnoringOrder(actual, expected)
def test_install_twice(self):
self.test_install(dummy=True)
output = self.installer.install(dummy=True)
with open(os.path.join(LOCAL_RESOURCES_DIR, 'install_twice.txt'), 'r') \
as f:
expected = f.read()
expected = self.escape_for_regex(self.replace_keywords(expected))
self.assertRegexpMatchesLineByLine(output.splitlines(),
expected.splitlines())
for container in self.cluster.all_hosts():
self.installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
class TestServerUpgrade(BaseProductTestCase):
def setUp(self):
super(TestServerUpgrade, self).setUp()
self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER)
self.dummy_installer = StandalonePrestoInstaller(
self, (os.path.join(prestoadmin.main_dir, 'tests', 'product',
'resources'), 'dummy-rpm.rpm'))
self.real_installer = StandalonePrestoInstaller(self)
def start_and_assert_started(self):
cmd_output = self.run_prestoadmin('server start')
process_per_host = self.get_process_per_host(cmd_output.splitlines())
self.assert_started(process_per_host)
#
# The dummy RPM is not guaranteed to have any functionality beyond not
# including any real payload and adding the random README file. It's a
# hacky one-off that satisfies the requirement of having *something* to
# upgrade to without downloading another copy of the real RPM. This is NOT
# the place to test functionality that the presto-server-rpm normally
# provides, because the dummy rpm probably doesn't provide it, or worse,
# provides an old and/or broken version of it.
#
def assert_upgraded_to_dummy_rpm(self, hosts):
for container in hosts:
# Still should have the same configs
self.dummy_installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
# However, dummy_rpm.rpm removes /usr/lib/presto/lib and
# /usr/lib/presto/lib/plugin
self.assert_path_removed(container, '/usr/lib/presto/lib')
self.assert_path_removed(container, '/usr/lib/presto/lib/plugin')
# And adds /usr/lib/presto/README.txt
self.assert_path_exists(container, '/usr/lib/presto/README.txt')
# And modifies the text of the readme in
# /usr/shared/doc/presto/README.txt
self.assert_file_content_regex(
container,
'/usr/shared/doc/presto/README.txt',
r'.*New line of text here.$'
)
@attr('smoketest')
def test_upgrade(self):
self.start_and_assert_started()
self.run_prestoadmin('configuration deploy')
for container in self.cluster.all_hosts():
self.real_installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
self.upgrade_and_assert_success(path_on_cluster)
def upgrade_and_assert_success(self, path_on_cluster, extra_arguments=''):
self.run_prestoadmin('server upgrade ' + path_on_cluster + extra_arguments)
self.assert_upgraded_to_dummy_rpm(self.cluster.all_hosts())
@docker_only
def test_rolling_upgrade(self):
# Test that if a node is down, and then you upgrade again, it works
self.run_prestoadmin('configuration deploy')
self.cluster.stop_host(self.cluster.slaves[0])
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
self.run_prestoadmin('server upgrade ' + path_on_cluster,
raise_error=False)
running_hosts = self.cluster.all_hosts()[:]
running_hosts.remove(self.cluster.slaves[0])
self.assert_upgraded_to_dummy_rpm(running_hosts)
self.cluster.start_host(self.cluster.slaves[0])
self.retry(lambda: self.upgrade_and_assert_success(path_on_cluster))
def copy_upgrade_rpm_to_cluster(self):
rpm_name = self.dummy_installer.copy_presto_rpm_to_master()
return os.path.join(self.cluster.mount_dir, rpm_name)
def test_upgrade_fails_given_directory(self):
dir_on_cluster = '/opt/prestoadmin'
self.assertRaisesRegexp(
OSError,
'RPM file not found at %s.' % dir_on_cluster,
self.run_prestoadmin,
'server upgrade ' + dir_on_cluster
)
def test_upgrade_works_with_symlink(self):
self.run_prestoadmin('configuration deploy')
for container in self.cluster.all_hosts():
self.real_installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
symlink = '/opt/prestoadmin/link.rpm'
self.cluster.exec_cmd_on_host(self.cluster.master, 'ln -s %s %s'
% (path_on_cluster, symlink))
self.upgrade_and_assert_success(symlink)
def test_configuration_preserved_on_upgrade(self):
self.run_prestoadmin('configuration deploy')
for container in self.cluster.all_hosts():
self.real_installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
self.add_dummy_properties_to_host(self.cluster.slaves[1])
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
symlink = '/opt/prestoadmin/link.rpm'
self.cluster.exec_cmd_on_host(self.cluster.master, 'ln -s %s %s'
% (path_on_cluster, symlink))
self.run_prestoadmin('server upgrade ' + path_on_cluster)
self.assert_dummy_properties(self.cluster.slaves[1])
def test_upgrade_non_root_user(self):
self.upload_topology(
{"coordinator": "master",
"workers": ["slave1", "slave2", "slave3"],
"username": "******"}
)
self.run_prestoadmin('configuration deploy -p password')
for container in self.cluster.all_hosts():
self.real_installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
self.upgrade_and_assert_success(path_on_cluster, extra_arguments=' -p password')
def add_dummy_properties_to_host(self, host):
self.cluster.write_content_to_host(
'com.facebook.presto=INFO',
'/etc/presto/log.properties',
host
)
self.cluster.write_content_to_host(
'dummy config file',
'/etc/presto/jvm.config',
host
)
def assert_dummy_properties(self, host):
# assert log properties file is there
self.assert_file_content(
host,
'/etc/presto/log.properties',
'com.facebook.presto=INFO'
)
# assert dummy jvm config is there too
self.assert_file_content(
host,
'/etc/presto/jvm.config',
'dummy config file'
)
class TestPackageInstall(BaseProductTestCase):
def setUp(self):
super(TestPackageInstall, self).setUp()
self.setup_cluster(self.PA_ONLY_CLUSTER)
self.upload_topology()
self.installer = StandalonePrestoInstaller(self)
@attr('smoketest')
def test_package_install(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
output = self.run_prestoadmin(
'package install '
'/mnt/presto-admin/%(rpm)s', rpm=rpm_name)
for container in self.cluster.all_hosts():
self.installer.assert_installed(self, container, msg=output)
def test_install_coord_using_dash_h(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
output = self.run_prestoadmin(
'package install /mnt/presto-admin/%(rpm)s -H %(master)s',
rpm=rpm_name)
self.installer.assert_installed(self, self.cluster.master)
for slave in self.cluster.slaves:
self.installer.assert_uninstalled(slave, msg=output)
def test_install_worker_using_dash_h(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
output = self.run_prestoadmin(
'package install /mnt/presto-admin/%(rpm)s -H %(slave1)s',
rpm=rpm_name)
self.installer.assert_installed(self,
self.cluster.slaves[0],
msg=output)
self.installer.assert_uninstalled(self.cluster.master, msg=output)
self.installer.assert_uninstalled(self.cluster.slaves[1], msg=output)
self.installer.assert_uninstalled(self.cluster.slaves[2], msg=output)
def test_install_workers_using_dash_h(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
output = self.run_prestoadmin(
'package install /mnt/presto-admin/'
'%(rpm)s -H %(slave1)s,%(slave2)s',
rpm=rpm_name)
self.installer.assert_installed(self,
self.cluster.slaves[0],
msg=output)
self.installer.assert_installed(self,
self.cluster.slaves[1],
msg=output)
self.installer.assert_uninstalled(self.cluster.master, msg=output)
self.installer.assert_uninstalled(self.cluster.slaves[2], msg=output)
def test_install_exclude_coord(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
output = self.run_prestoadmin(
'package install /mnt/presto-admin/'
'%(rpm)s -x %(master)s',
rpm=rpm_name)
self.installer.assert_uninstalled(self.cluster.master, msg=output)
for slave in self.cluster.slaves:
self.installer.assert_installed(self, slave, msg=output)
def test_install_exclude_worker(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
output = self.run_prestoadmin(
'package install /mnt/presto-admin/'
'%(rpm)s -x %(slave1)s',
rpm=rpm_name)
self.installer.assert_uninstalled(self.cluster.slaves[0], msg=output)
self.installer.assert_installed(self,
self.cluster.slaves[1],
msg=output)
self.installer.assert_installed(self, self.cluster.master, msg=output)
self.installer.assert_installed(self,
self.cluster.slaves[2],
msg=output)
def test_install_exclude_workers(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
output = self.run_prestoadmin(
'package install /mnt/presto-admin/'
'%(rpm)s -x %(slave1)s,%(slave2)s',
rpm=rpm_name)
self.installer.assert_uninstalled(self.cluster.slaves[0], msg=output)
self.installer.assert_uninstalled(self.cluster.slaves[1], msg=output)
self.installer.assert_installed(self, self.cluster.master, msg=output)
self.installer.assert_installed(self,
self.cluster.slaves[2],
msg=output)
def test_install_invalid_path(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
cmd_output = self.run_prestoadmin(
'package install /mnt/presto-admin'
'/invalid-path/presto.rpm',
rpm=rpm_name)
error = '\nFatal error: [%s] error: ' \
'/mnt/presto-admin/invalid-path/presto.rpm: open failed: ' \
'No such file or directory\n\nAborting.\n'
expected = ''
for host in self.cluster.all_internal_hosts():
expected += error % host
self.assertEqualIgnoringOrder(cmd_output, expected)
def test_install_no_path_arg(self):
self.installer.copy_presto_rpm_to_master()
output = self.run_prestoadmin('package install', raise_error=False)
self.assertEqual(
output, 'Incorrect number of arguments to task.\n\n'
'Displaying detailed information for task '
'\'package install\':\n\n'
' Install the rpm package on the cluster\n'
' \n Args:\n'
' local_path: Absolute path to the rpm'
' to be installed\n '
'--nodeps (optional): Flag to indicate if '
'rpm install\n'
' should ignore c'
'hecking package dependencies. Equivalent\n'
' to adding --nodeps flag to rpm '
'-i.\n\n')
def test_install_already_installed(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
self.run_prestoadmin(
'package install /mnt/presto-admin/%(rpm)s -H '
'%(master)s',
rpm=rpm_name)
self.installer.assert_installed(self, self.cluster.master)
cmd_output = self.run_prestoadmin(
'package install /mnt/presto-admin/%(rpm)s -H %(master)s',
rpm=rpm_name)
expected = self.escape_for_regex(
self.replace_keywords(
"""
Fatal error: [%(master)s] sudo() received nonzero return code 1 while \
executing!
Requested: rpm -i /opt/prestoadmin/packages/%(rpm)s
Executed: sudo -S -p 'sudo password:'******'package install '
'/etc/opt/prestoadmin/config.json')
error = """
Fatal error: \[%s\] error: (/etc/opt/prestoadmin/config.json: )?not an rpm \
package
Aborting.
"""
expected = ''
for host in self.cluster.all_internal_hosts():
expected += error % host
self.assertRegexpMatchesLineByLine(cmd_output.splitlines(),
expected.splitlines())
@docker_only
def test_install_rpm_with_missing_jdk(self):
rpm_name = self.installer.copy_presto_rpm_to_master(
rpm_dir=prestoadmin.main_dir,
rpm_name=self.installer.presto_rpm_filename)
self.cluster.exec_cmd_on_host(self.cluster.master,
'rpm -e jdk1.8.0_40-1.8.0_40-fcs')
self.assertRaisesRegexp(
OSError, 'package jdk1.8.0_40-1.8.0_40-fcs is not '
'installed', self.cluster.exec_cmd_on_host, self.cluster.master,
'rpm -q jdk1.8.0_40-1.8.0_40-fcs')
cmd_output = self.run_prestoadmin(
'package install /mnt/presto-admin/%(rpm)s -H %(master)s',
rpm=rpm_name)
self.assertRegexpMatchesLineByLine(
cmd_output.splitlines(),
self.jdk_not_found_error_message(rpm_name).splitlines())
def jdk_not_found_error_message(self, rpm_name):
with open(os.path.join(LOCAL_RESOURCES_DIR, 'jdk_not_found.txt')) as f:
jdk_not_found_error = f.read()
return self.escape_for_regex(
self.replace_keywords(jdk_not_found_error,
**self.installer.get_keywords(rpm_name)))
@docker_only
def test_install_rpm_missing_dependency(self):
rpm_name = self.installer.copy_presto_rpm_to_master()
self.cluster.exec_cmd_on_host(self.cluster.master,
'rpm -e --nodeps python-2.6.6')
self.assertRaisesRegexp(OSError,
'package python-2.6.6 is not installed',
self.cluster.exec_cmd_on_host,
self.cluster.master, 'rpm -q python-2.6.6')
cmd_output = self.run_prestoadmin(
'package install /mnt/presto-admin/%(rpm)s -H %(master)s',
rpm=rpm_name)
expected = self.replace_keywords(
"""
Fatal error: [%(master)s] sudo() received nonzero return code 1 while \
executing!
Requested: rpm -i /opt/prestoadmin/packages/%(rpm)s
Executed: sudo -S -p 'sudo password:'******'rpm -e --nodeps python-2.6.6')
self.assertRaisesRegexp(OSError,
'package python-2.6.6 is not installed',
self.cluster.exec_cmd_on_host,
self.cluster.master, 'rpm -q python-2.6.6')
cmd_output = self.run_prestoadmin(
'package install /mnt/presto-admin/%(rpm)s -H %(master)s --nodeps',
rpm=rpm_name)
expected = 'Deploying rpm on %(host)s...\n' \
'Package deployed successfully on: %(host)s\n' \
'Package installed successfully on: %(host)s' \
% {'host': self.cluster.internal_master}
self.assertEqualIgnoringOrder(expected, cmd_output)
class TestServerUpgrade(BaseProductTestCase):
def setUp(self):
super(TestServerUpgrade, self).setUp()
self.setup_cluster(NoHadoopBareImageProvider(),
self.STANDALONE_PRESTO_CLUSTER)
self.dummy_installer = StandalonePrestoInstaller(
self, (os.path.join(prestoadmin.main_dir, 'tests', 'product',
'resources'), 'dummy-rpm.rpm'))
self.real_installer = StandalonePrestoInstaller(self)
def start_and_assert_started(self):
cmd_output = self.run_prestoadmin('server start')
process_per_host = self.get_process_per_host(cmd_output.splitlines())
self.assert_started(process_per_host)
#
# The dummy RPM is not guaranteed to have any functionality beyond not
# including any real payload and adding the random README file. It's a
# hacky one-off that satisfies the requirement of having *something* to
# upgrade to without downloading another copy of the real RPM. This is NOT
# the place to test functionality that the presto-server-rpm normally
# provides, because the dummy rpm probably doesn't provide it, or worse,
# provides an old and/or broken version of it.
#
def assert_upgraded_to_dummy_rpm(self, hosts):
for container in hosts:
# Still should have the same configs
self.dummy_installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
# However, dummy_rpm.rpm removes /usr/lib/presto/lib and
# /usr/lib/presto/lib/plugin
self.assert_path_removed(container, '/usr/lib/presto/lib')
self.assert_path_removed(container, '/usr/lib/presto/lib/plugin')
# And adds /usr/lib/presto/README.txt
self.assert_path_exists(container, '/usr/lib/presto/README.txt')
# And modifies the text of the readme in
# /usr/shared/doc/presto/README.txt
self.assert_file_content_regex(
container,
'/usr/shared/doc/presto/README.txt',
r'.*New line of text here.$'
)
@attr('smoketest')
def test_upgrade(self):
self.start_and_assert_started()
self.run_prestoadmin('configuration deploy')
for container in self.cluster.all_hosts():
self.real_installer.assert_installed(self, container)
self.assert_has_default_config(container)
self.assert_has_default_connector(container)
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
self.upgrade_and_assert_success(path_on_cluster)
def upgrade_and_assert_success(self, path_on_cluster):
self.run_prestoadmin('server upgrade ' + path_on_cluster)
self.assert_upgraded_to_dummy_rpm(self.cluster.all_hosts())
@docker_only
def test_rolling_upgrade(self):
# Test that if a node is down, and then you upgrade again, it works
self.run_prestoadmin('configuration deploy')
self.cluster.stop_host(self.cluster.slaves[0])
path_on_cluster = self.copy_upgrade_rpm_to_cluster()
self.run_prestoadmin('server upgrade ' + path_on_cluster,
raise_error=False)
running_hosts = self.cluster.all_hosts()[:]
running_hosts.remove(self.cluster.slaves[0])
self.assert_upgraded_to_dummy_rpm(running_hosts)
self.cluster.start_host(self.cluster.slaves[0])
self.retry(lambda: self.upgrade_and_assert_success(path_on_cluster))
def copy_upgrade_rpm_to_cluster(self):
rpm_name = self.dummy_installer.copy_presto_rpm_to_master()
return os.path.join(self.cluster.mount_dir, rpm_name)
