def assert_common_configs(self, host):
     installer = StandalonePrestoInstaller(self)
     installer.assert_installed(self, host)
     self.assert_file_content(host, '/etc/presto/jvm.config',
                              self.default_jvm_config_)
     self.assert_node_config(host, self.default_node_properties_)
     self.assert_has_default_connector(host)
Esempio n. 2
0
    def test_catalog_add_lost_host(self):
        installer = StandalonePrestoInstaller(self)
        self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER)
        self.upload_topology()
        installer.install()
        self.run_prestoadmin('catalog remove tpch')

        self.cluster.stop_host(
            self.cluster.slaves[0])
        self.cluster.write_content_to_host(
            'connector.name=tpch',
            os.path.join(get_catalog_directory(), 'tpch.properties'),
            self.cluster.master
        )
        output = self.run_prestoadmin('catalog add tpch', raise_error=False)
        for host in self.cluster.all_internal_hosts():
            deploying_message = 'Deploying tpch.properties catalog configurations on: %s'
            self.assertTrue(deploying_message % host in output,
                            'expected %s \n actual %s'
                            % (deploying_message % host, output))
        self.assertRegexpMatches(
            output,
            self.down_node_connection_error(self.cluster.internal_slaves[0])
        )
        self.assertEqual(len(output.splitlines()),
                         len(self.cluster.all_hosts()) +
                         self.len_down_node_error)
        self.run_prestoadmin('server start', raise_error=False)

        for host in [self.cluster.master,
                     self.cluster.slaves[1],
                     self.cluster.slaves[2]]:
            self.assert_has_default_catalog(host)
        self._assert_catalogs_loaded([['system'], ['tpch']])
    def test_install_when_connector_json_exists(self):
        installer = StandalonePrestoInstaller(self, dummy=True)
        topology = {"coordinator": "master",
                    "workers": ["slave1"]}
        self.upload_topology(topology)
        self.cluster.write_content_to_host(
            'connector.name=jmx',
            os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'),
            self.cluster.master
        )

        cmd_output = installer.install()
        expected = ['Deploying rpm on master...',
                    'Deploying rpm on slave1...',
                    'Package deployed successfully on: slave1',
                    'Package installed successfully on: slave1',
                    'Package deployed successfully on: master',
                    'Package installed successfully on: master',
                    'Deploying configuration on: master',
                    'Deploying jmx.properties, tpch.properties '
                    'connector configurations on: master ',
                    'Deploying configuration on: slave1',
                    'Deploying jmx.properties, tpch.properties '
                    'connector configurations on: slave1 ']

        actual = cmd_output.splitlines()
        self.assertEqual(sorted(expected), sorted(actual))

        for container in [self.cluster.master,
                          self.cluster.slaves[0]]:
            installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_connector(container)
            self.assert_has_jmx_connector(container)
Esempio n. 4
0
    def test_connector_add_lost_host(self):
        installer = StandalonePrestoInstaller(self)
        self.setup_cluster(self.PA_ONLY_CLUSTER)
        self.upload_topology()
        installer.install()
        self.run_prestoadmin('connector remove tpch')

        self.cluster.stop_host(
            self.cluster.slaves[0])
        self.cluster.write_content_to_host(
            'connector.name=tpch',
            os.path.join(constants.CONNECTORS_DIR, 'tpch.properties'),
            self.cluster.master
        )
        output = self.run_prestoadmin('connector add tpch')
        for host in self.cluster.all_internal_hosts():
            deploying_message = 'Deploying tpch.properties connector ' \
                                'configurations on: %s'
            self.assertTrue(deploying_message % host in output,
                            'expected %s \n actual %s'
                            % (deploying_message % host, output))
        self.assertRegexpMatches(
            output,
            self.down_node_connection_error(self.cluster.internal_slaves[0])
        )
        self.assertEqual(len(output.splitlines()),
                         len(self.cluster.all_hosts()) +
                         self.len_down_node_error)
        self.run_prestoadmin('server start')

        for host in [self.cluster.master,
                     self.cluster.slaves[1],
                     self.cluster.slaves[2]]:
            self.assert_has_default_connector(host)
        self._assert_connectors_loaded([['system'], ['tpch']])
    def test_connection_to_coord_lost(self):
        installer = StandalonePrestoInstaller(self)
        down_node = self.cluster.internal_slaves[0]
        topology = {"coordinator": down_node,
                    "workers": [self.cluster.internal_master,
                                self.cluster.internal_slaves[1],
                                self.cluster.internal_slaves[2]]}
        self.upload_topology(topology=topology)
        self.cluster.stop_host(
            self.cluster.slaves[0])

        actual_out = installer.install(
            coordinator=down_node, pa_raise_error=False)

        self.assertRegexpMatches(
            actual_out,
            self.down_node_connection_error(down_node)
        )

        for host in [self.cluster.master,
                     self.cluster.slaves[1],
                     self.cluster.slaves[2]]:
            self.assert_common_configs(host)
            self.assert_file_content(
                host,
                '/etc/presto/config.properties',
                self.default_workers_config_with_slave1_
            )
Esempio n. 6
0
 def setUp(self):
     super(TestServerUpgrade, self).setUp()
     self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER)
     self.dummy_installer = StandalonePrestoInstaller(
         self, (os.path.join(prestoadmin.main_dir, 'tests', 'product',
                             'resources'), 'dummy-rpm.rpm'))
     self.real_installer = StandalonePrestoInstaller(self)
Esempio n. 7
0
    def test_install_interactive_with_hostnames(self):
        installer = StandalonePrestoInstaller(self)
        self.cluster.write_content_to_host(
            'connector.name=jmx',
            os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'),
            self.cluster.master
        )
        rpm_name = installer.copy_presto_rpm_to_master()
        self.write_test_configs(self.cluster)

        cmd_output = self.run_script_from_prestoadmin_dir(
            'echo -e "root\n22\n%(master)s\n%(slave1)s\n" | '
            './presto-admin server install /mnt/presto-admin/%(rpm)s ',
            rpm=rpm_name)

        actual = cmd_output.splitlines()
        expected = [r'Enter user name for SSH connection to all nodes: '
                    r'\[root\] '
                    r'Enter port number for SSH connections to all nodes: '
                    r'\[22\] '
                    r'Enter host name or IP address for coordinator node. '
                    r'Enter an external host name or ip address if this is a '
                    r'multi-node cluster: \[localhost\] '
                    r'Enter host names or IP addresses for worker nodes '
                    r'separated by spaces: '
                    r'\[localhost\] Using rpm_specifier as a local path',
                    r'Package deployed successfully on: ' +
                    self.cluster.internal_master,
                    r'Package installed successfully on: ' +
                    self.cluster.internal_master,
                    r'Package deployed successfully on: ' +
                    self.cluster.internal_slaves[0],
                    r'Package installed successfully on: ' +
                    self.cluster.internal_slaves[0],
                    r'Deploying configuration on: ' +
                    self.cluster.internal_master,
                    r'Deploying jmx.properties, tpch.properties connector '
                    r'configurations on: ' +
                    self.cluster.internal_master,
                    r'Deploying configuration on: ' +
                    self.cluster.internal_slaves[0],
                    r'Deploying jmx.properties, tpch.properties connector '
                    r'configurations on: ' +
                    self.cluster.internal_slaves[0],
                    r'Deploying rpm on .*\.\.\.',
                    r'Deploying rpm on .*\.\.\.',
                    r'Fetching local presto rpm at path: .*',
                    r'Found existing rpm at: .*'
                    ]

        self.assertRegexpMatchesLineByLine(actual, expected)
        for container in [self.cluster.master,
                          self.cluster.slaves[0]]:
            installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_connector(container)
            self.assert_has_jmx_connector(container)
Esempio n. 8
0
    def test_install_interactive_with_ips(self):
        installer = StandalonePrestoInstaller(self)
        ips = self.cluster.get_ip_address_dict()
        rpm_name = installer.copy_presto_rpm_to_master()
        self.write_test_configs(self.cluster)

        additional_keywords = {
            'rpm': rpm_name,
            'master_ip': ips[self.cluster.master],
            'slave1_ip': ips[self.cluster.slaves[0]]
        }
        cmd_output = self.run_script_from_prestoadmin_dir(
            'echo -e "root\n22\n%(master_ip)s\n%(slave1_ip)s\n" | '
            './presto-admin server install /mnt/presto-admin/%(rpm)s ',
            **additional_keywords).splitlines()
        expected = [r'Enter user name for SSH connection to all nodes: '
                    r'\[root\] '
                    r'Enter port number for SSH connections to all nodes: '
                    r'\[22\] '
                    r'Enter host name or IP address for coordinator node. '
                    r'Enter an external host name or ip address if this is a '
                    r'multi-node cluster: \[localhost\] '
                    r'Enter host names or IP addresses for worker nodes '
                    r'separated by spaces: '
                    r'\[localhost\] Using rpm_specifier as a local path',
                    r'Package deployed successfully on: ' +
                    ips[self.cluster.master],
                    r'Package installed successfully on: ' +
                    ips[self.cluster.master],
                    r'Package deployed successfully on: ' +
                    ips[self.cluster.slaves[0]],
                    r'Package installed successfully on: ' +
                    ips[self.cluster.slaves[0]],
                    r'Deploying configuration on: ' +
                    ips[self.cluster.master],
                    r'Deploying tpch.properties connector '
                    r'configurations on: ' +
                    ips[self.cluster.master] + r' ',
                    r'Deploying configuration on: ' +
                    ips[self.cluster.slaves[0]],
                    r'Deploying tpch.properties connector '
                    r'configurations on: ' +
                    ips[self.cluster.slaves[0]] + r' ',
                    r'Deploying rpm on .*\.\.\.',
                    r'Deploying rpm on .*\.\.\.',
                    r'Fetching local presto rpm at path: .*',
                    r'Found existing rpm at: .*']

        cmd_output.sort()
        expected.sort()
        for expected_regexp, actual_line in zip(expected, cmd_output):
            self.assertRegexpMatches(actual_line, expected_regexp)

        self.assert_installed_with_regex_configs(
            self.cluster.master,
            [self.cluster.slaves[0]])
Esempio n. 9
0
 def test_start_stop_restart_worker_down(self):
     installer = StandalonePrestoInstaller(self)
     self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER)
     topology = {"coordinator": "slave1",
                 "workers": ["master", "slave2", "slave3"]}
     self.upload_topology(topology=topology)
     installer.install(coordinator='slave1')
     self.assert_start_stop_restart_down_node(
         self.cluster.slaves[0],
         self.cluster.internal_slaves[0])
Esempio n. 10
0
 def test_system_info_pa_separate_node(self):
     installer = StandalonePrestoInstaller(self)
     self.setup_cluster(NoHadoopBareImageProvider(), self.PA_ONLY_CLUSTER)
     topology = {"coordinator": "slave1",
                 "workers": ["slave2", "slave3"]}
     self.upload_topology(topology=topology)
     installer.install(coordinator='slave1')
     self.test_basic_system_info(
         coordinator=self.cluster.internal_slaves[0],
         hosts=self.cluster.slaves)
class TestServerUninstall(BaseProductTestCase):
    def setUp(self):
        super(TestServerUninstall, self).setUp()
        self.installer = StandalonePrestoInstaller(self)

    @attr('smoketest')
    def test_uninstall(self):
        self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER)
        start_output = self.run_prestoadmin('server start')
        process_per_host = self.get_process_per_host(start_output.splitlines())
        self.assert_started(process_per_host)

        cmd_output = self.run_prestoadmin(
            'server uninstall', raise_error=False).splitlines()
        self.assert_stopped(process_per_host)
        expected = uninstall_output + self.expected_stop()[:]
        self.assertRegexpMatchesLineByLine(cmd_output, expected)

        for container in self.cluster.all_hosts():
            self.assert_uninstalled_dirs_removed(container)

    def assert_uninstalled_dirs_removed(self, container):
        self.installer.assert_uninstalled(container)
        self.assert_path_removed(container, '/etc/presto')
        self.assert_path_removed(container, '/usr/lib/presto')
        self.assert_path_removed(container, '/var/lib/presto')
        self.assert_path_removed(container, '/usr/shared/doc/presto')
        self.assert_path_removed(container, '/etc/init.d/presto')

    def test_uninstall_twice(self):
        self.test_uninstall()

        output = self.run_prestoadmin('server uninstall', raise_error=False)
        with open(os.path.join(LOCAL_RESOURCES_DIR, 'uninstall_twice.txt'),
                  'r') as f:
            expected = f.read()

        self.assertEqualIgnoringOrder(expected, output)

    def test_uninstall_lost_host(self):
        self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER)

        self.cluster.stop_host(
            self.cluster.slaves[0])

        expected = self.down_node_connection_error(
            self.cluster.internal_slaves[0])
        cmd_output = self.run_prestoadmin('server uninstall',
                                          raise_error=False)
        self.assertRegexpMatches(cmd_output, expected)

        for container in [self.cluster.internal_master,
                          self.cluster.internal_slaves[1],
                          self.cluster.internal_slaves[2]]:
            self.assert_uninstalled_dirs_removed(container)
Esempio n. 12
0
 def test_system_info_pa_separate_node(self):
     installer = StandalonePrestoInstaller(self)
     self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER)
     topology = {"coordinator": "slave1",
                 "workers": ["slave2", "slave3"]}
     self.upload_topology(topology=topology)
     installer.install(coordinator='slave1')
     self.run_prestoadmin('server start')
     actual = self.run_prestoadmin('collect system_info')
     self._test_basic_system_info(
         actual,
         coordinator=self.cluster.internal_slaves[0],
         hosts=self.cluster.slaves)
 def test_install_with_wrong_topology(self):
     installer = StandalonePrestoInstaller(self)
     rpm_name = installer.copy_presto_rpm_to_master()
     topology = {'coordinator': 'dummy_master', 'workers': ['slave1']}
     self.upload_topology(topology)
     expected = 'u\'dummy_master\' is not a valid ip address or' \
                ' host name.' \
                '  More detailed information can be found in ' \
                '/var/log/prestoadmin/presto-admin.log\n'
     self.assertRaisesRegexp(OSError,
                             expected,
                             self.run_prestoadmin,
                             'server install /mnt/presto-admin/%(rpm)s ',
                             rpm=rpm_name)
Esempio n. 14
0
    def test_install_with_malformed_topology(self):
        installer = StandalonePrestoInstaller(self)
        rpm_name = installer.copy_presto_rpm_to_master()
        topology = {'coordinator': 'master', 'workers': 'slave1' 'slave2'}
        self.upload_topology(topology)
        expected = 'Workers must be of type list.  Found <type \'unicode\'>.' \
                   '  More detailed information can be found in ' \
                   '/var/log/prestoadmin/presto-admin.log'

        self.assertRaisesRegexp(OSError,
                                expected,
                                self.run_prestoadmin,
                                'server install /mnt/presto-admin/%(rpm)s ',
                                rpm=rpm_name)
Esempio n. 15
0
    def test_install_twice(self):
        installer = StandalonePrestoInstaller(self)
        self.upload_topology()
        cmd_output = installer.install()
        expected = self.format_err_msgs_with_internal_hosts(installed_all_hosts_output)

        actual = cmd_output.splitlines()
        self.assertRegexpMatchesLineByLine(actual, expected)

        for container in self.cluster.all_hosts():
            installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_connector(container)

        output = installer.install(pa_raise_error=False)

        self.default_keywords.update(installer.get_keywords())

        with open(os.path.join(LOCAL_RESOURCES_DIR, 'install_twice.txt'),
                  'r') as f:
            expected = f.read()
        expected = self.escape_for_regex(
            self.replace_keywords(expected))

        self.assertRegexpMatchesLineByLine(output.splitlines(),
                                           expected.splitlines())
        for container in self.cluster.all_hosts():
            installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_connector(container)
Esempio n. 16
0
 def test_install_with_wrong_topology(self):
     installer = StandalonePrestoInstaller(self)
     rpm_name = installer.copy_presto_rpm_to_master()
     topology = {'coordinator': 'dummy_master', 'workers': ['slave1']}
     self.upload_topology(topology)
     expected = 'u\'dummy_master\' is not a valid ip address or' \
                ' host name.' \
                '  More detailed information can be found in ' \
                '/var/log/prestoadmin/presto-admin.log\n'
     self.assertRaisesRegexp(OSError,
                             expected,
                             self.run_prestoadmin,
                             'server install /mnt/presto-admin/%(rpm)s ',
                             rpm=rpm_name)
Esempio n. 17
0
    def test_install_ext_host_is_pa_master(self):
        installer = StandalonePrestoInstaller(self)
        topology = {"coordinator": "slave1", "workers": ["slave2", "slave3"]}
        self.upload_topology(topology)

        cmd_output = installer.install(coordinator='slave1')
        expected = install_with_ext_host_pa_master_out

        actual = cmd_output.splitlines()
        self.assertRegexpMatchesLineByLine(actual, expected)

        self.assert_installed_with_configs(
            self.cluster.slaves[0],
            [self.cluster.slaves[1], self.cluster.slaves[2]])
    def test_install_interactive_with_hostnames(self):
        installer = StandalonePrestoInstaller(self)
        self.cluster.write_content_to_host(
            'connector.name=jmx',
            os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'),
            self.cluster.master)
        rpm_name = installer.copy_presto_rpm_to_master()
        self.write_test_configs(self.cluster)

        cmd_output = self.run_script_from_prestoadmin_dir(
            'echo -e "root\n22\n%(master)s\n%(slave1)s\n" | '
            './presto-admin server install /mnt/presto-admin/%(rpm)s ',
            rpm=rpm_name)

        actual = cmd_output.splitlines()
        expected = [
            r'Enter user name for SSH connection to all nodes: '
            r'\[root\] '
            r'Enter port number for SSH connections to all nodes: '
            r'\[22\] '
            r'Enter host name or IP address for coordinator node. '
            r'Enter an external host name or ip address if this is a '
            r'multi-node cluster: \[localhost\] '
            r'Enter host names or IP addresses for worker nodes '
            r'separated by spaces: '
            r'\[localhost\] Deploying rpm on .*\.\.\.',
            r'Package deployed successfully on: ' +
            self.cluster.internal_master,
            r'Package installed successfully on: ' +
            self.cluster.internal_master,
            r'Package deployed successfully on: ' +
            self.cluster.internal_slaves[0],
            r'Package installed successfully on: ' +
            self.cluster.internal_slaves[0],
            r'Deploying configuration on: ' + self.cluster.internal_master,
            r'Deploying jmx.properties, tpch.properties connector '
            r'configurations on: ' + self.cluster.internal_master,
            r'Deploying configuration on: ' + self.cluster.internal_slaves[0],
            r'Deploying jmx.properties, tpch.properties connector '
            r'configurations on: ' + self.cluster.internal_slaves[0],
            r'Deploying rpm on .*\.\.\.'
        ]

        self.assertRegexpMatchesLineByLine(actual, expected)
        for container in [self.cluster.master, self.cluster.slaves[0]]:
            installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_connector(container)
            self.assert_has_jmx_connector(container)
Esempio n. 19
0
    def test_install_interactive_with_ips(self):
        installer = StandalonePrestoInstaller(self)
        ips = self.cluster.get_ip_address_dict()
        rpm_name = installer.copy_presto_rpm_to_master()
        self.write_test_configs(self.cluster)

        additional_keywords = {
            'rpm': rpm_name,
            'master_ip': ips[self.cluster.master],
            'slave1_ip': ips[self.cluster.slaves[0]]
        }
        cmd_output = self.run_script_from_prestoadmin_dir(
            'echo -e "root\n22\n%(master_ip)s\n%(slave1_ip)s\n" | '
            './presto-admin server install /mnt/presto-admin/%(rpm)s ',
            **additional_keywords).splitlines()
        expected = [
            r'Enter user name for SSH connection to all nodes: '
            r'\[root\] '
            r'Enter port number for SSH connections to all nodes: '
            r'\[22\] '
            r'Enter host name or IP address for coordinator node. '
            r'Enter an external host name or ip address if this is a '
            r'multi-node cluster: \[localhost\] '
            r'Enter host names or IP addresses for worker nodes '
            r'separated by spaces: '
            r'\[localhost\] Using rpm_specifier as a local path',
            r'Package deployed successfully on: ' + ips[self.cluster.master],
            r'Package installed successfully on: ' + ips[self.cluster.master],
            r'Package deployed successfully on: ' +
            ips[self.cluster.slaves[0]],
            r'Package installed successfully on: ' +
            ips[self.cluster.slaves[0]], r'Deploying configuration on: ' +
            ips[self.cluster.master], r'Deploying tpch.properties connector '
            r'configurations on: ' + ips[self.cluster.master] + r' ',
            r'Deploying configuration on: ' + ips[self.cluster.slaves[0]],
            r'Deploying tpch.properties connector '
            r'configurations on: ' + ips[self.cluster.slaves[0]] + r' ',
            r'Deploying rpm on .*\.\.\.', r'Deploying rpm on .*\.\.\.',
            r'Fetching local presto rpm at path: .*',
            r'Found existing rpm at: .*'
        ]

        cmd_output.sort()
        expected.sort()
        for expected_regexp, actual_line in zip(expected, cmd_output):
            self.assertRegexpMatches(actual_line, expected_regexp)

        self.assert_installed_with_regex_configs(self.cluster.master,
                                                 [self.cluster.slaves[0]])
    def test_install_with_malformed_topology(self):
        installer = StandalonePrestoInstaller(self)
        rpm_name = installer.copy_presto_rpm_to_master()
        topology = {'coordinator': 'master',
                    'workers': 'slave1' 'slave2'}
        self.upload_topology(topology)
        expected = 'Workers must be of type list.  Found <type \'unicode\'>.' \
                   '  More detailed information can be found in ' \
                   '/var/log/prestoadmin/presto-admin.log'

        self.assertRaisesRegexp(OSError,
                                expected,
                                self.run_prestoadmin,
                                'server install /mnt/presto-admin/%(rpm)s ',
                                rpm=rpm_name)
Esempio n. 21
0
    def test_install_with_malformed_connector(self):
        installer = StandalonePrestoInstaller(self)
        self.upload_topology()
        self.cluster.write_content_to_host(
            'connectr.typo:invalid',
            os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'),
            self.cluster.master)
        actual_out = installer.install(pa_raise_error=False)
        expected = 'Underlying exception:\n    Catalog configuration ' \
                   'jmx.properties does not contain connector.name'
        self.assertRegexpMatches(actual_out, expected)

        for container in self.cluster.all_hosts():
            installer.assert_installed(self, container)
            self.assert_has_default_config(container)
Esempio n. 22
0
    def test_install_when_topology_has_ips(self):
        installer = StandalonePrestoInstaller(self)
        ips = self.cluster.get_ip_address_dict()
        topology = {"coordinator": ips[self.cluster.master],
                    "workers": [ips[self.cluster.slaves[0]]]}
        self.upload_topology(topology)
        self.cluster.write_content_to_host(
            'connector.name=jmx',
            os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'),
            self.cluster.master
        )

        cmd_output = installer.install().splitlines()
        expected = [
            r'Deploying rpm on %s...' % ips[self.cluster.master],
            r'Deploying rpm on %s...' % ips[self.cluster.slaves[0]],
            r'Package deployed successfully on: ' + ips[
                self.cluster.master],
            r'Package installed successfully on: ' + ips[
                self.cluster.master],
            r'Package deployed successfully on: ' +
            ips[self.cluster.slaves[0]],
            r'Package installed successfully on: ' +
            ips[self.cluster.slaves[0]],
            r'Deploying configuration on: ' +
            ips[self.cluster.master],
            r'Deploying jmx.properties, tpch.properties '
            r'connector configurations on: ' +
            ips[self.cluster.master] + r' ',
            r'Deploying configuration on: ' +
            ips[self.cluster.slaves[0]],
            r'Deploying jmx.properties, tpch.properties '
            r'connector configurations on: ' +
            ips[self.cluster.slaves[0]] + r' ',
            r'Using rpm_specifier as a local path',
            r'Fetching local presto rpm at path: .*',
            r'Found existing rpm at: .*']

        cmd_output.sort()
        expected.sort()
        self.assertRegexpMatchesLineByLine(cmd_output, expected)

        self.assert_installed_with_regex_configs(
            self.cluster.master,
            [self.cluster.slaves[0]])
        for container in [self.cluster.master,
                          self.cluster.slaves[0]]:
            self.assert_has_jmx_connector(container)
    def test_install_when_topology_has_ips(self):
        installer = StandalonePrestoInstaller(self)
        ips = self.cluster.get_ip_address_dict()
        topology = {
            "coordinator": ips[self.cluster.internal_master],
            "workers": [ips[self.cluster.internal_slaves[0]]]
        }
        self.upload_topology(topology)
        self.cluster.write_content_to_host(
            'connector.name=jmx',
            os.path.join(get_catalog_directory(), 'jmx.properties'),
            self.cluster.master)

        cmd_output = installer.install().splitlines()
        expected = [
            r'Deploying rpm on %s...' % ips[self.cluster.internal_master],
            r'Deploying rpm on %s...' % ips[self.cluster.internal_slaves[0]],
            r'Package deployed successfully on: ' +
            ips[self.cluster.internal_master],
            r'Package installed successfully on: ' +
            ips[self.cluster.internal_master],
            r'Package deployed successfully on: ' +
            ips[self.cluster.internal_slaves[0]],
            r'Package installed successfully on: ' +
            ips[self.cluster.internal_slaves[0]],
            r'Deploying configuration on: ' +
            ips[self.cluster.internal_master],
            r'Deploying jmx.properties, tpch.properties '
            r'catalog configurations on: ' +
            ips[self.cluster.internal_master] + r' ',
            r'Deploying configuration on: ' +
            ips[self.cluster.internal_slaves[0]],
            r'Deploying jmx.properties, tpch.properties '
            r'catalog configurations on: ' +
            ips[self.cluster.internal_slaves[0]] + r' ',
            r'Using rpm_specifier as a local path',
            r'Fetching local presto rpm at path: .*',
            r'Found existing rpm at: .*'
        ]

        cmd_output.sort()
        expected.sort()
        self.assertRegexpMatchesLineByLine(cmd_output, expected)

        self.assert_installed_with_regex_configs(self.cluster.master,
                                                 [self.cluster.slaves[0]])
        for host in [self.cluster.master, self.cluster.slaves[0]]:
            self.assert_has_jmx_catalog(host)
Esempio n. 24
0
    def test_install_fails_java8_not_found(self):
        installer = StandalonePrestoInstaller(self)
        with relocate_jdk_directory(self.cluster, '/usr'):
            self.upload_topology()
            cmd_output = installer.install(pa_raise_error=False)
            actual = cmd_output.splitlines()
            num_failures = 0
            for line in enumerate(actual):
                if str(line).find('Error: Required Java version'
                                  ' could not be found') != -1:
                    num_failures += 1

            self.assertEqual(4, num_failures)

            for container in self.cluster.all_hosts():
                installer.assert_uninstalled(container)
Esempio n. 25
0
    def test_install(self, installer=None):
        if installer is None:
            installer = StandalonePrestoInstaller(self)

        self.upload_topology()

        cmd_output = installer.install()
        expected = installed_all_hosts_output

        actual = cmd_output.splitlines()
        self.assertRegexpMatchesLineByLine(actual, expected)

        for container in self.cluster.all_hosts():
            installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_connector(container)
Esempio n. 26
0
    def test_install_fails_java_not_found(self):
        installer = StandalonePrestoInstaller(self)
        with relocate_jdk_directory(self.cluster, '/usr'):
            self.upload_topology()
            cmd_output = installer.install(pa_raise_error=False)
            actual = cmd_output.splitlines()
            num_failures = 0
            for line in enumerate(actual):
                if str(line).find('Error: Required Java version'
                                  ' could not be found') != -1:
                    num_failures += 1

            self.assertEqual(4, num_failures)

            for container in self.cluster.all_hosts():
                installer.assert_uninstalled(container)
    def test_install(self, installer=None):
        if installer is None:
            installer = StandalonePrestoInstaller(self)

        self.upload_topology()

        cmd_output = installer.install()
        expected = installed_all_hosts_output

        actual = cmd_output.splitlines()
        self.assertEqual(sorted(expected), sorted(actual))

        for container in self.cluster.all_hosts():
            installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_connector(container)
    def test_install_with_malformed_connector(self):
        installer = StandalonePrestoInstaller(self, dummy=True)
        self.upload_topology()
        self.cluster.write_content_to_host(
            'connectr.typo:invalid',
            os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'),
            self.cluster.master
        )
        actual_out = installer.install(pa_raise_error=False)
        expected = 'Underlying exception:\n    Catalog configuration ' \
                   'jmx.properties does not contain connector.name'
        self.assertRegexpMatches(actual_out, expected)

        for container in self.cluster.all_hosts():
            installer.assert_installed(self, container)
            self.assert_has_default_config(container)
Esempio n. 29
0
    def test_install_ext_host_is_pa_master(self):
        installer = StandalonePrestoInstaller(self)
        topology = {"coordinator": "slave1",
                    "workers": ["slave2", "slave3"]}
        self.upload_topology(topology)

        cmd_output = installer.install(coordinator='slave1')
        expected = install_with_ext_host_pa_master_out

        actual = cmd_output.splitlines()
        self.assertRegexpMatchesLineByLine(actual, expected)

        self.assert_installed_with_configs(
            self.cluster.slaves[0],
            [self.cluster.slaves[1],
             self.cluster.slaves[2]])
    def test_install_worker_is_pa_master(self):
        installer = StandalonePrestoInstaller(self, dummy=True)
        topology = {"coordinator": "slave1",
                    "workers": ["master", "slave2", "slave3"]}
        self.upload_topology(topology)

        cmd_output = installer.install(coordinator='slave1')
        expected = install_with_worker_pa_master_out

        actual = cmd_output.splitlines()
        self.assertEqual(sorted(expected), sorted(actual))

        self.assert_installed_with_configs(
            self.cluster.slaves[0],
            [self.cluster.slaves[1],
             self.cluster.slaves[2],
             self.cluster.master])
Esempio n. 31
0
    def test_server_starts_java8_in_bin_java(self):
        installer = StandalonePrestoInstaller(self)
        for container in self.cluster.all_hosts():
            self.cluster.exec_cmd_on_host(container,
                                          'mv /usr/java/jdk1.8.0_40 /usr')
            self.cluster.exec_cmd_on_host(
                container, 'ln -s /usr/jdk1.8.0_40/bin/java '
                '/bin/java')
        self.upload_topology()

        installer.install()

        # starts successfully with java8_home set
        output = self.run_prestoadmin('server start')
        self.assertFalse(
            'Warning: No value found for JAVA8_HOME. Default Java will be '
            'used.' in output)
    def test_install_worker_is_pa_master(self):
        installer = StandalonePrestoInstaller(self)
        topology = {
            "coordinator": "slave1",
            "workers": ["master", "slave2", "slave3"]
        }
        self.upload_topology(topology)

        cmd_output = installer.install(coordinator='slave1')
        expected = install_with_worker_pa_master_out

        actual = cmd_output.splitlines()
        self.assertEqual(sorted(expected), sorted(actual))

        self.assert_installed_with_configs(self.cluster.slaves[0], [
            self.cluster.slaves[1], self.cluster.slaves[2], self.cluster.master
        ])
    def test_install_with_no_perm_to_local_path(self):
        installer = StandalonePrestoInstaller(self)
        rpm_name = installer.copy_presto_rpm_to_master()
        self.upload_topology()
        self.run_prestoadmin("configuration deploy")

        script = 'chmod 600 /mnt/presto-admin/%(rpm)s; su app-admin -c ' \
                 '"./presto-admin server install /mnt/presto-admin/%(rpm)s "'
        error_msg = '\nFatal error: [%(host)s] error: ' \
                    '/mnt/presto-admin/%(rpm)s: ' \
                    'open failed: Permission denied\n\nAborting.\n'
        expected = ''
        for host in self.cluster.all_internal_hosts():
            expected += error_msg % {'host': host, 'rpm': rpm_name}
        actual = self.run_script_from_prestoadmin_dir(script, rpm=rpm_name,
                                                      raise_error=False)
        self.assertEqualIgnoringOrder(actual, expected)
Esempio n. 34
0
    def test_server_starts_java8_in_bin_java(self):
        installer = StandalonePrestoInstaller(self)
        for container in self.cluster.all_hosts():
            self.cluster.exec_cmd_on_host(container,
                                          'mv /usr/java/jdk1.8.0_40 /usr')
            self.cluster.exec_cmd_on_host(container,
                                          'ln -s /usr/jdk1.8.0_40/bin/java '
                                          '/bin/java')
        self.upload_topology()

        installer.install()

        # starts successfully with java8_home set
        output = self.run_prestoadmin('server start')
        self.assertFalse(
            'Warning: No value found for JAVA8_HOME. Default Java will be '
            'used.' in output)
Esempio n. 35
0
    def test_install_non_root_user(self):
        installer = StandalonePrestoInstaller(self)
        self.upload_topology(
            {"coordinator": "master",
             "workers": ["slave1", "slave2", "slave3"],
             "username": "******"}
        )

        rpm_name = installer.copy_presto_rpm_to_master(cluster=self.cluster)
        self.write_test_configs(self.cluster)
        self.run_prestoadmin(
            'server install ' + os.path.join(self.cluster.mount_dir, rpm_name) + ' -p password'
        )

        for container in self.cluster.all_hosts():
            installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_connector(container)
Esempio n. 36
0
    def test_query_info_pa_separate_node(self):
        installer = StandalonePrestoInstaller(self)
        self.setup_cluster(NoHadoopBareImageProvider, STANDALONE_PA_CLUSTER)
        topology = {"coordinator": "slave1",
                    "workers": ["slave2", "slave3"]}
        self.upload_topology(topology=topology)
        installer.install(coordinator='slave1')
        self.run_prestoadmin('server start')
        sql_to_run = 'SELECT * FROM system.runtime.nodes WHERE 1234 = 1234'
        query_id = self.retry(
            lambda: self.get_query_id(sql_to_run, host=self.cluster.slaves[0]))

        actual = self.run_prestoadmin('collect query_info ' + query_id)
        query_info_file_name = path.join(TMP_PRESTO_DEBUG, 'query_info_' + query_id + '.json')

        expected = 'Gathered query information in file: ' + query_info_file_name + '\n'
        self.assert_path_exists(self.cluster.master, query_info_file_name)
        self.assertEqual(actual, expected)
    def test_install_failure_without_java8_home(self):
        installer = StandalonePrestoInstaller(self, dummy=True)
        for container in self.cluster.all_hosts():
            self.cluster.exec_cmd_on_host(container,
                                          "mv /usr/java/jdk1.8.0_40 /usr/")
        self.upload_topology()
        cmd_output = installer.install(pa_raise_error=False)
        actual = cmd_output.splitlines()
        num_failures = 0
        for line in enumerate(actual):
            if str(line).find("Error: Required Java version"
                              " could not be found") != -1:
                num_failures += 1

        self.assertEqual(4, num_failures)

        for container in self.cluster.all_hosts():
            installer.assert_uninstalled(container, dummy=True)
    def test_install_non_root_user(self):
        installer = StandalonePrestoInstaller(self)
        self.upload_topology({
            "coordinator": "master",
            "workers": ["slave1", "slave2", "slave3"],
            "username": "******"
        })

        rpm_name = installer.copy_presto_rpm_to_master(cluster=self.cluster)
        self.write_test_configs(self.cluster)
        self.run_prestoadmin(
            'server install {rpm_dir}/{name} -p password'.format(
                rpm_dir=self.cluster.rpm_cache_dir, name=rpm_name))

        for container in self.cluster.all_hosts():
            installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_catalog(container)
Esempio n. 39
0
    def test_install_non_root_user(self):
        installer = StandalonePrestoInstaller(self)
        self.upload_topology({
            "coordinator": "master",
            "workers": ["slave1", "slave2", "slave3"],
            "username": "******"
        })

        rpm_name = installer.copy_presto_rpm_to_master(cluster=self.cluster)
        self.write_test_configs(self.cluster)
        self.run_prestoadmin('server install ' +
                             os.path.join(self.cluster.mount_dir, rpm_name) +
                             ' -p password')

        for container in self.cluster.all_hosts():
            installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_connector(container)
Esempio n. 40
0
    def test_server_starts_java8_in_bin_java(self):
        installer = StandalonePrestoInstaller(self)

        with relocate_jdk_directory(self.cluster, '/usr') as new_java_home:
            java_bin = os.path.join(new_java_home, 'bin', 'java')

            for container in self.cluster.all_hosts():
                self.cluster.exec_cmd_on_host(
                    container, 'ln -s %s /bin/java' % (java_bin,))

            self.upload_topology()

            installer.install()

            # starts successfully with java8_home set
            output = self.run_prestoadmin('server start')
            self.assertFalse(
                'Warning: No value found for JAVA8_HOME. Default Java will be '
                'used.' in output)
Esempio n. 41
0
    def test_install_interactive(self):
        installer = StandalonePrestoInstaller(self)
        self.cluster.write_content_to_host(
            'connector.name=jmx',
            os.path.join(get_catalog_directory(), 'jmx.properties'),
            self.cluster.master)
        ips = self.cluster.get_ip_address_dict()
        self.upload_topology({
            "coordinator":
            ips[self.cluster.internal_master],
            "workers": [
                ips[self.cluster.internal_slaves[0]],
                ips[self.cluster.internal_slaves[1]],
                ips[self.cluster.internal_slaves[2]]
            ],
            "username":
            "******"
        })

        rpm_name = installer.copy_presto_rpm_to_master()
        self.write_test_configs(self.cluster,
                                coordinator=ips[self.cluster.internal_master])

        additional_keywords = {
            'user': "******",
            'rpm_dir': self.cluster.rpm_cache_dir,
            'rpm': rpm_name
        }

        self.run_script_from_prestoadmin_dir(
            'echo -e "%(user)s\n22\n%(master)s\n%(slave1)s\n" | '
            './presto-admin server install %(rpm_dir)s/%(rpm)s -p password',
            **additional_keywords)

        self.assert_installed_with_regex_configs(self.cluster.master, [
            self.cluster.slaves[0], self.cluster.slaves[1],
            self.cluster.slaves[2]
        ])

        for container in self.cluster.all_hosts():
            installer.assert_installed(self, container)
            self.assert_has_default_catalog(container)
            self.assert_has_jmx_catalog(container)
    def test_install_twice(self):
        installer = StandalonePrestoInstaller(self, dummy=True)
        self.test_install(installer=installer)
        output = installer.install(pa_raise_error=False)

        self.default_keywords.update(installer.get_keywords())

        with open(os.path.join(LOCAL_RESOURCES_DIR, 'install_twice.txt'), 'r') \
                as f:
            expected = f.read()
        expected = self.escape_for_regex(
            self.replace_keywords(expected))

        self.assertRegexpMatchesLineByLine(output.splitlines(),
                                           expected.splitlines())
        for container in self.cluster.all_hosts():
            installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_connector(container)
    def test_install_with_java8_home(self):
        installer = StandalonePrestoInstaller(self)

        with relocate_jdk_directory(self.cluster, '/usr') as new_java_home:
            topology = {"coordinator": "master",
                        "workers": ["slave1", "slave2", "slave3"],
                        "java8_home": new_java_home}
            self.upload_topology(topology)

            cmd_output = installer.install()
            expected = self.format_err_msgs_with_internal_hosts(installed_all_hosts_output)

            actual = cmd_output.splitlines()
            self.assertRegexpMatchesLineByLine(actual, expected)

            for host in self.cluster.all_hosts():
                installer.assert_installed(self, host)
                self.assert_has_default_config(host)
                self.assert_has_default_catalog(host)
Esempio n. 44
0
    def test_query_info_pa_separate_node(self):
        installer = StandalonePrestoInstaller(self)
        self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER)
        topology = {"coordinator": "slave1", "workers": ["slave2", "slave3"]}
        self.upload_topology(topology=topology)
        installer.install(coordinator='slave1')
        self.run_prestoadmin('server start')
        sql_to_run = 'SELECT * FROM system.runtime.nodes WHERE 1234 = 1234'
        with settings(roledefs={'coordinator': ['slave1']}):
            query_id = self.retry(lambda: self.get_query_id(
                sql_to_run, host=self.cluster.slaves[0]))

        actual = self.run_prestoadmin('collect query_info ' + query_id)
        query_info_file_name = path.join(TMP_PRESTO_DEBUG,
                                         'query_info_' + query_id + '.json')

        expected = 'Gathered query information in file: ' + query_info_file_name + '\n'
        self.assert_path_exists(self.cluster.master, query_info_file_name)
        self.assertEqual(actual, expected)
Esempio n. 45
0
    def test_server_starts_java_in_bin_java(self):
        installer = StandalonePrestoInstaller(self)

        with relocate_jdk_directory(self.cluster, '/usr') as new_java_home:
            java_bin = os.path.join(new_java_home, 'bin', 'java')

            for container in self.cluster.all_hosts():
                self.cluster.exec_cmd_on_host(
                    container, 'ln -s %s /bin/java' % (java_bin, ))

            self.upload_topology()

            installer.install()

            # starts successfully with java_home set
            output = self.run_prestoadmin('server start')
            self.assertFalse(
                'Warning: No value found for JAVA_HOME. Default Java will be '
                'used.' in output)
Esempio n. 46
0
    def test_install_with_java8_home(self):
        installer = StandalonePrestoInstaller(self)

        with relocate_jdk_directory(self.cluster, '/usr') as new_java_home:
            topology = {"coordinator": "master",
                        "workers": ["slave1", "slave2", "slave3"],
                        "java8_home": new_java_home}
            self.upload_topology(topology)

            cmd_output = installer.install()
            expected = self.format_err_msgs_with_internal_hosts(installed_all_hosts_output)

            actual = cmd_output.splitlines()
            self.assertRegexpMatchesLineByLine(actual, expected)

            for host in self.cluster.all_hosts():
                installer.assert_installed(self, host)
                self.assert_has_default_config(host)
                self.assert_has_default_connector(host)
Esempio n. 47
0
    def test_install_with_java8_home(self):
        installer = StandalonePrestoInstaller(self)
        new_java_home = relocate_default_java(self.cluster, '/usr')

        topology = {"coordinator": "master",
                    "workers": ["slave1", "slave2", "slave3"],
                    "java8_home": new_java_home}
        self.upload_topology(topology)

        cmd_output = installer.install()
        expected = installed_all_hosts_output

        actual = cmd_output.splitlines()
        self.assertRegexpMatchesLineByLine(actual, expected)

        for container in self.cluster.all_hosts():
            installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_connector(container)
class TestServerUninstall(BaseProductTestCase):
    def setUp(self):
        super(TestServerUninstall, self).setUp()
        self.installer = StandalonePrestoInstaller(self)

    @attr('smoketest')
    def test_uninstall(self):
        self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER)
        start_output = self.run_prestoadmin('server start')
        process_per_host = self.get_process_per_host(start_output.splitlines())
        self.assert_started(process_per_host)

        self.run_prestoadmin(
            'server uninstall', raise_error=False).splitlines()
        self.assert_stopped(process_per_host)

        for container in self.cluster.all_hosts():
            self.assert_uninstalled_dirs_removed(container)

    def assert_uninstalled_dirs_removed(self, container):
        self.installer.assert_uninstalled(container)
        self.assert_path_removed(container, '/usr/lib/presto')
        self.assert_path_removed(container, '/var/lib/presto')
        self.assert_path_removed(container, '/usr/shared/doc/presto')
        self.assert_path_removed(container, '/etc/init.d/presto')

    def test_uninstall_lost_host(self):
        self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PRESTO_CLUSTER)

        self.cluster.stop_host(
            self.cluster.slaves[0])

        expected = self.down_node_connection_error(
            self.cluster.internal_slaves[0])
        cmd_output = self.run_prestoadmin('server uninstall',
                                          raise_error=False)
        self.assertRegexpMatches(cmd_output, expected)

        for container in [self.cluster.internal_master,
                          self.cluster.internal_slaves[1],
                          self.cluster.internal_slaves[2]]:
            self.assert_uninstalled_dirs_removed(container)
Esempio n. 49
0
    def test_install_with_java8_home(self):
        installer = StandalonePrestoInstaller(self)
        new_java_home = relocate_default_java(self.cluster, '/usr')

        topology = {
            "coordinator": "master",
            "workers": ["slave1", "slave2", "slave3"],
            "java8_home": new_java_home
        }
        self.upload_topology(topology)

        cmd_output = installer.install()
        expected = installed_all_hosts_output

        actual = cmd_output.splitlines()
        self.assertRegexpMatchesLineByLine(actual, expected)

        for container in self.cluster.all_hosts():
            installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_connector(container)
    def test_install_with_java8_home(self):
        installer = StandalonePrestoInstaller(self)
        for container in self.cluster.all_hosts():
            self.cluster.exec_cmd_on_host(container,
                                          "mv /usr/java/jdk1.8.0_40 /usr/")
        topology = {
            "coordinator": "master",
            "workers": ["slave1", "slave2", "slave3"],
            "java8_home": "/usr/jdk1.8.0_40/jre"
        }
        self.upload_topology(topology)

        cmd_output = installer.install()
        expected = installed_all_hosts_output

        actual = cmd_output.splitlines()
        self.assertEqual(sorted(expected), sorted(actual))

        for container in self.cluster.all_hosts():
            installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_connector(container)
    def test_install_when_topology_has_ips(self):
        installer = StandalonePrestoInstaller(self)
        ips = self.cluster.get_ip_address_dict()
        topology = {
            "coordinator": ips[self.cluster.master],
            "workers": [ips[self.cluster.slaves[0]]]
        }
        self.upload_topology(topology)
        self.cluster.write_content_to_host(
            'connector.name=jmx',
            os.path.join(constants.CONNECTORS_DIR, 'jmx.properties'),
            self.cluster.master)

        cmd_output = installer.install().splitlines()
        expected = [
            r'Deploying rpm on %s...' % ips[self.cluster.master],
            r'Deploying rpm on %s...' % ips[self.cluster.slaves[0]],
            r'Package deployed successfully on: ' + ips[self.cluster.master],
            r'Package installed successfully on: ' + ips[self.cluster.master],
            r'Package deployed successfully on: ' +
            ips[self.cluster.slaves[0]],
            r'Package installed successfully on: ' +
            ips[self.cluster.slaves[0]],
            r'Deploying configuration on: ' + ips[self.cluster.master],
            r'Deploying jmx.properties, tpch.properties '
            r'connector configurations on: ' + ips[self.cluster.master],
            r'Deploying configuration on: ' + ips[self.cluster.slaves[0]],
            r'Deploying jmx.properties, tpch.properties '
            r'connector configurations on: ' + ips[self.cluster.slaves[0]]
        ]

        cmd_output.sort()
        expected.sort()
        self.assertRegexpMatchesLineByLine(expected, cmd_output)

        self.assert_installed_with_regex_configs(self.cluster.master,
                                                 [self.cluster.slaves[0]])
        for container in [self.cluster.master, self.cluster.slaves[0]]:
            self.assert_has_jmx_connector(container)
    def test_install_when_catalog_json_exists(self):
        installer = StandalonePrestoInstaller(self)
        topology = {"coordinator": "master",
                    "workers": ["slave1"]}
        self.upload_topology(topology)
        self.cluster.write_content_to_host(
            'connector.name=jmx',
            os.path.join(get_catalog_directory(), 'jmx.properties'),
            self.cluster.master
        )

        cmd_output = installer.install()
        expected = ['Deploying rpm on master...',
                    'Deploying rpm on slave1...',
                    'Package deployed successfully on: slave1',
                    'Package installed successfully on: slave1',
                    'Package deployed successfully on: master',
                    'Package installed successfully on: master',
                    'Deploying configuration on: master',
                    'Deploying jmx.properties, tpch.properties '
                    'catalog configurations on: master ',
                    'Deploying configuration on: slave1',
                    'Deploying jmx.properties, tpch.properties '
                    'catalog configurations on: slave1 ',
                    'Using rpm_specifier as a local path',
                    'Fetching local presto rpm at path: .*',
                    'Found existing rpm at: .*']

        actual = cmd_output.splitlines()
        self.assertRegexpMatchesLineByLine(actual, expected)

        for container in [self.cluster.master,
                          self.cluster.slaves[0]]:
            installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_catalog(container)
            self.assert_has_jmx_catalog(container)
Esempio n. 53
0
    def test_install_twice(self):
        installer = StandalonePrestoInstaller(self)
        self.test_install(installer=installer)
        output = installer.install(pa_raise_error=False)

        self.default_keywords.update(installer.get_keywords())

        with open(os.path.join(LOCAL_RESOURCES_DIR, 'install_twice.txt'),
                  'r') as f:
            expected = f.read()
        expected = self.escape_for_regex(self.replace_keywords(expected))

        self.assertRegexpMatchesLineByLine(output.splitlines(),
                                           expected.splitlines())
        for container in self.cluster.all_hosts():
            installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_connector(container)
Esempio n. 54
0
 def setUp(self):
     super(TestStatus, self).setUp()
     self.installer = StandalonePrestoInstaller(self)
Esempio n. 55
0
class TestStatus(BaseProductTestCase):
    def setUp(self):
        super(TestStatus, self).setUp()
        self.installer = StandalonePrestoInstaller(self)

    def test_status_uninstalled(self):
        self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER)
        self.upload_topology()
        status_output = self._server_status_with_retries()
        self.check_status(status_output, self.not_installed_status())

    def test_status_not_started(self):
        self.setup_cluster(NoHadoopBareImageProvider(),
                           STANDALONE_PRESTO_CLUSTER)
        status_output = self._server_status_with_retries()
        self.check_status(status_output, self.not_started_status())

    @attr('smoketest')
    def test_status_happy_path(self):
        self.setup_cluster(NoHadoopBareImageProvider(),
                           STANDALONE_PRESTO_CLUSTER)
        self.run_prestoadmin('server start')
        status_output = self._server_status_with_retries(check_connectors=True)
        self.check_status(status_output, self.base_status())

    def test_status_only_coordinator(self):
        self.setup_cluster(NoHadoopBareImageProvider(),
                           STANDALONE_PRESTO_CLUSTER)

        self.run_prestoadmin('server start -H master')
        # don't run with retries because it won't be able to query the
        # coordinator because the coordinator is set to not be a worker
        status_output = self.run_prestoadmin('server status')
        self.check_status(
            status_output,
            self.single_node_up_status(self.cluster.internal_master))

    def test_status_only_worker(self):
        self.setup_cluster(NoHadoopBareImageProvider(),
                           STANDALONE_PRESTO_CLUSTER)

        self.run_prestoadmin('server start -H slave1')
        status_output = self._server_status_with_retries()
        self.check_status(
            status_output,
            self.single_node_up_status(self.cluster.internal_slaves[0]))

        # Check that the slave sees that it's stopped, even though the
        # discovery server is not up.
        self.run_prestoadmin('server stop')
        status_output = self._server_status_with_retries()
        self.check_status(status_output, self.not_started_status())

    def test_connection_to_coordinator_lost(self):
        self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER)
        topology = {
            "coordinator": "slave1",
            "workers": ["master", "slave2", "slave3"]
        }
        self.upload_topology(topology=topology)
        self.installer.install(coordinator='slave1')
        self.run_prestoadmin('server start')
        self.cluster.stop_host(self.cluster.slaves[0])
        topology = {
            "coordinator": self.cluster.get_down_hostname("slave1"),
            "workers": ["master", "slave2", "slave3"]
        }
        status_output = self._server_status_with_retries()
        statuses = self.node_not_available_status(
            topology, self.cluster.internal_slaves[0], coordinator_down=True)
        self.check_status(status_output, statuses)

    def test_connection_to_worker_lost(self):
        self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER)
        topology = {
            "coordinator": "slave1",
            "workers": ["master", "slave2", "slave3"]
        }
        self.upload_topology(topology=topology)
        self.installer.install(coordinator='slave1')
        self.run_prestoadmin('server start')
        self.cluster.stop_host(self.cluster.slaves[1])
        topology = {
            "coordinator":
            "slave1",
            "workers":
            ["master",
             self.cluster.get_down_hostname("slave2"), "slave3"]
        }
        status_output = self._server_status_with_retries(check_connectors=True)
        statuses = self.node_not_available_status(
            topology, self.cluster.internal_slaves[1])
        self.check_status(status_output, statuses)

    def test_status_port_not_8080(self):
        self.setup_cluster(NoHadoopBareImageProvider(), STANDALONE_PA_CLUSTER)
        self.upload_topology()

        port_config = """discovery.uri=http://master:8090
http-server.http.port=8090"""

        self.installer.install(extra_configs=port_config)
        self.run_prestoadmin('server start')
        status_output = self._server_status_with_retries(check_connectors=True)

        self.check_status(status_output, self.base_status(), 8090)

    def test_status_non_root_user(self):
        self.setup_cluster(NoHadoopBareImageProvider(),
                           STANDALONE_PRESTO_CLUSTER)
        self.upload_topology({
            "coordinator": "master",
            "workers": ["slave1", "slave2", "slave3"],
            "username": "******"
        })
        self.run_prestoadmin('server start -p password')
        status_output = self._server_status_with_retries(
            check_connectors=True, extra_arguments=' -p password')
        self.check_status(status_output, self.base_status())

    def base_status(self, topology=None):
        ips = self.cluster.get_ip_address_dict()
        if not topology:
            topology = {
                'coordinator':
                self.cluster.internal_master,
                'workers': [
                    self.cluster.internal_slaves[0],
                    self.cluster.internal_slaves[1],
                    self.cluster.internal_slaves[2]
                ]
            }
        statuses = []
        hosts_in_status = [topology['coordinator']] + topology['workers'][:]
        for host in hosts_in_status:
            role = 'coordinator' if host is topology['coordinator']\
                else 'worker'
            status = {
                'host': host,
                'role': role,
                'ip': ips[host],
                'is_running': 'Running'
            }
            statuses += [status]
        return statuses

    def not_started_status(self):
        statuses = self.base_status()
        for status in statuses:
            status['ip'] = 'Unknown'
            status['is_running'] = 'Not Running'
            status['error_message'] = '\tNo information available: ' \
                                      'unable to query coordinator'
        return statuses

    def not_installed_status(self):
        statuses = self.base_status()
        for status in statuses:
            status['ip'] = 'Unknown'
            status['is_running'] = 'Not Running'
            status['error_message'] = '\tPresto is not installed.'
        return statuses

    def single_node_up_status(self, node):
        statuses = self.not_started_status()
        for status in statuses:
            if status['host'] is node:
                status['is_running'] = 'Running'
        return statuses

    def node_not_available_status(self,
                                  topology,
                                  node,
                                  coordinator_down=False):
        statuses = self.base_status(topology)
        for status in statuses:
            if status['host'] == node:
                status['is_running'] = 'Not Running'
                status['error_message'] = \
                    self.status_node_connection_error(node)
                status['ip'] = 'Unknown'
                status['host'] = self.cluster.get_down_hostname(node)
            elif coordinator_down:
                status['error_message'] = '\tNo information available: ' \
                                          'unable to query coordinator'
                status['ip'] = 'Unknown'

        return statuses

    def status_fail_msg(self, actual_output, expected_regexp):
        log_tail = self.fetch_log_tail(lines=100)

        return ('=== ACTUAL OUTPUT ===\n%s\n=== DID NOT MATCH REGEXP ===\n%s\n'
                '=== LOG FOR DEBUGGING ===\n%s=== END OF LOG ===' %
                (actual_output, expected_regexp, log_tail))

    def check_status(self, cmd_output, statuses, port=8080):
        expected_output = []
        for status in statuses:
            expected_output += \
                ['Server Status:',
                 '\t%s\(IP: %s, Roles: %s\): %s' %
                 (status['host'], status['ip'], status['role'],
                  status['is_running'])]
            if 'error_message' in status and status['error_message']:
                expected_output += [status['error_message']]
            elif status['is_running'] is 'Running':
                expected_output += \
                    ['\tNode URI\(http\): http://%s:%s' % (status['ip'],
                                                           str(port)),
                     '\tPresto Version: ' + PRESTO_VERSION,
                     '\tNode status:    active',
                     '\tConnectors:     system, tpch']

        expected_regex = '\n'.join(expected_output)
        # The status command is written such that there are a couple ways that
        # the presto client can fail that result in partial output from the
        # command, but errors in the logs. If we fail to match, we include the
        # log information in the assertion message to make determining exactly
        # what failed easier. Grab the logs lazily so that we don't incur the
        # cost of getting them when they aren't needed. The status tests are
        # slow enough already.
        self.assertLazyMessage(
            lambda: self.status_fail_msg(cmd_output, expected_regex),
            self.assertRegexpMatches, cmd_output, expected_regex)

    def _server_status_with_retries(self,
                                    check_connectors=False,
                                    extra_arguments=''):
        try:
            return self.retry(
                lambda: self._get_status_until_coordinator_updated(
                    check_connectors, extra_arguments=extra_arguments), 180, 0)
        except PrestoError as e:
            self.assertLazyMessage(
                self.status_fail_msg(e.message,
                                     "Ran out of time retrying status"),
                self.fail("PrestoError %s" % (e.message, )))

    def _get_status_until_coordinator_updated(self,
                                              check_connectors=False,
                                              extra_arguments=''):
        status_output = self.run_prestoadmin('server status' + extra_arguments)
        if 'the coordinator has not yet discovered this node' in status_output:
            raise PrestoError('Coordinator has not discovered all nodes yet: '
                              '%s' % status_output)
        if 'Roles: coordinator): Running\n\tNo information available: ' \
           'unable to query coordinator' in status_output:
            raise PrestoError('Coordinator not started up properly yet.'
                              '\nOutput: %s' % status_output)
        if check_connectors and 'Connectors:' not in status_output:
            raise PrestoError('Connectors not loaded yet: %s' % status_output)
        return status_output
Esempio n. 56
0
 def setUp(self):
     super(TestPackageInstall, self).setUp()
     self.setup_cluster(self.PA_ONLY_CLUSTER)
     self.upload_topology()
     self.installer = StandalonePrestoInstaller(self)
Esempio n. 57
0
class TestPackageInstall(BaseProductTestCase):
    def setUp(self):
        super(TestPackageInstall, self).setUp()
        self.setup_cluster(self.PA_ONLY_CLUSTER)
        self.upload_topology()
        self.installer = StandalonePrestoInstaller(self)

    @attr('smoketest')
    def test_package_install(self):
        rpm_name = self.installer.copy_presto_rpm_to_master()
        output = self.run_prestoadmin(
            'package install '
            '/mnt/presto-admin/%(rpm)s', rpm=rpm_name)
        for container in self.cluster.all_hosts():
            self.installer.assert_installed(self, container, msg=output)

    def test_install_coord_using_dash_h(self):
        rpm_name = self.installer.copy_presto_rpm_to_master()
        output = self.run_prestoadmin(
            'package install /mnt/presto-admin/%(rpm)s -H %(master)s',
            rpm=rpm_name)
        self.installer.assert_installed(self, self.cluster.master)
        for slave in self.cluster.slaves:
            self.installer.assert_uninstalled(slave, msg=output)

    def test_install_worker_using_dash_h(self):
        rpm_name = self.installer.copy_presto_rpm_to_master()
        output = self.run_prestoadmin(
            'package install /mnt/presto-admin/%(rpm)s -H %(slave1)s',
            rpm=rpm_name)

        self.installer.assert_installed(self,
                                        self.cluster.slaves[0],
                                        msg=output)
        self.installer.assert_uninstalled(self.cluster.master, msg=output)
        self.installer.assert_uninstalled(self.cluster.slaves[1], msg=output)
        self.installer.assert_uninstalled(self.cluster.slaves[2], msg=output)

    def test_install_workers_using_dash_h(self):
        rpm_name = self.installer.copy_presto_rpm_to_master()
        output = self.run_prestoadmin(
            'package install /mnt/presto-admin/'
            '%(rpm)s -H %(slave1)s,%(slave2)s',
            rpm=rpm_name)

        self.installer.assert_installed(self,
                                        self.cluster.slaves[0],
                                        msg=output)
        self.installer.assert_installed(self,
                                        self.cluster.slaves[1],
                                        msg=output)
        self.installer.assert_uninstalled(self.cluster.master, msg=output)
        self.installer.assert_uninstalled(self.cluster.slaves[2], msg=output)

    def test_install_exclude_coord(self):
        rpm_name = self.installer.copy_presto_rpm_to_master()
        output = self.run_prestoadmin(
            'package install /mnt/presto-admin/'
            '%(rpm)s -x %(master)s',
            rpm=rpm_name)

        self.installer.assert_uninstalled(self.cluster.master, msg=output)
        for slave in self.cluster.slaves:
            self.installer.assert_installed(self, slave, msg=output)

    def test_install_exclude_worker(self):
        rpm_name = self.installer.copy_presto_rpm_to_master()
        output = self.run_prestoadmin(
            'package install /mnt/presto-admin/'
            '%(rpm)s -x %(slave1)s',
            rpm=rpm_name)
        self.installer.assert_uninstalled(self.cluster.slaves[0], msg=output)
        self.installer.assert_installed(self,
                                        self.cluster.slaves[1],
                                        msg=output)
        self.installer.assert_installed(self, self.cluster.master, msg=output)
        self.installer.assert_installed(self,
                                        self.cluster.slaves[2],
                                        msg=output)

    def test_install_exclude_workers(self):
        rpm_name = self.installer.copy_presto_rpm_to_master()
        output = self.run_prestoadmin(
            'package install /mnt/presto-admin/'
            '%(rpm)s -x %(slave1)s,%(slave2)s',
            rpm=rpm_name)

        self.installer.assert_uninstalled(self.cluster.slaves[0], msg=output)
        self.installer.assert_uninstalled(self.cluster.slaves[1], msg=output)
        self.installer.assert_installed(self, self.cluster.master, msg=output)
        self.installer.assert_installed(self,
                                        self.cluster.slaves[2],
                                        msg=output)

    def test_install_invalid_path(self):
        rpm_name = self.installer.copy_presto_rpm_to_master()
        cmd_output = self.run_prestoadmin(
            'package install /mnt/presto-admin'
            '/invalid-path/presto.rpm',
            rpm=rpm_name)
        error = '\nFatal error: [%s] error: ' \
                '/mnt/presto-admin/invalid-path/presto.rpm: open failed: ' \
                'No such file or directory\n\nAborting.\n'
        expected = ''
        for host in self.cluster.all_internal_hosts():
            expected += error % host

        self.assertEqualIgnoringOrder(cmd_output, expected)

    def test_install_no_path_arg(self):
        self.installer.copy_presto_rpm_to_master()
        output = self.run_prestoadmin('package install', raise_error=False)
        self.assertEqual(
            output, 'Incorrect number of arguments to task.\n\n'
            'Displaying detailed information for task '
            '\'package install\':\n\n'
            '    Install the rpm package on the cluster\n'
            '    \n    Args:\n'
            '        local_path: Absolute path to the rpm'
            ' to be installed\n        '
            '--nodeps (optional): Flag to indicate if '
            'rpm install\n'
            '            should ignore c'
            'hecking package dependencies. Equivalent\n'
            '            to adding --nodeps flag to rpm '
            '-i.\n\n')

    def test_install_already_installed(self):
        rpm_name = self.installer.copy_presto_rpm_to_master()
        self.run_prestoadmin(
            'package install /mnt/presto-admin/%(rpm)s -H '
            '%(master)s',
            rpm=rpm_name)
        self.installer.assert_installed(self, self.cluster.master)
        cmd_output = self.run_prestoadmin(
            'package install /mnt/presto-admin/%(rpm)s -H %(master)s',
            rpm=rpm_name)
        expected = self.escape_for_regex(
            self.replace_keywords(
                """
Fatal error: [%(master)s] sudo() received nonzero return code 1 while \
executing!

Requested: rpm -i /opt/prestoadmin/packages/%(rpm)s
Executed: sudo -S -p 'sudo password:'******'package install '
                                          '/etc/opt/prestoadmin/config.json')

        error = """
Fatal error: \[%s\] error: (/etc/opt/prestoadmin/config.json: )?not an rpm \
package

Aborting.
"""
        expected = ''
        for host in self.cluster.all_internal_hosts():
            expected += error % host

        self.assertRegexpMatchesLineByLine(cmd_output.splitlines(),
                                           expected.splitlines())

    @docker_only
    def test_install_rpm_with_missing_jdk(self):
        rpm_name = self.installer.copy_presto_rpm_to_master(
            rpm_dir=prestoadmin.main_dir,
            rpm_name=self.installer.presto_rpm_filename)
        self.cluster.exec_cmd_on_host(self.cluster.master,
                                      'rpm -e jdk1.8.0_40-1.8.0_40-fcs')
        self.assertRaisesRegexp(
            OSError, 'package jdk1.8.0_40-1.8.0_40-fcs is not '
            'installed', self.cluster.exec_cmd_on_host, self.cluster.master,
            'rpm -q jdk1.8.0_40-1.8.0_40-fcs')

        cmd_output = self.run_prestoadmin(
            'package install /mnt/presto-admin/%(rpm)s -H %(master)s',
            rpm=rpm_name)
        self.assertRegexpMatchesLineByLine(
            cmd_output.splitlines(),
            self.jdk_not_found_error_message(rpm_name).splitlines())

    def jdk_not_found_error_message(self, rpm_name):
        with open(os.path.join(LOCAL_RESOURCES_DIR, 'jdk_not_found.txt')) as f:
            jdk_not_found_error = f.read()
        return self.escape_for_regex(
            self.replace_keywords(jdk_not_found_error,
                                  **self.installer.get_keywords(rpm_name)))

    @docker_only
    def test_install_rpm_missing_dependency(self):
        rpm_name = self.installer.copy_presto_rpm_to_master()
        self.cluster.exec_cmd_on_host(self.cluster.master,
                                      'rpm -e --nodeps python-2.6.6')
        self.assertRaisesRegexp(OSError,
                                'package python-2.6.6 is not installed',
                                self.cluster.exec_cmd_on_host,
                                self.cluster.master, 'rpm -q python-2.6.6')

        cmd_output = self.run_prestoadmin(
            'package install /mnt/presto-admin/%(rpm)s -H %(master)s',
            rpm=rpm_name)
        expected = self.replace_keywords(
            """
Fatal error: [%(master)s] sudo() received nonzero return code 1 while \
executing!

Requested: rpm -i /opt/prestoadmin/packages/%(rpm)s
Executed: sudo -S -p 'sudo password:'******'rpm -e --nodeps python-2.6.6')
        self.assertRaisesRegexp(OSError,
                                'package python-2.6.6 is not installed',
                                self.cluster.exec_cmd_on_host,
                                self.cluster.master, 'rpm -q python-2.6.6')

        cmd_output = self.run_prestoadmin(
            'package install /mnt/presto-admin/%(rpm)s -H %(master)s --nodeps',
            rpm=rpm_name)
        expected = 'Deploying rpm on %(host)s...\n' \
                   'Package deployed successfully on: %(host)s\n' \
                   'Package installed successfully on: %(host)s' \
                   % {'host': self.cluster.internal_master}

        self.assertEqualIgnoringOrder(expected, cmd_output)
    def test_install_interactive(self):
        installer = StandalonePrestoInstaller(self)
        self.cluster.write_content_to_host(
            'connector.name=jmx',
            os.path.join(get_catalog_directory(), 'jmx.properties'),
            self.cluster.master
        )
        rpm_name = installer.copy_presto_rpm_to_master()
        self.write_test_configs(self.cluster)

        additional_keywords = {
            'user': self.cluster.user,
            'rpm_dir': self.cluster.rpm_cache_dir,
            'rpm': rpm_name
        }

        cmd_output = self.run_script_from_prestoadmin_dir(
            'echo -e "%(user)s\n22\n%(master)s\n%(slave1)s\n" | '
            './presto-admin server install %(rpm_dir)s/%(rpm)s ',
            **additional_keywords)

        actual = cmd_output.splitlines()
        expected = [r'Enter user name for SSH connection to all nodes: '
                    r'\[root\] '
                    r'Enter port number for SSH connections to all nodes: '
                    r'\[22\] '
                    r'Enter host name or IP address for coordinator node. '
                    r'Enter an external host name or ip address if this is a '
                    r'multi-node cluster: \[localhost\] '
                    r'Enter host names or IP addresses for worker nodes '
                    r'separated by spaces: '
                    r'\[localhost\] Using rpm_specifier as a local path',
                    r'Package deployed successfully on: ' +
                    self.cluster.internal_master,
                    r'Package installed successfully on: ' +
                    self.cluster.internal_master,
                    r'Package deployed successfully on: ' +
                    self.cluster.internal_slaves[0],
                    r'Package installed successfully on: ' +
                    self.cluster.internal_slaves[0],
                    r'Deploying configuration on: ' +
                    self.cluster.internal_master,
                    r'Deploying jmx.properties, tpch.properties catalog '
                    r'configurations on: ' +
                    self.cluster.internal_master,
                    r'Deploying configuration on: ' +
                    self.cluster.internal_slaves[0],
                    r'Deploying jmx.properties, tpch.properties catalog '
                    r'configurations on: ' +
                    self.cluster.internal_slaves[0],
                    r'Deploying rpm on .*\.\.\.',
                    r'Deploying rpm on .*\.\.\.',
                    r'Fetching local presto rpm at path: .*',
                    r'Found existing rpm at: .*'
                    ]

        self.assertRegexpMatchesLineByLine(actual, expected)
        for container in [self.cluster.master,
                          self.cluster.slaves[0]]:
            installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_catalog(container)
            self.assert_has_jmx_catalog(container)
class TestServerUpgrade(BaseProductTestCase):
    def setUp(self):
        super(TestServerUpgrade, self).setUp()
        self.setup_cluster(NoHadoopBareImageProvider(),
                           STANDALONE_PRESTO_CLUSTER)
        self.dummy_installer = StandalonePrestoInstaller(
            self, (os.path.join(prestoadmin.main_dir, 'tests', 'product',
                                'resources'), 'dummy-rpm.rpm'))
        self.real_installer = StandalonePrestoInstaller(self)

    def start_and_assert_started(self):
        cmd_output = self.run_prestoadmin('server start')
        process_per_host = self.get_process_per_host(cmd_output.splitlines())
        self.assert_started(process_per_host)

    #
    # The dummy RPM is not guaranteed to have any functionality beyond not
    # including any real payload and adding the random README file. It's a
    # hacky one-off that satisfies the requirement of having *something* to
    # upgrade to without downloading another copy of the real RPM. This is NOT
    # the place to test functionality that the presto-server-rpm normally
    # provides, because the dummy rpm probably doesn't provide it, or worse,
    # provides an old and/or broken version of it.
    #
    def assert_upgraded_to_dummy_rpm(self, hosts):
        for container in hosts:
            # Still should have the same configs
            self.dummy_installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_catalog(container)

            # However, dummy_rpm.rpm removes /usr/lib/presto/lib and
            # /usr/lib/presto/lib/plugin
            self.assert_path_removed(container, '/usr/lib/presto/lib')
            self.assert_path_removed(container, '/usr/lib/presto/lib/plugin')

            # And modifies the text of the readme in
            # /usr/shared/doc/presto/README.txt
            self.assert_file_content_regex(
                container, '/usr/shared/doc/presto/README.txt',
                r'.*New line of text here.$')

    @attr('smoketest')
    def test_upgrade(self):
        self.start_and_assert_started()

        self.run_prestoadmin('configuration deploy')
        for container in self.cluster.all_hosts():
            self.real_installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_catalog(container)

        path_on_cluster = self.copy_upgrade_rpm_to_cluster()
        self.upgrade_and_assert_success(path_on_cluster)

    def upgrade_and_assert_success(self, path_on_cluster, extra_arguments=''):
        self.run_prestoadmin('server upgrade ' + path_on_cluster +
                             extra_arguments)
        self.assert_upgraded_to_dummy_rpm(self.cluster.all_hosts())

    def copy_upgrade_rpm_to_cluster(self):
        rpm_name = self.dummy_installer.copy_presto_rpm_to_master()
        return os.path.join(self.cluster.rpm_cache_dir, rpm_name)

    def test_upgrade_fails_given_directory(self):
        dir_on_cluster = '/opt/prestoadmin'
        self.assertRaisesRegexp(OSError,
                                'RPM file not found at %s.' % dir_on_cluster,
                                self.run_prestoadmin,
                                'server upgrade ' + dir_on_cluster)

    def test_upgrade_works_with_symlink(self):
        self.run_prestoadmin('configuration deploy')
        for container in self.cluster.all_hosts():
            self.real_installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_catalog(container)

        path_on_cluster = self.copy_upgrade_rpm_to_cluster()
        symlink = os.path.join(get_install_directory(), 'link.rpm')
        self.cluster.exec_cmd_on_host(
            self.cluster.master, 'ln -s %s %s' % (path_on_cluster, symlink))
        self.upgrade_and_assert_success(symlink)

    def test_configuration_preserved_on_upgrade(self):
        book_content = 'Call me Ishmael ... FINIS'
        book_path = '/etc/presto/moby_dick_abridged'
        self.run_prestoadmin('configuration deploy')
        big_files = {}
        for container in self.cluster.all_hosts():
            self.real_installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_catalog(container)

            big_file = self.cluster.exec_cmd_on_host(
                container, "find /usr -size +2M -ls | "
                "sort -nk7 | "
                "tail -1 | "
                "awk '{print $NF}'").strip()

            self.cluster.exec_cmd_on_host(container,
                                          "cp %s /etc/presto" % (big_file, ),
                                          invoke_sudo=True)
            big_files[container] = os.path.join("/etc/presto",
                                                os.path.basename(big_file))

            self.cluster.write_content_to_host(book_content,
                                               book_path,
                                               host=container)
            self.cluster.exec_cmd_on_host(container,
                                          "chown presto:games %s" %
                                          (book_path, ),
                                          invoke_sudo=True)
            self.cluster.exec_cmd_on_host(container,
                                          "chmod 272 %s" % (book_path, ),
                                          invoke_sudo=True)
            self.assert_file_content(container, book_path, book_content)
            self.assert_file_perm_owner(container, book_path, '--w-rwx-w-',
                                        'presto', 'games')
            self.assert_path_exists(container, big_files[container])

        self.add_dummy_properties_to_host(self.cluster.slaves[1])
        path_on_cluster = self.copy_upgrade_rpm_to_cluster()
        symlink = os.path.join(get_install_directory(), 'link.rpm')
        self.cluster.exec_cmd_on_host(
            self.cluster.master, 'ln -s %s %s' % (path_on_cluster, symlink))

        self.run_prestoadmin('server upgrade ' + path_on_cluster)
        self.assert_dummy_properties(self.cluster.slaves[1])

        for container in self.cluster.all_hosts():
            self.assert_file_content(container, book_path, book_content)
            self.assert_file_perm_owner(container, book_path, '--w-rwx-w-',
                                        'presto', 'games')

            self.assert_path_exists(container, big_files[container])

    def test_upgrade_non_root_user(self):
        self.upload_topology({
            "coordinator": "master",
            "workers": ["slave1", "slave2", "slave3"],
            "username": "******"
        })
        self.run_prestoadmin('configuration deploy -p password')
        for container in self.cluster.all_hosts():
            self.real_installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_catalog(container)

        path_on_cluster = self.copy_upgrade_rpm_to_cluster()
        self.upgrade_and_assert_success(path_on_cluster,
                                        extra_arguments=' -p password')

    def add_dummy_properties_to_host(self, host):
        self.cluster.write_content_to_host('io.prestosql=INFO',
                                           '/etc/presto/log.properties', host)
        self.cluster.write_content_to_host('dummy config file',
                                           '/etc/presto/jvm.config', host)

    def assert_dummy_properties(self, host):
        # assert log properties file is there
        self.assert_file_content(host, '/etc/presto/log.properties',
                                 'io.prestosql=INFO')

        # assert dummy jvm config is there too
        self.assert_file_content(host, '/etc/presto/jvm.config',
                                 'dummy config file')
Esempio n. 60
0
class TestServerUpgrade(BaseProductTestCase):
    def setUp(self):
        super(TestServerUpgrade, self).setUp()
        self.setup_cluster(NoHadoopBareImageProvider(),
                           STANDALONE_PRESTO_CLUSTER)
        self.dummy_installer = StandalonePrestoInstaller(
            self, (os.path.join(prestoadmin.main_dir, 'tests', 'product',
                                'resources'), 'dummy-rpm.rpm'))
        self.real_installer = StandalonePrestoInstaller(self)

    def start_and_assert_started(self):
        cmd_output = self.run_prestoadmin('server start')
        process_per_host = self.get_process_per_host(cmd_output.splitlines())
        self.assert_started(process_per_host)

    #
    # The dummy RPM is not guaranteed to have any functionality beyond not
    # including any real payload and adding the random README file. It's a
    # hacky one-off that satisfies the requirement of having *something* to
    # upgrade to without downloading another copy of the real RPM. This is NOT
    # the place to test functionality that the presto-server-rpm normally
    # provides, because the dummy rpm probably doesn't provide it, or worse,
    # provides an old and/or broken version of it.
    #
    def assert_upgraded_to_dummy_rpm(self, hosts):
        for container in hosts:
            # Still should have the same configs
            self.dummy_installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_connector(container)

            # However, dummy_rpm.rpm removes /usr/lib/presto/lib and
            # /usr/lib/presto/lib/plugin
            self.assert_path_removed(container, '/usr/lib/presto/lib')
            self.assert_path_removed(container, '/usr/lib/presto/lib/plugin')

            # And adds /usr/lib/presto/README.txt
            self.assert_path_exists(container, '/usr/lib/presto/README.txt')

            # And modifies the text of the readme in
            # /usr/shared/doc/presto/README.txt
            self.assert_file_content_regex(
                container, '/usr/shared/doc/presto/README.txt',
                r'.*New line of text here.$')

    @attr('smoketest')
    def test_upgrade(self):
        self.start_and_assert_started()

        self.run_prestoadmin('configuration deploy')
        for container in self.cluster.all_hosts():
            self.real_installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_connector(container)

        path_on_cluster = self.copy_upgrade_rpm_to_cluster()
        self.upgrade_and_assert_success(path_on_cluster)

    def upgrade_and_assert_success(self, path_on_cluster, extra_arguments=''):
        self.run_prestoadmin('server upgrade ' + path_on_cluster +
                             extra_arguments)
        self.assert_upgraded_to_dummy_rpm(self.cluster.all_hosts())

    @docker_only
    def test_rolling_upgrade(self):
        # Test that if a node is down, and then you upgrade again, it works
        self.run_prestoadmin('configuration deploy')

        self.cluster.stop_host(self.cluster.slaves[0])
        path_on_cluster = self.copy_upgrade_rpm_to_cluster()
        self.run_prestoadmin('server upgrade ' + path_on_cluster,
                             raise_error=False)
        running_hosts = self.cluster.all_hosts()[:]
        running_hosts.remove(self.cluster.slaves[0])
        self.assert_upgraded_to_dummy_rpm(running_hosts)

        self.cluster.start_host(self.cluster.slaves[0])
        self.retry(lambda: self.upgrade_and_assert_success(path_on_cluster))

    def copy_upgrade_rpm_to_cluster(self):
        rpm_name = self.dummy_installer.copy_presto_rpm_to_master()
        return os.path.join(self.cluster.mount_dir, rpm_name)

    def test_upgrade_fails_given_directory(self):
        dir_on_cluster = '/opt/prestoadmin'
        self.assertRaisesRegexp(OSError,
                                'RPM file not found at %s.' % dir_on_cluster,
                                self.run_prestoadmin,
                                'server upgrade ' + dir_on_cluster)

    def test_upgrade_works_with_symlink(self):
        self.run_prestoadmin('configuration deploy')
        for container in self.cluster.all_hosts():
            self.real_installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_connector(container)

        path_on_cluster = self.copy_upgrade_rpm_to_cluster()
        symlink = '/opt/prestoadmin/link.rpm'
        self.cluster.exec_cmd_on_host(
            self.cluster.master, 'ln -s %s %s' % (path_on_cluster, symlink))
        self.upgrade_and_assert_success(symlink)

    def test_configuration_preserved_on_upgrade(self):
        self.run_prestoadmin('configuration deploy')
        for container in self.cluster.all_hosts():
            self.real_installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_connector(container)

        self.add_dummy_properties_to_host(self.cluster.slaves[1])
        path_on_cluster = self.copy_upgrade_rpm_to_cluster()
        symlink = '/opt/prestoadmin/link.rpm'
        self.cluster.exec_cmd_on_host(
            self.cluster.master, 'ln -s %s %s' % (path_on_cluster, symlink))

        self.run_prestoadmin('server upgrade ' + path_on_cluster)
        self.assert_dummy_properties(self.cluster.slaves[1])

    def test_upgrade_non_root_user(self):
        self.upload_topology({
            "coordinator": "master",
            "workers": ["slave1", "slave2", "slave3"],
            "username": "******"
        })
        self.run_prestoadmin('configuration deploy -p password')
        for container in self.cluster.all_hosts():
            self.real_installer.assert_installed(self, container)
            self.assert_has_default_config(container)
            self.assert_has_default_connector(container)

        path_on_cluster = self.copy_upgrade_rpm_to_cluster()
        self.upgrade_and_assert_success(path_on_cluster,
                                        extra_arguments=' -p password')

    def add_dummy_properties_to_host(self, host):
        self.cluster.write_content_to_host('com.facebook.presto=INFO',
                                           '/etc/presto/log.properties', host)
        self.cluster.write_content_to_host('dummy config file',
                                           '/etc/presto/jvm.config', host)

    def assert_dummy_properties(self, host):
        # assert log properties file is there
        self.assert_file_content(host, '/etc/presto/log.properties',
                                 'com.facebook.presto=INFO')

        # assert dummy jvm config is there too
        self.assert_file_content(host, '/etc/presto/jvm.config',
                                 'dummy config file')