Exemplo n.º 1
0
  def test_perform_maintenance_hosts(self):
    mock_options = self.make_mock_options()
    mock_options.post_drain_script = None
    mock_options.groups_per_batch = '1'
    mock_options.grouping = 'by_host'

    def host_status_results(hostnames):
      if isinstance(hostnames, Hosts):
        return self.create_drained_status_result(hostnames)
      return self.create_maintenance_status_result()

    mock_api, mock_scheduler_proxy = self.create_mock_api()
    mock_scheduler_proxy.endMaintenance.return_value = self.create_end_maintenance_result()
    mock_scheduler_proxy.maintenanceStatus.side_effect = host_status_results
    mock_scheduler_proxy.startMaintenance.return_value = self.create_start_maintenance_result()
    mock_scheduler_proxy.drainHosts.return_value = self.create_start_maintenance_result()

    with contextlib.nested(
        patch('time.sleep'),
        patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
        patch('apache.aurora.client.commands.maintenance.CLUSTERS', new=self.TEST_CLUSTERS),
        patch('twitter.common.app.get_options', return_value=mock_options)) as (
            mock_sleep,
            mock_scheduler_proxy_class,
            mock_clusters_maintenancepatch,
            options):
      perform_maintenance_hosts([self.TEST_CLUSTER])

      mock_scheduler_proxy.startMaintenance.assert_called_with(Hosts(set(self.HOSTNAMES)))
      #TODO(jsmith): Consider not mocking out sleep and instead refactoring
      assert mock_sleep.call_count == 3
      assert mock_scheduler_proxy.maintenanceStatus.call_count == 7
      assert mock_scheduler_proxy.drainHosts.call_count == 3
      assert mock_scheduler_proxy.endMaintenance.call_count == 4
Exemplo n.º 2
0
  def test_perform_maintenance_hosts_failed_default_sla(self):
    with temporary_file() as fp:
      mock_options = self.make_mock_options()
      mock_options.post_drain_script = None
      mock_options.grouping = 'by_host'
      mock_options.unsafe_hosts_filename = fp.name

      def host_status_results(hostnames):
        if isinstance(hostnames, Hosts):
          return self.create_drained_status_result(hostnames)
        return self.create_maintenance_status_result()

      mock_api, mock_scheduler_proxy = self.create_mock_api()
      mock_scheduler_proxy.endMaintenance.return_value = self.create_end_maintenance_result()
      mock_scheduler_proxy.maintenanceStatus.side_effect = host_status_results
      mock_scheduler_proxy.startMaintenance.return_value = self.create_start_maintenance_result()
      mock_scheduler_proxy.drainHosts.return_value = self.create_start_maintenance_result()
      mock_vector = self.create_mock_probe_hosts_vector([
          self.create_probe_hosts(self.HOSTNAMES[0], 95, False, None),
          self.create_probe_hosts(self.HOSTNAMES[1], 95, False, None),
          self.create_probe_hosts(self.HOSTNAMES[2], 95, False, None)
      ])

      with contextlib.nested(
          patch('time.sleep'),
          patch('apache.aurora.client.api.SchedulerProxy', return_value=mock_scheduler_proxy),
          patch('apache.aurora.client.api.sla.Sla.get_domain_uptime_vector',
                return_value=mock_vector),
          patch('apache.aurora.client.commands.maintenance.CLUSTERS', new=self.TEST_CLUSTERS),
          patch('twitter.common.app.get_options', return_value=mock_options)):
        perform_maintenance_hosts([self.TEST_CLUSTER])

        mock_scheduler_proxy.startMaintenance.assert_called_with(Hosts(set(self.HOSTNAMES)))
        assert mock_scheduler_proxy.endMaintenance.call_count == len(self.HOSTNAMES)
    def test_perform_maintenance_hosts_reason_missing(self):
        mock_options = self.make_mock_options()
        mock_options.grouping = "by_host"
        mock_options.percentage = 50
        mock_options.duration = "10m"

        with contextlib.nested(patch("twitter.common.app.get_options", return_value=mock_options)):
            try:
                perform_maintenance_hosts([self.TEST_CLUSTER])
            except SystemExit:
                pass
            else:
                assert "Expected error is not raised."
    def test_perform_maintenance_hosts_failed_custom_sla(self):
        with temporary_file() as fp:
            mock_options = self.make_mock_options()
            mock_options.post_drain_script = None
            mock_options.grouping = "by_host"
            mock_options.percentage = 50
            mock_options.duration = "10m"
            mock_options.reason = "Test overrides"
            mock_options.unsafe_hosts_filename = fp.name

            def host_status_results(hostnames):
                if isinstance(hostnames, Hosts):
                    return self.create_drained_status_result(hostnames)
                return self.create_maintenance_status_result()

            mock_api, mock_scheduler_proxy = self.create_mock_api()
            mock_scheduler_proxy.endMaintenance.return_value = self.create_end_maintenance_result()
            mock_scheduler_proxy.maintenanceStatus.side_effect = host_status_results
            mock_scheduler_proxy.startMaintenance.return_value = self.create_start_maintenance_result()
            mock_scheduler_proxy.drainHosts.return_value = self.create_start_maintenance_result()
            mock_vector = self.create_mock_probe_hosts_vector(
                [
                    self.create_probe_hosts(self.HOSTNAMES[0], 95, False, None),
                    self.create_probe_hosts(self.HOSTNAMES[1], 95, False, None),
                    self.create_probe_hosts(self.HOSTNAMES[2], 95, False, None),
                ]
            )

            with contextlib.nested(
                patch("time.sleep"),
                patch("apache.aurora.client.api.SchedulerProxy", return_value=mock_scheduler_proxy),
                patch("apache.aurora.client.api.sla.Sla.get_domain_uptime_vector", return_value=mock_vector),
                patch("apache.aurora.client.commands.maintenance.CLUSTERS", new=self.TEST_CLUSTERS),
                patch("apache.aurora.client.commands.maintenance.log_admin_message"),
                patch("twitter.common.app.get_options", return_value=mock_options),
            ) as (_, _, _, _, log, _):
                perform_maintenance_hosts([self.TEST_CLUSTER])

                assert "Test overrides" in log.call_args[0][1]
                mock_scheduler_proxy.startMaintenance.assert_called_with(Hosts(set(self.HOSTNAMES)))
                assert mock_scheduler_proxy.endMaintenance.call_count == len(self.HOSTNAMES)
    def test_perform_maintenance_hosts_multiple_sla_groups_failure(self):
        mock_options = self.make_mock_options()
        mock_options.post_drain_script = None
        mock_options.grouping = "by_host"
        mock_options.unsafe_hosts_filename = None

        def host_status_results(hostnames):
            if isinstance(hostnames, Hosts):
                return self.create_drained_status_result(hostnames)
            return self.create_maintenance_status_result()

        mock_api, mock_scheduler_proxy = self.create_mock_api()
        mock_scheduler_proxy.endMaintenance.return_value = self.create_end_maintenance_result()
        mock_scheduler_proxy.maintenanceStatus.side_effect = host_status_results
        mock_scheduler_proxy.startMaintenance.return_value = self.create_start_maintenance_result()
        mock_scheduler_proxy.drainHosts.return_value = self.create_start_maintenance_result()

        def create_multiple_sla_results():
            mock_vector = Mock()
            mock_vector.probe_hosts.return_value = self.HOSTNAMES
            return mock_vector

        with contextlib.nested(
            patch("time.sleep"),
            patch("apache.aurora.client.api.SchedulerProxy", return_value=mock_scheduler_proxy),
            patch(
                "apache.aurora.client.api.sla.Sla.get_domain_uptime_vector", return_value=create_multiple_sla_results()
            ),
            patch("apache.aurora.client.commands.maintenance.CLUSTERS", new=self.TEST_CLUSTERS),
            patch("twitter.common.app.get_options", return_value=mock_options),
        ) as (mock_sleep, mock_scheduler_proxy_class, mock_vector_class, mock_clusters_maintenancepatch, options):

            perform_maintenance_hosts([self.TEST_CLUSTER])

            mock_scheduler_proxy.startMaintenance.assert_called_with(Hosts(set(self.HOSTNAMES)))
            assert mock_sleep.call_count == 0
            assert mock_scheduler_proxy.maintenanceStatus.call_count == 3
            assert mock_scheduler_proxy.drainHosts.call_count == 0
            assert mock_scheduler_proxy.endMaintenance.call_count == 3
    def test_perform_maintenance_hosts_unknown_hosts_skipped(self):
        mock_options = self.make_mock_options()
        mock_options.post_drain_script = None
        mock_options.grouping = "by_host"

        def host_status_results(hostnames):
            if isinstance(hostnames, Hosts):
                return self.create_drained_status_result(hostnames)
            return self.create_maintenance_status_result()

        mock_api, mock_scheduler_proxy = self.create_mock_api()
        mock_scheduler_proxy.endMaintenance.return_value = self.create_end_maintenance_result()
        mock_scheduler_proxy.maintenanceStatus.side_effect = host_status_results
        mock_scheduler_proxy.startMaintenance.return_value = self.create_start_maintenance_result(
            skip_hosts=["us-grf-20"]
        )
        mock_scheduler_proxy.drainHosts.return_value = self.create_start_maintenance_result()
        mock_vector = self.create_mock_probe_hosts_vector(
            [
                self.create_probe_hosts(self.HOSTNAMES[0], 95, True, None),
                self.create_probe_hosts(self.HOSTNAMES[1], 95, True, None),
                self.create_probe_hosts(self.HOSTNAMES[2], 95, True, None),
            ]
        )

        with contextlib.nested(
            patch("time.sleep"),
            patch("apache.aurora.client.api.SchedulerProxy", return_value=mock_scheduler_proxy),
            patch("apache.aurora.client.api.sla.Sla.get_domain_uptime_vector", return_value=mock_vector),
            patch("apache.aurora.client.commands.maintenance.CLUSTERS", new=self.TEST_CLUSTERS),
            patch("twitter.common.app.get_options", return_value=mock_options),
        ) as (mock_sleep, mock_scheduler_proxy_class, mock_vector_class, mock_clusters_maintenancepatch, options):
            perform_maintenance_hosts([self.TEST_CLUSTER])

            mock_scheduler_proxy.startMaintenance.assert_called_with(Hosts(set(self.HOSTNAMES)))
            assert mock_sleep.call_count == 2
            assert mock_scheduler_proxy.maintenanceStatus.call_count == 4
            assert mock_scheduler_proxy.drainHosts.call_count == 2
            assert mock_scheduler_proxy.endMaintenance.call_count == 2