def setup_method(self):
"""Setup test environment."""
# pylint: disable=attribute-defined-outside-init
self.remote = mock.MagicMock(spec_set=Remote)
self.ganeti = Ganeti(username="******",
password="******",
timeout=10,
remote=self.remote)
self.cluster = "ganeti01.svc.eqiad.wmnet"
self.instance = "test.example.com"
self.cluster_base_url = {
cluster: RAPI_URL_FORMAT.format(cluster=cluster) + "/2"
for cluster in CLUSTERS_AND_ROWS
}
self.base_url = self.cluster_base_url[self.cluster]
self.instance_url = f"{self.base_url}/instances/{self.instance}"
# load test fixtures
with open(get_fixture_path("ganeti", "info.json"),
encoding="utf-8") as info_json:
self.info = info_json.read()
with open(get_fixture_path("ganeti", "404.json"),
encoding="utf-8") as fourohfour_json:
self.fourohfour = fourohfour_json.read()
with open(get_fixture_path("ganeti", "instance.json"),
encoding="utf-8") as instance_json:
self.instance_info = instance_json.read()
with open(get_fixture_path("ganeti", "bogus.json"),
encoding="utf-8") as bogus_json:
self.bogus_data = bogus_json.read()
def test_services_downtimed(self, mocked_time):
"""It should downtime the hosts on Icinga, yield and delete the downtime once done."""
with open(get_fixture_path("icinga",
"status_with_services.json")) as before:
with open(
get_fixture_path(
"icinga",
"status_with_services_downtimed.json")) as after:
set_mocked_icinga_host_outputs(
self.mocked_icinga_host,
[before.read(), "", after.read(), ""])
with self.icinga_hosts.services_downtimed("service.*", self.reason):
assert_has_service_downtime_calls(self.mocked_icinga_host,
[("host1", "service1"),
("host1", "service2")],
self.reason)
self.mocked_icinga_host.run_sync.reset_mock()
self.mocked_icinga_host.run_sync.assert_called_with(
'bash -c \'echo -n "[1514764800] DEL_DOWNTIME_BY_HOST_NAME;host1;service1" > '
"/var/lib/icinga/rw/icinga.cmd '",
'bash -c \'echo -n "[1514764800] DEL_DOWNTIME_BY_HOST_NAME;host1;service2" > '
"/var/lib/icinga/rw/icinga.cmd '",
print_output=False,
print_progress_bars=False,
)
assert mocked_time.called
def test_spicerack(mocked_dns_resolver, mocked_remote_query, monkeypatch):
"""An instance of Spicerack should allow to access all the library features."""
monkeypatch.setenv("SUDO_USER", "user1")
verbose = True
dry_run = False
proxy = "http://proxy.example.com:8080"
spicerack = Spicerack(verbose=verbose,
dry_run=dry_run,
http_proxy=proxy,
**SPICERACK_TEST_PARAMS)
assert spicerack.verbose is verbose
assert spicerack.dry_run is dry_run
assert spicerack.username == "user1"
assert spicerack.config_dir == get_fixture_path()
assert spicerack.http_proxy == proxy
assert spicerack.requests_proxies == {"http": proxy, "https": proxy}
assert isinstance(spicerack.irc_logger, logging.Logger)
assert isinstance(spicerack.actions, ActionsDict)
assert isinstance(spicerack.remote(), Remote)
assert isinstance(spicerack.remote(installer=True), Remote)
assert isinstance(spicerack.confctl("discovery"), ConftoolEntity)
assert isinstance(spicerack.confctl("mwconfig"), ConftoolEntity)
assert isinstance(spicerack.dns(), Dns)
assert isinstance(spicerack.discovery("discovery-record"), Discovery)
assert isinstance(spicerack.mediawiki(), MediaWiki)
assert isinstance(spicerack.mysql(), Mysql)
assert isinstance(spicerack.mysql_legacy(), MysqlLegacy)
assert isinstance(spicerack.redis_cluster("cluster"), RedisCluster)
assert isinstance(
spicerack.elasticsearch_clusters("search_eqiad", ("some_core_dc", )),
ElasticsearchClusters,
)
assert isinstance(
spicerack.admin_reason("Reason message", task_id="T12345"), Reason)
assert isinstance(spicerack.puppet(mock.MagicMock(spec_set=RemoteHosts)),
PuppetHosts)
assert isinstance(
spicerack.phabricator(get_fixture_path("phabricator", "valid.conf")),
Phabricator,
)
assert isinstance(spicerack.prometheus(), Prometheus)
assert isinstance(spicerack.debmonitor(), Debmonitor)
assert isinstance(spicerack.ganeti(), Ganeti)
assert isinstance(spicerack.requests_session("name"), Session)
assert isinstance(
spicerack.etcdctl(remote_host=mock.MagicMock(spec_set=RemoteHosts)),
EtcdctlController,
)
assert isinstance(spicerack.kafka(), Kafka)
assert mocked_remote_query.called
assert mocked_dns_resolver.Resolver.called
def setup_method(self):
"""Setup a Confctl instance with a mocked conftool backend and driver."""
# pylint: disable=attribute-defined-outside-init
self.conftool_backend = MockBackend({})
confctl.kvobject.KVObject.backend = self.conftool_backend
confctl.kvobject.KVObject.config = confctl.configuration.Config(driver="")
config = get_fixture_path("confctl", "config.yaml")
schema = get_fixture_path("confctl", "schema.yaml")
with mock.patch("spicerack.confctl.kvobject.KVObject.setup"):
self.confctl = confctl.Confctl(config=config, schema=schema, dry_run=False)
self.entity = self.confctl._schema.entities["discovery"] # pylint: disable=protected-access
self.entity.query = mock.MagicMock(return_value=[self.entity("test", "dnsdisc")])
self.discovery = confctl.ConftoolEntity(self.entity, dry_run=False)
self.discovery_dry_run = confctl.ConftoolEntity(self.entity)
def setup_method(self, _, mocked_redis):
"""Initialize the test environment for RedisCluster."""
config_dir = get_fixture_path("redis_cluster")
# pylint: disable=attribute-defined-outside-init
self.mocked_redis = mocked_redis
self.redis_cluster = RedisCluster("cluster", config_dir, dry_run=False)
self.redis_cluster_dry_run = RedisCluster("cluster", config_dir)
def setup_method(self, _, mocked_transports):
"""Initialize the test environment for MysqlLegacy."""
# pylint: disable=attribute-defined-outside-init
self.config = Config(get_fixture_path("remote", "config.yaml"))
self.mocked_transports = mocked_transports
self.mocked_remote = mock.MagicMock(spec_set=Remote)
self.mysql = mysql_legacy.MysqlLegacy(self.mocked_remote, dry_run=False)
def test_wait_for_optimal_ok(self, mocked_sleep):
"""It should return immediately if host is optimal."""
with open(get_fixture_path("icinga", "status_valid.json")) as f:
set_mocked_icinga_host_output(self.mocked_icinga_host, f.read(), 2)
self.icinga_hosts.wait_for_optimal()
assert not mocked_sleep.called
def test_downtime_default_params(self, _mocked_time):
"""It should downtime the hosts on the Icinga server with the default params."""
with open(get_fixture_path("icinga", "status_valid.json")) as f:
set_mocked_icinga_host_outputs(self.mocked_icinga_host,
[f.read(), "", "", ""])
self.icinga_hosts.downtime(self.reason)
assert_has_downtime_calls(self.mocked_icinga_host, ["host1"],
self.reason)
def test_get_status_parse_fail(self):
"""It should raise IcingaStatusParseError if unable to parse the JSON payload."""
with open(get_fixture_path("icinga", "status_invalid.json")) as f:
set_mocked_icinga_host_output(self.mocked_icinga_host, f.read())
with pytest.raises(icinga.IcingaStatusParseError,
match="Unable to parse Icinga status"):
self.icinga_hosts.get_status()
def test_remove_service_downtimes_not_downtimed(self):
"""It should do nothing if the services exist but aren't downtimed."""
with open(get_fixture_path("icinga",
"status_with_services.json")) as f:
set_mocked_icinga_host_outputs(self.mocked_icinga_host,
[f.read(), "", "", ""])
self.icinga_hosts.remove_service_downtimes(r"service\d")
self.mocked_icinga_host.run_sync.assert_called_once(
) # Only the icinga-status call, no downtimes.
def setup_method(self, _, mocked_transports):
"""Setup the test environment."""
# pylint: disable=attribute-defined-outside-init
self.config = Config(get_fixture_path("remote", "config.yaml"))
self.mocked_transports = mocked_transports
self.mysql_remote_hosts = mysql_legacy.MysqlLegacyRemoteHosts(
RemoteHosts(self.config, NodeSet("host[1-9]"), dry_run=False)
)
self.expected = [(NodeSet("host1"), "output1")]
def test_get_status_missing_hosts(self):
"""It should raise IcingaStatusNotFoundError if any host is missing its status."""
with open(get_fixture_path("icinga", "status_missing.json")) as f:
set_mocked_icinga_host_output(self.mocked_icinga_host, f.read())
with pytest.raises(
icinga.IcingaStatusNotFoundError,
match="Host host2 was not found in Icinga status",
):
self.icinga_hosts.get_status()
def test_downtime_custom_duration(self, _mocked_time):
"""It should downtime the hosts for the given duration on the Icinga server."""
with open(get_fixture_path("icinga", "status_valid.json")) as f:
set_mocked_icinga_host_outputs(self.mocked_icinga_host,
[f.read(), "", "", ""])
self.icinga_hosts.downtime(self.reason, duration=timedelta(minutes=30))
assert_has_downtime_calls(self.mocked_icinga_host, ["host1"],
self.reason,
duration=1800)
def setup_method(self):
"""Setup the test environment."""
# pylint: disable=attribute-defined-outside-init
config = get_fixture_path("remote", "config.yaml")
self.hosts = NodeSet("host[1-10]")
# We want to mock out ConftoolEntity completely here. As far as we're concerned it's just an interface
self.conftool = mock.MagicMock(spec=confctl.ConftoolEntity)
self.remote_hosts = remote.RemoteHosts(config, self.hosts, dry_run=False)
self.remote_hosts.run_async = mock.MagicMock()
self.lbcluster = remote.LBRemoteCluster(config, self.remote_hosts, self.conftool)
def test_remove_service_downtimes_unknown_service(self):
"""It should raise IcingaError if no services match the regex."""
with open(
get_fixture_path(
"icinga", "status_with_no_matching_services.json")) as f:
set_mocked_icinga_host_outputs(self.mocked_icinga_host,
[f.read(), "", "", ""])
with pytest.raises(icinga.IcingaError,
match=r'No services on host1 matched "service\\d"'):
self.icinga_hosts.remove_service_downtimes(r"service\d")
def test_offset_transfer_dry_run(self, consumer_patch, _, func_arguments):
"""It should read but not transfer offsets between consumer groups."""
kafka = Kafka(kafka_config=load_yaml_config(
get_fixture_path("kafka", "config.yaml")),
dry_run=True)
to_consumer_mock = self._setup_consumer_mocks(
consumer_patch, _answer_offsets_for_times,
_answer_partitions_for_topic)
kafka.transfer_consumer_position(*func_arguments[0])
assert not to_consumer_mock.commit.called
def test_downtime_with_apostrophe_in_reason(self, _mocked_time):
"""It should correctly quote the apostrophe in the reason string."""
with open(get_fixture_path("icinga", "status_valid.json")) as f:
set_mocked_icinga_host_outputs(self.mocked_icinga_host,
[f.read(), "", "", ""])
reason = Reason("An apostrophe's here",
"user1",
"orchestration-host",
task_id="T12345")
self.icinga_hosts.downtime(reason)
assert_has_downtime_calls(self.mocked_icinga_host, ["host1"], reason)
def test_downtime_is_kept_when_exception_is_raised(self, _mocked_time):
"""Downtime should not be removed if an exception is raised."""
with open(get_fixture_path("icinga", "status_valid.json")) as f:
set_mocked_icinga_host_outputs(self.mocked_icinga_host,
[f.read(), "", "", ""])
with pytest.raises(ValueError):
with self.icinga_hosts.downtimed(self.reason):
assert_has_downtime_calls(self.mocked_icinga_host, ["host1"],
self.reason)
self.mocked_icinga_host.run_sync.reset_mock()
raise ValueError()
assert not self.mocked_icinga_host.run_sync.called
def test_downtime_services_default_params(self, _mocked_time, caplog):
"""It should downtime the services on the Icinga server with the default params."""
caplog.set_level(logging.INFO)
with open(get_fixture_path("icinga",
"status_with_services.json")) as f:
set_mocked_icinga_host_outputs(self.mocked_icinga_host,
[f.read(), "", "", ""])
self.icinga_hosts.downtime_services(r"service\d", self.reason)
assert_has_service_downtime_calls(self.mocked_icinga_host,
[("host1", "service1"),
("host1", "service2")], self.reason)
assert r'for services "service\d" for host: host1 (matched 2 unique service names on 1 host)' in caplog.text
def test_wait_for_optimal_timeout(self, mocked_sleep):
"""It should raise icinga.IcingaError if host is optimal in the required time."""
with open(
get_fixture_path("icinga",
"status_with_failed_services.json")) as f:
set_mocked_icinga_host_output(self.mocked_icinga_host, f.read(),
20)
with pytest.raises(icinga.IcingaError,
match="Not all services are recovered"):
self.icinga_hosts.wait_for_optimal()
assert mocked_sleep.called
def test_recheck_failed_services_optimal(self):
"""It should force a recheck of all services for the hosts on the Icinga server."""
with open(get_fixture_path("icinga",
"status_with_services.json")) as f:
set_mocked_icinga_host_output(self.mocked_icinga_host, f.read())
self.icinga_hosts.recheck_failed_services()
# This also ensures that we are not making an additional call of run_sync in the recheck method
self.mocked_icinga_host.run_sync.assert_called_with(
Command('/usr/local/bin/icinga-status -j "host1"', ok_codes=[]),
is_safe=True,
print_output=False,
print_progress_bars=False,
)
def test_downtime_services_custom_duration(self, _mocked_time):
"""It should downtime the services for the given duration on the Icinga server."""
with open(get_fixture_path("icinga",
"status_with_services.json")) as f:
set_mocked_icinga_host_outputs(self.mocked_icinga_host,
[f.read(), "", "", ""])
self.icinga_hosts.downtime_services(r"service\d",
self.reason,
duration=timedelta(minutes=30))
assert_has_service_downtime_calls(self.mocked_icinga_host,
[("host1", "service1"),
("host1", "service2")],
self.reason,
duration=1800)
def test_services_downtime_is_removed_when_exception_is_raised(
self, _mocked_time):
"""Downtime should not be removed if an exception is raised."""
with open(get_fixture_path("icinga",
"status_with_services.json")) as before:
with open(
get_fixture_path(
"icinga",
"status_with_services_downtimed.json")) as after:
set_mocked_icinga_host_outputs(
self.mocked_icinga_host,
[before.read(), "", after.read(), ""])
with pytest.raises(ValueError):
with self.icinga_hosts.services_downtimed("service.*",
self.reason,
remove_on_error=True):
assert_has_service_downtime_calls(self.mocked_icinga_host,
[("host1", "service1"),
("host1", "service2")],
self.reason)
self.mocked_icinga_host.run_sync.reset_mock()
raise ValueError()
assert self.mocked_icinga_host.run_sync.called
def test_get_status_with_services(self):
"""It should parse the JSON payload and return an instance of HostsStatus with service status."""
with open(get_fixture_path("icinga",
"status_with_services.json")) as f:
set_mocked_icinga_host_output(self.mocked_icinga_host, f.read())
status = self.icinga_hosts.get_status(service_re=r"service\d")
assert r"--services 'service\d'" in self.mocked_icinga_host.run_sync.call_args[
0][0].command
assert status.optimal
assert {service["name"]
for service in status["host1"].services
} == {"service1", "service2"}
assert status.failed_hosts == []
def test_no_topic_partitions(self, consumer_patch, _):
"""It should read but not transfer offsets between consumer groups."""
kafka = Kafka(kafka_config=load_yaml_config(
get_fixture_path("kafka", "config.yaml")),
dry_run=False)
self._setup_consumer_mocks(consumer_patch, _answer_offsets_for_times,
lambda _: None)
with pytest.raises(
expected_exception=KafkaError,
match="Partitions not found for topic eqiad.wikidata."):
kafka.transfer_consumer_position(
["wikidata"],
ConsumerDefinition("eqiad", "jumbo", "consumer_jumbo_1"),
ConsumerDefinition("eqiad", "jumbo", "consumer_jumbo_2"),
)
def test_get_status_ok(self):
"""It should parse the JSON payload and return an instance of HostsStatus."""
with open(
get_fixture_path("icinga",
"status_with_failed_services.json")) as f:
set_mocked_icinga_host_output(self.mocked_icinga_host, f.read())
status = self.icinga_hosts.get_status()
assert "--services" not in self.mocked_icinga_host.run_sync.call_args[
0][0].command
assert not status.optimal
assert status.non_optimal_hosts == ["host2"]
assert status.failed_services == {
"host2": ["check_name1", "check_name2"]
}
assert status.failed_hosts == []
def test_recheck_failed_services_failed(self, mocked_time):
"""It should force a recheck of all services for the hosts on the Icinga server."""
with open(
get_fixture_path("icinga",
"status_with_failed_services.json")) as f:
set_mocked_icinga_host_output(self.mocked_icinga_host, f.read())
self.icinga_hosts.recheck_failed_services()
self.mocked_icinga_host.run_sync.assert_called_with(
'bash -c \'echo -n "[1514764800] SCHEDULE_FORCED_SVC_CHECK;host2;check_name1;1514764800" > '
"/var/lib/icinga/rw/icinga.cmd '",
'bash -c \'echo -n "[1514764800] SCHEDULE_FORCED_SVC_CHECK;host2;check_name2;1514764800" > '
"/var/lib/icinga/rw/icinga.cmd '",
print_output=False,
print_progress_bars=False,
)
assert mocked_time.called
def test_remove_service_downtimes(self, mocked_time):
"""It should remove the downtime for the hosts on the Icinga server."""
with open(
get_fixture_path("icinga",
"status_with_services_downtimed.json")) as f:
set_mocked_icinga_host_outputs(self.mocked_icinga_host,
[f.read(), "", "", ""])
self.icinga_hosts.remove_service_downtimes(r"service\d")
self.mocked_icinga_host.run_sync.assert_called_with(
*[
f'bash -c \'echo -n "[1514764800] DEL_DOWNTIME_BY_HOST_NAME;host1;{service}" '
f"> /var/lib/icinga/rw/icinga.cmd '"
for service in ["service1", "service2"]
],
print_output=False,
print_progress_bars=False,
)
assert mocked_time.called
def test_downtimed(self, mocked_time):
"""It should downtime the hosts on Icinga, yield and delete the downtime once done."""
with open(get_fixture_path("icinga", "status_valid.json")) as f:
set_mocked_icinga_host_outputs(self.mocked_icinga_host,
[f.read(), "", "", ""])
with self.icinga_hosts.downtimed(self.reason):
assert_has_downtime_calls(self.mocked_icinga_host, ["host1"],
self.reason)
self.mocked_icinga_host.run_sync.reset_mock()
self.mocked_icinga_host.run_sync.assert_has_calls([
mock.call(
'bash -c \'echo -n "[1514764800] DEL_DOWNTIME_BY_HOST_NAME;host1" > '
"/var/lib/icinga/rw/icinga.cmd '",
print_output=False,
print_progress_bars=False,
)
])
assert mocked_time.called
def setup_method(self):
"""Set up the module under test."""
# pylint: disable=attribute-defined-outside-init
self.kafka = Kafka(kafka_config=load_yaml_config(
get_fixture_path("kafka", "config.yaml")),
dry_run=False)
