def test_raw_data_exception(self, duration):
raw_data: result.Result[AgentRawData,
Exception] = result.Error(Exception("zomg!"))
message = FetcherMessage.from_raw_data(raw_data, duration,
FetcherType.TCP)
assert isinstance(message.raw_data.error, Exception)
assert str(message.raw_data.error) == "zomg!"
def test_multiple_sources_from_the_same_host(
self,
hostname,
ipaddress,
config_cache,
host_config,
):
sources = [
ProgramSource.ds(hostname, ipaddress, template=""),
TCPSource(hostname, ipaddress),
]
host_sections = _collect_host_sections(
sources=sources,
file_cache_max_age=file_cache.MaxAge.none(),
fetcher_messages=[
FetcherMessage.from_raw_data(
result.OK(source.default_raw_data),
Snapshot.null(),
source.fetcher_type,
) for source in sources
],
selected_sections=NO_SELECTION,
)[0]
assert len(host_sections) == 1
key = HostKey(hostname, ipaddress, SourceType.HOST)
assert key in host_sections
section = host_sections[key]
assert len(section.sections) == 1
assert section.sections[SectionName(
"section_name_%s" %
hostname)] == len(sources) * [["section_content"]]
def test_no_sources(self, cluster, nodes, config_cache, host_config):
made_nodes = make_nodes(config_cache, host_config, None, sources=())
host_sections = _collect_host_sections(
nodes=made_nodes,
file_cache_max_age=file_cache.MaxAge.none(),
fetcher_messages=[
# We do not pass sources explicitly but still append Piggyback.
FetcherMessage.from_raw_data(
result.OK(AgentRawData(b"")),
Snapshot.null(),
FetcherType.PIGGYBACK,
) for _n in made_nodes
],
selected_sections=NO_SELECTION,
)[0]
assert len(host_sections) == len(nodes)
key_clu = HostKey(cluster, None, SourceType.HOST)
assert key_clu not in host_sections
for hostname, addr in nodes.items():
key = HostKey(hostname, addr, SourceType.HOST)
assert key in host_sections
section = host_sections[key]
# yapf: disable
assert (section.sections[SectionName("section_name_%s" % hostname)]
== [["section_content_%s" % hostname]])
assert not section.cache_info
assert not section.piggybacked_raw_data
def test_multiple_sources_from_different_hosts(self, hostname, ipaddress, config_cache, host_config):
sources = [
ProgramSource.ds(hostname + "0", ipaddress, template=""),
TCPSource(hostname + "1", ipaddress),
TCPSource(hostname + "2", ipaddress),
]
nodes = make_nodes(config_cache, host_config, ipaddress, sources=sources)
host_sections = _collect_host_sections(
nodes=nodes,
file_cache_max_age=file_cache.MaxAge.none(),
fetcher_messages=[
FetcherMessage.from_raw_data(
result.OK(source.default_raw_data),
Snapshot.null(),
source.fetcher_type,
)
for _h, _i, sources in nodes for source in sources
],
selected_sections=NO_SELECTION,
)[0]
assert len(host_sections) == 1
key = HostKey(hostname, ipaddress, SourceType.HOST)
assert key in host_sections
section = host_sections[key]
assert len(section.sections) == len(sources)
for source in sources:
# yapf: disable
assert (
section.sections[SectionName("section_name_%s" % source.hostname)]
== [["section_content"]])
def fetch_all(
nodes: Iterable[Tuple[HostName, Optional[HostAddress], Sequence[Source]]],
*,
max_cachefile_age: int,
host_config: HostConfig,
) -> Iterator[FetcherMessage]:
console.verbose("%s+%s %s\n", tty.yellow, tty.normal,
"Fetching data".upper())
# TODO(ml): It is not clear to me in which case it is possible for the following to hold true
# for any source in nodes:
# - hostname != source.hostname
# - ipaddress != source.ipaddress
# If this is impossible, then we do not need the Tuple[HostName, HostAddress, ...].
for _hostname, _ipaddress, sources in nodes:
for source in sources:
console.vverbose(" Source: %s/%s\n" %
(source.source_type, source.fetcher_type))
source.file_cache_max_age = max_cachefile_age
with CPUTracker() as tracker:
raw_data = source.fetch()
yield FetcherMessage.from_raw_data(
raw_data,
tracker.duration,
source.fetcher_type,
)
def test_result_answer(self, count):
fetcher_payload = AgentResultMessage(AgentRawData(69 * b"\xff"))
fetcher_stats = ResultStats(Snapshot.null())
fetcher_message = FetcherMessage(
FetcherHeader(
FetcherType.TCP,
PayloadType.AGENT,
status=42,
payload_length=len(fetcher_payload),
stats_length=len(fetcher_stats),
),
fetcher_payload,
fetcher_stats,
)
fetcher_messages = list(repeat(fetcher_message, count))
timeout = 7
message = CMCMessage.result_answer(fetcher_messages, timeout,
Snapshot.null())
assert isinstance(repr(message), str)
assert CMCMessage.from_bytes(bytes(message)) == message
assert message.header.name == "fetch"
assert message.header.state == CMCHeader.State.RESULT
assert message.header.log_level.strip() == ""
assert message.header.payload_length == len(message) - len(
message.header)
assert message.header.payload_length == len(message.payload)
def fetch_all(
sources: Iterable[Source],
*,
file_cache_max_age: file_cache.MaxAge,
mode: Mode,
) -> Sequence[Tuple[Source, FetcherMessage]]:
console.verbose("%s+%s %s\n", tty.yellow, tty.normal, "Fetching data".upper())
out: List[Tuple[Source, FetcherMessage]] = []
for source in sources:
console.vverbose(" Source: %s/%s\n" % (source.source_type, source.fetcher_type))
source.file_cache_max_age = file_cache_max_age
with CPUTracker() as tracker:
raw_data = source.fetch(mode)
out.append(
(
source,
FetcherMessage.from_raw_data(
raw_data,
tracker.duration,
source.fetcher_type,
),
)
)
return out
def test_no_sources(self, cluster, nodes, config_cache, host_config):
sources = make_cluster_sources(config_cache, host_config)
host_sections = _collect_host_sections(
fetched=[
(
source,
FetcherMessage.from_raw_data(
result.OK(AgentRawData(b"")),
Snapshot.null(),
FetcherType.PIGGYBACK,
),
)
for source in sources
],
file_cache_max_age=file_cache.MaxAge.none(),
selected_sections=NO_SELECTION,
)[0]
assert len(host_sections) == len(nodes)
key_clu = HostKey(cluster, None, SourceType.HOST)
assert key_clu not in host_sections
for hostname, addr in nodes.items():
key = HostKey(hostname, addr, SourceType.HOST)
assert key in host_sections
section = host_sections[key]
assert section.sections[SectionName("section_name_%s" % hostname)] == [
["section_content_%s" % hostname]
]
assert not section.cache_info
assert not section.piggybacked_raw_data
def test_multiple_sources_from_different_hosts(
self, hostname, ipaddress, config_cache, host_config
):
sources = [
ProgramSource.ds(HostName(f"{hostname}0"), ipaddress, template=""),
TCPSource(HostName(f"{hostname}1"), ipaddress),
TCPSource(HostName(f"{hostname}2"), ipaddress),
]
host_sections = _collect_host_sections(
fetched=[
(
source,
FetcherMessage.from_raw_data(
result.OK(source.default_raw_data),
Snapshot.null(),
source.fetcher_type,
),
)
for source in sources
],
file_cache_max_age=file_cache.MaxAge.none(),
selected_sections=NO_SELECTION,
)[0]
assert set(host_sections) == {
HostKey(HostName(f"{hostname}0"), ipaddress, SourceType.HOST),
HostKey(HostName(f"{hostname}1"), ipaddress, SourceType.HOST),
HostKey(HostName(f"{hostname}2"), ipaddress, SourceType.HOST),
}
for source in sources:
assert host_sections[
HostKey(source.hostname, source.ipaddress, SourceType.HOST)
].sections[SectionName(f"section_name_{source.hostname}")] == [["section_content"]]
def test_one_snmp_source(self, hostname, ipaddress, config_cache, host_config):
raw_data: SNMPRawData = {}
host_sections = _collect_host_sections(
fetched=[
(
SNMPSource.snmp(
hostname,
ipaddress,
selected_sections=NO_SELECTION,
force_cache_refresh=False,
on_scan_error=OnError.RAISE,
),
FetcherMessage.from_raw_data(
result.OK(raw_data),
Snapshot.null(),
FetcherType.SNMP,
),
)
],
file_cache_max_age=file_cache.MaxAge.none(),
selected_sections=NO_SELECTION,
)[0]
assert len(host_sections) == 1
key = HostKey(hostname, ipaddress, SourceType.HOST)
assert key in host_sections
section = host_sections[key]
assert len(section.sections) == 1
assert section.sections[SectionName("section_name_%s" % hostname)] == [["section_content"]]
def test_raw_data_tcp_standard(self, agent_raw_data, duration,
fetcher_type):
raw_data: result.Result[AgentRawData,
Exception] = result.OK(agent_raw_data)
message = FetcherMessage.from_raw_data(raw_data, duration,
fetcher_type)
assert message.raw_data == raw_data
def test_one_nonsnmp_source(self, hostname, ipaddress, config_cache,
host_config, source):
source = source(hostname, ipaddress)
assert source.source_type is SourceType.HOST
host_sections = _collect_host_sections(
nodes=make_nodes(
config_cache,
host_config,
ipaddress,
sources=[source],
),
file_cache_max_age=file_cache.MaxAge.none(),
fetcher_messages=[
FetcherMessage.from_raw_data(
result.OK(source.default_raw_data),
Snapshot.null(),
source.fetcher_type,
),
],
selected_sections=NO_SELECTION,
)[0]
assert len(host_sections) == 1
key = HostKey(hostname, ipaddress, source.source_type)
assert key in host_sections
section = host_sections[key]
assert len(section.sections) == 1
assert section.sections[SectionName("section_name_%s" %
hostname)] == [["section_content"]]
def test_from_raw_data_snmp(self, snmp_raw_data, duration):
raw_data: result.Result[SNMPRawData,
Exception] = result.OK(snmp_raw_data)
message = FetcherMessage.from_raw_data(raw_data, duration,
FetcherType.SNMP)
assert message.header.fetcher_type is FetcherType.SNMP
assert message.header.payload_type is PayloadType.SNMP
assert message.raw_data == raw_data
def test_from_raw_data_exception(self, duration):
error: result.Result[AgentRawData, Exception] = result.Error(ValueError("zomg!"))
message = FetcherMessage.from_raw_data(error, duration, FetcherType.TCP)
assert message.header.fetcher_type is FetcherType.TCP
assert message.header.payload_type is PayloadType.ERROR
# Comparison of exception is "interesting" in Python so we check the type and args.
assert type(message.raw_data.error) is type(error.error)
assert message.raw_data.error.args == error.error.args
def test_from_raw_data_standard(self, agent_raw_data, duration,
fetcher_type):
raw_data: result.Result[AgentRawData,
Exception] = result.OK(agent_raw_data)
message = FetcherMessage.from_raw_data(raw_data, duration,
fetcher_type)
assert message.header.fetcher_type is fetcher_type
assert message.header.payload_type is PayloadType.AGENT
assert message.raw_data == raw_data
def test_multiple_sources_from_the_same_host(
self,
hostname,
ipaddress,
mode,
config_cache,
host_config,
):
sources = [
ProgramSource.ds(
hostname,
ipaddress,
mode=mode,
template="",
),
TCPSource(
hostname,
ipaddress,
mode=mode,
),
]
broker = ParsedSectionsBroker()
update_host_sections(
broker,
make_nodes(
config_cache,
host_config,
ipaddress,
mode=mode,
sources=sources,
),
max_cachefile_age=0,
host_config=host_config,
fetcher_messages=[
FetcherMessage.from_raw_data(
result.OK(source.default_raw_data),
Snapshot.null(),
source.fetcher_type,
) for source in sources
],
selected_sections=NO_SELECTION,
)
assert len(broker) == 1
key = HostKey(hostname, ipaddress, SourceType.HOST)
assert key in broker
section = broker[key]
assert len(section.sections) == 1
# yapf: disable
assert (section.sections[SectionName("section_name_%s" % hostname)]
== len(sources) * [["section_content"]])
def fetcher_message(self, fetcher_payload, fetcher_stats):
return FetcherMessage(
FetcherHeader(
FetcherType.TCP,
PayloadType.AGENT,
status=42,
payload_length=len(fetcher_payload),
stats_length=len(fetcher_stats),
),
fetcher_payload,
fetcher_stats,
)
def messages(self):
msg = []
for payload, stats in (
(AgentResultMessage(AgentRawData(42 * b"\0")), ResultStats(Snapshot.null())),
(AgentResultMessage(AgentRawData(12 * b"\0")), ResultStats(Snapshot.null())),
):
msg.append(
FetcherMessage(
FetcherHeader(
FetcherType.TCP,
PayloadType.AGENT,
status=69,
payload_length=len(payload),
stats_length=len(stats),
),
payload,
stats,
)
)
return msg
def test_one_snmp_source(self, hostname, ipaddress, mode, config_cache,
host_config):
broker = ParsedSectionsBroker()
update_host_sections(
broker,
make_nodes(
config_cache,
host_config,
ipaddress,
mode=mode,
sources=[
SNMPSource.snmp(
hostname,
ipaddress,
mode=mode,
selected_sections=NO_SELECTION,
on_scan_error="raise",
),
],
),
max_cachefile_age=0,
host_config=host_config,
fetcher_messages=[
FetcherMessage.from_raw_data(
result.OK({}),
Snapshot.null(),
FetcherType.SNMP,
),
],
selected_sections=NO_SELECTION,
)
assert len(broker) == 1
key = HostKey(hostname, ipaddress, SourceType.HOST)
assert key in broker
section = broker[key]
assert len(section.sections) == 1
assert section.sections[SectionName("section_name_%s" %
hostname)] == [["section_content"]]
def test_serialization(self, count):
fetcher_payload = AgentResultMessage(AgentRawData(69 * b"\xff"))
fetcher_stats = ResultStats(Snapshot.null())
fetcher_message = FetcherMessage(
FetcherHeader(
FetcherType.TCP,
PayloadType.AGENT,
status=42,
payload_length=len(fetcher_payload),
stats_length=len(fetcher_stats),
),
fetcher_payload,
fetcher_stats,
)
fetcher_messages = list(repeat(fetcher_message, count))
serial = 1337
host_name = "my_host_name"
timeout = 7
message = CMCMessage.result_answer(
fetcher_messages,
serial=serial,
host_name=host_name,
timeout=timeout,
duration=Snapshot.null(),
)
other = CMCMessage.from_bytes(bytes(message))
assert other == message
assert isinstance(repr(other), str)
assert other.header.name == "fetch"
assert other.header.state == CMCHeader.State.RESULT
assert other.header.log_level.strip() == ""
assert other.header.payload_length == len(message) - len(
message.header)
assert other.header.payload_length == len(message.payload)
assert isinstance(other.payload, CMCResults)
assert other.payload.id_.serial == serial
assert other.payload.id_.host_name == host_name
assert other.payload.stats.timeout == timeout
assert other.payload.stats.duration == Snapshot.null()
def fetch_all(
*,
sources: Iterable[Source],
file_cache_max_age: file_cache.MaxAge,
mode: Mode,
) -> Iterator[FetcherMessage]:
console.verbose("%s+%s %s\n", tty.yellow, tty.normal,
"Fetching data".upper())
for source in sources:
console.vverbose(" Source: %s/%s\n" %
(source.source_type, source.fetcher_type))
source.file_cache_max_age = file_cache_max_age
with CPUTracker() as tracker:
raw_data = source.fetch(mode)
yield FetcherMessage.from_raw_data(
raw_data,
tracker.duration,
source.fetcher_type,
)
def test_no_sources(self, cluster, nodes, config_cache, host_config, mode):
made_nodes = make_nodes(
config_cache,
host_config,
None,
mode=mode,
sources=(),
)
broker = ParsedSectionsBroker()
update_host_sections(
broker,
made_nodes,
max_cachefile_age=0,
host_config=host_config,
fetcher_messages=[
# We do not pass sources explicitly but still append Piggyback.
FetcherMessage.from_raw_data(
result.OK(AgentRawData(b"")),
Snapshot.null(),
FetcherType.PIGGYBACK,
) for _n in made_nodes
],
selected_sections=NO_SELECTION,
)
assert len(broker) == len(nodes)
key_clu = HostKey(cluster, None, SourceType.HOST)
assert key_clu not in broker
for hostname, addr in nodes.items():
key = HostKey(hostname, addr, SourceType.HOST)
assert key in broker
section = broker[key]
# yapf: disable
assert (section.sections[SectionName("section_name_%s" % hostname)]
== [["section_content_%s" % hostname]])
assert not section.cache_info
assert not section.piggybacked_raw_data
def test_one_snmp_source(self, hostname, ipaddress, mode, config_cache,
host_config):
host_sections = _collect_host_sections(
nodes=make_nodes(
config_cache,
host_config,
ipaddress,
mode=mode,
sources=[
SNMPSource.snmp(
hostname,
ipaddress,
mode=mode,
selected_sections=NO_SELECTION,
force_cache_refresh=False,
on_scan_error="raise",
),
],
),
file_cache_max_age=0,
fetcher_messages=[
FetcherMessage.from_raw_data(
result.OK({}),
Snapshot.null(),
FetcherType.SNMP,
),
],
selected_sections=NO_SELECTION,
)[0]
assert len(host_sections) == 1
key = HostKey(hostname, ipaddress, SourceType.HOST)
assert key in host_sections
section = host_sections[key]
assert len(section.sections) == 1
assert section.sections[SectionName("section_name_%s" %
hostname)] == [["section_content"]]
def test_one_nonsnmp_source(self, hostname, ipaddress, mode, config_cache,
host_config, source):
source = source(hostname, ipaddress, mode=mode)
assert source.source_type is SourceType.HOST
broker = ParsedSectionsBroker()
update_host_sections(
broker,
make_nodes(
config_cache,
host_config,
ipaddress,
mode=mode,
sources=[source],
),
max_cachefile_age=0,
host_config=host_config,
fetcher_messages=[
FetcherMessage.from_raw_data(
result.OK(source.default_raw_data),
Snapshot.null(),
source.fetcher_type,
),
],
selected_sections=NO_SELECTION,
)
assert len(broker) == 1
key = HostKey(hostname, ipaddress, source.source_type)
assert key in broker
section = broker[key]
assert len(section.sections) == 1
assert section.sections[SectionName("section_name_%s" %
hostname)] == [["section_content"]]
def test_raw_data_snmp(self, snmp_raw_data, duration):
raw_data: result.Result[SNMPRawData,
Exception] = result.OK(snmp_raw_data)
message = FetcherMessage.from_raw_data(raw_data, duration,
FetcherType.SNMP)
assert message.raw_data == raw_data
def test_from_bytes_failure(self):
with pytest.raises(ValueError):
FetcherMessage.from_bytes(b"random bytes")
def test_from_bytes_success(self, message):
assert FetcherMessage.from_bytes(bytes(message) + 42 * b"*") == message
def message(self, header, payload, stats):
return FetcherMessage(header, payload, stats)
def test_get_host_sections_cluster(monkeypatch, mocker):
hostname = "testhost"
hosts = {
"host0": "10.0.0.0",
"host1": "10.0.0.1",
"host2": "10.0.0.2",
}
address = "1.2.3.4"
tags = {"agent": "no-agent"}
section_name = SectionName("test_section")
config_cache = make_scenario(hostname, tags).apply(monkeypatch)
host_config = config.HostConfig.make_host_config(hostname)
def fake_lookup_ip_address(host_config, family=None):
return hosts[host_config.hostname]
def check(_, *args, **kwargs):
return result.OK(AgentHostSections(sections={section_name: [[str(section_name)]]}))
monkeypatch.setattr(
config,
"lookup_ip_address",
fake_lookup_ip_address,
)
monkeypatch.setattr(
Source,
"parse",
check,
)
mocker.patch.object(
cmk.utils.piggyback,
"remove_source_status_file",
autospec=True,
)
mocker.patch.object(
cmk.utils.piggyback,
"_store_status_file_of",
autospec=True,
)
# Create a cluster
host_config.nodes = list(hosts.keys())
nodes = make_nodes(
config_cache,
host_config,
address,
sources=make_sources(host_config, address),
)
host_sections = _collect_host_sections(
nodes=nodes,
file_cache_max_age=host_config.max_cachefile_age,
fetcher_messages=[
FetcherMessage.from_raw_data(
result.OK(source.default_raw_data),
Snapshot.null(),
source.fetcher_type,
)
for _h, _i, sources in nodes for source in sources
],
selected_sections=NO_SELECTION,
)[0]
assert len(host_sections) == len(hosts) == 3
cmk.utils.piggyback._store_status_file_of.assert_not_called() # type: ignore[attr-defined]
assert cmk.utils.piggyback.remove_source_status_file.call_count == 3 # type: ignore[attr-defined]
for host, addr in hosts.items():
remove_source_status_file = cmk.utils.piggyback.remove_source_status_file
remove_source_status_file.assert_any_call(host) # type: ignore[attr-defined]
key = HostKey(host, addr, SourceType.HOST)
assert key in host_sections
section = host_sections[key]
assert len(section.sections) == 1
assert next(iter(section.sections)) == section_name
assert not section.cache_info
assert not section.piggybacked_raw_data