def fetch_all(
nodes: Iterable[Tuple[HostName, Optional[HostAddress], Sequence[Source]]],
*,
max_cachefile_age: int,
host_config: HostConfig,
selected_raw_sections: Optional[SelectedRawSections],
) -> Iterator[FetcherMessage]:
console.verbose("%s+%s %s\n", tty.yellow, tty.normal, "Fetching data".upper())
# TODO(ml): It is not clear to me in which case it is possible for the following to hold true
# for any source in nodes:
# - hostname != source.hostname
# - ipaddress != source.ipaddress
# If this is impossible, then we do not need the Tuple[HostName, HostAddress, ...].
for _hostname, _ipaddress, sources in nodes:
for source in sources:
console.vverbose(" Source: %s/%s\n" % (source.source_type, source.fetcher_type))
if host_config.nodes is None:
source.selected_raw_sections = selected_raw_sections
else:
source.selected_raw_sections = _make_piggybacked_sections(host_config)
source.file_cache_max_age = max_cachefile_age
with CPUTracker() as tracker:
raw_data = source.fetch()
yield FetcherMessage.from_raw_data(
raw_data,
L3Stats(tracker.duration),
source.fetcher_type,
)
def test_multiple_sources_from_the_same_host(
self,
hostname,
ipaddress,
mode,
config_cache,
host_config,
):
sources = [
ProgramSource.ds(
hostname,
ipaddress,
mode=mode,
template="",
),
TCPSource(
hostname,
ipaddress,
mode=mode,
),
]
mhs = MultiHostSections()
update_host_sections(
mhs,
make_nodes(
config_cache,
host_config,
ipaddress,
mode=mode,
sources=sources,
),
max_cachefile_age=0,
selected_raw_sections=None,
host_config=host_config,
fetcher_messages=[
FetcherMessage.from_raw_data(
result.OK(source.default_raw_data),
L3Stats(CPUTracker()),
source.fetcher_type,
) for source in sources
],
)
assert len(mhs) == 1
key = HostKey(hostname, ipaddress, SourceType.HOST)
assert key in mhs
section = mhs[key]
assert isinstance(section, AgentHostSections)
assert len(section.sections) == 1
# yapf: disable
assert (section.sections[SectionName("section_name_%s" % hostname)]
== len(sources) * [["section_content"]])
def test_controller_success(self):
payload = AgentPayload(69 * b"\0")
stats = L3Stats(CPUTracker())
header = FetcherHeader(
FetcherType.TCP,
PayloadType.AGENT,
status=42,
payload_length=len(payload),
stats_length=len(stats),
)
message = FetcherMessage(header, payload, stats)
assert make_payload_answer(message) == (b"fetch:SUCCESS: :240 :" + header +
payload + stats)
def test_one_snmp_source(self, hostname, ipaddress, mode, config_cache,
host_config):
mhs = MultiHostSections()
update_host_sections(
mhs,
make_nodes(
config_cache,
host_config,
ipaddress,
mode=mode,
sources=[
SNMPSource.snmp(
hostname,
ipaddress,
mode=mode,
),
],
),
max_cachefile_age=0,
selected_raw_sections=None,
host_config=host_config,
fetcher_messages=[
FetcherMessage.from_raw_data(
result.OK({}),
L3Stats(CPUTracker()),
FetcherType.SNMP,
),
],
)
assert len(mhs) == 1
key = HostKey(hostname, ipaddress, SourceType.HOST)
assert key in mhs
section = mhs[key]
assert isinstance(section, SNMPHostSections)
assert len(section.sections) == 1
assert section.sections[SectionName("section_name_%s" %
hostname)] == [["section_content"]]
def test_no_sources(self, cluster, nodes, config_cache, host_config, mode):
mhs = MultiHostSections()
update_host_sections(
mhs,
make_nodes(
config_cache,
host_config,
None,
mode=mode,
sources=(),
),
max_cachefile_age=0,
selected_raw_sections=None,
host_config=host_config,
fetcher_messages=[
# We do not pass sources explicitly but still append Piggyback.
FetcherMessage.from_raw_data(
result.OK(b""),
L3Stats(CPUTracker()),
FetcherType.PIGGYBACK,
),
],
)
assert len(mhs) == len(nodes)
key_clu = HostKey(cluster, None, SourceType.HOST)
assert key_clu not in mhs
for hostname, addr in nodes.items():
key = HostKey(hostname, addr, SourceType.HOST)
assert key in mhs
section = mhs[key]
# yapf: disable
assert (section.sections[SectionName("section_name_%s" % hostname)]
== [["section_content"]])
assert not section.cache_info
assert not section.piggybacked_raw_data
assert not section.persisted_sections
def test_one_nonsnmp_source(self, hostname, ipaddress, mode, config_cache,
host_config, source):
source = source(hostname, ipaddress, mode=mode)
assert source.source_type is SourceType.HOST
mhs = MultiHostSections()
update_host_sections(
mhs,
make_nodes(
config_cache,
host_config,
ipaddress,
mode=mode,
sources=[source],
),
max_cachefile_age=0,
selected_raw_sections=None,
host_config=host_config,
fetcher_messages=[
FetcherMessage.from_raw_data(
result.OK(source.default_raw_data),
L3Stats(CPUTracker()),
source.fetcher_type,
),
],
)
assert len(mhs) == 1
key = HostKey(hostname, ipaddress, source.source_type)
assert key in mhs
section = mhs[key]
assert isinstance(section, AgentHostSections)
assert len(section.sections) == 1
assert section.sections[SectionName("section_name_%s" %
hostname)] == [["section_content"]]
def test_get_host_sections_cluster(mode, monkeypatch, mocker):
hostname = "testhost"
hosts = {
"host0": "10.0.0.0",
"host1": "10.0.0.1",
"host2": "10.0.0.2",
}
address = "1.2.3.4"
tags = {"agent": "no-agent"}
section_name = SectionName("test_section")
config_cache = make_scenario(hostname, tags).apply(monkeypatch)
host_config = config.HostConfig.make_host_config(hostname)
def lookup_ip_address(host_config, family=None, for_mgmt_board=False):
return hosts[host_config.hostname]
def make_piggybacked_sections(hc):
if hc.nodes == host_config.nodes:
return {section_name: True}
return {}
def check(_, *args, **kwargs):
return result.OK(AgentHostSections(sections={section_name: [[str(section_name)]]}))
monkeypatch.setattr(
ip_lookup,
"lookup_ip_address",
lookup_ip_address,
)
monkeypatch.setattr(
_checkers,
"_make_piggybacked_sections",
make_piggybacked_sections,
)
monkeypatch.setattr(
Source,
"parse",
check,
)
mocker.patch.object(
cmk.utils.piggyback,
"remove_source_status_file",
autospec=True,
)
mocker.patch.object(
cmk.utils.piggyback,
"_store_status_file_of",
autospec=True,
)
# Create a cluster
host_config.nodes = list(hosts.keys())
mhs = MultiHostSections()
sources = make_sources(host_config, address, mode=mode)
update_host_sections(
mhs,
make_nodes(
config_cache,
host_config,
address,
mode=mode,
sources=sources,
),
max_cachefile_age=host_config.max_cachefile_age,
selected_raw_sections=None,
host_config=host_config,
fetcher_messages=[
FetcherMessage.from_raw_data(
result.OK(source.default_raw_data),
L3Stats(CPUTracker()),
source.fetcher_type,
)
for source in sources
],
)
assert len(mhs) == len(hosts) == 3
cmk.utils.piggyback._store_status_file_of.assert_not_called() # type: ignore[attr-defined]
assert cmk.utils.piggyback.remove_source_status_file.call_count == 3 # type: ignore[attr-defined]
for host, addr in hosts.items():
remove_source_status_file = cmk.utils.piggyback.remove_source_status_file
remove_source_status_file.assert_any_call(host) # type: ignore[attr-defined]
key = HostKey(host, addr, SourceType.HOST)
assert key in mhs
section = mhs[key]
assert len(section.sections) == 1
assert next(iter(section.sections)) == section_name
assert not section.cache_info
assert not section.piggybacked_raw_data
assert not section.persisted_sections
def stats(self):
return L3Stats(Snapshot.null())
def test_encode_decode(self, l3stats):
assert L3Stats.from_bytes(bytes(l3stats)) == l3stats
def stats_length(self):
return len(L3Stats(Snapshot.null()))
def stats(self):
return L3Stats(CPUTracker())
def l3stats(self, tracker):
return L3Stats(tracker)
def stats_length(self):
return len(L3Stats(CPUTracker()))