def test_multiple_sources_from_different_hosts(self, hostname, ipaddress, mode, config_cache, host_config):
sources = [
ProgramSource.ds(hostname + "0", ipaddress, mode=mode, template=""),
TCPSource(hostname + "1", ipaddress, mode=mode),
TCPSource(hostname + "2", ipaddress, mode=mode),
]
nodes = make_nodes(
config_cache,
host_config,
ipaddress,
mode=mode,
sources=sources,
)
mhs = MultiHostSections()
update_host_sections(
mhs,
nodes,
max_cachefile_age=0,
host_config=host_config,
fetcher_messages=[
FetcherMessage.from_raw_data(
result.OK(source.default_raw_data),
Snapshot.null(),
source.fetcher_type,
)
for _h, _i, sources in nodes for source in sources
],
selected_sections=NO_SELECTION,
)
assert len(mhs) == 1
key = HostKey(hostname, ipaddress, SourceType.HOST)
assert key in mhs
section = mhs[key]
assert len(section.sections) == len(sources)
for source in sources:
# yapf: disable
assert (
section.sections[SectionName("section_name_%s" % source.hostname)]
== [["section_content"]])
def test_get_section_content(monkeypatch, hostname, nodes, host_entries,
cluster_mapping, service_descr, expected_result):
_set_up(monkeypatch, hostname, nodes, cluster_mapping)
multi_host_sections = MultiHostSections()
for nodename, node_section_content in host_entries:
multi_host_sections.setdefault(
HostKey(nodename, "127.0.0.1", SourceType.HOST),
AgentHostSections(
sections={
SectionName("section_plugin_name"): node_section_content
}),
)
section_content = multi_host_sections.get_section_content(
HostKey(hostname, "127.0.0.1", SourceType.HOST),
check_api_utils.HOST_ONLY,
"section_plugin_name",
False,
service_description=service_descr,
)
assert expected_result == section_content,\
"Section content: Expected '%s' but got '%s'" % (expected_result, section_content)
section_content = multi_host_sections.get_section_content(
HostKey(hostname, "127.0.0.1", SourceType.HOST),
check_api_utils.HOST_PRECEDENCE,
"section_plugin_name",
False,
service_description=service_descr,
)
assert expected_result == section_content,\
"Section content: Expected '%s' but got '%s'" % (expected_result, section_content)
section_content = multi_host_sections.get_section_content(
HostKey(hostname, "127.0.0.1", SourceType.MANAGEMENT),
check_api_utils.MGMT_ONLY,
"section_plugin_name",
False,
service_description=service_descr,
)
assert section_content is None, \
"Section content: Expected 'None' but got '%s'" % (section_content,)
def from_headerline(cls, headerline: bytes) -> "SectionMarker":
def parse_options(elems: Iterable[str]) -> Iterable[Tuple[str, str]]:
for option in elems:
if "(" not in option:
continue
name, value = option.split("(", 1)
assert value[-1] == ")", value
yield name, value[:-1]
if not SectionMarker.is_header(headerline):
raise ValueError(headerline)
headerparts = headerline[3:-3].decode().split(":")
options = dict(parse_options(headerparts[1:]))
cached: Optional[Tuple[int, int]]
try:
cached_ = tuple(map(int, options["cached"].split(",")))
cached = cached_[0], cached_[1]
except KeyError:
cached = None
encoding = options.get("encoding", "utf-8")
nostrip = options.get("nostrip") is not None
persist: Optional[int]
try:
persist = int(options["persist"])
except KeyError:
persist = None
separator: Optional[str]
try:
separator = chr(int(options["sep"]))
except KeyError:
separator = None
return SectionMarker(
name=SectionName(headerparts[0]),
cached=cached,
encoding=encoding,
nostrip=nostrip,
persist=persist,
separator=separator,
)
def test_fetch_from_io_partially_empty(self, monkeypatch, fetcher):
section_name = SectionName('pum')
monkeypatch.setattr(
fetcher, "sections", {
section_name: SectionMeta(
checking=True,
disabled=False,
redetect=False,
fetch_interval=None,
),
})
table = [['1']]
monkeypatch.setattr(
snmp_table,
"get_snmp_table",
lambda tree, **__: table
if tree.base == fetcher.plugin_store[section_name].trees[0].base else [],
)
assert fetcher.fetch(Mode.CHECKING) == result.OK({section_name: [table, []]})
def test_multiple_sources_from_the_same_host(
self,
hostname,
ipaddress,
mode,
config_cache,
host_config,
):
sources = [
ProgramDataSource(configurator=ProgramConfigurator.ds(
hostname,
ipaddress,
mode=mode,
template="",
),),
TCPDataSource(configurator=TCPConfigurator(
hostname,
ipaddress,
mode=mode,
),),
]
mhs = make_host_sections(
config_cache,
host_config,
ipaddress,
mode=mode,
sources=sources,
max_cachefile_age=0,
selected_raw_sections=None,
)
assert len(mhs) == 1
key = HostKey(hostname, ipaddress, SourceType.HOST)
assert key in mhs
section = mhs[key]
assert isinstance(section, AgentHostSections)
assert len(section.sections) == 1
# yapf: disable
assert (section.sections[SectionName("section_name_%s" % hostname)]
== len(sources) * [["section_content"]])
def test_create_snmp_section_plugin_from_legacy():
plugin = section_plugins_legacy.create_snmp_section_plugin_from_legacy(
"norris",
{
'parse_function': old_school_parse_function,
'inventory_function': old_school_discover_function,
},
old_school_scan_function,
(".1.2.3.4.5", ["2", 3]),
)
assert plugin.name == SectionName("norris")
assert plugin.parsed_section_name == ParsedSectionName("norris")
assert plugin.parse_function.__name__ == "old_school_parse_function"
assert plugin.host_label_function.__name__ == "host_label_function"
assert plugin.supersedes == []
assert plugin.detect_spec == [[(".1.2.3.4.5", "norris.*", True)]]
assert plugin.trees == [SNMPTree(base=".1.2.3.4.5", oids=["2", "3"])]
def test_get_simple_snmp_table(backend):
oid_info = SNMPTree(
base=".1.3.6.1.2.1.1",
oids=["1.0", "2.0", "5.0"],
)
table = snmp_table.get_snmp_table(
section_name=SectionName("my_Section"),
oid_info=oid_info,
backend=backend,
)
assert table == [
[
u'Linux zeus 4.8.6.5-smp #2 SMP Sun Nov 13 14:58:11 CDT 2016 i686',
u'.1.3.6.1.4.1.8072.3.2.10',
u'new system name',
],
]
assert isinstance(table[0][0], str)
def patch_io(self, monkeypatch):
class DummyHostSection(HostSections):
def _extend_section(self, section_name, section_content):
pass
for fetcher in (IPMIFetcher, PiggybackFetcher, ProgramFetcher, SNMPFetcher, TCPFetcher):
monkeypatch.setattr(fetcher, "__enter__", lambda self: self)
monkeypatch.setattr(fetcher, "fetch", lambda self, mode, fetcher=fetcher: {} if fetcher is SNMPFetcher else b"",)
monkeypatch.setattr(
Source,
"parse",
lambda self, *args, **kwargs: result.OK(DummyHostSection(
sections={SectionName("section_name_%s" % self.hostname): [["section_content_%s" % self.hostname]]},
cache_info={},
piggybacked_raw_data={},
),
),
)
def test_get_simple_snmp_table_with_hex_str(backend):
oid_info = BackendSNMPTree(
base=".1.3.6.1.2.1.2.2.1",
oids=[BackendOIDSpec("6", "string", False)],
)
table = snmp_table.get_snmp_table(
section_name=SectionName("my_Section"),
tree=oid_info,
walk_cache={},
backend=backend,
)
assert table == [
[""],
[
"\x00\x12yb\xf9@",
],
]
def test_get_simple_snmp_table_not_resolvable(backend):
if backend.config.is_usewalk_host:
pytest.skip("Not relevant")
backend.config = backend.config.update(ipaddress="bla.local")
# TODO: Unify different error messages
if backend.config.snmp_backend == SNMPBackendEnum.INLINE:
exc_match = "Failed to initiate SNMP"
else:
exc_match = "Unknown host"
with pytest.raises(MKSNMPError, match=exc_match):
snmp_table.get_snmp_table(
section_name=SectionName("my_Section"),
tree=INFO_TREE,
walk_cache={},
backend=backend,
)
def test_docker_container_diskstat_discovery(
section_name: str,
plugin_name: str,
mocker,
discovery_mode,
string_table_0,
fix_register,
expected_item,
) -> None:
agent_section = fix_register.agent_sections[SectionName(section_name)]
plugin = fix_register.check_plugins[CheckPluginName(plugin_name)]
assert plugin
section_0_seconds = agent_section.parse_function(string_table_0)
assert list(
plugin.discovery_function(
[discovery_mode],
section_diskstat=section_0_seconds,
section_multipath=None)) == [Service(item=expected_item)]
def create_snmp_section_plugin(
*,
name: str,
parsed_section_name: Optional[str] = None,
parse_function: SNMPParseFunction,
host_label_function: Optional[HostLabelFunction] = None,
supersedes: Optional[List[str]] = None,
detect_spec: SNMPDetectSpec,
trees: List[SNMPTree],
module: Optional[str] = None,
) -> SNMPSectionPlugin:
"""Return an SNMPSectionPlugin object after validating and converting the arguments one by one
For a detailed description of the parameters please refer to the exposed function in the
'register' namespace of the API.
"""
# TODO (mo): Well, implement it, and remove pragma below!
if supersedes is not None:
raise NotImplementedError("supersedes is not yet available")
if parsed_section_name is not None:
raise NotImplementedError("parsed_section_name is not yet available")
section_name = SectionName(name)
if any(isinstance(oid, OIDBytes) for tree in trees for oid in tree.oids):
expected_annotation: Tuple[Type, str] = (SNMPStringByteTable, "SNMPStringByteTable")
else:
expected_annotation = (SNMPStringTable, "SNMPStringTable")
_validate_parse_function(parse_function, expected_annotation=expected_annotation)
_validate_detect_spec(detect_spec)
_validate_snmp_trees(trees)
return SNMPSectionPlugin(
section_name,
ParsedSectionName(parsed_section_name if parsed_section_name else str(section_name)),
parse_function,
_create_host_label_function(host_label_function),
_create_supersedes(supersedes),
detect_spec,
trees,
module,
)
def test_update_with_empty_store_and_raw_data(self):
raw_data = AgentRawData(b"<<<fresh>>>")
section_store = MockStore(PersistedSections[AgentRawDataSection]({}))
parser = AgentParser(
"testhost",
section_store,
check_interval=0,
keep_outdated=True,
translation={},
encoding_fallback="ascii",
simulation=False,
logger=logging.getLogger("test"),
)
ahs = parser.parse(raw_data, selection=NO_SELECTION)
assert ahs.sections == {SectionName("fresh"): []}
assert ahs.cache_info == {}
assert ahs.piggybacked_raw_data == {}
assert section_store.load() == {}
def test_get_simple_snmp_table_wrong_credentials(backend):
if backend.config.is_usewalk_host:
pytest.skip("Not relevant")
backend.config = backend.config.update(credentials="dingdong")
# TODO: Unify different error messages
if backend.config.snmp_backend == SNMPBackendEnum.INLINE:
exc_match = "SNMP query timed out"
else:
exc_match = "Timeout: No Response from"
with pytest.raises(MKSNMPError, match=exc_match):
snmp_table.get_snmp_table(
section_name=SectionName("my_Section"),
tree=INFO_TREE,
walk_cache={},
backend=backend,
)
def patch_io(self, monkeypatch):
class DummyHostSection(ABCHostSections):
def _extend_section(self, section_name, section_content):
pass
for fetcher in (IPMIFetcher, PiggybackFetcher, ProgramFetcher, SNMPFetcher, TCPFetcher):
monkeypatch.setattr(fetcher, "__enter__", lambda self: self)
monkeypatch.setattr(fetcher, "fetch", lambda self, mode, fetcher=fetcher: {} if fetcher is SNMPFetcher else b"",)
monkeypatch.setattr(
ABCChecker,
"check",
lambda self, *args, **kwargs: DummyHostSection(
sections={SectionName("section_name_%s" % self.configurator.hostname): [["section_content"]]},
cache_info={},
piggybacked_raw_data={},
persisted_sections="",
),
)
def test_one_snmp_source(self, hostname, ipaddress, config_cache, host_config):
mhs = make_host_sections(
config_cache,
host_config,
ipaddress,
sources=[SNMPDataSource(hostname, ipaddress)],
max_cachefile_age=0,
selected_raw_sections=None,
)
assert len(mhs) == 1
key = HostKey(hostname, ipaddress, SourceType.HOST)
assert key in mhs
section = mhs[key]
assert isinstance(section, SNMPHostSections)
assert len(section.sections) == 1
assert section.sections[SectionName("section_name_%s" % hostname)] == [["section_content"]]
def __call__(
self,
snmp_config: SNMPHostConfig,
on_error: str,
do_snmp_scan: bool,
for_mgmt_board: bool,
) -> Set[SectionName]:
"""Returns a list of raw sections that shall be processed by this source.
The logic is only processed once. Once processed, the answer is cached.
"""
if self._filter_function is None:
raise MKGeneralException(
"The check type filter function has not been set")
if self._cached_result is not None:
return self._cached_result
# Make hostname globally available for scan functions.
# This is rarely used, but e.g. the scan for if/if64 needs
# this to evaluate if_disabled_if64_checks.
check_api_utils.set_hostname(snmp_config.hostname)
found_plugins = self._filter_function(
self.sections(),
on_error=on_error,
do_snmp_scan=do_snmp_scan,
binary_host=config.get_config_cache().in_binary_hostlist(
snmp_config.hostname,
config.snmp_without_sys_descr,
),
backend=factory.backend(snmp_config),
)
self._cached_result = {
SectionName(s)
for s in config.filter_by_management_board(
snmp_config.hostname,
found_plugins,
for_mgmt_board=for_mgmt_board,
for_discovery=True,
)
}
return self._cached_result
def test_get_simple_snmp_table_bulkwalk(backend, bulk):
backend.config = backend.config.update(is_bulkwalk_host=bulk)
oid_info: OIDWithColumns = (
".1.3.6.1.2.1.1",
["1.0", "2.0", "5.0"],
)
table = snmp_table.get_snmp_table(
section_name=SectionName("my_Section"),
oid_info=oid_info,
backend=backend,
)
assert table == [
[
u'Linux zeus 4.8.6.5-smp #2 SMP Sun Nov 13 14:58:11 CDT 2016 i686',
u'.1.3.6.1.4.1.8072.3.2.10',
u'new system name',
],
]
assert isinstance(table[0][0], str)
def test_get_simple_snmp_table_oid_end(backend):
oid_info = BackendSNMPTree(
base=".1.3.6.1.2.1.2.2.1",
oids=[
BackendOIDSpec("1", "string", False),
BackendOIDSpec("2", "string", False),
BackendOIDSpec("3", "string", False),
BackendOIDSpec(SpecialColumn.END, "string", False),
],
)
table = snmp_table.get_snmp_table(
section_name=SectionName("my_Section"),
oid_info=oid_info,
backend=backend,
)
assert table == [
[u'1', u'lo', u'24', u'1'],
[u'2', u'eth0', u'6', u'2'],
]
def _from_json(cls, serialized: Dict[str, Any]) -> 'SNMPFetcher':
# The SNMPv3 configuration is represented by a tuple of different lengths (see
# SNMPCredentials). Since we just deserialized from JSON, we have to convert the
# list used by JSON back to a tuple.
# SNMPv1/v2 communities are represented by a string: Leave it untouched.
if isinstance(serialized["snmp_config"]["credentials"], list):
serialized["snmp_config"]["credentials"] = tuple(
serialized["snmp_config"]["credentials"])
return cls(
file_cache=SNMPFileCache.from_json(serialized.pop("file_cache")),
sections={
SectionName(s): SectionMeta.deserialize(m)
for s, m in serialized["sections"].items()
},
on_error=serialized["on_error"],
missing_sys_description=serialized["missing_sys_description"],
do_status_data_inventory=serialized["do_status_data_inventory"],
snmp_config=SNMPHostConfig.deserialize(serialized["snmp_config"]),
)
def test_get_simple_snmp_table_with_hex_str(backend):
oid_info: OIDWithColumns = (
".1.3.6.1.2.1.2.2.1",
[
"6",
],
)
table = snmp_table.get_snmp_table(
section_name=SectionName("my_Section"),
oid_info=oid_info,
backend=backend,
)
assert table == [
[u''],
[
u'\x00\x12yb\xf9@',
],
]
def patch_io(self, monkeypatch):
class DummyHostSection(ABCHostSections):
def _extend_section(self, section_name, section_content):
pass
monkeypatch.setattr(ABCFetcher, "fetch", lambda self: None)
monkeypatch.setattr(CachedSNMPDetector, "__call__", lambda *args, **kwargs: frozenset())
monkeypatch.setattr(
ABCChecker,
"check",
lambda self, raw_data: DummyHostSection(
sections={
SectionName("section_name_%s" % self.configurator.hostname):
[["section_content"]]
},
cache_info={},
piggybacked_raw_data={},
persisted_sections="",
),
)
def test_get_section_content(hostname, host_entries, cluster_node_keys, expected_result):
parsed_sections_broker = ParsedSectionsBroker({
HostKey(nodename, "127.0.0.1",
SourceType.HOST): SectionsParser(host_sections=AgentHostSections(
sections={SectionName("section_plugin_name"): node_section_content}))
for nodename, node_section_content in host_entries
})
mhs = _MultiHostSections(parsed_sections_broker)
section_content = mhs.get_section_content(
HostKey(hostname, "127.0.0.1", SourceType.HOST),
HOST_ONLY,
"section_plugin_name",
False,
cluster_node_keys=cluster_node_keys,
check_legacy_info={}, # only for parse_function lookup, not needed in this test
)
assert expected_result == section_content
section_content = mhs.get_section_content(
HostKey(hostname, "127.0.0.1", SourceType.HOST),
HOST_PRECEDENCE,
"section_plugin_name",
False,
cluster_node_keys=cluster_node_keys,
check_legacy_info={}, # only for parse_function lookup, not needed in this test
)
assert expected_result == section_content
section_content = mhs.get_section_content(
HostKey(hostname, "127.0.0.1", SourceType.MANAGEMENT),
MGMT_ONLY,
"section_plugin_name",
False,
cluster_node_keys=None if cluster_node_keys is None else
[HostKey(hn, ip, SourceType.MANAGEMENT) for (hn, ip, _st) in cluster_node_keys],
check_legacy_info={}, # only for parse_function lookup, not needed in this test
)
assert section_content is None
def test_one_snmp_source(self, hostname, ipaddress, mode, config_cache,
host_config):
broker = ParsedSectionsBroker()
update_host_sections(
broker,
make_nodes(
config_cache,
host_config,
ipaddress,
mode=mode,
sources=[
SNMPSource.snmp(
hostname,
ipaddress,
mode=mode,
selected_sections=NO_SELECTION,
on_scan_error="raise",
),
],
),
max_cachefile_age=0,
host_config=host_config,
fetcher_messages=[
FetcherMessage.from_raw_data(
result.OK({}),
Snapshot.null(),
FetcherType.SNMP,
),
],
selected_sections=NO_SELECTION,
)
assert len(broker) == 1
key = HostKey(hostname, ipaddress, SourceType.HOST)
assert key in broker
section = broker[key]
assert len(section.sections) == 1
assert section.sections[SectionName("section_name_%s" %
hostname)] == [["section_content"]]
def test_get_simple_snmp_table_oid_bin(backend):
oid_info = BackendSNMPTree(
base=".1.3.6.1.2.1.2.2.1",
# deprecated with checkmk version 2.0
oids=[
BackendOIDSpec("1", "string", False),
BackendOIDSpec("2", "string", False),
BackendOIDSpec("3", "string", False),
BackendOIDSpec(SpecialColumn.BIN, "string", False),
],
)
table = snmp_table.get_snmp_table(
section_name=SectionName("my_Section"),
oid_info=oid_info,
backend=backend,
)
assert table == [
[u'1', u'lo', u'24', u'\x01\x03\x06\x01\x02\x01\x02\x02\x01\x01\x01'],
[u'2', u'eth0', u'6', u'\x01\x03\x06\x01\x02\x01\x02\x02\x01\x01\x02'],
]
def test_get_simple_snmp_table_oid_end(backend):
oid_info = BackendSNMPTree(
base=".1.3.6.1.2.1.2.2.1",
oids=[
BackendOIDSpec("1", "string", False),
BackendOIDSpec("2", "string", False),
BackendOIDSpec("3", "string", False),
BackendOIDSpec(SpecialColumn.END, "string", False),
],
)
table = snmp_table.get_snmp_table(
section_name=SectionName("my_Section"),
tree=oid_info,
walk_cache={},
backend=backend,
)
assert table == [
["1", "lo", "24", "1"],
["2", "eth0", "6", "2"],
]
def test_options_decode_values(self):
section_header = SectionMarker.from_headerline(
b"<<<"
+ b":".join(
(
b"name",
b"cached(1,2)",
b"encoding(ascii)",
b"nostrip()",
b"persist(42)",
b"sep(124)",
)
)
+ b">>>"
)
assert section_header.name == SectionName("name")
assert section_header.cached == (1, 2)
assert section_header.encoding == "ascii"
assert section_header.nostrip is True
assert section_header.persist == 42
assert section_header.separator == "|"
def test_one_snmp_source(self, hostname, ipaddress, mode, config_cache,
host_config):
mhs = MultiHostSections()
update_host_sections(
mhs,
make_nodes(
config_cache,
host_config,
ipaddress,
mode=mode,
sources=[
SNMPSource.snmp(
hostname,
ipaddress,
mode=mode,
),
],
),
max_cachefile_age=0,
selected_raw_sections=None,
host_config=host_config,
fetcher_messages=[
FetcherMessage.from_raw_data(
result.OK({}),
L3Stats(CPUTracker()),
FetcherType.SNMP,
),
],
)
assert len(mhs) == 1
key = HostKey(hostname, ipaddress, SourceType.HOST)
assert key in mhs
section = mhs[key]
assert isinstance(section, SNMPHostSections)
assert len(section.sections) == 1
assert section.sections[SectionName("section_name_%s" %
hostname)] == [["section_content"]]
def test_one_snmp_source(self, hostname, ipaddress, mode, config_cache,
host_config):
mhs = MultiHostSections()
update_host_sections(
mhs,
make_nodes(
config_cache,
host_config,
ipaddress,
mode=mode,
sources=[
SNMPSource.snmp(
hostname,
ipaddress,
mode=mode,
selected_sections=NO_SELECTION,
),
],
),
max_cachefile_age=0,
host_config=host_config,
fetcher_messages=[
FetcherMessage.from_raw_data(
result.OK(SNMPRawData({})),
Snapshot.null(),
FetcherType.SNMP,
),
],
selected_sections=NO_SELECTION,
)
assert len(mhs) == 1
key = HostKey(hostname, ipaddress, SourceType.HOST)
assert key in mhs
section = mhs[key]
assert len(section.sections) == 1
assert section.sections[SectionName("section_name_%s" %
hostname)] == [["section_content"]]
def test_no_sources(self, cluster, nodes, config_cache, host_config, mode):
made_nodes = make_nodes(
config_cache,
host_config,
None,
mode=mode,
sources=(),
)
broker = ParsedSectionsBroker()
update_host_sections(
broker,
made_nodes,
max_cachefile_age=0,
host_config=host_config,
fetcher_messages=[
# We do not pass sources explicitly but still append Piggyback.
FetcherMessage.from_raw_data(
result.OK(AgentRawData(b"")),
Snapshot.null(),
FetcherType.PIGGYBACK,
) for _n in made_nodes
],
selected_sections=NO_SELECTION,
)
assert len(broker) == len(nodes)
key_clu = HostKey(cluster, None, SourceType.HOST)
assert key_clu not in broker
for hostname, addr in nodes.items():
key = HostKey(hostname, addr, SourceType.HOST)
assert key in broker
section = broker[key]
# yapf: disable
assert (section.sections[SectionName("section_name_%s" % hostname)]
== [["section_content_%s" % hostname]])
assert not section.cache_info
assert not section.piggybacked_raw_data