def test_structured_data_StructuredDataTree_equal_numerations():
tree_addresses_ordered = StructuredDataTree().load_from(
"%s/tree_addresses_ordered" % TEST_DIR)
tree_addresses_unordered = StructuredDataTree().load_from(
"%s/tree_addresses_unordered" % TEST_DIR)
assert tree_addresses_ordered.is_equal(tree_addresses_unordered)
assert tree_addresses_unordered.is_equal(tree_addresses_ordered)
def _save_inventory_tree(hostname, inventory_tree):
# type: (HostName, StructuredDataTree) -> Optional[StructuredDataTree]
store.makedirs(cmk.utils.paths.inventory_output_dir)
filepath = cmk.utils.paths.inventory_output_dir + "/" + hostname
if inventory_tree.is_empty():
# Remove empty inventory files. Important for host inventory icon
if os.path.exists(filepath):
os.remove(filepath)
if os.path.exists(filepath + ".gz"):
os.remove(filepath + ".gz")
return None
old_tree = StructuredDataTree().load_from(filepath)
old_tree.normalize_nodes()
if old_tree.is_equal(inventory_tree):
console.verbose("Inventory was unchanged\n")
return None
if old_tree.is_empty():
console.verbose("New inventory tree\n")
else:
console.verbose("Inventory tree has changed\n")
old_time = os.stat(filepath).st_mtime
arcdir = "%s/%s" % (cmk.utils.paths.inventory_archive_dir, hostname)
store.makedirs(arcdir)
os.rename(filepath, arcdir + ("/%d" % old_time))
inventory_tree.save_to(cmk.utils.paths.inventory_output_dir, hostname)
return old_tree
def _do_inv_for(sources, multi_host_sections, host_config, ipaddress,
do_status_data_inv):
# type: (data_sources.DataSources, data_sources.MultiHostSections, config.HostConfig, Optional[str], bool) -> Tuple[Optional[float], StructuredDataTree, StructuredDataTree]
hostname = host_config.hostname
_initialize_inventory_tree()
inventory_tree = g_inv_tree
status_data_tree = StructuredDataTree()
node = inventory_tree.get_dict("software.applications.check_mk.cluster.")
if host_config.is_cluster:
node["is_cluster"] = True
_do_inv_for_cluster(host_config, inventory_tree)
else:
node["is_cluster"] = False
_do_inv_for_realhost(host_config, sources, multi_host_sections,
hostname, ipaddress, inventory_tree,
status_data_tree)
inventory_tree.normalize_nodes()
old_timestamp = _save_inventory_tree(hostname, inventory_tree)
_run_inventory_export_hooks(host_config, inventory_tree)
success_msg = [
"Found %s%s%d%s inventory entries" %
(tty.bold, tty.yellow, inventory_tree.count_entries(), tty.normal)
]
console.section_success(", ".join(success_msg))
if do_status_data_inv:
status_data_tree.normalize_nodes()
_save_status_data_tree(hostname, status_data_tree)
console.section_success("Found %s%s%d%s status entries" %
(tty.bold, tty.yellow,
status_data_tree.count_entries(), tty.normal))
return old_timestamp, inventory_tree, status_data_tree
def _do_inv_for(
config_cache: config.ConfigCache,
host_config: config.HostConfig,
ipaddress: Optional[HostAddress],
*,
sources: Sequence[ABCSource],
multi_host_sections: Optional[MultiHostSections],
) -> Tuple[StructuredDataTree, StructuredDataTree, Sequence[Tuple[
ABCSource, Result[ABCHostSections, Exception]]]]:
hostname = host_config.hostname
initialize_inventory_tree()
inventory_tree = g_inv_tree
status_data_tree = StructuredDataTree()
node = inventory_tree.get_dict("software.applications.check_mk.cluster.")
results: Sequence[Tuple[ABCSource, Result[ABCHostSections,
Exception]]] = []
if host_config.is_cluster:
node["is_cluster"] = True
_do_inv_for_cluster(host_config, inventory_tree)
else:
node["is_cluster"] = False
results = _do_inv_for_realhost(
config_cache,
host_config,
sources,
multi_host_sections,
hostname,
ipaddress,
inventory_tree,
status_data_tree,
)
inventory_tree.normalize_nodes()
status_data_tree.normalize_nodes()
return inventory_tree, status_data_tree, results
def _load_inventory_tree(hostname):
# Load data of a host, cache it in the current HTTP request
if not hostname:
return
inventory_tree_cache = g.setdefault("inventory", {})
if hostname in inventory_tree_cache:
inventory_tree = inventory_tree_cache[hostname]
else:
if '/' in hostname:
# just for security reasons
return
cache_path = "%s/inventory/%s" % (cmk.utils.paths.var_dir, hostname)
inventory_tree = StructuredDataTree().load_from(cache_path)
inventory_tree_cache[hostname] = inventory_tree
return inventory_tree
def _do_inv_for_cluster(host_config: config.HostConfig) -> InventoryTrees:
inventory_tree = StructuredDataTree()
_set_cluster_property(inventory_tree, host_config)
if not host_config.nodes:
return InventoryTrees(inventory_tree, StructuredDataTree())
inv_node = inventory_tree.get_list("software.applications.check_mk.cluster.nodes:")
for node_name in host_config.nodes:
inv_node.append({
"name": node_name,
})
inventory_tree.normalize_nodes()
return InventoryTrees(inventory_tree, StructuredDataTree())
def _run_inventory_export_hooks(host_config: config.HostConfig,
inventory_tree: StructuredDataTree) -> None:
import cmk.base.inventory_plugins as inventory_plugins # pylint: disable=import-outside-toplevel
hooks = host_config.inventory_export_hooks
if not hooks:
return
section.section_step("Execute inventory export hooks")
for hookname, params in hooks:
console.verbose("Execute export hook: %s%s%s%s" %
(tty.blue, tty.bold, hookname, tty.normal))
try:
func = inventory_plugins.inv_export[hookname]["export_function"]
func(host_config.hostname, params, inventory_tree.get_raw_tree())
except Exception as e:
if cmk.utils.debug.enabled():
raise
raise MKGeneralException("Failed to execute export hook %s: %s" % (hookname, e))
def _load_inventory_tree(hostname):
# Load data of a host, cache it in the current HTTP request
if not hostname:
return
inventory_tree_cache = g.setdefault("inventory", {})
if hostname in inventory_tree_cache:
inventory_tree = inventory_tree_cache[hostname]
else:
if '/' in hostname:
# just for security reasons
return
cache_path = "%s/inventory/%s" % (cmk.utils.paths.var_dir, hostname)
try:
inventory_tree = StructuredDataTree().load_from(cache_path)
except Exception as e:
if config.debug:
html.show_warning(e)
raise LoadStructuredDataError()
inventory_tree_cache[hostname] = inventory_tree
return inventory_tree
def _load_structured_data_tree(
tree_type: Literal["inventory", "status_data"],
hostname: Optional[HostName]) -> Optional[StructuredDataTree]:
"""Load data of a host, cache it in the current HTTP request"""
if not hostname:
return None
inventory_tree_cache = g.setdefault(tree_type, {})
if hostname in inventory_tree_cache:
inventory_tree = inventory_tree_cache[hostname]
else:
if '/' in hostname:
# just for security reasons
return None
cache_path = "%s/%s" % (cmk.utils.paths.inventory_output_dir if tree_type == "inventory" \
else cmk.utils.paths.status_data_dir, hostname)
try:
inventory_tree = StructuredDataTree().load_from(cache_path)
except Exception as e:
if config.debug:
html.show_warning("%s" % e)
raise LoadStructuredDataError()
inventory_tree_cache[hostname] = inventory_tree
return inventory_tree
def test_structured_data_StructuredDataTree_get_list():
with pytest.raises(MKGeneralException) as e:
StructuredDataTree().get_list("")
assert 'Empty tree path or zero' in "%s" % e
with pytest.raises(MKGeneralException) as e:
StructuredDataTree().get_list(0) # type: ignore[arg-type]
assert 'Empty tree path or zero' in "%s" % e
with pytest.raises(MKGeneralException) as e:
StructuredDataTree().get_list(100) # type: ignore[arg-type]
assert 'Wrong tree path format' in "%s" % e
with pytest.raises(MKGeneralException) as e:
StructuredDataTree().get_list("a?")
assert 'No valid tree path' in "%s" % e
with pytest.raises(MKGeneralException) as e:
StructuredDataTree().get_list("a$.")
assert 'Specified tree path contains unexpected characters' in "%s" % e
assert StructuredDataTree().get_list("a:") == []
for_mgmt_board=for_mgmt_board)
#.
# .--Inventory Tree------------------------------------------------------.
# | ___ _ _____ |
# | |_ _|_ ____ _____ _ __ | |_ ___ _ __ _ _ |_ _| __ ___ ___ |
# | | || '_ \ \ / / _ \ '_ \| __/ _ \| '__| | | | | || '__/ _ \/ _ \ |
# | | || | | \ V / __/ | | | || (_) | | | |_| | | || | | __/ __/ |
# | |___|_| |_|\_/ \___|_| |_|\__\___/|_| \__, | |_||_| \___|\___| |
# | |___/ |
# +----------------------------------------------------------------------+
# | Managing the inventory tree of a host |
# '----------------------------------------------------------------------'
g_inv_tree = StructuredDataTree(
) # TODO Remove one day. Deprecated with version 1.5.0i3??
def _initialize_inventory_tree(
): # TODO Remove one day. Deprecated with version 1.5.0i3??
# type: () -> None
global g_inv_tree
g_inv_tree = StructuredDataTree()
# Dict based
def inv_tree(path): # TODO Remove one day. Deprecated with version 1.5.0i3??
# type: (str) -> Dict
return g_inv_tree.get_dict(path)
def test_delta_structured_data_tree_serialization(zipped_trees):
old_tree = StructuredDataTree()
new_tree = StructuredDataTree()
old_filename, new_filename = zipped_trees
old_tree.load_from(old_filename)
new_tree.load_from(new_filename)
_, __, ___, delta_tree = old_tree.compare_with(new_tree)
new_delta_tree = StructuredDataTree()
new_delta_tree.create_tree_from_raw_tree(delta_tree.get_raw_tree())
assert delta_tree.is_equal(new_delta_tree)
def test_structured_data_StructuredDataTree_building_tree():
def plugin_dict():
node = struct_tree.get_dict("level0_0.level1_dict.")
for a, b in [("d1", "D1"), ("d2", "D2")]:
node.setdefault(a, b)
def plugin_list():
node = struct_tree.get_list("level0_1.level1_list:")
for a, b in [("l1", "L1"), ("l2", "L2")]:
node.append({a: b})
def plugin_nested_list():
node = struct_tree.get_list("level0_2.level1_nested_list:")
for index in range(10):
array: Dict[str, List[Dict[str, str]]] = {"foo": []}
for a, b in [("nl1", "NL1"), ("nl2", "NL2")]:
array["foo"].append({a: "%s-%s" % (b, index)})
node.append(array)
struct_tree = StructuredDataTree()
plugin_dict()
plugin_list()
plugin_nested_list()
struct_tree.normalize_nodes()
assert struct_tree.has_edge("level0_0")
assert struct_tree.has_edge("level0_1")
assert struct_tree.has_edge("level0_2")
assert not struct_tree.has_edge("foobar")
level1_dict = struct_tree.get_sub_attributes(["level0_0", "level1_dict"])
level1_list = struct_tree.get_sub_numeration(["level0_1", "level1_list"])
level1_nested_list_con = struct_tree.get_sub_container(
["level0_2", "level1_nested_list"])
level1_nested_list_num = struct_tree.get_sub_numeration(
["level0_2", "level1_nested_list"])
level1_nested_list_att = struct_tree.get_sub_attributes(
["level0_2", "level1_nested_list"])
assert isinstance(level1_dict, Attributes)
assert 'd1' in level1_dict.get_child_data()
assert 'd2' in level1_dict.get_child_data()
assert isinstance(level1_list, Numeration)
known_keys = [key for row in level1_list.get_child_data() for key in row]
assert 'l1' in known_keys
assert 'l2' in known_keys
assert level1_nested_list_num is None
assert level1_nested_list_att is None
assert isinstance(level1_nested_list_con, Container)
assert list(
level1_nested_list_con._nodes) == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
def test_structured_data_StructuredDataTree_is_empty():
assert StructuredDataTree().is_empty() is True
def test_delta_structured_data_tree_serialization(two_tree_filenames):
old_tree = StructuredDataTree()
new_tree = StructuredDataTree()
old_filename, new_filename = two_tree_filenames
old_tree.load_from(old_filename)
new_tree.load_from(new_filename)
_, __, ___, delta_tree = old_tree.compare_with(new_tree)
raw_delta_tree = delta_tree.get_raw_tree()
new_delta_tree = StructuredDataTree()
new_delta_tree.create_tree_from_raw_tree(raw_delta_tree)
new_raw_delta_tree = new_delta_tree.get_raw_tree()
assert raw_delta_tree == new_raw_delta_tree
def _save_status_data_tree(hostname: HostName, status_data_tree: StructuredDataTree) -> None:
if status_data_tree and not status_data_tree.is_empty():
store.makedirs(cmk.utils.paths.status_data_dir)
status_data_tree.save_to(cmk.utils.paths.status_data_dir, hostname)
def _set_cluster_property(
inventory_tree: StructuredDataTree,
host_config: config.HostConfig,
) -> None:
inventory_tree.get_dict(
"software.applications.check_mk.cluster.")["is_cluster"] = host_config.is_cluster
def _create_tree_from_raw_tree(raw_tree):
if raw_tree:
return StructuredDataTree().create_tree_from_raw_tree(
ast.literal_eval(raw_tree))
return
def test_structured_data_StructuredDataTree_save_to():
with pytest.raises(TypeError):
StructuredDataTree().save_to()
with pytest.raises(TypeError):
StructuredDataTree().save_to("x")
def _create_tree_from_raw_tree(
raw_tree: bytes) -> Optional[StructuredDataTree]:
if raw_tree:
return StructuredDataTree().create_tree_from_raw_tree(
ast.literal_eval(raw_tree.decode("utf-8")))
return None
def test_structured_data_StructuredDataTree_building_tree():
def plugin_dict():
node = struct_tree.get_dict("level0_0.level1_dict.")
for a, b in [("d1", "D1"), ("d2", "D2")]:
node.setdefault(a, b)
def plugin_list():
node = struct_tree.get_list("level0_1.level1_list:")
for a, b in [("l1", "L1"), ("l2", "L2")]:
node.append({a: b})
def plugin_nested_list():
node = struct_tree.get_list("level0_2.level1_nested_list:")
for index in xrange(10):
array = {"foo": []}
for a, b in [("nl1", "NL1"), ("nl2", "NL2")]:
array["foo"].append({a: "%s-%s" % (b, index)})
node.append(array)
struct_tree = StructuredDataTree()
plugin_dict()
plugin_list()
plugin_nested_list()
struct_tree.normalize_nodes()
assert struct_tree.has_edge("level0_0")
assert struct_tree.has_edge("level0_1")
assert struct_tree.has_edge("level0_2")
assert not struct_tree.has_edge("foobar")
level1_dict = struct_tree.get_sub_attributes(["level0_0", "level1_dict"])
level1_list = struct_tree.get_sub_numeration(["level0_1", "level1_list"])
level1_nested_list_con = struct_tree.get_sub_container(
["level0_2", "level1_nested_list"])
level1_nested_list_num = struct_tree.get_sub_numeration(
["level0_2", "level1_nested_list"])
level1_nested_list_att = struct_tree.get_sub_attributes(
["level0_2", "level1_nested_list"])
assert 'd1' in level1_dict.get_child_data().keys()
assert 'd2' in level1_dict.get_child_data().keys()
assert ['l1'] in [x.keys() for x in level1_list.get_child_data()]
assert ['l2'] in [x.keys() for x in level1_list.get_child_data()]
assert level1_nested_list_num is None
assert level1_nested_list_att is None
assert level1_nested_list_con._edges.keys() == [
0, 1, 2, 3, 4, 5, 6, 7, 8, 9
]
def do_inv_check(hostname, options):
_inv_hw_changes = options.get("hw-changes", 0)
_inv_sw_changes = options.get("sw-changes", 0)
_inv_sw_missing = options.get("sw-missing", 0)
_inv_fail_status = options.get(
"inv-fail-status", 1) # State in case of an error (default: WARN)
config_cache = config.get_config_cache()
host_config = config_cache.get_host_config(hostname)
if host_config.is_cluster:
ipaddress = None
else:
ipaddress = ip_lookup.lookup_ip_address(hostname)
status, infotexts, long_infotexts, perfdata = 0, [], [], []
sources = data_sources.DataSources(hostname, ipaddress)
old_timestamp, inventory_tree, status_data_tree = _do_inv_for(
sources,
multi_host_sections=None,
host_config=host_config,
ipaddress=ipaddress,
do_status_data_inv=host_config.do_status_data_inventory,
)
if inventory_tree.is_empty() and status_data_tree.is_empty():
infotexts.append("Found no data")
else:
infotexts.append("Found %d inventory entries" %
inventory_tree.count_entries())
# Node 'software' is always there because _do_inv_for creates this node for cluster info
if not inventory_tree.get_sub_container(['software']).has_edge('packages')\
and _inv_sw_missing:
infotexts.append("software packages information is missing" +
check_api_utils.state_markers[_inv_sw_missing])
status = max(status, _inv_sw_missing)
if old_timestamp:
path = "%s/%s/%d" % (cmk.utils.paths.inventory_archive_dir,
hostname, old_timestamp)
old_tree = StructuredDataTree().load_from(path)
if not old_tree.is_equal(inventory_tree, edges=["software"]):
infotext = "software changes"
if _inv_sw_changes:
status = max(status, _inv_sw_changes)
infotext += check_api_utils.state_markers[_inv_sw_changes]
infotexts.append(infotext)
if not old_tree.is_equal(inventory_tree, edges=["hardware"]):
infotext = "hardware changes"
if _inv_hw_changes:
status = max(status, _inv_hw_changes)
infotext += check_api_utils.state_markers[_inv_hw_changes]
infotexts.append(infotext)
if not status_data_tree.is_empty():
infotexts.append("Found %s status entries" %
status_data_tree.count_entries())
for source in sources.get_data_sources():
source_state, source_output, _source_perfdata = source.get_summary_result_for_inventory(
)
# Do not output informational (state = 0) things. These information are shown by the "Check_MK" service
if source_state != 0:
status = max(source_state, status)
infotexts.append("[%s] %s" % (source.id(), source_output))
return status, infotexts, long_infotexts, perfdata
def labels(request):
if request.param == "host":
return DiscoveredHostLabels(StructuredDataTree())
return DiscoveredServiceLabels()
def _initialize_inventory_tree(
): # TODO Remove one day. Deprecated with version 1.5.0i3??
# type: () -> None
global g_inv_tree
g_inv_tree = StructuredDataTree()
def test_structured_data_StructuredDataTree_load_from(tree_name):
with pytest.raises(TypeError):
StructuredDataTree().load_from()
StructuredDataTree().load_from(tree_name)
},
}
tree = StructuredDataTree().create_tree_from_raw_tree(raw_tree)
tree.save_to(tmp_path, filename)
assert target.exists()
gzip_filepath = target.with_suffix('.gz')
assert gzip_filepath.exists()
with gzip.open(str(gzip_filepath), 'rb') as f:
f.read()
tree_old_addresses_arrays_memory = StructuredDataTree().load_from(
tree_name_old_addresses_arrays_memory)
tree_old_addresses = StructuredDataTree().load_from(tree_name_old_addresses)
tree_old_arrays = StructuredDataTree().load_from(tree_name_old_arrays)
tree_old_interfaces = StructuredDataTree().load_from(tree_name_old_interfaces)
tree_old_memory = StructuredDataTree().load_from(tree_name_old_memory)
tree_old_heute = StructuredDataTree().load_from(tree_name_old_heute)
tree_new_addresses_arrays_memory = StructuredDataTree().load_from(
tree_name_new_addresses_arrays_memory)
tree_new_addresses = StructuredDataTree().load_from(tree_name_new_addresses)
tree_new_arrays = StructuredDataTree().load_from(tree_name_new_arrays)
tree_new_interfaces = StructuredDataTree().load_from(tree_name_new_interfaces)
tree_new_memory = StructuredDataTree().load_from(tree_name_new_memory)
tree_new_heute = StructuredDataTree().load_from(tree_name_new_heute)
# Must have same order as tree_new
def __init__(self):
self.trees = InventoryTrees(
inventory=StructuredDataTree(),
status_data=StructuredDataTree(),
)
self._index_cache = {}
def test_structured_data_StructuredDataTree_load_from(tree_name):
StructuredDataTree().load_from(tree_name)
def get_history_deltas(hostname, search_timestamp=None):
if '/' in hostname:
return None, [] # just for security reasons
inventory_path = "%s/inventory/%s" % (cmk.utils.paths.var_dir, hostname)
if not os.path.exists(inventory_path):
return [], []
latest_timestamp = str(int(os.stat(inventory_path).st_mtime))
inventory_archive_dir = "%s/inventory_archive/%s" % (
cmk.utils.paths.var_dir, hostname)
try:
archived_timestamps = sorted(os.listdir(inventory_archive_dir))
except OSError:
return [], []
all_timestamps = archived_timestamps + [latest_timestamp]
previous_timestamp = None
if not search_timestamp:
required_timestamps = all_timestamps
else:
new_timestamp_idx = all_timestamps.index(search_timestamp)
if new_timestamp_idx == 0:
required_timestamps = [search_timestamp]
else:
previous_timestamp = all_timestamps[new_timestamp_idx - 1]
required_timestamps = [search_timestamp]
tree_lookup = {}
def get_tree(timestamp):
if timestamp is None:
return StructuredDataTree()
if timestamp in tree_lookup:
return tree_lookup[timestamp]
if timestamp == latest_timestamp:
inventory_tree = load_filtered_inventory_tree(hostname)
if inventory_tree is None:
return
tree_lookup[timestamp] = inventory_tree
else:
inventory_archive_path = "%s/%s" % (inventory_archive_dir,
timestamp)
tree_lookup[timestamp] = _filter_tree(
StructuredDataTree().load_from(inventory_archive_path))
return tree_lookup[timestamp]
corrupted_history_files = []
delta_history = []
for _idx, timestamp in enumerate(required_timestamps):
cached_delta_path = os.path.join(
cmk.utils.paths.var_dir, "inventory_delta_cache", hostname,
"%s_%s" % (previous_timestamp, timestamp))
cached_data = None
try:
cached_data = store.load_object_from_file(cached_delta_path)
except MKGeneralException:
pass
if cached_data:
new, changed, removed, delta_tree_data = cached_data
delta_tree = StructuredDataTree()
delta_tree.create_tree_from_raw_tree(delta_tree_data)
delta_history.append(
(timestamp, (new, changed, removed, delta_tree)))
previous_timestamp = timestamp
continue
try:
previous_tree = get_tree(previous_timestamp)
current_tree = get_tree(timestamp)
delta_data = current_tree.compare_with(previous_tree)
new, changed, removed, delta_tree = delta_data
if new or changed or removed:
store.save_file(
cached_delta_path,
repr((new, changed, removed, delta_tree.get_raw_tree())),
)
delta_history.append((timestamp, delta_data))
except RequestTimeout:
raise
except LoadStructuredDataError:
corrupted_history_files.append(
str(get_short_inventory_history_filepath(hostname, timestamp)))
previous_timestamp = timestamp
return delta_history, corrupted_history_files
def _show_inventory_results_on_console(inventory_tree: StructuredDataTree,
status_data_tree: StructuredDataTree) -> None:
section.section_success("Found %s%s%d%s inventory entries" %
(tty.bold, tty.yellow, inventory_tree.count_entries(), tty.normal))
section.section_success("Found %s%s%d%s status entries" %
(tty.bold, tty.yellow, status_data_tree.count_entries(), tty.normal))
