def test_structured_data_StructuredDataTree_equal_numerations(): tree_addresses_ordered = StructuredDataTree().load_from( "%s/tree_addresses_ordered" % TEST_DIR) tree_addresses_unordered = StructuredDataTree().load_from( "%s/tree_addresses_unordered" % TEST_DIR) assert tree_addresses_ordered.is_equal(tree_addresses_unordered) assert tree_addresses_unordered.is_equal(tree_addresses_ordered)
def _save_inventory_tree(hostname, inventory_tree): # type: (HostName, StructuredDataTree) -> Optional[StructuredDataTree] store.makedirs(cmk.utils.paths.inventory_output_dir) filepath = cmk.utils.paths.inventory_output_dir + "/" + hostname if inventory_tree.is_empty(): # Remove empty inventory files. Important for host inventory icon if os.path.exists(filepath): os.remove(filepath) if os.path.exists(filepath + ".gz"): os.remove(filepath + ".gz") return None old_tree = StructuredDataTree().load_from(filepath) old_tree.normalize_nodes() if old_tree.is_equal(inventory_tree): console.verbose("Inventory was unchanged\n") return None if old_tree.is_empty(): console.verbose("New inventory tree\n") else: console.verbose("Inventory tree has changed\n") old_time = os.stat(filepath).st_mtime arcdir = "%s/%s" % (cmk.utils.paths.inventory_archive_dir, hostname) store.makedirs(arcdir) os.rename(filepath, arcdir + ("/%d" % old_time)) inventory_tree.save_to(cmk.utils.paths.inventory_output_dir, hostname) return old_tree
def _do_inv_for(sources, multi_host_sections, host_config, ipaddress, do_status_data_inv): # type: (data_sources.DataSources, data_sources.MultiHostSections, config.HostConfig, Optional[str], bool) -> Tuple[Optional[float], StructuredDataTree, StructuredDataTree] hostname = host_config.hostname _initialize_inventory_tree() inventory_tree = g_inv_tree status_data_tree = StructuredDataTree() node = inventory_tree.get_dict("software.applications.check_mk.cluster.") if host_config.is_cluster: node["is_cluster"] = True _do_inv_for_cluster(host_config, inventory_tree) else: node["is_cluster"] = False _do_inv_for_realhost(host_config, sources, multi_host_sections, hostname, ipaddress, inventory_tree, status_data_tree) inventory_tree.normalize_nodes() old_timestamp = _save_inventory_tree(hostname, inventory_tree) _run_inventory_export_hooks(host_config, inventory_tree) success_msg = [ "Found %s%s%d%s inventory entries" % (tty.bold, tty.yellow, inventory_tree.count_entries(), tty.normal) ] console.section_success(", ".join(success_msg)) if do_status_data_inv: status_data_tree.normalize_nodes() _save_status_data_tree(hostname, status_data_tree) console.section_success("Found %s%s%d%s status entries" % (tty.bold, tty.yellow, status_data_tree.count_entries(), tty.normal)) return old_timestamp, inventory_tree, status_data_tree
def _do_inv_for( config_cache: config.ConfigCache, host_config: config.HostConfig, ipaddress: Optional[HostAddress], *, sources: Sequence[ABCSource], multi_host_sections: Optional[MultiHostSections], ) -> Tuple[StructuredDataTree, StructuredDataTree, Sequence[Tuple[ ABCSource, Result[ABCHostSections, Exception]]]]: hostname = host_config.hostname initialize_inventory_tree() inventory_tree = g_inv_tree status_data_tree = StructuredDataTree() node = inventory_tree.get_dict("software.applications.check_mk.cluster.") results: Sequence[Tuple[ABCSource, Result[ABCHostSections, Exception]]] = [] if host_config.is_cluster: node["is_cluster"] = True _do_inv_for_cluster(host_config, inventory_tree) else: node["is_cluster"] = False results = _do_inv_for_realhost( config_cache, host_config, sources, multi_host_sections, hostname, ipaddress, inventory_tree, status_data_tree, ) inventory_tree.normalize_nodes() status_data_tree.normalize_nodes() return inventory_tree, status_data_tree, results
def _load_inventory_tree(hostname): # Load data of a host, cache it in the current HTTP request if not hostname: return inventory_tree_cache = g.setdefault("inventory", {}) if hostname in inventory_tree_cache: inventory_tree = inventory_tree_cache[hostname] else: if '/' in hostname: # just for security reasons return cache_path = "%s/inventory/%s" % (cmk.utils.paths.var_dir, hostname) inventory_tree = StructuredDataTree().load_from(cache_path) inventory_tree_cache[hostname] = inventory_tree return inventory_tree
def _do_inv_for_cluster(host_config: config.HostConfig) -> InventoryTrees: inventory_tree = StructuredDataTree() _set_cluster_property(inventory_tree, host_config) if not host_config.nodes: return InventoryTrees(inventory_tree, StructuredDataTree()) inv_node = inventory_tree.get_list("software.applications.check_mk.cluster.nodes:") for node_name in host_config.nodes: inv_node.append({ "name": node_name, }) inventory_tree.normalize_nodes() return InventoryTrees(inventory_tree, StructuredDataTree())
def _run_inventory_export_hooks(host_config: config.HostConfig, inventory_tree: StructuredDataTree) -> None: import cmk.base.inventory_plugins as inventory_plugins # pylint: disable=import-outside-toplevel hooks = host_config.inventory_export_hooks if not hooks: return section.section_step("Execute inventory export hooks") for hookname, params in hooks: console.verbose("Execute export hook: %s%s%s%s" % (tty.blue, tty.bold, hookname, tty.normal)) try: func = inventory_plugins.inv_export[hookname]["export_function"] func(host_config.hostname, params, inventory_tree.get_raw_tree()) except Exception as e: if cmk.utils.debug.enabled(): raise raise MKGeneralException("Failed to execute export hook %s: %s" % (hookname, e))
def _load_inventory_tree(hostname): # Load data of a host, cache it in the current HTTP request if not hostname: return inventory_tree_cache = g.setdefault("inventory", {}) if hostname in inventory_tree_cache: inventory_tree = inventory_tree_cache[hostname] else: if '/' in hostname: # just for security reasons return cache_path = "%s/inventory/%s" % (cmk.utils.paths.var_dir, hostname) try: inventory_tree = StructuredDataTree().load_from(cache_path) except Exception as e: if config.debug: html.show_warning(e) raise LoadStructuredDataError() inventory_tree_cache[hostname] = inventory_tree return inventory_tree
def _load_structured_data_tree( tree_type: Literal["inventory", "status_data"], hostname: Optional[HostName]) -> Optional[StructuredDataTree]: """Load data of a host, cache it in the current HTTP request""" if not hostname: return None inventory_tree_cache = g.setdefault(tree_type, {}) if hostname in inventory_tree_cache: inventory_tree = inventory_tree_cache[hostname] else: if '/' in hostname: # just for security reasons return None cache_path = "%s/%s" % (cmk.utils.paths.inventory_output_dir if tree_type == "inventory" \ else cmk.utils.paths.status_data_dir, hostname) try: inventory_tree = StructuredDataTree().load_from(cache_path) except Exception as e: if config.debug: html.show_warning("%s" % e) raise LoadStructuredDataError() inventory_tree_cache[hostname] = inventory_tree return inventory_tree
def test_structured_data_StructuredDataTree_get_list(): with pytest.raises(MKGeneralException) as e: StructuredDataTree().get_list("") assert 'Empty tree path or zero' in "%s" % e with pytest.raises(MKGeneralException) as e: StructuredDataTree().get_list(0) # type: ignore[arg-type] assert 'Empty tree path or zero' in "%s" % e with pytest.raises(MKGeneralException) as e: StructuredDataTree().get_list(100) # type: ignore[arg-type] assert 'Wrong tree path format' in "%s" % e with pytest.raises(MKGeneralException) as e: StructuredDataTree().get_list("a?") assert 'No valid tree path' in "%s" % e with pytest.raises(MKGeneralException) as e: StructuredDataTree().get_list("a$.") assert 'Specified tree path contains unexpected characters' in "%s" % e assert StructuredDataTree().get_list("a:") == []
for_mgmt_board=for_mgmt_board) #. # .--Inventory Tree------------------------------------------------------. # | ___ _ _____ | # | |_ _|_ ____ _____ _ __ | |_ ___ _ __ _ _ |_ _| __ ___ ___ | # | | || '_ \ \ / / _ \ '_ \| __/ _ \| '__| | | | | || '__/ _ \/ _ \ | # | | || | | \ V / __/ | | | || (_) | | | |_| | | || | | __/ __/ | # | |___|_| |_|\_/ \___|_| |_|\__\___/|_| \__, | |_||_| \___|\___| | # | |___/ | # +----------------------------------------------------------------------+ # | Managing the inventory tree of a host | # '----------------------------------------------------------------------' g_inv_tree = StructuredDataTree( ) # TODO Remove one day. Deprecated with version 1.5.0i3?? def _initialize_inventory_tree( ): # TODO Remove one day. Deprecated with version 1.5.0i3?? # type: () -> None global g_inv_tree g_inv_tree = StructuredDataTree() # Dict based def inv_tree(path): # TODO Remove one day. Deprecated with version 1.5.0i3?? # type: (str) -> Dict return g_inv_tree.get_dict(path)
def test_delta_structured_data_tree_serialization(zipped_trees): old_tree = StructuredDataTree() new_tree = StructuredDataTree() old_filename, new_filename = zipped_trees old_tree.load_from(old_filename) new_tree.load_from(new_filename) _, __, ___, delta_tree = old_tree.compare_with(new_tree) new_delta_tree = StructuredDataTree() new_delta_tree.create_tree_from_raw_tree(delta_tree.get_raw_tree()) assert delta_tree.is_equal(new_delta_tree)
def test_structured_data_StructuredDataTree_building_tree(): def plugin_dict(): node = struct_tree.get_dict("level0_0.level1_dict.") for a, b in [("d1", "D1"), ("d2", "D2")]: node.setdefault(a, b) def plugin_list(): node = struct_tree.get_list("level0_1.level1_list:") for a, b in [("l1", "L1"), ("l2", "L2")]: node.append({a: b}) def plugin_nested_list(): node = struct_tree.get_list("level0_2.level1_nested_list:") for index in range(10): array: Dict[str, List[Dict[str, str]]] = {"foo": []} for a, b in [("nl1", "NL1"), ("nl2", "NL2")]: array["foo"].append({a: "%s-%s" % (b, index)}) node.append(array) struct_tree = StructuredDataTree() plugin_dict() plugin_list() plugin_nested_list() struct_tree.normalize_nodes() assert struct_tree.has_edge("level0_0") assert struct_tree.has_edge("level0_1") assert struct_tree.has_edge("level0_2") assert not struct_tree.has_edge("foobar") level1_dict = struct_tree.get_sub_attributes(["level0_0", "level1_dict"]) level1_list = struct_tree.get_sub_numeration(["level0_1", "level1_list"]) level1_nested_list_con = struct_tree.get_sub_container( ["level0_2", "level1_nested_list"]) level1_nested_list_num = struct_tree.get_sub_numeration( ["level0_2", "level1_nested_list"]) level1_nested_list_att = struct_tree.get_sub_attributes( ["level0_2", "level1_nested_list"]) assert isinstance(level1_dict, Attributes) assert 'd1' in level1_dict.get_child_data() assert 'd2' in level1_dict.get_child_data() assert isinstance(level1_list, Numeration) known_keys = [key for row in level1_list.get_child_data() for key in row] assert 'l1' in known_keys assert 'l2' in known_keys assert level1_nested_list_num is None assert level1_nested_list_att is None assert isinstance(level1_nested_list_con, Container) assert list( level1_nested_list_con._nodes) == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
def test_structured_data_StructuredDataTree_is_empty(): assert StructuredDataTree().is_empty() is True
def test_delta_structured_data_tree_serialization(two_tree_filenames): old_tree = StructuredDataTree() new_tree = StructuredDataTree() old_filename, new_filename = two_tree_filenames old_tree.load_from(old_filename) new_tree.load_from(new_filename) _, __, ___, delta_tree = old_tree.compare_with(new_tree) raw_delta_tree = delta_tree.get_raw_tree() new_delta_tree = StructuredDataTree() new_delta_tree.create_tree_from_raw_tree(raw_delta_tree) new_raw_delta_tree = new_delta_tree.get_raw_tree() assert raw_delta_tree == new_raw_delta_tree
def _save_status_data_tree(hostname: HostName, status_data_tree: StructuredDataTree) -> None: if status_data_tree and not status_data_tree.is_empty(): store.makedirs(cmk.utils.paths.status_data_dir) status_data_tree.save_to(cmk.utils.paths.status_data_dir, hostname)
def _set_cluster_property( inventory_tree: StructuredDataTree, host_config: config.HostConfig, ) -> None: inventory_tree.get_dict( "software.applications.check_mk.cluster.")["is_cluster"] = host_config.is_cluster
def _create_tree_from_raw_tree(raw_tree): if raw_tree: return StructuredDataTree().create_tree_from_raw_tree( ast.literal_eval(raw_tree)) return
def test_structured_data_StructuredDataTree_save_to(): with pytest.raises(TypeError): StructuredDataTree().save_to() with pytest.raises(TypeError): StructuredDataTree().save_to("x")
def _create_tree_from_raw_tree( raw_tree: bytes) -> Optional[StructuredDataTree]: if raw_tree: return StructuredDataTree().create_tree_from_raw_tree( ast.literal_eval(raw_tree.decode("utf-8"))) return None
def test_structured_data_StructuredDataTree_building_tree(): def plugin_dict(): node = struct_tree.get_dict("level0_0.level1_dict.") for a, b in [("d1", "D1"), ("d2", "D2")]: node.setdefault(a, b) def plugin_list(): node = struct_tree.get_list("level0_1.level1_list:") for a, b in [("l1", "L1"), ("l2", "L2")]: node.append({a: b}) def plugin_nested_list(): node = struct_tree.get_list("level0_2.level1_nested_list:") for index in xrange(10): array = {"foo": []} for a, b in [("nl1", "NL1"), ("nl2", "NL2")]: array["foo"].append({a: "%s-%s" % (b, index)}) node.append(array) struct_tree = StructuredDataTree() plugin_dict() plugin_list() plugin_nested_list() struct_tree.normalize_nodes() assert struct_tree.has_edge("level0_0") assert struct_tree.has_edge("level0_1") assert struct_tree.has_edge("level0_2") assert not struct_tree.has_edge("foobar") level1_dict = struct_tree.get_sub_attributes(["level0_0", "level1_dict"]) level1_list = struct_tree.get_sub_numeration(["level0_1", "level1_list"]) level1_nested_list_con = struct_tree.get_sub_container( ["level0_2", "level1_nested_list"]) level1_nested_list_num = struct_tree.get_sub_numeration( ["level0_2", "level1_nested_list"]) level1_nested_list_att = struct_tree.get_sub_attributes( ["level0_2", "level1_nested_list"]) assert 'd1' in level1_dict.get_child_data().keys() assert 'd2' in level1_dict.get_child_data().keys() assert ['l1'] in [x.keys() for x in level1_list.get_child_data()] assert ['l2'] in [x.keys() for x in level1_list.get_child_data()] assert level1_nested_list_num is None assert level1_nested_list_att is None assert level1_nested_list_con._edges.keys() == [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 ]
def do_inv_check(hostname, options): _inv_hw_changes = options.get("hw-changes", 0) _inv_sw_changes = options.get("sw-changes", 0) _inv_sw_missing = options.get("sw-missing", 0) _inv_fail_status = options.get( "inv-fail-status", 1) # State in case of an error (default: WARN) config_cache = config.get_config_cache() host_config = config_cache.get_host_config(hostname) if host_config.is_cluster: ipaddress = None else: ipaddress = ip_lookup.lookup_ip_address(hostname) status, infotexts, long_infotexts, perfdata = 0, [], [], [] sources = data_sources.DataSources(hostname, ipaddress) old_timestamp, inventory_tree, status_data_tree = _do_inv_for( sources, multi_host_sections=None, host_config=host_config, ipaddress=ipaddress, do_status_data_inv=host_config.do_status_data_inventory, ) if inventory_tree.is_empty() and status_data_tree.is_empty(): infotexts.append("Found no data") else: infotexts.append("Found %d inventory entries" % inventory_tree.count_entries()) # Node 'software' is always there because _do_inv_for creates this node for cluster info if not inventory_tree.get_sub_container(['software']).has_edge('packages')\ and _inv_sw_missing: infotexts.append("software packages information is missing" + check_api_utils.state_markers[_inv_sw_missing]) status = max(status, _inv_sw_missing) if old_timestamp: path = "%s/%s/%d" % (cmk.utils.paths.inventory_archive_dir, hostname, old_timestamp) old_tree = StructuredDataTree().load_from(path) if not old_tree.is_equal(inventory_tree, edges=["software"]): infotext = "software changes" if _inv_sw_changes: status = max(status, _inv_sw_changes) infotext += check_api_utils.state_markers[_inv_sw_changes] infotexts.append(infotext) if not old_tree.is_equal(inventory_tree, edges=["hardware"]): infotext = "hardware changes" if _inv_hw_changes: status = max(status, _inv_hw_changes) infotext += check_api_utils.state_markers[_inv_hw_changes] infotexts.append(infotext) if not status_data_tree.is_empty(): infotexts.append("Found %s status entries" % status_data_tree.count_entries()) for source in sources.get_data_sources(): source_state, source_output, _source_perfdata = source.get_summary_result_for_inventory( ) # Do not output informational (state = 0) things. These information are shown by the "Check_MK" service if source_state != 0: status = max(source_state, status) infotexts.append("[%s] %s" % (source.id(), source_output)) return status, infotexts, long_infotexts, perfdata
def labels(request): if request.param == "host": return DiscoveredHostLabels(StructuredDataTree()) return DiscoveredServiceLabels()
def _initialize_inventory_tree( ): # TODO Remove one day. Deprecated with version 1.5.0i3?? # type: () -> None global g_inv_tree g_inv_tree = StructuredDataTree()
def test_structured_data_StructuredDataTree_load_from(tree_name): with pytest.raises(TypeError): StructuredDataTree().load_from() StructuredDataTree().load_from(tree_name)
}, } tree = StructuredDataTree().create_tree_from_raw_tree(raw_tree) tree.save_to(tmp_path, filename) assert target.exists() gzip_filepath = target.with_suffix('.gz') assert gzip_filepath.exists() with gzip.open(str(gzip_filepath), 'rb') as f: f.read() tree_old_addresses_arrays_memory = StructuredDataTree().load_from( tree_name_old_addresses_arrays_memory) tree_old_addresses = StructuredDataTree().load_from(tree_name_old_addresses) tree_old_arrays = StructuredDataTree().load_from(tree_name_old_arrays) tree_old_interfaces = StructuredDataTree().load_from(tree_name_old_interfaces) tree_old_memory = StructuredDataTree().load_from(tree_name_old_memory) tree_old_heute = StructuredDataTree().load_from(tree_name_old_heute) tree_new_addresses_arrays_memory = StructuredDataTree().load_from( tree_name_new_addresses_arrays_memory) tree_new_addresses = StructuredDataTree().load_from(tree_name_new_addresses) tree_new_arrays = StructuredDataTree().load_from(tree_name_new_arrays) tree_new_interfaces = StructuredDataTree().load_from(tree_name_new_interfaces) tree_new_memory = StructuredDataTree().load_from(tree_name_new_memory) tree_new_heute = StructuredDataTree().load_from(tree_name_new_heute) # Must have same order as tree_new
def __init__(self): self.trees = InventoryTrees( inventory=StructuredDataTree(), status_data=StructuredDataTree(), ) self._index_cache = {}
def test_structured_data_StructuredDataTree_load_from(tree_name): StructuredDataTree().load_from(tree_name)
def get_history_deltas(hostname, search_timestamp=None): if '/' in hostname: return None, [] # just for security reasons inventory_path = "%s/inventory/%s" % (cmk.utils.paths.var_dir, hostname) if not os.path.exists(inventory_path): return [], [] latest_timestamp = str(int(os.stat(inventory_path).st_mtime)) inventory_archive_dir = "%s/inventory_archive/%s" % ( cmk.utils.paths.var_dir, hostname) try: archived_timestamps = sorted(os.listdir(inventory_archive_dir)) except OSError: return [], [] all_timestamps = archived_timestamps + [latest_timestamp] previous_timestamp = None if not search_timestamp: required_timestamps = all_timestamps else: new_timestamp_idx = all_timestamps.index(search_timestamp) if new_timestamp_idx == 0: required_timestamps = [search_timestamp] else: previous_timestamp = all_timestamps[new_timestamp_idx - 1] required_timestamps = [search_timestamp] tree_lookup = {} def get_tree(timestamp): if timestamp is None: return StructuredDataTree() if timestamp in tree_lookup: return tree_lookup[timestamp] if timestamp == latest_timestamp: inventory_tree = load_filtered_inventory_tree(hostname) if inventory_tree is None: return tree_lookup[timestamp] = inventory_tree else: inventory_archive_path = "%s/%s" % (inventory_archive_dir, timestamp) tree_lookup[timestamp] = _filter_tree( StructuredDataTree().load_from(inventory_archive_path)) return tree_lookup[timestamp] corrupted_history_files = [] delta_history = [] for _idx, timestamp in enumerate(required_timestamps): cached_delta_path = os.path.join( cmk.utils.paths.var_dir, "inventory_delta_cache", hostname, "%s_%s" % (previous_timestamp, timestamp)) cached_data = None try: cached_data = store.load_object_from_file(cached_delta_path) except MKGeneralException: pass if cached_data: new, changed, removed, delta_tree_data = cached_data delta_tree = StructuredDataTree() delta_tree.create_tree_from_raw_tree(delta_tree_data) delta_history.append( (timestamp, (new, changed, removed, delta_tree))) previous_timestamp = timestamp continue try: previous_tree = get_tree(previous_timestamp) current_tree = get_tree(timestamp) delta_data = current_tree.compare_with(previous_tree) new, changed, removed, delta_tree = delta_data if new or changed or removed: store.save_file( cached_delta_path, repr((new, changed, removed, delta_tree.get_raw_tree())), ) delta_history.append((timestamp, delta_data)) except RequestTimeout: raise except LoadStructuredDataError: corrupted_history_files.append( str(get_short_inventory_history_filepath(hostname, timestamp))) previous_timestamp = timestamp return delta_history, corrupted_history_files
def _show_inventory_results_on_console(inventory_tree: StructuredDataTree, status_data_tree: StructuredDataTree) -> None: section.section_success("Found %s%s%d%s inventory entries" % (tty.bold, tty.yellow, inventory_tree.count_entries(), tty.normal)) section.section_success("Found %s%s%d%s status entries" % (tty.bold, tty.yellow, status_data_tree.count_entries(), tty.normal))