def _make_updater( self, retention_key: RetentionKey, retention_info: RetentionInfo, previous_tree: StructuredDataNode, ) -> NodeUpdater: node_path, node_name = retention_key path = list(node_path) inv_node = self._inv_tree.get_node(path) previous_node = previous_tree.get_node(path) if previous_node is None: previous_node = StructuredDataNode() if inv_node is None: inv_node = self._inv_tree.setdefault_node(path) if node_name == ATTRIBUTES_KEY: return AttributesUpdater( retention_info, inv_node, previous_node, ) if node_name == TABLE_KEY: return TableUpdater( retention_info, inv_node, previous_node, ) raise NotImplementedError()
def _create_inventory_history() -> None: hostname = "inv-host" # history cmk.utils.store.save_object_to_file( Path(cmk.utils.paths.inventory_archive_dir, hostname, "0"), StructuredDataNode.deserialize({ "inv": "attr-0" }).serialize(), ) cmk.utils.store.save_object_to_file( Path(cmk.utils.paths.inventory_archive_dir, hostname, "1"), StructuredDataNode.deserialize({ "inv": "attr-1" }).serialize(), ) cmk.utils.store.save_object_to_file( Path(cmk.utils.paths.inventory_archive_dir, hostname, "2"), StructuredDataNode.deserialize({ "inv-2": "attr" }).serialize(), ) cmk.utils.store.save_object_to_file( Path(cmk.utils.paths.inventory_archive_dir, hostname, "3"), StructuredDataNode.deserialize({ "inv": "attr-3" }).serialize(), ) # current tree cmk.utils.store.save_object_to_file( Path(cmk.utils.paths.inventory_output_dir, hostname), StructuredDataNode.deserialize({ "inv": "attr" }).serialize(), )
def test_updater_merge_previous_tables_outdated(filter_func: SDFilterFunc) -> None: previous_tree, _inv_tree = _make_trees( {}, { ("Ident 1",): {"old": (1, 2, 3)}, ("Ident 2",): {"old": (1, 2, 3)}, }, ) inv_tree = StructuredDataNode() previous_node = previous_tree.get_node(("path-to", "node")) assert isinstance(previous_node, StructuredDataNode) updater = TableUpdater( RetentionInfo( filter_func, RetentionIntervals(-1, -2, -3), ), inv_tree.setdefault_node(("path-to", "node")), previous_node, ) result = updater.filter_and_merge(1000) assert not result.save_tree assert not result.reason inv_node = inv_tree.get_node(("path-to", "node")) assert inv_node is not None assert inv_node.table.retentions == {}
def test_updater_merge_previous_attributes( filter_func: SDFilterFunc, expected_retentions: Dict, ): previous_tree, _inv_tree = _make_trees({"old": (1, 2, 3)}, {}) inv_tree = StructuredDataNode() previous_node = previous_tree.get_node(("path-to", "node")) assert isinstance(previous_node, StructuredDataNode) updater = AttributesUpdater( RetentionInfo( filter_func, RetentionIntervals(-1, -2, -3), ), inv_tree.setdefault_node(("path-to", "node")), previous_node, ) result = updater.filter_and_merge(-1) if expected_retentions: assert result.save_tree assert result.reason else: assert not result.save_tree assert not result.reason inv_node = inv_tree.get_node(("path-to", "node")) assert inv_node is not None assert inv_node.attributes.retentions == expected_retentions if expected_retentions: assert "old" in inv_node.attributes.pairs
def test_set_path_sub_nodes(): root = _create_empty_tree() nta = root.get_node(("path", "to", "nta")) sub_node = StructuredDataNode(name="node") sub_node.setdefault_node(("sub-path-to", "sub-node")) nta.add_node(sub_node) path_to_node = root.get_node(["path", "to", "nta", "node", "sub-path-to"]) assert path_to_node is not None assert path_to_node.attributes.path == ("path", "to", "nta", "node", "sub-path-to") assert path_to_node.table.path == ("path", "to", "nta", "node", "sub-path-to") assert path_to_node.path == ("path", "to", "nta", "node", "sub-path-to") path_to_sub_node = root.get_node( ["path", "to", "nta", "node", "sub-path-to", "sub-node"]) assert path_to_sub_node is not None assert path_to_sub_node.attributes.path == ( "path", "to", "nta", "node", "sub-path-to", "sub-node", ) assert path_to_sub_node.table.path == ("path", "to", "nta", "node", "sub-path-to", "sub-node") assert path_to_sub_node.path == ("path", "to", "nta", "node", "sub-path-to", "sub-node")
def _make_trees( previous_attributes_retentions: Dict, previous_table_retentions: Dict ) -> Tuple[StructuredDataNode, StructuredDataNode]: previous_tree = StructuredDataNode.deserialize( { "Attributes": {}, "Table": {}, "Nodes": { "path-to": { "Attributes": {}, "Table": {}, "Nodes": { "node": { "Attributes": { "Pairs": {"old": "Key", "keys": "Previous Keys"}, "Retentions": previous_attributes_retentions, }, "Table": { "KeyColumns": ["ident"], "Rows": [ {"ident": "Ident 1", "old": "Key 1", "keys": "Previous Keys 1"}, {"ident": "Ident 2", "old": "Key 2", "keys": "Previous Keys 2"}, ], "Retentions": previous_table_retentions, }, "Nodes": {}, } }, } }, } ) inv_tree = StructuredDataNode.deserialize( { "Attributes": {}, "Table": {}, "Nodes": { "path-to": { "Attributes": {}, "Table": {}, "Nodes": { "node": { "Attributes": { "Pairs": {"new": "Key", "keys": "New Keys"}, }, "Table": { "KeyColumns": ["ident"], "Rows": [ {"ident": "Ident 1", "new": "Key 1", "keys": "New Keys 1"}, {"ident": "Ident 2", "new": "Key 2", "keys": "New Keys 2"}, ], }, "Nodes": {}, } }, } }, } ) return previous_tree, inv_tree
def test_load_latest_delta_tree_one_archive_and_inv_tree() -> None: hostname = "inv-host" expected_delta_tree = StructuredDataNode.deserialize( {"inv": ("attr-0", "attr")}) # history cmk.utils.store.save_object_to_file( Path(cmk.utils.paths.inventory_archive_dir, hostname, "0"), StructuredDataNode.deserialize({ "inv": "attr-0" }).serialize(), ) # current tree cmk.utils.store.save_object_to_file( Path(cmk.utils.paths.inventory_output_dir, hostname), StructuredDataNode.deserialize({ "inv": "attr" }).serialize(), ) delta_tree = cmk.gui.inventory.load_latest_delta_tree(hostname) assert delta_tree is not None assert delta_tree.is_equal(expected_delta_tree)
def __init__(self): self.trees = InventoryTrees( inventory=StructuredDataNode(), status_data=StructuredDataNode(), ) self._index_cache = {} self._class_mutex = {}
def test_set_path_sub_nodes_error(): root = _create_empty_tree() nta = root.get_node(("path", "to", "nta")) sub_node = StructuredDataNode() sub_node.setdefault_node(("sub-path", "sub-to", "sub-node")) with pytest.raises(ValueError): nta.add_node(sub_node)
def _merge_inventory_and_status_data_tree(inventory_tree, status_data_tree): if inventory_tree is None and status_data_tree is None: return if inventory_tree is None: inventory_tree = StructuredDataNode() if status_data_tree is not None: inventory_tree.merge_with(status_data_tree) return inventory_tree
def test_add_attributes(): path = ("path-to", "node") retentions = {"key": RetentionIntervals(1, 2, 3)} node = StructuredDataNode(name="node", path=path) attributes = Attributes(retentions=retentions) node.add_attributes(attributes) assert node.attributes.path == path assert node.attributes.retentions == retentions
def _do_inv_for_cluster(host_config: config.HostConfig) -> InventoryTrees: inventory_tree = StructuredDataNode() _set_cluster_property(inventory_tree, host_config) if not host_config.nodes: return InventoryTrees(inventory_tree, StructuredDataNode()) node = inventory_tree.setdefault_node( ("software", "applications", "check_mk", "cluster", "nodes") ) node.table.add_key_columns(["name"]) node.table.add_rows([{"name": node_name} for node_name in host_config.nodes]) return InventoryTrees(inventory_tree, StructuredDataNode())
def test_updater_null_obj_tables_outdated() -> None: inv_tree = StructuredDataNode() updater = TableUpdater( RetentionInfo( lambda key: True, RetentionIntervals(1, 2, 3), ), inv_tree, StructuredDataNode(), ) result = updater.filter_and_merge(1000) assert not result.save_tree assert not result.reason assert inv_tree.get_node(("path-to", "node")) is None
def get_inventory_data(inventory_tree: StructuredDataNode, tree_path: RawInventoryPath) -> InventoryData: invdata = None parsed_path, attribute_keys = parse_tree_path(tree_path) if attribute_keys == []: table = inventory_tree.get_table(parsed_path) if table is not None: invdata = table.data elif attribute_keys: attributes = inventory_tree.get_attributes(parsed_path) if attributes is not None: # In paint_host_inventory_tree we parse invpath and get # a path and attribute_keys which may be either None, [], or ["KEY"]. invdata = attributes.data.get(attribute_keys[-1]) return invdata
def test_updater_null_obj_attributes(): inv_tree = StructuredDataNode() updater = AttributesUpdater( RetentionInfo( lambda key: True, RetentionIntervals(1, 2, 3), ), inv_tree, StructuredDataNode(), ) result = updater.filter_and_merge(-1) assert not result.save_tree assert not result.reason assert inv_tree.get_node(["path-to", "node"]) is None
def get_inventory_table(tree: StructuredDataNode, raw_path: SDRawPath) -> Optional[InventoryRows]: inventory_path = InventoryPath.parse(raw_path) if inventory_path.source != TreeSource.table: return None table = tree.get_table(inventory_path.path) return None if table is None else table.rows
def test_updater_handle_inv_tables_outdated( filter_func, path, expected_retentions, ): _previous_tree, inv_tree = _make_trees({}, {}) updater = TableUpdater( RetentionInfo( filter_func, RetentionIntervals(1, 2, 3), ), inv_tree.get_node(path), StructuredDataNode(), ) result = updater.filter_and_merge(1000) if expected_retentions: assert result.save_tree assert result.reason else: assert not result.save_tree assert not result.reason inv_node = inv_tree.get_node(["path-to", "node"]) assert inv_node is not None assert inv_node.table.retentions == expected_retentions
def test_delta_structured_data_tree_serialization(zipped_trees): old_tree = StructuredDataNode() new_tree = StructuredDataNode() old_filename, new_filename = zipped_trees old_tree = load_tree_from(old_filename) new_tree = load_tree_from(new_filename) delta_result = old_tree.compare_with(new_tree) delta_raw_tree = delta_result.delta.get_raw_tree() assert isinstance(delta_raw_tree, dict) new_delta_tree = StructuredDataNode().create_tree_from_raw_tree( delta_raw_tree) assert delta_result.delta.is_equal(new_delta_tree)
def get_inventory_table(tree: StructuredDataNode, raw_path: SDRawPath) -> Optional[InventoryRows]: parsed_path, attribute_keys = parse_tree_path(raw_path) if attribute_keys != []: return None table = tree.get_table(parsed_path) return None if table is None else table.rows
def test_empty_but_different_structure(): root = _create_empty_tree() nt = root.get_node(["path", "to", "nta", "nt"]) na = root.get_node(["path", "to", "nta", "na"]) ta = root.get_node(["path", "to", "nta", "ta"]) assert nt.attributes.pairs == {} assert nt.attributes.is_empty() assert nt.table._rows == {} assert nt.table.rows == [] assert nt.table.is_empty() assert na.attributes.pairs == {} assert na.attributes.is_empty() assert na.table._rows == {} assert na.table.rows == [] assert na.table.is_empty() assert ta.attributes.pairs == {} assert ta.attributes.is_empty() assert ta.table._rows == {} assert ta.table.rows == [] assert ta.table.is_empty() assert root.is_empty() assert root.count_entries() == 0 assert not root.is_equal(StructuredDataNode())
def _set_cluster_property( inventory_tree: StructuredDataNode, host_config: config.HostConfig, ) -> None: node = inventory_tree.setdefault_node( ["software", "applications", "check_mk", "cluster"]) node.attributes.add_pairs({"is_cluster": host_config.is_cluster})
def _save_inventory_tree( hostname: HostName, inventory_tree: StructuredDataNode, ) -> Optional[StructuredDataNode]: store.makedirs(cmk.utils.paths.inventory_output_dir) filepath = cmk.utils.paths.inventory_output_dir + "/" + hostname if inventory_tree.is_empty(): # Remove empty inventory files. Important for host inventory icon if os.path.exists(filepath): os.remove(filepath) if os.path.exists(filepath + ".gz"): os.remove(filepath + ".gz") return None old_tree = load_tree_from(filepath) old_tree.normalize_nodes() if old_tree.is_equal(inventory_tree): console.verbose("Inventory was unchanged\n") return None if old_tree.is_empty(): console.verbose("New inventory tree\n") else: console.verbose("Inventory tree has changed\n") old_time = os.stat(filepath).st_mtime arcdir = "%s/%s" % (cmk.utils.paths.inventory_archive_dir, hostname) store.makedirs(arcdir) os.rename(filepath, arcdir + ("/%d" % old_time)) save_tree_to(inventory_tree, cmk.utils.paths.inventory_output_dir, hostname) return old_tree
def test_add_node(): root = _create_filled_tree() sub_node = StructuredDataNode(name="node") sub_node.attributes.add_pairs({"sn0": "SN 0", "sn1": "SN 1"}) sub_node.table.add_key_columns(["sn0"]) sub_node.table.add_rows([ { "sn0": "SN 00", "sn1": "SN 01" }, { "sn0": "SN 10", "sn1": "SN 11" }, ]) node = root.get_node(["path", "to", "nta"]).add_node(sub_node) # Do not modify orig node. assert sub_node.attributes.path == tuple() assert sub_node.table.path == tuple() assert sub_node.path == tuple() assert node.attributes.path == tuple(["path", "to", "nta", "node"]) assert node.table.key_columns == ["sn0"] assert node.table.path == tuple(["path", "to", "nta", "node"]) assert node.path == tuple(["path", "to", "nta", "node"]) assert not root.is_empty() assert root.count_entries() == 18
def test_updater_handle_inv_tables_outdated( filter_func: SDFilterFunc, expected_retentions: Dict, ) -> None: _previous_tree, inv_tree = _make_trees({}, {}) fst_inv_node = inv_tree.get_node(("path-to", "node")) assert isinstance(fst_inv_node, StructuredDataNode) updater = TableUpdater( RetentionInfo( filter_func, RetentionIntervals(1, 2, 3), ), fst_inv_node, StructuredDataNode(), ) result = updater.filter_and_merge(1000) if expected_retentions: assert result.save_tree assert result.reason else: assert not result.save_tree assert not result.reason inv_node = inv_tree.get_node(("path-to", "node")) assert inv_node is not None assert inv_node.table.retentions == expected_retentions
def get_inventory_attribute(tree: StructuredDataNode, raw_path: SDRawPath) -> InventoryValue: inventory_path = InventoryPath.parse(raw_path) if inventory_path.source != TreeSource.attributes or not inventory_path.key: return None attributes = tree.get_attributes(inventory_path.path) return None if attributes is None else attributes.pairs.get( inventory_path.key)
def get_inventory_attribute(tree: StructuredDataNode, raw_path: SDRawPath) -> InventoryValue: parsed_path, attribute_keys = parse_tree_path(raw_path) if not attribute_keys: return None attributes = tree.get_attributes(parsed_path) return None if attributes is None else attributes.pairs.get( attribute_keys[-1])
def test__load_status_data_tree(monkeypatch, hostname, row, expected_tree): monkeypatch.setattr( cmk.gui.inventory, "_load_structured_data_tree", lambda t, hostname: StructuredDataNode.deserialize({"loaded": "tree"}), ) status_data_tree = cmk.gui.inventory._load_status_data_tree(hostname, row) assert status_data_tree is not None assert status_data_tree.is_equal(expected_tree)
def _load_status_data_tree(hostname: Optional[HostName], row: Row) -> Optional[StructuredDataNode]: # If no data from livestatus could be fetched (CRE) try to load from cache # or status dir raw_status_data_tree = row.get("host_structured_status") if not raw_status_data_tree: return _load_structured_data_tree("status_data", hostname) return StructuredDataNode.deserialize( ast.literal_eval(raw_status_data_tree.decode("utf-8")))
def get_tree(self, filepath: Path) -> StructuredDataNode: if filepath == _DEFAULT_PATH_TO_TREE: return StructuredDataNode() if filepath in self._lookup: return self._lookup[filepath] return self._lookup.setdefault(filepath, self._load_tree_from_file(filepath))
def test_add_table(): path = ("path-to", "node") key_columns = ["key-0"] retentions: TableRetentions = { ("Value 0", ): { "key-1": RetentionIntervals(1, 2, 3) }, } node = StructuredDataNode(name="node", path=path) table = Table( key_columns=key_columns, retentions=retentions, ) node.add_table(table) assert node.table.path == path assert node.table.key_columns == key_columns assert node.table.retentions == retentions