def _make_updater(
self,
retention_key: RetentionKey,
retention_info: RetentionInfo,
previous_tree: StructuredDataNode,
) -> NodeUpdater:
node_path, node_name = retention_key
path = list(node_path)
inv_node = self._inv_tree.get_node(path)
previous_node = previous_tree.get_node(path)
if previous_node is None:
previous_node = StructuredDataNode()
if inv_node is None:
inv_node = self._inv_tree.setdefault_node(path)
if node_name == ATTRIBUTES_KEY:
return AttributesUpdater(
retention_info,
inv_node,
previous_node,
)
if node_name == TABLE_KEY:
return TableUpdater(
retention_info,
inv_node,
previous_node,
)
raise NotImplementedError()
def _create_inventory_history() -> None:
hostname = "inv-host"
# history
cmk.utils.store.save_object_to_file(
Path(cmk.utils.paths.inventory_archive_dir, hostname, "0"),
StructuredDataNode.deserialize({
"inv": "attr-0"
}).serialize(),
)
cmk.utils.store.save_object_to_file(
Path(cmk.utils.paths.inventory_archive_dir, hostname, "1"),
StructuredDataNode.deserialize({
"inv": "attr-1"
}).serialize(),
)
cmk.utils.store.save_object_to_file(
Path(cmk.utils.paths.inventory_archive_dir, hostname, "2"),
StructuredDataNode.deserialize({
"inv-2": "attr"
}).serialize(),
)
cmk.utils.store.save_object_to_file(
Path(cmk.utils.paths.inventory_archive_dir, hostname, "3"),
StructuredDataNode.deserialize({
"inv": "attr-3"
}).serialize(),
)
# current tree
cmk.utils.store.save_object_to_file(
Path(cmk.utils.paths.inventory_output_dir, hostname),
StructuredDataNode.deserialize({
"inv": "attr"
}).serialize(),
)
def test_updater_merge_previous_tables_outdated(filter_func: SDFilterFunc) -> None:
previous_tree, _inv_tree = _make_trees(
{},
{
("Ident 1",): {"old": (1, 2, 3)},
("Ident 2",): {"old": (1, 2, 3)},
},
)
inv_tree = StructuredDataNode()
previous_node = previous_tree.get_node(("path-to", "node"))
assert isinstance(previous_node, StructuredDataNode)
updater = TableUpdater(
RetentionInfo(
filter_func,
RetentionIntervals(-1, -2, -3),
),
inv_tree.setdefault_node(("path-to", "node")),
previous_node,
)
result = updater.filter_and_merge(1000)
assert not result.save_tree
assert not result.reason
inv_node = inv_tree.get_node(("path-to", "node"))
assert inv_node is not None
assert inv_node.table.retentions == {}
def test_updater_merge_previous_attributes(
filter_func: SDFilterFunc,
expected_retentions: Dict,
):
previous_tree, _inv_tree = _make_trees({"old": (1, 2, 3)}, {})
inv_tree = StructuredDataNode()
previous_node = previous_tree.get_node(("path-to", "node"))
assert isinstance(previous_node, StructuredDataNode)
updater = AttributesUpdater(
RetentionInfo(
filter_func,
RetentionIntervals(-1, -2, -3),
),
inv_tree.setdefault_node(("path-to", "node")),
previous_node,
)
result = updater.filter_and_merge(-1)
if expected_retentions:
assert result.save_tree
assert result.reason
else:
assert not result.save_tree
assert not result.reason
inv_node = inv_tree.get_node(("path-to", "node"))
assert inv_node is not None
assert inv_node.attributes.retentions == expected_retentions
if expected_retentions:
assert "old" in inv_node.attributes.pairs
def test_set_path_sub_nodes():
root = _create_empty_tree()
nta = root.get_node(("path", "to", "nta"))
sub_node = StructuredDataNode(name="node")
sub_node.setdefault_node(("sub-path-to", "sub-node"))
nta.add_node(sub_node)
path_to_node = root.get_node(["path", "to", "nta", "node", "sub-path-to"])
assert path_to_node is not None
assert path_to_node.attributes.path == ("path", "to", "nta", "node",
"sub-path-to")
assert path_to_node.table.path == ("path", "to", "nta", "node",
"sub-path-to")
assert path_to_node.path == ("path", "to", "nta", "node", "sub-path-to")
path_to_sub_node = root.get_node(
["path", "to", "nta", "node", "sub-path-to", "sub-node"])
assert path_to_sub_node is not None
assert path_to_sub_node.attributes.path == (
"path",
"to",
"nta",
"node",
"sub-path-to",
"sub-node",
)
assert path_to_sub_node.table.path == ("path", "to", "nta", "node",
"sub-path-to", "sub-node")
assert path_to_sub_node.path == ("path", "to", "nta", "node",
"sub-path-to", "sub-node")
def _make_trees(
previous_attributes_retentions: Dict, previous_table_retentions: Dict
) -> Tuple[StructuredDataNode, StructuredDataNode]:
previous_tree = StructuredDataNode.deserialize(
{
"Attributes": {},
"Table": {},
"Nodes": {
"path-to": {
"Attributes": {},
"Table": {},
"Nodes": {
"node": {
"Attributes": {
"Pairs": {"old": "Key", "keys": "Previous Keys"},
"Retentions": previous_attributes_retentions,
},
"Table": {
"KeyColumns": ["ident"],
"Rows": [
{"ident": "Ident 1", "old": "Key 1", "keys": "Previous Keys 1"},
{"ident": "Ident 2", "old": "Key 2", "keys": "Previous Keys 2"},
],
"Retentions": previous_table_retentions,
},
"Nodes": {},
}
},
}
},
}
)
inv_tree = StructuredDataNode.deserialize(
{
"Attributes": {},
"Table": {},
"Nodes": {
"path-to": {
"Attributes": {},
"Table": {},
"Nodes": {
"node": {
"Attributes": {
"Pairs": {"new": "Key", "keys": "New Keys"},
},
"Table": {
"KeyColumns": ["ident"],
"Rows": [
{"ident": "Ident 1", "new": "Key 1", "keys": "New Keys 1"},
{"ident": "Ident 2", "new": "Key 2", "keys": "New Keys 2"},
],
},
"Nodes": {},
}
},
}
},
}
)
return previous_tree, inv_tree
def test_load_latest_delta_tree_one_archive_and_inv_tree() -> None:
hostname = "inv-host"
expected_delta_tree = StructuredDataNode.deserialize(
{"inv": ("attr-0", "attr")})
# history
cmk.utils.store.save_object_to_file(
Path(cmk.utils.paths.inventory_archive_dir, hostname, "0"),
StructuredDataNode.deserialize({
"inv": "attr-0"
}).serialize(),
)
# current tree
cmk.utils.store.save_object_to_file(
Path(cmk.utils.paths.inventory_output_dir, hostname),
StructuredDataNode.deserialize({
"inv": "attr"
}).serialize(),
)
delta_tree = cmk.gui.inventory.load_latest_delta_tree(hostname)
assert delta_tree is not None
assert delta_tree.is_equal(expected_delta_tree)
def __init__(self):
self.trees = InventoryTrees(
inventory=StructuredDataNode(),
status_data=StructuredDataNode(),
)
self._index_cache = {}
self._class_mutex = {}
def test_set_path_sub_nodes_error():
root = _create_empty_tree()
nta = root.get_node(("path", "to", "nta"))
sub_node = StructuredDataNode()
sub_node.setdefault_node(("sub-path", "sub-to", "sub-node"))
with pytest.raises(ValueError):
nta.add_node(sub_node)
def _merge_inventory_and_status_data_tree(inventory_tree, status_data_tree):
if inventory_tree is None and status_data_tree is None:
return
if inventory_tree is None:
inventory_tree = StructuredDataNode()
if status_data_tree is not None:
inventory_tree.merge_with(status_data_tree)
return inventory_tree
def test_add_attributes():
path = ("path-to", "node")
retentions = {"key": RetentionIntervals(1, 2, 3)}
node = StructuredDataNode(name="node", path=path)
attributes = Attributes(retentions=retentions)
node.add_attributes(attributes)
assert node.attributes.path == path
assert node.attributes.retentions == retentions
def _do_inv_for_cluster(host_config: config.HostConfig) -> InventoryTrees:
inventory_tree = StructuredDataNode()
_set_cluster_property(inventory_tree, host_config)
if not host_config.nodes:
return InventoryTrees(inventory_tree, StructuredDataNode())
node = inventory_tree.setdefault_node(
("software", "applications", "check_mk", "cluster", "nodes")
)
node.table.add_key_columns(["name"])
node.table.add_rows([{"name": node_name} for node_name in host_config.nodes])
return InventoryTrees(inventory_tree, StructuredDataNode())
def test_updater_null_obj_tables_outdated() -> None:
inv_tree = StructuredDataNode()
updater = TableUpdater(
RetentionInfo(
lambda key: True,
RetentionIntervals(1, 2, 3),
),
inv_tree,
StructuredDataNode(),
)
result = updater.filter_and_merge(1000)
assert not result.save_tree
assert not result.reason
assert inv_tree.get_node(("path-to", "node")) is None
def get_inventory_data(inventory_tree: StructuredDataNode,
tree_path: RawInventoryPath) -> InventoryData:
invdata = None
parsed_path, attribute_keys = parse_tree_path(tree_path)
if attribute_keys == []:
table = inventory_tree.get_table(parsed_path)
if table is not None:
invdata = table.data
elif attribute_keys:
attributes = inventory_tree.get_attributes(parsed_path)
if attributes is not None:
# In paint_host_inventory_tree we parse invpath and get
# a path and attribute_keys which may be either None, [], or ["KEY"].
invdata = attributes.data.get(attribute_keys[-1])
return invdata
def test_updater_null_obj_attributes():
inv_tree = StructuredDataNode()
updater = AttributesUpdater(
RetentionInfo(
lambda key: True,
RetentionIntervals(1, 2, 3),
),
inv_tree,
StructuredDataNode(),
)
result = updater.filter_and_merge(-1)
assert not result.save_tree
assert not result.reason
assert inv_tree.get_node(["path-to", "node"]) is None
def get_inventory_table(tree: StructuredDataNode,
raw_path: SDRawPath) -> Optional[InventoryRows]:
inventory_path = InventoryPath.parse(raw_path)
if inventory_path.source != TreeSource.table:
return None
table = tree.get_table(inventory_path.path)
return None if table is None else table.rows
def test_updater_handle_inv_tables_outdated(
filter_func,
path,
expected_retentions,
):
_previous_tree, inv_tree = _make_trees({}, {})
updater = TableUpdater(
RetentionInfo(
filter_func,
RetentionIntervals(1, 2, 3),
),
inv_tree.get_node(path),
StructuredDataNode(),
)
result = updater.filter_and_merge(1000)
if expected_retentions:
assert result.save_tree
assert result.reason
else:
assert not result.save_tree
assert not result.reason
inv_node = inv_tree.get_node(["path-to", "node"])
assert inv_node is not None
assert inv_node.table.retentions == expected_retentions
def test_delta_structured_data_tree_serialization(zipped_trees):
old_tree = StructuredDataNode()
new_tree = StructuredDataNode()
old_filename, new_filename = zipped_trees
old_tree = load_tree_from(old_filename)
new_tree = load_tree_from(new_filename)
delta_result = old_tree.compare_with(new_tree)
delta_raw_tree = delta_result.delta.get_raw_tree()
assert isinstance(delta_raw_tree, dict)
new_delta_tree = StructuredDataNode().create_tree_from_raw_tree(
delta_raw_tree)
assert delta_result.delta.is_equal(new_delta_tree)
def get_inventory_table(tree: StructuredDataNode,
raw_path: SDRawPath) -> Optional[InventoryRows]:
parsed_path, attribute_keys = parse_tree_path(raw_path)
if attribute_keys != []:
return None
table = tree.get_table(parsed_path)
return None if table is None else table.rows
def test_empty_but_different_structure():
root = _create_empty_tree()
nt = root.get_node(["path", "to", "nta", "nt"])
na = root.get_node(["path", "to", "nta", "na"])
ta = root.get_node(["path", "to", "nta", "ta"])
assert nt.attributes.pairs == {}
assert nt.attributes.is_empty()
assert nt.table._rows == {}
assert nt.table.rows == []
assert nt.table.is_empty()
assert na.attributes.pairs == {}
assert na.attributes.is_empty()
assert na.table._rows == {}
assert na.table.rows == []
assert na.table.is_empty()
assert ta.attributes.pairs == {}
assert ta.attributes.is_empty()
assert ta.table._rows == {}
assert ta.table.rows == []
assert ta.table.is_empty()
assert root.is_empty()
assert root.count_entries() == 0
assert not root.is_equal(StructuredDataNode())
def _set_cluster_property(
inventory_tree: StructuredDataNode,
host_config: config.HostConfig,
) -> None:
node = inventory_tree.setdefault_node(
["software", "applications", "check_mk", "cluster"])
node.attributes.add_pairs({"is_cluster": host_config.is_cluster})
def _save_inventory_tree(
hostname: HostName,
inventory_tree: StructuredDataNode,
) -> Optional[StructuredDataNode]:
store.makedirs(cmk.utils.paths.inventory_output_dir)
filepath = cmk.utils.paths.inventory_output_dir + "/" + hostname
if inventory_tree.is_empty():
# Remove empty inventory files. Important for host inventory icon
if os.path.exists(filepath):
os.remove(filepath)
if os.path.exists(filepath + ".gz"):
os.remove(filepath + ".gz")
return None
old_tree = load_tree_from(filepath)
old_tree.normalize_nodes()
if old_tree.is_equal(inventory_tree):
console.verbose("Inventory was unchanged\n")
return None
if old_tree.is_empty():
console.verbose("New inventory tree\n")
else:
console.verbose("Inventory tree has changed\n")
old_time = os.stat(filepath).st_mtime
arcdir = "%s/%s" % (cmk.utils.paths.inventory_archive_dir, hostname)
store.makedirs(arcdir)
os.rename(filepath, arcdir + ("/%d" % old_time))
save_tree_to(inventory_tree, cmk.utils.paths.inventory_output_dir,
hostname)
return old_tree
def test_add_node():
root = _create_filled_tree()
sub_node = StructuredDataNode(name="node")
sub_node.attributes.add_pairs({"sn0": "SN 0", "sn1": "SN 1"})
sub_node.table.add_key_columns(["sn0"])
sub_node.table.add_rows([
{
"sn0": "SN 00",
"sn1": "SN 01"
},
{
"sn0": "SN 10",
"sn1": "SN 11"
},
])
node = root.get_node(["path", "to", "nta"]).add_node(sub_node)
# Do not modify orig node.
assert sub_node.attributes.path == tuple()
assert sub_node.table.path == tuple()
assert sub_node.path == tuple()
assert node.attributes.path == tuple(["path", "to", "nta", "node"])
assert node.table.key_columns == ["sn0"]
assert node.table.path == tuple(["path", "to", "nta", "node"])
assert node.path == tuple(["path", "to", "nta", "node"])
assert not root.is_empty()
assert root.count_entries() == 18
def test_updater_handle_inv_tables_outdated(
filter_func: SDFilterFunc,
expected_retentions: Dict,
) -> None:
_previous_tree, inv_tree = _make_trees({}, {})
fst_inv_node = inv_tree.get_node(("path-to", "node"))
assert isinstance(fst_inv_node, StructuredDataNode)
updater = TableUpdater(
RetentionInfo(
filter_func,
RetentionIntervals(1, 2, 3),
),
fst_inv_node,
StructuredDataNode(),
)
result = updater.filter_and_merge(1000)
if expected_retentions:
assert result.save_tree
assert result.reason
else:
assert not result.save_tree
assert not result.reason
inv_node = inv_tree.get_node(("path-to", "node"))
assert inv_node is not None
assert inv_node.table.retentions == expected_retentions
def get_inventory_attribute(tree: StructuredDataNode,
raw_path: SDRawPath) -> InventoryValue:
inventory_path = InventoryPath.parse(raw_path)
if inventory_path.source != TreeSource.attributes or not inventory_path.key:
return None
attributes = tree.get_attributes(inventory_path.path)
return None if attributes is None else attributes.pairs.get(
inventory_path.key)
def get_inventory_attribute(tree: StructuredDataNode,
raw_path: SDRawPath) -> InventoryValue:
parsed_path, attribute_keys = parse_tree_path(raw_path)
if not attribute_keys:
return None
attributes = tree.get_attributes(parsed_path)
return None if attributes is None else attributes.pairs.get(
attribute_keys[-1])
def test__load_status_data_tree(monkeypatch, hostname, row, expected_tree):
monkeypatch.setattr(
cmk.gui.inventory,
"_load_structured_data_tree",
lambda t, hostname: StructuredDataNode.deserialize({"loaded": "tree"}),
)
status_data_tree = cmk.gui.inventory._load_status_data_tree(hostname, row)
assert status_data_tree is not None
assert status_data_tree.is_equal(expected_tree)
def _load_status_data_tree(hostname: Optional[HostName],
row: Row) -> Optional[StructuredDataNode]:
# If no data from livestatus could be fetched (CRE) try to load from cache
# or status dir
raw_status_data_tree = row.get("host_structured_status")
if not raw_status_data_tree:
return _load_structured_data_tree("status_data", hostname)
return StructuredDataNode.deserialize(
ast.literal_eval(raw_status_data_tree.decode("utf-8")))
def get_tree(self, filepath: Path) -> StructuredDataNode:
if filepath == _DEFAULT_PATH_TO_TREE:
return StructuredDataNode()
if filepath in self._lookup:
return self._lookup[filepath]
return self._lookup.setdefault(filepath,
self._load_tree_from_file(filepath))
def test_add_table():
path = ("path-to", "node")
key_columns = ["key-0"]
retentions: TableRetentions = {
("Value 0", ): {
"key-1": RetentionIntervals(1, 2, 3)
},
}
node = StructuredDataNode(name="node", path=path)
table = Table(
key_columns=key_columns,
retentions=retentions,
)
node.add_table(table)
assert node.table.path == path
assert node.table.key_columns == key_columns
assert node.table.retentions == retentions
