def test_updater_merge_previous_attributes(
filter_func: SDFilterFunc,
expected_retentions: Dict,
):
previous_tree, _inv_tree = _make_trees({"old": (1, 2, 3)}, {})
inv_tree = StructuredDataNode()
previous_node = previous_tree.get_node(("path-to", "node"))
assert isinstance(previous_node, StructuredDataNode)
updater = AttributesUpdater(
RetentionInfo(
filter_func,
RetentionIntervals(-1, -2, -3),
),
inv_tree.setdefault_node(("path-to", "node")),
previous_node,
)
result = updater.filter_and_merge(-1)
if expected_retentions:
assert result.save_tree
assert result.reason
else:
assert not result.save_tree
assert not result.reason
inv_node = inv_tree.get_node(("path-to", "node"))
assert inv_node is not None
assert inv_node.attributes.retentions == expected_retentions
if expected_retentions:
assert "old" in inv_node.attributes.pairs
def test_updater_merge_previous_tables_outdated(filter_func: SDFilterFunc) -> None:
previous_tree, _inv_tree = _make_trees(
{},
{
("Ident 1",): {"old": (1, 2, 3)},
("Ident 2",): {"old": (1, 2, 3)},
},
)
inv_tree = StructuredDataNode()
previous_node = previous_tree.get_node(("path-to", "node"))
assert isinstance(previous_node, StructuredDataNode)
updater = TableUpdater(
RetentionInfo(
filter_func,
RetentionIntervals(-1, -2, -3),
),
inv_tree.setdefault_node(("path-to", "node")),
previous_node,
)
result = updater.filter_and_merge(1000)
assert not result.save_tree
assert not result.reason
inv_node = inv_tree.get_node(("path-to", "node"))
assert inv_node is not None
assert inv_node.table.retentions == {}
def _make_intervals(
now: int,
raw_cache_info: Optional[RawCacheInfo],
retention_interval: int,
) -> RetentionIntervals:
if raw_cache_info:
return RetentionIntervals(
cached_at=raw_cache_info[0],
cache_interval=raw_cache_info[1],
retention_interval=retention_interval,
)
return RetentionIntervals(
cached_at=now,
cache_interval=0,
retention_interval=retention_interval,
)
def test_updater_merge_attributes_outdated(
filter_func,
expected_retentions,
):
previous_tree, inv_tree = _make_trees(
{
"old": (1, 2, 3),
"keys": (1, 2, 3),
},
{},
)
previous_node = previous_tree.get_node(["path-to", "node"])
assert previous_node is not None
inv_node = inv_tree.get_node(["path-to", "node"])
assert inv_node is not None
updater = AttributesUpdater(
RetentionInfo(
filter_func,
RetentionIntervals(4, 5, 6),
),
inv_node,
previous_node,
)
result = updater.filter_and_merge(1000)
if expected_retentions:
assert result.save_tree
assert result.reason
else:
assert not result.save_tree
assert not result.reason
assert inv_node.attributes.retentions == expected_retentions
def test_updater_handle_inv_tables_outdated(
filter_func,
path,
expected_retentions,
):
_previous_tree, inv_tree = _make_trees({}, {})
updater = TableUpdater(
RetentionInfo(
filter_func,
RetentionIntervals(1, 2, 3),
),
inv_tree.get_node(path),
StructuredDataNode(),
)
result = updater.filter_and_merge(1000)
if expected_retentions:
assert result.save_tree
assert result.reason
else:
assert not result.save_tree
assert not result.reason
inv_node = inv_tree.get_node(["path-to", "node"])
assert inv_node is not None
assert inv_node.table.retentions == expected_retentions
def test_updater_merge_tables_outdated(
filter_func: SDFilterFunc,
expected_retentions: Dict,
) -> None:
previous_tree, inv_tree = _make_trees(
{},
{
("Ident 1",): {"old": (1, 2, 3), "keys": (1, 2, 3)},
("Ident 2",): {"old": (1, 2, 3), "keys": (1, 2, 3)},
},
)
previous_node = previous_tree.get_node(("path-to", "node"))
assert previous_node is not None
inv_node = inv_tree.get_node(("path-to", "node"))
assert inv_node is not None
updater = TableUpdater(
RetentionInfo(
filter_func,
RetentionIntervals(4, 5, 6),
),
inv_node,
previous_node,
)
result = updater.filter_and_merge(1000)
if expected_retentions:
assert result.save_tree
assert result.reason
else:
assert not result.save_tree
assert not result.reason
assert inv_node.table.retentions == expected_retentions
def test_updater_handle_inv_tables_outdated(
filter_func: SDFilterFunc,
expected_retentions: Dict,
) -> None:
_previous_tree, inv_tree = _make_trees({}, {})
fst_inv_node = inv_tree.get_node(("path-to", "node"))
assert isinstance(fst_inv_node, StructuredDataNode)
updater = TableUpdater(
RetentionInfo(
filter_func,
RetentionIntervals(1, 2, 3),
),
fst_inv_node,
StructuredDataNode(),
)
result = updater.filter_and_merge(1000)
if expected_retentions:
assert result.save_tree
assert result.reason
else:
assert not result.save_tree
assert not result.reason
inv_node = inv_tree.get_node(("path-to", "node"))
assert inv_node is not None
assert inv_node.table.retentions == expected_retentions
def test_add_attributes():
path = ("path-to", "node")
retentions = {"key": RetentionIntervals(1, 2, 3)}
node = StructuredDataNode(name="node", path=path)
attributes = Attributes(retentions=retentions)
node.add_attributes(attributes)
assert node.attributes.path == path
assert node.attributes.retentions == retentions
def test_updater_merge_previous_tables(
filter_func,
expected_retentions,
):
previous_tree, _inv_tree = _make_trees(
{},
{
("Ident 1", ): {
"old": (1, 2, 3)
},
("Ident 2", ): {
"old": RetentionIntervals(1, 2, 3)
},
},
)
inv_tree = StructuredDataNode()
updater = TableUpdater(
RetentionInfo(
filter_func,
RetentionIntervals(-1, -2, -3),
),
inv_tree.setdefault_node(["path-to", "node"]),
previous_tree.get_node(["path-to", "node"]),
)
result = updater.filter_and_merge(-1)
if expected_retentions:
assert result.save_tree
assert result.reason
else:
assert not result.save_tree
assert not result.reason
inv_node = inv_tree.get_node(["path-to", "node"])
assert inv_node is not None
assert inv_node.table.retentions == expected_retentions
if expected_retentions:
for row in inv_node.table.rows:
assert "old" in row
def test_updater_null_obj_tables_outdated() -> None:
inv_tree = StructuredDataNode()
updater = TableUpdater(
RetentionInfo(
lambda key: True,
RetentionIntervals(1, 2, 3),
),
inv_tree,
StructuredDataNode(),
)
result = updater.filter_and_merge(1000)
assert not result.save_tree
assert not result.reason
assert inv_tree.get_node(("path-to", "node")) is None
def test_updater_null_obj_attributes():
inv_tree = StructuredDataNode()
updater = AttributesUpdater(
RetentionInfo(
lambda key: True,
RetentionIntervals(1, 2, 3),
),
inv_tree,
StructuredDataNode(),
)
result = updater.filter_and_merge(-1)
assert not result.save_tree
assert not result.reason
assert inv_tree.get_node(["path-to", "node"]) is None
def test_updater_merge_previous_attributes_outdated(filter_func):
previous_tree, _inv_tree = _make_trees({"old": (1, 2, 3)}, {})
inv_tree = StructuredDataNode()
updater = AttributesUpdater(
RetentionInfo(
filter_func,
RetentionIntervals(-1, -2, -3),
),
inv_tree.setdefault_node(["path-to", "node"]),
previous_tree.get_node(["path-to", "node"]),
)
result = updater.filter_and_merge(1000)
assert not result.save_tree
assert not result.reason
inv_node = inv_tree.get_node(["path-to", "node"])
assert inv_node is not None
assert inv_node.attributes.retentions == {}
def test_updater_merge_tables(filter_func, expected_retentions):
previous_tree, inv_tree = _make_trees(
{},
{
("Ident 1", ): {
"old": (1, 2, 3),
"keys": (1, 2, 3),
},
("Ident 2", ): {
"old": (1, 2, 3),
"keys": (1, 2, 3),
},
},
)
previous_node = previous_tree.get_node(["path-to", "node"])
assert previous_node is not None
inv_node = inv_tree.get_node(["path-to", "node"])
assert inv_node is not None
updater = TableUpdater(
RetentionInfo(
filter_func,
RetentionIntervals(4, 5, 6),
),
inv_node,
previous_node,
)
result = updater.filter_and_merge(-1)
if expected_retentions:
assert result.save_tree
assert result.reason
else:
assert not result.save_tree
assert not result.reason
assert inv_node.table.retentions == expected_retentions
if expected_retentions:
for row in inv_node.table.rows:
assert "old" in row
assert row.get("keys").startswith("New Keys")
def test_add_table():
path = ("path-to", "node")
key_columns = ["key-0"]
retentions: TableRetentions = {
("Value 0", ): {
"key-1": RetentionIntervals(1, 2, 3)
},
}
node = StructuredDataNode(name="node", path=path)
table = Table(
key_columns=key_columns,
retentions=retentions,
)
node.add_table(table)
assert node.table.path == path
assert node.table.key_columns == key_columns
assert node.table.retentions == retentions
def test_updater_merge_attributes(
filter_func: SDFilterFunc,
expected_retentions: Dict,
) -> None:
previous_tree, inv_tree = _make_trees(
{
"old": (1, 2, 3),
"keys": (1, 2, 3),
},
{},
)
previous_node = previous_tree.get_node(("path-to", "node"))
assert previous_node is not None
inv_node = inv_tree.get_node(("path-to", "node"))
assert inv_node is not None
updater = AttributesUpdater(
RetentionInfo(
filter_func,
RetentionIntervals(4, 5, 6),
),
inv_node,
previous_node,
)
result = updater.filter_and_merge(-1)
if expected_retentions:
assert result.save_tree
assert result.reason
else:
assert not result.save_tree
assert not result.reason
assert inv_node.attributes.retentions == expected_retentions
if expected_retentions:
assert "old" in inv_node.attributes.pairs
assert inv_node.attributes.pairs.get("keys") == "New Keys"
"raw_intervals, node_name, path, raw_cache_info, expected_intervals, match_some_keys, match_other_keys",
[
# === Attributes ===
# config, right path, all attributes
(
[{
"interval": 3,
"visible_raw_path": "path-to.node",
"attributes": "all",
}],
"Attributes",
["path-to", "node"],
None,
RetentionIntervals(
cached_at=100,
cache_interval=0,
retention_interval=3,
),
True,
True,
),
(
[{
"interval": 3,
"visible_raw_path": "path-to.node",
"attributes": "all",
}],
"Attributes",
["path-to", "node"],
(1, 2),
RetentionIntervals(
[
# === Attributes ===
# config, right path, all attributes
(
[
{
"interval": 3,
"visible_raw_path": "path-to.node",
"attributes": "all",
}
],
"Attributes",
None,
RetentionIntervals(
cached_at=100,
cache_interval=0,
retention_interval=3,
),
True,
True,
),
(
[
{
"interval": 3,
"visible_raw_path": "path-to.node",
"attributes": "all",
}
],
"Attributes",
(1, 2),