def run_cmpids_for_query_tests(hmodel: HubitModel):
# Two components match query path
qpath = HubitQueryPath("first_coor[:].second_coor[:].value")
result = hmodel._cmpids_for_query(qpath, check_intersection=True)
expected_result = [
"cmp0@./components/comp0.move_number",
"cmp1@./components/comp0.move_number",
]
assert result == expected_result
# Only the second components match query path
qpath = HubitQueryPath("first_coor[:].second_coor[1].value")
result = hmodel._cmpids_for_query(qpath, check_intersection=True)
expected_result = [
"cmp1@./components/comp0.move_number",
]
assert result == expected_result
# Both components match the query path since we don't check the intersection
qpath = HubitQueryPath("first_coor[:].second_coor[1].value")
result = hmodel._cmpids_for_query(qpath, check_intersection=False)
expected_result = [
"cmp0@./components/comp0.move_number",
"cmp1@./components/comp0.move_number",
]
assert result == expected_result
def test_get_bindings(self):
"""
Get bindings for query where model path is fully specified
"""
bindings = [
HubitBinding.from_cfg(
{
"name": "inflow",
"path": "inlets[0@IDX_INLET].tanks[2@IDX_TANK].inflow",
}
)
]
# The query path mathces the model path
querypath = HubitQueryPath("inlets[0].tanks[2].inflow")
path_for_name, idxval_for_idxid = _Worker.get_bindings(bindings, querypath)
expected_idxval_for_idxid = {"IDX_INLET": "0", "IDX_TANK": "2"}
self.assertDictEqual(expected_idxval_for_idxid, idxval_for_idxid)
expected_path_for_name = {
"inflow": HubitModelPath("inlets[0@IDX_INLET].tanks[2@IDX_TANK].inflow")
}
self.assertDictEqual(expected_path_for_name, path_for_name)
querypath = HubitQueryPath("inlets.1.tanks.2.inflow")
with self.assertRaises(HubitWorkerError):
_Worker.get_bindings(bindings, querypath)
def test_init(self):
# Success
qpath = HubitQueryPath("lines[:].tanks[:].vol_outlet_flow")
mpaths = [
"lines[IDX_LINE].tanks[0@IDX_TANK].vol_outlet_flow",
"lines[IDX_LINE].tanks[1@IDX_TANK].vol_outlet_flow",
"lines[IDX_LINE].tanks[2@IDX_TANK].vol_outlet_flow",
]
mpaths = [HubitModelPath(mpath) for mpath in mpaths]
# Use a dummy tree. It will not be used since the path normalization is bypassed
tree = DummyLengthTree()
cmps = get_mock_components(mpaths)
_QueryExpansion(qpath, mpaths, tree, cmps)
# mpaths cannot have different index contexts
qpath = HubitQueryPath("lines[:].tanks[:].vol_outlet_flow")
mpaths = [
"lines[IDX_LINE].tanks[0@IDX_TANK].vol_outlet_flow",
"lines[IDX_LINE].tanks[1@IDX_OTHER].vol_outlet_flow",
]
mpaths = [HubitModelPath(mpath) for mpath in mpaths]
cmps = get_mock_components(mpaths)
with pytest.raises(HubitModelQueryError):
_QueryExpansion(qpath, mpaths, tree, cmps)
# No mpaths i.e. no provider
qpath = HubitQueryPath("lines[:].tanks[:].vol_outlet_flow")
mpaths = []
cmps = get_mock_components(mpaths)
with pytest.raises(HubitModelQueryError):
_QueryExpansion(qpath, mpaths, tree, cmps)
def run_mpaths_for_qpath_fields_only(hmodel: HubitModel):
expected_mpaths = [
"first_coor[IDX1].second_coor[0@IDX2].value",
"first_coor[IDX1].second_coor[1@IDX2].value",
]
expected_cmp_ids = [
"cmp0@./components/comp0.move_number",
"cmp1@./components/comp0.move_number",
]
# Two components match query path
qpath = HubitQueryPath("first_coor[:].second_coor[:].value")
mpaths, cmp_ids = hmodel._mpaths_for_qpath_fields_only(qpath)
assert cmp_ids == expected_cmp_ids
assert mpaths == expected_mpaths
# Only the second components match query path
qpath = HubitQueryPath("first_coor[:].second_coor[1].value")
mpaths, cmp_ids = hmodel._mpaths_for_qpath_fields_only(qpath)
assert cmp_ids == expected_cmp_ids
assert mpaths == expected_mpaths
# Both components match the query path since we don't check the intersection
qpath = HubitQueryPath("first_coor[:].second_coor[1].value")
mpaths, cmp_ids = hmodel._mpaths_for_qpath_fields_only(qpath)
assert cmp_ids == expected_cmp_ids
assert mpaths == expected_mpaths
def test_number_of_workers_multiple_levels(self):
"""Query level-1 attribute. Should deploy 2 level-0 workers
and 2 level-1 workers.
"""
queries = [(
HubitQueryPath("list[0].some_attr.two_x_numbers_x_factor"),
HubitQueryPath("list[1].some_attr.two_x_numbers_x_factor"),
)]
worker_counts = self.get_worker_counts(queries)
expected_worker_counts = [4]
self.assertListEqual(worker_counts, expected_worker_counts)
def test_balanced(self):
path = "segments[0].layers[17]"
result = HubitQueryPath.balanced(path)
self.assertTrue(result)
path = "segments[44].layers[76"
result = HubitQueryPath.balanced(path)
self.assertFalse(result)
path = "segments]44[.layers[76"
result = HubitQueryPath.balanced(path)
self.assertFalse(result)
def test_number_of_workers_case6(self):
"""Query multiple attributes that are actually supplied by the
same component. Therefore, only one worker should be deployed.
"""
queries = [[
HubitQueryPath("list[0].some_attr.inner_list[0].yval"),
HubitQueryPath("list[0].some_attr.inner_list[1].yval"),
]]
worker_counts = self.get_worker_counts(queries)
expected_worker_counts = [1]
self.assertListEqual(worker_counts, expected_worker_counts)
def test_has_slice_range(self):
path = HubitQueryPath("hi.how.2.you")
assert path.has_slice_range() == False
path = HubitQueryPath("hi.how[2].are.you")
assert path.has_slice_range() == False
path = HubitQueryPath("hi.how[:].are.you")
assert path.has_slice_range() == True
def test_is_path_described_query_paths(self):
print(self.tree)
# Path with no ranges is not described by a LengthTree
assert not self.tree.is_path_described(HubitQueryPath("i.dont.exist"))
assert self.tree.is_path_described(
HubitQueryPath("segments[0].layers[0].test.positions[0]"))
assert not self.tree.is_path_described(
HubitQueryPath("segments[0].layers[0].test.positions[1]"))
assert not self.tree.is_path_described(
HubitQueryPath("segments[7].layers[0].test.positions[1]"))
assert self.tree.is_path_described(
HubitQueryPath("segments[1].layers[0].test.positions[4]"))
assert not self.tree.is_path_described(
HubitQueryPath("segments[1].layers[0].test.positions[5]"))
# At least one segment has 4 positions on layer 0
assert self.tree.is_path_described(
HubitQueryPath("segments[:].layers[0].test.positions[4]"))
# No segment has 5 positions on layer 0
assert not self.tree.is_path_described(
HubitQueryPath("segments[:].layers[0].test.positions[5]"))
def _get_qexp():
path = HubitModelPath(
"lines[IDX_LINE].tanks[0@IDX_TANK].vol_outlet_flow")
yml_input = """
lines:
- tanks:
- tank1: 1
- tank2: 2
- tank3: 3
- tanks:
- tank1: 1
- tank2: 2
- tank3: 3
- tank4: 4
"""
input_data = yaml.load(yml_input, Loader=yaml.FullLoader)
tree = LengthTree.from_data(path, input_data)
qpath = HubitQueryPath("lines[:].tanks[:].vol_outlet_flow")
mpaths = [
"lines[IDX_LINE].tanks[0@IDX_TANK].vol_outlet_flow",
"lines[IDX_LINE].tanks[1@IDX_TANK].vol_outlet_flow",
"lines[IDX_LINE].tanks[2@IDX_TANK].vol_outlet_flow",
]
mpaths = [HubitModelPath(mpath) for mpath in mpaths]
cmps = get_mock_components(mpaths)
qexp = _QueryExpansion(qpath, mpaths, tree, cmps)
return qexp
def test_2(self):
"""
Initialize a simple worker with no idxids
"""
func = None
version = None
cfg = {
"path": "dummy",
"func_name": "dummy",
"provides_results": [
{"name": "attr1", "path": "shared.results.attr1.path"},
{"name": "attr2", "path": "shared.results.attr2.path"},
],
"consumes_input": [{"name": "attr", "path": "shared.input.attr.path"}],
}
component = HubitModelComponent.from_cfg(cfg, 0)
# No index IDs in model
tree_for_idxcontext = {"": DummyLengthTree()}
# Query something known to exist
querystring = HubitQueryPath(component.provides_results[0].path)
w = _Worker(
lambda x: x,
lambda x: x,
component,
querystring,
func,
version,
tree_for_idxcontext,
dryrun=True,
)
def test_status(self):
"""Check the status string representation completes"""
self.get_worker_counts([[self.querystr_level0]])
flat_results = FlatData({"hello.world": 5})
self.qr.flat_results = flat_results
expanded_paths = [HubitQueryPath("hello.world")]
self.qr._status(flat_results, expanded_paths)
def test_1(self):
"""
Fails since query does not match.
"""
func = None
version = None
cfg = {
"path": "dummy",
"func_name": "dummy",
"provides_results": [
{"name": "attr1", "path": "shared.results.attr1.path"},
{"name": "attr2", "path": "shared.results.attr2.path"},
],
"consumes_input": [{"name": "attr", "path": "shared.input.attr.path"}],
}
component = HubitModelComponent.from_cfg(cfg, 0)
# No index IDs in model
tree = DummyLengthTree()
querystring = HubitQueryPath("shared.attr.path")
with self.assertRaises(HubitWorkerError) as context:
w = _Worker(
lambda x: x,
lambda x: x,
component,
querystring,
func,
version,
tree,
dryrun=True,
)
def test_6(self):
"""
Get bindings for query with two location IDs and component
bindings with one index ID and one index wildcard.
The index wildcard is left as is since it is handled by
the expansion
"""
bindings = [
HubitBinding.from_cfg(
{
"name": "k_therm",
"path": "segments[IDX_SEG].layers[:@IDX_LAY].k_therm",
}
)
]
querystring = HubitQueryPath("segments[0].layers[0].k_therm")
path_for_name, _ = _Worker.get_bindings(bindings, querystring)
# This is what will be provided for the query: The attribute 'k_therm'
# for all layers for the specific index ID _IDX=0
# expected_path_for_name = {"k_therm": "segments.0.layers.:.k_therm"}
expected_path_for_name = {
"k_therm": HubitModelPath("segments[0@IDX_SEG].layers[:@IDX_LAY].k_therm")
}
self.assertDictEqual(expected_path_for_name, path_for_name)
self.assertTrue(
all([type(item) == HubitModelPath for item in path_for_name.values()])
)
def test_expand_path2(self):
"""Expand path"""
paths = (
HubitModelPath("segments[:@IDX_SEG].layers[:@IDX_LAY].test"),
HubitQueryPath("segments[:].layers[:].test"),
)
# path = "segments.:@IDX_SEG.layers.:@IDX_LAY.test"
seg_node = LengthNode(2)
lay_nodes = LengthNode(2), LengthNode(2)
seg_node.set_children(lay_nodes)
nodes = [seg_node]
nodes.extend(lay_nodes)
level_names = "IDX_SEG", "IDX_LAY"
tree = LengthTree(nodes, level_names)
# 2 values for segment 0 and 2 values for segment 1
expected_paths = [
[
"segments[0].layers[0].test",
"segments[0].layers[1].test",
],
[
"segments[1].layers[0].test",
"segments[1].layers[1].test",
],
]
for path in paths:
with self.subTest(path=path):
expanded_paths = tree.prune_from_path(path).expand_path(path)
self.assertSequenceEqual(expanded_paths, expected_paths)
def test_get_matches(self):
"""Test that we can find the provider strings
that match the query
"""
qpath = HubitQueryPath("segs[42].walls[3].temps")
provides = HubitModelPath("segs[IDXSEG].walls[IDXWALL].temps")
mpaths = (
HubitModelPath("price"),
provides,
HubitModelPath("segs[IDXSEG].walls.thicknesses"),
HubitModelPath(qpath),
HubitModelPath("segs[IDXSEG].walls[IDXWALL].thicknesses"),
HubitModelPath("segs[IDXSEG].walls[IDXWALL]"),
)
idxs_match_expected = (1, 3)
idxs_match = qpath.idxs_for_matches(mpaths)
self.assertSequenceEqual(idxs_match, idxs_match_expected)
def test_5(self):
"""
Initialize worker with ILOC locations in
query and ILOC wildcards in bindings
"""
func = None
version = None
cfg = """
path: dummy,
func_name: dummy,
provides_results:
- name: k_therm
path: segments[IDX_SEG].layers[:@IDX_LAY].k_therm
consumes_input:
- name: material
path: segments[IDX_SEG].layers[:@IDX_LAY].material
"""
component = HubitModelComponent.from_cfg(
yaml.load(cfg, Loader=yaml.FullLoader), 0
)
# Query something known to exist
querystr = HubitQueryPath("segments[0].layers[0].k_therm")
seg_node = LengthNode(2)
lay_nodes = LengthNode(2), LengthNode(2)
seg_node.set_children(lay_nodes)
nodes = [seg_node]
nodes.extend(lay_nodes)
level_names = "IDX_SEG", "IDX_LAY"
tree = LengthTree(nodes, level_names)
_tree_for_idxcontext = {tree.index_context: tree}
querystr = HubitQueryPath(querystr)
w = _Worker(
lambda x: x,
lambda x: x,
component,
querystr,
func,
version,
_tree_for_idxcontext,
dryrun=True,
)
def test_2a(self):
"""Outside bounds top level index"""
path = HubitModelPath(
"segments[2@IDX_SEG].layers[:@IDX_LAY].test.positions[:@IDX_POS]")
with self.assertRaises(HubitIndexError) as context:
self.tree.prune_from_path(path)
path = HubitQueryPath("segments[2].layers[:].test.positions[:]")
with self.assertRaises(HubitIndexError) as context:
self.tree.prune_from_path(path)
def setUp(self):
cfg = yaml.load(model, Loader=yaml.FullLoader)
self.model_cfg = HubitModelConfig.from_cfg(cfg, base_path=THIS_DIR)
self.hmodel = HubitModel(
self.model_cfg,
name="My model",
output_path=REL_TMP_DIR,
)
use_multi_processing = False
self.qr = query_runner_factory(use_multi_processing, self.hmodel)
self.input = yaml.load(yml_input, Loader=yaml.FullLoader)
self.hmodel.set_input(self.input)
# Query which does not consume results
self.idx = 1
self.querystr_level0 = HubitQueryPath(
"list[{}].some_attr.two_x_numbers".format(self.idx))
self.querystr_level1 = HubitQueryPath(
"list[{}].some_attr.two_x_numbers_x_factor".format(self.idx))
def test_2(self):
"""Top level index fixed to 1"""
expected_lengths = [1, 4, [5, 1, 2, 4]]
path = HubitModelPath(
"segments[1@IDX_SEG].layers[:@IDX_LAY].test.positions[:@IDX_POS]")
pruned_tree = self.tree.prune_from_path(path, inplace=False)
self.assertListEqual(pruned_tree.to_list(), expected_lengths)
path = HubitQueryPath("segments[1].layers[:].test.positions[:]")
pruned_tree = self.tree.prune_from_path(path, inplace=False)
self.assertListEqual(pruned_tree.to_list(), expected_lengths)
def _make_worker():
qrunner = Mock()
qrunner.check_cache.return_value = None
cname = None
func = dummy_function
version = None
cfg = {
"path": "dummy",
"func_name": "dummy_fun",
"provides_results": [
{
"name": "attrs1",
"path": "items_outer[:@IDX_OUTER].attr.items_inner[:@IDX_INNER].path1",
}
],
"consumes_input": [
{
"name": "attrs",
"path": "items_outer[:@IDX_OUTER].attr.items_inner[:@IDX_INNER].path",
},
{"name": "number", "path": "some_number"},
],
"consumes_results": [
{"name": "dependency", "path": "value"},
{"name": "dependency2", "path": "items_outer[:@IDX_OUTER].value"},
],
}
component = HubitModelComponent.from_cfg(cfg, 0)
# Required for shape inference. TODO: change when shapes are defined in model
inputdata = {
"items_outer": [
{"attr": {"items_inner": [{"path": 2}, {"path": 1}]}},
{"attr": {"items_inner": [{"path": 3}, {"path": 4}]}},
],
"some_number": 33,
}
querystring = HubitQueryPath("items_outer[1].attr.items_inner[0].path1")
_tree_for_idxcontext = tree_for_idxcontext([component], inputdata)
w = _Worker(
lambda x: x,
lambda x: x,
component,
querystring,
func,
version,
_tree_for_idxcontext,
dryrun=True, # Use dryrun to easily predict the result
)
return w
def test_5(self):
"""Two indices fixed"""
expected_lengths = [1, 4, [1, 1, 1, 1]]
path = HubitModelPath(
"segments[1@IDX_SEG].layers[:@IDX_LAY].test.positions[0@IDX_POS]")
pruned_tree = self.tree.prune_from_path(path, inplace=False)
self.assertListEqual(pruned_tree.to_list(), expected_lengths)
path = HubitQueryPath("segments[1].layers[:].test.positions[0]")
pruned_tree = self.tree.prune_from_path(path, inplace=False)
self.assertListEqual(pruned_tree.to_list(), expected_lengths)
def test_4(self):
"""In bounds for all bottom-most paths."""
expected_lengths = [2, [3, 4], [[1, 1, 1], [1, 1, 1, 1]]]
path = HubitModelPath(
"segments[:@IDX_SEG].layers[:@IDX_LAY].test.positions[0@IDX_POS]")
pruned_tree = self.tree.prune_from_path(path, inplace=False)
self.assertListEqual(pruned_tree.to_list(), expected_lengths)
path = HubitQueryPath("segments[:].layers[:].test.positions[0]")
pruned_tree = self.tree.prune_from_path(path, inplace=False)
self.assertListEqual(pruned_tree.to_list(), expected_lengths)
def test_3(self):
"""Middle index fixed"""
expected_lengths = [2, [1, 1], [[3], [1]]]
path = HubitModelPath(
"segments[:@IDX_SEG].layers[1@IDX_LAY].test.positions[:@IDX_POS]")
pruned_tree = self.tree.prune_from_path(path, inplace=False)
self.assertListEqual(pruned_tree.to_list(), expected_lengths)
path = HubitQueryPath("segments[:].layers[1].test.positions[:]")
pruned_tree = self.tree.prune_from_path(path)
self.assertListEqual(pruned_tree.to_list(), expected_lengths)
def test_6(self):
"""Out of bounds for all paths
The tree is modified even when an error is raised if inplace=True
Thats is OK since Hubit will raise an error and stop execution
"""
path = HubitModelPath(
"segments[:@IDX_SEG].layers[:@IDX_LAY].test.positions1[7@IDX_POS]")
with self.assertRaises(HubitIndexError) as context:
self.tree.prune_from_path(path, inplace=False)
path = HubitQueryPath("segments[:].layers[:].test.positions1[7]")
with self.assertRaises(HubitIndexError) as context:
self.tree.prune_from_path(path, inplace=False)
def test_7(self):
"""Queries should be expanded (location specific)
otherwise a HubitWorkerError is raised
"""
provides_results = [
HubitBinding.from_cfg(
{
"name": "k_therm",
"path": "segments[IDX_SEG].layers[:@IDX_LAY].k_therm",
}
)
]
querystring = HubitQueryPath("segments[0].layers[:].k_therm")
with self.assertRaises(HubitWorkerError):
_Worker.get_bindings(provides_results, querystring)
def test_5a(self):
"""Out of bounds for all but two paths
[['IDX_SEG', 2*], -> 1
['IDX_LAY', [3, 4*]], -> 2
['IDX_POS', [[1, 3, 2], [5*, 1, 2, 4*]]]] -> [1, 1]
"""
expected_lengths = [1, 2, [1, 1]]
path = HubitModelPath(
"segments[:@IDX_SEG].layers[:@IDX_LAY].test.positions[3@IDX_POS]")
pruned_tree = self.tree.prune_from_path(path, inplace=False)
self.assertListEqual(pruned_tree.to_list(), expected_lengths)
path = HubitQueryPath("segments[:].layers[:].test.positions[3]")
pruned_tree = self.tree.prune_from_path(path, inplace=False)
self.assertListEqual(pruned_tree.to_list(), expected_lengths)
def test_expand_path_count_from_back(self):
"""Expand path with fixed index set to negative number"""
path = HubitQueryPath("segments[:].layers[-2].test")
seg_node = LengthNode(3)
lay_nodes = LengthNode(2), LengthNode(4), LengthNode(3)
seg_node.set_children(lay_nodes)
nodes = [seg_node]
nodes.extend(lay_nodes)
level_names = "IDX_SEG", "IDX_LAY"
tree = LengthTree(nodes, level_names)
expected_paths = [
"segments[0].layers[0].test",
"segments[1].layers[2].test",
"segments[2].layers[1].test",
]
# Since the tree is not pruned we must use flat=True
expanded_paths = tree.prune_from_path(path, inplace=False).expand_path(
path, flat=True)
self.assertSequenceEqual(expanded_paths, expected_paths)
# Index error. There are 2 layers on segment at index 0.
path = HubitQueryPath("segments[:].layers[-3].test")
with self.assertRaises(HubitIndexError) as context:
expanded_paths = tree.prune_from_path(
path, inplace=False).expand_path(path, flat=True)
tree, _ = _get_data()
qpath = HubitQueryPath("sites[1].lines[0].tanks[-1].Q_yield")
expanded_paths = tree.prune_from_path(
qpath, inplace=False).expand_path(qpath, flat=True)
assert expanded_paths == ["sites[1].lines[0].tanks[3].Q_yield"]
qpath = HubitQueryPath("sites[0].lines[-1].tanks[-1].Q_yield")
expanded_paths = tree.prune_from_path(
qpath, inplace=False).expand_path(qpath, flat=True)
assert expanded_paths == ["sites[0].lines[0].tanks[2].Q_yield"]
qpath = HubitQueryPath("sites[:].lines[-1].tanks[-1].Q_yield")
expanded_paths = tree.prune_from_path(
qpath, inplace=False).expand_path(qpath, flat=True)
assert expanded_paths == [
"sites[0].lines[0].tanks[2].Q_yield",
"sites[1].lines[0].tanks[3].Q_yield",
]
qpath = HubitQueryPath("sites[:].lines[:].tanks[:].Q_yield")
expanded_paths = tree.prune_from_path(
qpath, inplace=False).expand_path(qpath, flat=True)
def test_mpaths_for_qpath_fields_only(self):
# Test the default model
qpath = HubitQueryPath("first_coor[:].second_coor[:].value")
mpaths, cmp_ids = self.hmodel._mpaths_for_qpath_fields_only(qpath)
expected_cmp_ids = [
"cmp0@./components/comp0.move_number",
]
assert cmp_ids == expected_cmp_ids
expected_mpaths = ["first_coor[IDX1].second_coor[IDX2].value"]
assert mpaths == expected_mpaths
model = get_model_with_explicit_indices()
model_cfg = HubitModelConfig.from_cfg(yaml.load(
model, Loader=yaml.FullLoader),
base_path=THIS_DIR)
hmodel = HubitModel(model_cfg, name="TEST", output_path=REL_TMP_DIR)
TestModel.run_mpaths_for_qpath_fields_only(hmodel)
def test_8(self):
"""
Test compression of indices. The query does not include any indices
"""
func = None
version = None
comp_yml = """
path: dummy,
func_name: dummy,
provides_results:
- name: mylist
path: factors
consumes_input:
- name: factors
path: list[:@IDX1].some_attr.factors
"""
component = HubitModelComponent.from_cfg(
yaml.load(comp_yml, Loader=yaml.FullLoader), 0
)
# Query something known to exist
querystr = "factors"
idx1_node = LengthNode(2)
nodes = [idx1_node]
level_names = ("IDX1",)
tree = LengthTree(nodes, level_names)
dummy_tree = DummyLengthTree()
_tree_for_idxcontext = {"": dummy_tree, tree.index_context: tree}
querystr = HubitQueryPath(querystr)
w = _Worker(
lambda x: x,
lambda x: x,
component,
querystr,
func,
version,
_tree_for_idxcontext,
dryrun=True,
)