def test_ids_assigned_in_change(self):
manager = Manager(
excel_path=[
(DATA_DIRECTORY / "Composition Example 2 Model Baseline.xlsx"),
(DATA_DIRECTORY / "Composition Example 2 Model Changed.xlsx"),
],
json_path=[(PATTERNS / "Composition.json")],
)
eval_base = manager.evaluators[0]
eval_change = manager.evaluators[1]
eval_base.rename_df_columns()
eval_base.add_missing_columns()
eval_base.to_property_di_graph()
eval_change.rename_df_columns()
eval_change.add_missing_columns()
eval_change.to_property_di_graph()
self.assertTrue(
set(eval_base.translator.uml_id.keys()).issubset(
set(eval_change.translator.uml_id.keys())))
for key in eval_base.translator.uml_id.keys():
if key != "count":
assert (eval_base.translator.uml_id[key] ==
eval_change.translator.uml_id[key])
def test_create_evaluators(self):
manager = Manager(
excel_path=[
DATA_DIRECTORY / "Composition Example.xlsx" for i in range(2)
],
json_path=[PATTERNS / "Composition.json"],
)
# weak test: create_evaluators() run during init
self.assertEqual(2, len(manager.evaluators))
for eval in manager.evaluators:
self.assertIsInstance(eval, Evaluator)
def test_get_json_data(self):
manager = Manager(
excel_path=[
DATA_DIRECTORY / "Composition Example.xlsx" for i in range(2)
],
json_path=[PATTERNS / "Composition.json"],
)
expected_keys = [
"Columns to Navigation Map",
"Pattern Graph Edges",
"Root Node",
"Vertex MetaTypes",
"Vertex Settings",
"Vertex Stereotypes",
]
assert expected_keys == list(manager.json_data[0].keys())
def compare_md_model(inputs, input_patterns="", output_path=""):
"""
Produces difference files (JSON and Excel) for the original file to
each change file provided.
Parameters
----------
inputs : list of strs
List of one or more file paths parsed from the command line.
This can be a path to one or more Excel files or a path to a
directory of Excel files.
input_patterns : list of str
List of paths to pattern file provided by the user.
output_path : str
String of the desired location for the output. This is optional
and if omitted then the output files will be placed in the same
directory as the input files.
Returns
-------
output_json : JSON file
Generates a JSON file as output that the Player Piano digests to
Generates a JSON file as output that the Player Piano digests to
update the original Model.
output_excel : Excel file
Generates an Excel file that lists the confident changes (ones
made by the JSON) and the unstable pairs so the user can make the
determination on those changes on their own.
"""
provided_paths = inputs
wkbk_paths = []
here = Path(os.getcwd())
for counter, path in enumerate(provided_paths):
p = Path(path)
if not p.is_absolute():
p = here / p
if p.is_dir():
p = list(p.glob("*.xlsx"))
for path in p:
if counter != 0 and path.name == p[0].name:
p.remove(path)
else:
p = [p]
wkbk_paths.extend(p)
if output_path:
output_path = Path(output_path)
if not output_path.is_absolute():
if output_path.parts[-1] == here.parts[-1]:
output_path = here
else:
output_path = here / output_path
if not output_path.is_dir():
raise RuntimeError("Please provide an output directory")
outpath = output_path
else:
outpath = wkbk_paths[0].parent
for wkbk in wkbk_paths:
if wkbk.parts[-1].split(".")[-1] != "xlsx":
msg = ("\n" + "This program only supports Excel Files." +
' "{0}" was skipped, not an Excel File').format(
wkbk.parts[-1])
warnings.warn(msg)
continue
json_patterns = {
pattern_path.name.split(".")[0].lower(): pattern_path
for pattern_path in PATTERNS.glob("*.json")
}
if input_patterns:
for in_pat in map(Path, input_patterns):
if in_pat.is_dir():
new_pats = {
inp.name.split(".")[0].lower(): inp
for inp in in_pat.glob("*.json")
}
else:
new_pats = {in_pat.name.split(".")[0].lower(): in_pat}
json_patterns.update(new_pats)
xl = pd.ExcelFile(wkbk_paths[0])
not_found = 0
pattern_sheet = ""
for sheet in xl.sheet_names:
if sheet.lower() not in json_patterns.keys():
not_found += 1
if not_found == len(xl.sheet_names):
warn_msg = (
'The Excel File "{0}" cannot be processed as none of the worksheets match a '
+ "supported pattern type.").format(wkbk.parts[-1])
patterns_msg = ("The currently supported " +
"patterns are: {0}".format([*json_patterns]))
patts = ("New patterns may be added in the" +
" ingrid/src/model_processing/patterns directory")
warnings.warn("\n" + warn_msg + "\n" + patterns_msg + "\n" +
patts)
break
else:
continue
else:
pattern_sheet = sheet.lower()
break
if not pattern_sheet:
raise RuntimeError("No matching pattern found nor provided." +
" Check the sheet names and try again.")
pattern = json_patterns[pattern_sheet]
manager = Manager(excel_path=wkbk_paths, json_path=[pattern])
for evaluator in manager.evaluators:
evaluator.rename_df_columns()
evaluator.add_missing_columns()
evaluator.to_property_di_graph()
manager.get_pattern_graph_diff(out_directory=outpath)
manager.changes_to_excel(out_directory=outpath)
print("Comparison Complete")
def test_match_changes(self):
manager = Manager(
excel_path=[
(DATA_DIRECTORY / "Composition_Diff_JSON_Baseline.xlsx"),
(DATA_DIRECTORY / "Composition_Diff_JSON_Changed.xlsx"),
],
json_path=[(PATTERNS / "Composition.json")],
)
orig_data = {
"Component": [
"Thruster Cluster Assembly",
"Propellant Isolation Assembly",
"Spacecraft",
],
"Position": ["Thruster-1", "LV-3", "ME"],
"Part": ["Small Thruster", "Latch Valve", "Main Engine"],
}
derived_A_lv3 = (
"A_propellant isolation assembly qua lv-3 context_lv-3")
derived_lv3 = "propellant isolation assembly qua lv-3 context"
orig_ids = {
"Element Name": [
"Thruster Cluster Assembly",
"Propellant Isolation Assembly",
"Spacecraft",
"Thruster-1",
"LV-3",
"ME",
"Small Thruster",
"Latch Valve",
"Main Engine",
derived_A_lv3,
derived_lv3,
],
"ID": ["_{0}".format(num) for num in range(100, 111)],
}
change_data = {
"Component": [
"Thruster Cluster Assembly",
"Propellant Isolation Assembly",
"Space Ship",
],
"Position": ["Thruster-1", "SV-5", "ME"],
"Part": ["Big Thruster", "Solenoid Valve", "Main Engine"],
}
change_renm_data = {"new name": ["SV-5"], "old name": ["LV-3"]}
eval = manager.evaluators[0]
eval1 = manager.evaluators[-1]
eval.df = pd.DataFrame(data=orig_data)
eval.df_ids = pd.DataFrame(data=orig_ids)
eval.rename_df_columns()
eval.add_missing_columns()
eval.to_property_di_graph()
pdg = eval.prop_di_graph
eval1.df = pd.DataFrame(data=change_data)
eval1.df_ids = pd.DataFrame(data=orig_ids)
eval1.df_renames = pd.DataFrame(data=change_renm_data)
eval1.df_renames.set_index("new name", inplace=True)
eval1.rename_df_columns()
eval1.add_missing_columns()
eval1.to_property_di_graph()
pdg1 = eval1.prop_di_graph
add_edge = DiEdge(
source=Vertex(name="b", id="200"),
target=Vertex(name="c", id="201"),
edge_attribute="orange",
)
del_edge = DiEdge(
source=Vertex(name="song"),
target=Vertex(name="tiger"),
edge_attribute="blue",
)
eval_1_e_dict = pdg.edge_dict
eval_1_e_dict.update({del_edge.named_edge_triple: del_edge})
eval_2_e_dict = pdg1.edge_dict
eval_2_e_dict.update({add_edge.named_edge_triple: add_edge})
edge_set_one = eval.edge_set # get baseline edge set
edge_set_one.add(del_edge)
edge_set_two = eval1.edge_set # get the changed edge set
edge_set_two.add(add_edge)
# remove common edges
# have to do this with named edges.
edge_set_one_set = {edge.named_edge_triple for edge in edge_set_one}
edge_set_two_set = {edge.named_edge_triple for edge in edge_set_two}
# Remove edges common to each but preserve set integrity for
# each evaluator
eval_one_unmatched_named = list(
edge_set_one_set.difference(edge_set_two_set))
eval_two_unmatched_named = list(
edge_set_two_set.difference(edge_set_one_set))
# Organize edges in dictionary based on type (this goes on for
# multiple lines)
eval_one_unmatched = [
eval_1_e_dict[edge] for edge in eval_one_unmatched_named
]
eval_two_unmatched = [
eval_2_e_dict[edge] for edge in eval_two_unmatched_named
]
eval_one_unmatch_map = dict(
(edge.edge_attribute, list()) for edge in eval_one_unmatched)
eval_two_unmatch_map = dict(
(edge.edge_attribute, list()) for edge in eval_two_unmatched)
for edge in eval_one_unmatched:
eval_one_unmatch_map[edge.edge_attribute].append(edge)
for edge in eval_two_unmatched:
eval_two_unmatch_map[edge.edge_attribute].append(edge)
eval_one_unmatch_pref = {}
eval_two_unmatch_pref = {}
ance_keys_not_in_base = set(eval_two_unmatch_map.keys()).difference(
set(eval_one_unmatch_map))
eval_one_unmatch_pref["Added"] = []
eval_one_unmatch_pref["Deleted"] = []
for edge_type in ance_keys_not_in_base:
eval_one_unmatch_pref["Added"].extend(
eval_two_unmatch_map[edge_type])
for edge in eval_one_unmatched:
if edge.edge_attribute not in eval_two_unmatch_map.keys():
eval_one_unmatch_pref["Deleted"].append(edge)
else:
eval_one_unmatch_pref[edge] = copy(
eval_two_unmatch_map[edge.edge_attribute])
for edge in eval_two_unmatched:
if edge.edge_attribute not in eval_one_unmatch_map.keys():
eval_two_unmatch_pref[edge] = []
else:
eval_two_unmatch_pref[edge] = copy(
eval_one_unmatch_map[edge.edge_attribute])
match_dict = match_changes(change_dict=eval_one_unmatch_pref)
orig = [
(
"A_propellant isolation assembly qua lv-3 context_lv-3",
"propellant isolation assembly qua lv-3 context",
"memberEnd",
),
("LV-3", "Propellant Isolation Assembly", "owner"),
(
"propellant isolation assembly qua lv-3 context",
"A_propellant isolation assembly qua lv-3 context_lv-3",
"owner",
),
("A_spacecraft qua me context_me", "ME", "memberEnd"),
("ME", "Spacecraft", "owner"),
("Thruster-1", "Small Thruster", "type"),
(
"propellant isolation assembly qua lv-3 context",
"Propellant Isolation Assembly",
"type",
),
("LV-3", "Latch Valve", "type"),
(
"A_propellant isolation assembly qua lv-3 context_lv-3",
"LV-3",
"memberEnd",
),
]
change = [
(
"A_propellant isolation assembly qua sv-5 context_sv-5",
"propellant isolation assembly qua sv-5 context",
"memberEnd",
),
("SV-5", "Propellant Isolation Assembly", "owner"),
(
"propellant isolation assembly qua sv-5 context",
"A_propellant isolation assembly qua sv-5 context_sv-5",
"owner",
),
("A_space ship qua me context_me", "ME", "memberEnd"),
("ME", "Space Ship", "owner"),
("Thruster-1", "Big Thruster", "type"),
(
"propellant isolation assembly qua sv-5 context",
"Propellant Isolation Assembly",
"type",
),
("SV-5", "Solenoid Valve", "type"),
(
"A_propellant isolation assembly qua sv-5 context_sv-5",
"SV-5",
"memberEnd",
),
]
expected_matches = {z[0]: z[1] for z in zip(orig, change)}
expected_matches.update({
"Added": [("b", "c", "orange")],
"Deleted": [("song", "tiger", "blue")],
})
expected_unstable = {
("s1", "t1", "type"): [
("as1", "t1", "type"),
("s1", "at1", "type"),
]
}
pairings = match_dict[0]
unstable_pairs = match_dict[1]
pairings_str = {}
pairings_str.update({"Deleted": []})
pairings_str.update({"Added": []})
unstable_keys = set(unstable_pairs.keys()).intersection(
set(pairings.keys()))
for key in pairings.keys():
if key in unstable_keys:
continue
elif key not in ("Deleted", "Added"):
pairings_str.update({
key.named_edge_triple:
pairings[key][0].named_edge_triple
})
else:
for edge in pairings[key]:
pairings_str[key].append(edge.named_edge_triple)
self.assertDictEqual(expected_matches, pairings_str)
for key in unstable_keys:
unstable_key_vals = {
edge.named_edge_triple
for edge in unstable_pairs[key]
}
self.assertEqual(
set(expected_unstable[key.named_edge_triple]),
unstable_key_vals,
)
def test_get_pattern_graph_diff(self):
manager = Manager(
excel_path=[
DATA_DIRECTORY / "Composition Example.xlsx" for i in range(2)
],
json_path=[PATTERNS / "Composition.json"],
)
# Create the actual graph object because get_pattern_graph_diff
# employs the graph object properties
# with 2 different original edges of the same type I can induce a
# match based on rename and an unstable pair.
og_eval = manager.evaluators[0]
og_graph = PropertyDiGraph()
og_eval.prop_di_graph = og_graph
ch_eval = manager.evaluators[1]
ch_graph = PropertyDiGraph()
ch_eval.prop_di_graph = ch_graph
with tempfile.TemporaryDirectory() as tmpdir:
tmpdir = Path(tmpdir)
orig_edge = DiEdge(
source=Vertex(name="Car", id="_001"),
target=Vertex(name="car", id="_002"),
edge_attribute="type",
)
renm_source = DiEdge(
source=Vertex(
name="Subaru",
id="_001",
original_name="Car",
original_id="_001",
node_types=["Atomic Thing"],
),
target=Vertex(name="car", id="_002"),
edge_attribute="type",
)
orig_edge2 = DiEdge(
source=Vertex(name="Car", id="_001"),
target=Vertex(name="Vehicle", id="_003"),
edge_attribute="type",
)
unstab_edge1 = DiEdge(
source=Vertex(name="Car", id="_001"),
target=Vertex(name="Not Car", id="_100"),
edge_attribute="type",
)
unstab_edge2 = DiEdge(
source=Vertex(name="Cup", id="_101"),
target=Vertex(name="Vehicle", id="_003"),
edge_attribute="type",
)
added_edge = DiEdge(
source=Vertex(
name="New Source",
id=uuid.uuid4(),
node_types=["Atomic Thing"],
),
target=Vertex(
name="New Target",
id=uuid.uuid4(),
node_types=["Atomic Thing"],
),
edge_attribute="newEdge",
)
del_edge = DiEdge(
source=Vertex(name="Old Source", id="_010"),
target=Vertex(name="Old Target", id="_011"),
edge_attribute="oldEdge",
)
original_edges = [orig_edge, orig_edge2, del_edge]
change_edge = [
renm_source,
unstab_edge1,
unstab_edge2,
added_edge,
]
orig_attrs = [{
"diedge": edge,
"edge_attribute": edge.edge_attribute
} for edge in original_edges]
change_attrs = [{
"diedge": edge,
"edge_attribute": edge.edge_attribute
} for edge in change_edge]
for edge in zip(original_edges, orig_attrs):
og_graph.add_node(edge[0].source.name,
**{edge[0].source.name: edge[0].source})
og_graph.add_node(edge[0].target.name,
**{edge[0].target.name: edge[0].target})
og_graph.add_edge(edge[0].source.name, edge[0].target.name,
**edge[1])
for edge in zip(change_edge, change_attrs):
ch_graph.add_node(edge[0].source.name,
**{edge[0].source.name: edge[0].source})
ch_graph.add_node(edge[0].target.name,
**{edge[0].target.name: edge[0].target})
ch_graph.add_edge(edge[0].source.name, edge[0].target.name,
**edge[1])
ch_dict = manager.get_pattern_graph_diff(out_directory=tmpdir)
ch_dict = ch_dict["0-1"]
changes = ch_dict["Changes"]
add = changes["Added"] # a list
deld = changes["Deleted"] # a list
unstab = ch_dict["Unstable Pairs"] # DiEdge: [DiEdge ...]
unstab[orig_edge2] = set(unstab[orig_edge2])
change = changes[orig_edge]
assert change[0] == renm_source
# TODO: Find new edges if type is not found in original and if
# the edge is composed of at least one new model element.
assert add == [added_edge, added_edge]
assert deld == [del_edge]
assert unstab == {
orig_edge2: {unstab_edge1, renm_source, unstab_edge2}
}
def test_graph_difference_to_json(self):
manager = Manager(
excel_path=[
DATA_DIRECTORY / "Composition Example.xlsx" for i in range(2)
],
json_path=[PATTERNS / "Composition.json"],
)
tr = manager.translator[0]
with tempfile.TemporaryDirectory() as tmpdir:
tmpdir = Path(tmpdir)
orig_edge = DiEdge(
source=Vertex(name="Car", id="_001"),
target=Vertex(name="car", id="_002"),
edge_attribute="type",
)
renm_source = DiEdge(
source=Vertex(
name="Subaru",
id="_001",
original_name="Car",
original_id="_001",
node_types=["Atomic Thing"],
),
target=Vertex(name="car", id="_002"),
edge_attribute="type",
)
orig_edge2 = DiEdge(
source=Vertex(name="Car", id="_001"),
target=Vertex(name="Vehicle", id="_003"),
edge_attribute="type",
)
renm_target = DiEdge(
source=Vertex(name="Car", id="_001"),
target=Vertex(
name="vehicle",
id="_003",
original_name="Vehicle",
original_id="_003",
node_types=["Composite Thing"],
),
edge_attribute="type",
)
orig_edge3 = DiEdge(
source=Vertex(name="subaru", id="_004"),
target=Vertex(name="Vehicle", id="_005"),
edge_attribute="type",
)
renm_both = DiEdge(
source=Vertex(
name="Subaru",
id="_004",
original_name="subaru",
original_id="_004",
node_types=["composite owner"],
),
target=Vertex(
name="vehicle",
id="_005",
original_name="Vehicle",
original_id="_005",
node_types=["Atomic Thing"],
),
edge_attribute="type",
)
orig_edge4 = DiEdge(
source=Vertex(name="subaru", id="_004"),
target=Vertex(name="car", id="_002"),
edge_attribute="type",
)
new_source = DiEdge(
source=Vertex(
name="Subaru",
id=uuid.uuid4(),
node_types=["Composite Thing"],
),
target=Vertex(name="car", id="_002"),
edge_attribute="type",
)
orig_edge5 = DiEdge(
source=Vertex(name="Car", id="_001"),
target=Vertex(name="Vehicle", id="_005"),
edge_attribute="type",
)
new_target = DiEdge(
source=Vertex(name="Car", id="_001"),
target=Vertex(
name="vehicle",
id=uuid.uuid4(),
node_types=["Atomic Thing"],
),
edge_attribute="type",
)
orig_edge6 = DiEdge(
source=Vertex(name="Car", id="_007"),
target=Vertex(name="Vehicle", id="_005"),
edge_attribute="type",
)
new_sub = Vertex(name="Subaru",
id=uuid.uuid4(),
node_types=["Composite Thing"])
sub_cons = {
"successors": [{
"source": "Subaru",
"target": "Car",
"edge_attribute": "type",
}]
}
new_sub.successors = sub_cons
new_both = DiEdge(
source=Vertex(
name="Subaru",
id=uuid.uuid4(),
node_types=["Composite Thing"],
),
target=Vertex(
name="vehicle",
id=uuid.uuid4(),
node_types=["Atomic Thing"],
),
edge_attribute="type",
)
added_edge = DiEdge(
source=Vertex(
name="New Source",
id=uuid.uuid4(),
node_types=["Atomic Thing"],
),
target=Vertex(
name="New Target",
id=uuid.uuid4(),
node_types=["Atomic Thing"],
),
edge_attribute="newEdge",
)
del_edge = DiEdge(
source=Vertex(name="Old Source", id="_010"),
target=Vertex(name="Old Target", id="_011"),
edge_attribute="oldEdge",
)
change_dict = {
orig_edge: [renm_source],
orig_edge2: [renm_target],
orig_edge3: [renm_both],
orig_edge4: [new_source],
orig_edge5: [new_target],
orig_edge6: [new_both],
"Added": [added_edge],
"Deleted": [del_edge],
}
changes = manager.graph_difference_to_json(
change_dict=change_dict,
evaluators="0-1",
translator=tr,
out_directory=tmpdir,
)
rename = 0
replace = 0
create = 0
delete = 0
fall_through_ops = []
for item in changes:
op = item["ops"][0]["op"]
if op == "create":
create += 1
elif op == "replace":
replace += 1
elif op == "rename":
rename += 1
elif op == "delete":
delete += 1
else:
fall_through_ops.append(op)
# expect 4 node Renames
# expect 7 edge replaces (1 is from add edge)
# expect 6 node creates
# expect 1 delete
assert (rename == 4 and replace == 7 and create == 6
and delete == 1)
assert not fall_through_ops
def test_changes_to_excel(self):
manager = Manager(
excel_path=[
DATA_DIRECTORY / "Composition Example.xlsx" for i in range(1)
],
json_path=[PATTERNS / "Composition.json"],
)
og_edge = DiEdge(
source=Vertex(name="green"),
target=Vertex(name="apple"),
edge_attribute="fruit",
)
change_edge = DiEdge(
source=Vertex(name="gala"),
target=Vertex(name="apple"),
edge_attribute="fruit",
)
added_edge = DiEdge(
source=Vertex(name="blueberry"),
target=Vertex(name="berry"),
edge_attribute="bush",
)
deleted_edge = DiEdge(
source=Vertex(name="yellow"),
target=Vertex(name="delicious"),
edge_attribute="apple",
)
unstable_key = DiEdge(
source=Vertex(name="tomato"),
target=Vertex(name="fruit"),
edge_attribute="fruit",
)
unstable_one = DiEdge(
source=Vertex(name="tomato"),
target=Vertex(name="vegetable"),
edge_attribute="fruit",
)
unstable_two = DiEdge(
source=Vertex(name="tomahto"),
target=Vertex(name="fruit"),
edge_attribute="fruit",
)
fake_datas = {
"0-1": {
"Changes": {
"Added": [added_edge],
"Deleted": [deleted_edge],
og_edge: [change_edge],
},
"Unstable Pairs": {
unstable_key: [unstable_one, unstable_two]
},
}
}
manager.evaluator_change_dict = fake_datas
with tempfile.TemporaryDirectory() as tmpdir:
outdir = Path(tmpdir)
manager.changes_to_excel(out_directory=outdir)
created_file_name = list(outdir.glob("*.xlsx"))[0]
created_file = OUTPUT_DIRECTORY / created_file_name
created_df = pd.read_excel(created_file)
created_dict = created_df.to_dict()
expected_data = {
"Edit 1": ["('green', 'apple', 'fruit')"],
"Edit 2": ["('gala', 'apple', 'fruit')"],
"Unstable Matches Original": [
"('tomato', 'fruit', 'fruit')",
"('tomato', 'fruit', 'fruit')",
],
"Unstable Matches Change": [
"('tomato', 'vegetable', 'fruit')",
"('tomahto', 'fruit', 'fruit')",
],
"Added": ["('blueberry', 'berry', 'bush')"],
"Deleted": ["('yellow', 'delicious', 'apple')"],
}
expected_df = pd.DataFrame(
data=dict([(k, pd.Series(v))
for k, v in expected_data.items()]))
expected_dict = expected_df.to_dict()
self.assertDictEqual(expected_dict, created_dict)
self.assertTrue(expected_df.equals(created_df))