def TrimGraphToAllowedActions(root_node: ActionNode,
allowed_actions: Dict[str, Action]) -> None:
new_children = {}
for child in root_node.children.values():
if child.action.name in allowed_actions:
new_children[child.action.name] = child
root_node.children = new_children
new_state_check_actions = {}
for action in root_node.state_check_actions.values():
if action.name in allowed_actions:
new_state_check_actions[action.name] = action
root_node.state_check_actions = new_state_check_actions
for child in root_node.children.values():
TrimGraphToAllowedActions(child, allowed_actions)
def PruneNonKeepNodes(node: ActionNode):
children = {}
for child in node.children.values():
if child.keep:
children[child.action.name] = child
PruneNonKeepNodes(child)
node.children = children
return node
def CreateFullCoverageActionGraph(root_node: ActionNode,
tests: List[CoverageTest]) -> ActionNode:
assert isinstance(root_node, ActionNode)
assert isinstance(tests, list)
for test in tests:
assert isinstance(test, CoverageTest)
parent = root_node
for action in test.actions:
assert isinstance(action, Action)
assert parent is not None
if action.is_state_check:
parent.AddStateCheckAction(action, ActionCoverage.FULL)
else:
node = None
if action.name in parent.children:
node = parent.children[action.name]
else:
node = ActionNode(action)
parent.AddChild(node)
node.coverage = ActionCoverage.FULL
node.full_coverage_tests.add(test)
parent = node
def AddActionListToGraph(parent: ActionNode, current: ActionNode,
action_list: List[Action], end: ActionNode,
partial_tests: Set[CoverageTest]) -> None:
nonlocal root_node
assert isinstance(parent, ActionNode)
assert isinstance(end, ActionNode)
assert len(action_list) > 0
only_state_check_actions = True
current.AssertValidity()
end.AssertValidity()
for action in action_list:
assert isinstance(action, Action)
if action.is_state_check:
parent.AddStateCheckAction(action, ActionCoverage.PARTIAL)
parent.partial_coverage_tests.update(partial_tests)
continue
only_state_check_actions = False
if action.name in parent.children:
current = parent.children[action.name]
else:
current = ActionNode(action)
current.coverage = ActionCoverage.PARTIAL
parent.AddChild(current)
current.partial_coverage_tests.update(partial_tests)
parent = current
if current == end:
return
if only_state_check_actions:
# If only state_check actions were added, then no need to add a new
# edge.
return
logging.info("Merging children of\n{" + parent.GetGraphPathStr() +
"}\n{" + end.GetGraphPathStr() + "}")
if current.action.name is end.action.name:
MergeNodes(current, end)
else:
MergeChildren(current, end)
root_node.AssertChildrenValidity()
def main():
parser = argparse.ArgumentParser(
description='WebApp Integration Test Analysis CLI Tool')
parser.add_argument('-v',
dest='v',
action='store_true',
help='Include info logging.',
required=False)
script_dir = os.path.dirname(os.path.realpath(__file__))
parser.add_argument(
'--test_dir',
dest='test_dir',
action='store',
help=('Specify a directory to find all required files, instead of ' +
'specifying each file individually. Overrides those options.'),
required=False)
parser.add_argument(
'--coverage_required',
dest='coverage_required',
action='store',
default=(script_dir + '/data/coverage_required.csv'),
help=(
'Test list csv file, which lists all integration tests that would '
+ 'give the required full coverage of the system. The first two ' +
'lines are skipped.'),
required=False)
parser.add_argument('--coverage_test_row',
dest='coverage_test_row',
action='append',
help='Individually select a coverage test row.',
required=False)
parser.add_argument(
'--partial_coverage_paths',
dest='partial_coverage_paths',
action='store',
default=(script_dir + '/data/partial_coverage_paths.csv'),
help=('File with path replacement descriptions to create partial ' +
'coverage paths in the tree.'),
required=False)
parser.add_argument('--actions',
dest='actions',
action='store',
default=(script_dir + '/data/actions.csv'),
help='Actions csv file, defining all actions.',
required=False)
parser.add_argument('--framework_actions',
dest='framework_actions',
default=(script_dir +
'/data/framework_actions_linux.csv'),
help=('Framework actions csv file, enumerating ' +
'all actions supported by the framework'),
action='store',
required=False)
parser.add_argument(
'--tests',
dest='tests',
action='append',
help=(
'Test csv files, enumerating all existing tests for coverage ' +
'calculations. First line is skipped, and first column is skipped.'
),
required=False)
subparsers = parser.add_subparsers(dest="cmd", required=True)
subparsers.add_parser('list_actions')
subparsers.add_parser('list_coverage_tests')
subparsers.add_parser('list_tests')
subparsers.add_parser('list_partial_paths')
subparsers.add_parser('coverage_required_graph')
framework_parse = subparsers.add_parser('generate_framework_tests')
framework_parse.add_argument('--graph_framework_tests',
dest='graph_framework_tests',
default=False,
action='store_true')
coverage_parse = subparsers.add_parser('generate_test_coverage')
coverage_parse.add_argument('--graph_test_coverage',
dest='graph_test_coverage',
default=False,
action='store_true')
options = parser.parse_args()
actions_file = options.actions
coverage_required_file = options.coverage_required
partial_paths_file = options.partial_coverage_paths
framework_actions_file = options.framework_actions
if options.test_dir:
actions_file = options.test_dir + "/actions.csv"
coverage_required_file = options.test_dir + "/coverage_required.csv"
partial_paths_file = options.test_dir + "/partial_coverage_paths.csv"
framework_actions_file = (options.test_dir +
"/framework_actions_linux.csv")
logging.basicConfig(level=logging.INFO if options.v else logging.WARN,
format='[%(asctime)s %(levelname)s] %(message)s',
datefmt='%H:%M:%S')
logging.info('Script directory: ' + script_dir)
actions_csv = csv.reader(open(actions_file), delimiter=',')
(actions, action_base_name_to_default_param,
action_base_name_to_all_params) = ReadActionsFile(actions_csv)
if options.cmd == 'list_actions':
for action in actions.values():
print(action)
return
if options.cmd == 'list_tests':
tests = []
for tests_file in options.tests:
tests_csv = csv.reader(open(tests_file), delimiter=',')
tests.extend(
ReadNamedTestsFile(tests_csv, actions,
action_base_name_to_default_param))
for test in tests:
print(test)
return
if options.cmd == 'list_coverage_tests':
coverage_csv = csv.reader(open(coverage_required_file), delimiter=',')
required_coverage_tests = ReadCoverageTestsFile(
coverage_csv, actions, action_base_name_to_default_param)
required_coverage_tests = MaybeFilterCoverageTests(
required_coverage_tests, options.coverage_test_row)
for test in required_coverage_tests:
print(test)
return
if options.cmd == 'list_partial_paths':
partial_csv = csv.reader(open(partial_paths_file), delimiter=',')
partial_paths = ReadPartialCoveragePathsFile(
partial_csv, actions, action_base_name_to_all_params)
for partial_path in partial_paths:
print(partial_path)
return
if options.cmd == 'coverage_required_graph':
coverage_csv = csv.reader(open(coverage_required_file), delimiter=',')
required_coverage_tests = ReadCoverageTestsFile(
coverage_csv, actions, action_base_name_to_default_param)
required_coverage_tests = MaybeFilterCoverageTests(
required_coverage_tests, options.coverage_test_row)
coverage_root_node = ActionNode(Action("root", "root", False))
CreateFullCoverageActionGraph(coverage_root_node,
required_coverage_tests)
partial_csv = csv.reader(open(partial_paths_file), delimiter=',')
partial_paths = ReadPartialCoveragePathsFile(
partial_csv, actions, action_base_name_to_all_params)
AddPartialPaths(coverage_root_node, partial_paths)
graph_file = GenerateGraphvizDotFile(coverage_root_node)
print(graph_file)
return
if options.cmd == 'generate_framework_tests':
coverage_csv = csv.reader(open(coverage_required_file), delimiter=',')
framework_actions_csv = csv.reader(open(framework_actions_file),
delimiter=',')
required_coverage_tests = ReadCoverageTestsFile(
coverage_csv, actions, action_base_name_to_default_param)
required_coverage_tests = MaybeFilterCoverageTests(
required_coverage_tests, options.coverage_test_row)
framework_actions = ReadFrameworkActions(
framework_actions_csv, actions, action_base_name_to_default_param)
coverage_root_node = ActionNode(Action("root", "root", False))
CreateFullCoverageActionGraph(coverage_root_node,
required_coverage_tests)
partial_csv = csv.reader(open(partial_paths_file), delimiter=',')
partial_paths = ReadPartialCoveragePathsFile(
partial_csv, actions, action_base_name_to_all_params)
AddPartialPaths(coverage_root_node, partial_paths)
TrimGraphToAllowedActions(coverage_root_node, framework_actions)
if options.graph_framework_tests:
return GenerateGraphvizDotFile(coverage_root_node)
lines = ["# This is a generated file."]
paths = GenerateFrameworkTests(coverage_root_node,
required_coverage_tests)
for path in paths:
all_actions_in_path = []
for node in path[1:]: # Skip the root node
all_actions_in_path.append(node.action)
all_actions_in_path.extend(node.state_check_actions.values())
lines.append(
"," + ",".join([action.name
for action in all_actions_in_path]))
print("\n".join(lines))
return
if options.cmd == 'generate_test_coverage':
coverage_csv = csv.reader(open(coverage_required_file), delimiter=',')
required_coverage_tests = ReadCoverageTestsFile(
coverage_csv, actions, action_base_name_to_default_param)
required_coverage_tests = MaybeFilterCoverageTests(
required_coverage_tests, options.coverage_test_row)
partial_csv = csv.reader(open(partial_paths_file), delimiter=',')
partial_paths = ReadPartialCoveragePathsFile(
partial_csv, actions, action_base_name_to_all_params)
for path in partial_paths:
path.Reverse()
tests = []
for tests_file in options.tests:
tests_csv = csv.reader(open(tests_file), delimiter=',')
tests.extend(
ReadNamedTestsFile(tests_csv, actions,
action_base_name_to_default_param))
tests_root_node = ActionNode(Action("root", "root", False))
CreateFullCoverageActionGraph(tests_root_node, tests)
AddPartialPaths(tests_root_node, partial_paths)
if options.graph_test_coverage:
print(GenerateGraphvizDotFile(tests_root_node))
return
(coverage_file, _,
_) = GenerateCoverageFileAndPercents(required_coverage_tests,
tests_root_node)
print(coverage_file)
def MergeNodes(node_a: ActionNode, node_b: ActionNode) -> ActionNode:
assert not node_a.dead
assert not node_b.dead
if node_a == node_b:
return node_a
assert node_a.action.name == node_b.action.name
logging.info("Merging nodes with paths:\n{" +
node_a.GetGraphPathStr() + "},\n{" +
node_b.GetGraphPathStr() + "}")
new_node = ActionNode(node_a.action)
new_node.coverage = (ActionCoverage.FULL
if node_a.coverage == ActionCoverage.FULL
or node_b.coverage == ActionCoverage.FULL else
ActionCoverage.PARTIAL)
new_node.state_check_actions = node_a.state_check_actions
new_node.state_check_actions.update(node_b.state_check_actions)
new_node.state_check_actions_coverage = MergeStateCheckCoverage(
node_a.state_check_actions_coverage,
node_b.state_check_actions_coverage)
new_node.full_coverage_tests = node_a.full_coverage_tests
new_node.full_coverage_tests.update(node_b.full_coverage_tests)
new_node.partial_coverage_tests = node_a.partial_coverage_tests
new_node.partial_coverage_tests.update(node_b.partial_coverage_tests)
children_a = node_a.children.copy()
children_b = node_b.children.copy()
parents_a = node_a.parents.copy()
parents_b = node_b.parents.copy()
# Fully remove the merging nodes from the graph
for child in list(children_a.values()) + list(children_b.values()):
if new_node.action.name in child.parents:
del child.parents[new_node.action.name]
for parent in list(parents_a.values()) + list(parents_b.values()):
if new_node.action.name in parent.children:
del parent.children[new_node.action.name]
# Merge children.
# Start by adding the non-intersecting children to the dictionary.
children = ({
k: children_a.get(k, children_b.get(k)).ResolveToAliveNode()
for k in (children_a.keys() ^ children_b.keys())
})
# For all children that are the same, they must be merged.
children.update({
k: MergeNodes(children_a[k].ResolveToAliveNode(),
children_b[k].ResolveToAliveNode())
for k in (children_a.keys() & children_b.keys())
})
# Merge parents.
# Start by adding the non-intersecting parents to the dictionary.
parents = ({
k: parents_a.get(k, parents_b.get(k)).ResolveToAliveNode()
for k in (parents_a.keys() ^ parents_b.keys())
})
# For all parents that are the same, they must be merged.
parents.update({
k: MergeNodes(parents_a[k].ResolveToAliveNode(),
parents_b[k].ResolveToAliveNode())
for k in (parents_a.keys() & parents_b.keys())
})
# Re-add the node back into the graph.
for child in children.values():
child = child.ResolveToAliveNode()
new_node.AddChild(child)
for parent in parents.values():
parent = parent.ResolveToAliveNode()
parent.AddChild(new_node)
node_a.dead = new_node
node_b.dead = new_node
new_node.AssertValidity()
return new_node
def GenerateFrameworkTests(
root_node: ActionNode,
tests: List[CoverageTest]) -> List[List[ActionNode]]:
assert isinstance(root_node, ActionNode)
def GetAllPaths(node: ActionNode) -> List[List[ActionNode]]:
assert node is not None
assert isinstance(node, ActionNode)
paths = []
for child in node.children.values():
for path in GetAllPaths(child):
assert path is not None
assert isinstance(path, list)
assert bool(path)
paths.append([node] + path)
if len(paths) == 0:
paths = [[node]]
return paths
def FindLongestFullCoveragePath(paths: List[List[ActionNode]],
test: CoverageTest):
best = None
best_length = 0
for path in paths:
assert isinstance(path, list)
len = 0
for node in path:
if test in node.full_coverage_tests:
len += 100
elif test in node.partial_coverage_tests:
len += 1
if len > best_length:
best_length = len
best = path
return best
def PruneNonKeepNodes(node: ActionNode):
children = {}
for child in node.children.values():
if child.keep:
children[child.action.name] = child
PruneNonKeepNodes(child)
node.children = children
return node
if len(tests) == 0:
return []
all_paths = GetAllPaths(root_node)
root_node.keep = True
for test in tests:
best = FindLongestFullCoveragePath(all_paths, test)
if best is None:
continue
if (logging.getLogger().isEnabledFor(logging.INFO)):
path_str = ", ".join([node.action.name for node in best])
logging.info(f"Best path for test {test.name}:\n{path_str}")
for node in best:
node.keep = True
PruneNonKeepNodes(root_node)
return GetAllPaths(root_node)
def MergeChildren(node_a: ActionNode, node_b: ActionNode) -> None:
assert not node_a.dead
assert not node_b.dead
assert node_a.action.name is not node_b.action.name
node_a.AssertValidity()
node_b.AssertValidity()
node_a_name = node_a.action.name
node_b_name = node_b.action.name
children_a = list(node_a.children.copy().values())
children_b = list(node_b.children.copy().values())
for child in children_a + children_b:
child = child.ResolveToAliveNode()
child.AssertValidity()
child_name = child.action.name
if (child_name in node_a.children
and child_name in node_b.children):
MergeNodes(node_a.children[child_name],
node_b.children[child_name])
elif child_name in node_a.children:
if (node_b_name in child.parents):
node_b = MergeNodes(node_b, child.parents[node_b_name])
else:
node_b.AddChild(child)
elif child_name in node_b.children:
if (node_a_name in child.parents):
node_a = MergeNodes(node_a, child.parents[node_a_name])
else:
node_a.AddChild(child)
else:
assert False
node_a = node_a.ResolveToAliveNode()
node_b = node_b.ResolveToAliveNode()
resolved_children = ({
k: node_a.children.get(k).ResolveToAliveNode()
for k in node_a.children.keys()
})
node_a.children = resolved_children.copy()
node_b.children = resolved_children
merged_state_check_coverage = MergeStateCheckCoverage(
node_a.state_check_actions_coverage,
node_b.state_check_actions_coverage)
node_a.state_check_actions.update(node_b.state_check_actions)
node_a.state_check_actions_coverage = merged_state_check_coverage
node_b.state_check_actions.update(node_a.state_check_actions)
node_b.state_check_actions_coverage = merged_state_check_coverage.copy(
)
node_a.AssertValidity()
node_b.AssertValidity()
def main():
parser = argparse.ArgumentParser(description='WebApp Test List Processor')
parser.add_argument('-v',
dest='v',
action='store_true',
help='Include info logging.',
required=False)
parser.add_argument('--graphs',
dest='graphs',
action='store_true',
help='Output dot graphs from all steps.',
required=False)
parser.add_argument(
'--ignore_audited',
dest='ignore_audited',
action='store_true',
help='Ignore the audited manual and automated test data',
required=False)
options = parser.parse_args()
logging.basicConfig(level=logging.INFO if options.v else logging.WARN,
format='[%(asctime)s %(levelname)s] %(message)s',
datefmt='%H:%M:%S')
script_dir = os.path.dirname(os.path.realpath(__file__))
output_dir = script_dir + "/output"
actions_file = script_dir + "/data/actions.csv"
coverage_required_file = script_dir + "/data/coverage_required.csv"
partial_coverage_file = script_dir + "/data/partial_coverage_paths.csv"
audited_manual_tests_file = script_dir + "/data/manual_tests.csv"
audited_automated_tests_file = script_dir + "/data/automated_tests.csv"
partial_coverage_file = script_dir + "/data/partial_coverage_paths.csv"
framework_actions_file_base = script_dir + "/data/framework_actions_"
actions_csv = csv.reader(open(actions_file), delimiter=',')
coverage_csv = csv.reader(open(coverage_required_file), delimiter=',')
partial_csv = csv.reader(open(partial_coverage_file), delimiter=',')
(actions, action_base_name_to_default_param,
action_base_name_to_all_params) = ReadActionsFile(actions_csv)
required_coverage_tests = ReadCoverageTestsFile(
coverage_csv, actions, action_base_name_to_default_param)
partial_paths = ReadPartialCoveragePathsFile(
partial_csv, actions, action_base_name_to_all_params)
partial_csv = csv.reader(open(partial_coverage_file), delimiter=',')
reversed_partial_paths = ReadPartialCoveragePathsFile(
partial_csv, actions, action_base_name_to_all_params)
for path in reversed_partial_paths:
path.Reverse()
if options.graphs:
coverage_root_node = ActionNode(Action("root", "root", False))
CreateFullCoverageActionGraph(coverage_root_node,
required_coverage_tests)
AddPartialPaths(coverage_root_node, partial_paths)
coverage_graph = GenerateGraphvizDotFile(coverage_root_node)
output_coverage_graph_file_name = (output_dir +
"/coverage_required_graph.dot")
coverage_graph_file = open(output_coverage_graph_file_name, 'w')
coverage_graph_file.write(GENERATED_FILE_HEADER + "\n")
coverage_graph_file.write(coverage_graph)
coverage_graph_file.close()
for platform in ['linux', 'mac', 'cros', 'win']:
framework_actions_csv = csv.reader(open(framework_actions_file_base +
platform + ".csv"),
delimiter=',')
framework_actions = ReadFrameworkActions(
framework_actions_csv, actions, action_base_name_to_default_param)
# Create the coverage graph, then prune all unsupported actions for
# this platform.
coverage_root_node = ActionNode(Action("root", "root", False))
CreateFullCoverageActionGraph(coverage_root_node,
required_coverage_tests)
AddPartialPaths(coverage_root_node, partial_paths)
TrimGraphToAllowedActions(coverage_root_node, framework_actions)
if options.graphs:
graph = GenerateGraphvizDotFile(coverage_root_node)
output_graph_file_name = (output_dir + "/framework_test_graph_" +
platform + ".dot")
graph_file = open(output_graph_file_name, 'w')
graph_file.write(GENERATED_FILE_HEADER + "\n")
graph_file.write(graph)
graph_file.close()
# Write the framework tests. All tests that involve 'sync' actions must
# be in a separate file.
output_tests_file_name = (output_dir + "/framework_tests_" + platform +
".csv")
output_tests_sync_file_name = (output_dir + "/framework_tests_sync_" +
platform + ".csv")
framework_file = open(output_tests_file_name, 'w')
framework_sync_file = open(output_tests_sync_file_name, 'w')
framework_file.write(GENERATED_FILE_HEADER + "\n")
framework_sync_file.write(GENERATED_FILE_HEADER + "\n")
paths = GenerateFrameworkTests(coverage_root_node,
required_coverage_tests)
for path in paths:
all_actions_in_path = []
for node in path[1:]: # Skip the root node
all_actions_in_path.append(node.action)
all_actions_in_path.extend(node.state_check_actions.values())
action_names = [action.name for action in all_actions_in_path]
file = framework_file
for action_name in action_names:
if action_name in SYNC_ACTIONS:
file = framework_sync_file
break
file.write("," + ",".join(action_names) + "\n")
framework_file.close()
framework_sync_file.close()
# Analyze all tests to generate coverage
test_files = [output_tests_file_name, output_tests_sync_file_name]
if not options.ignore_audited:
test_files.extend(
[audited_automated_tests_file, audited_manual_tests_file])
tests = []
for tests_file in test_files:
tests_csv = csv.reader(open(tests_file), delimiter=',')
tests.extend(
ReadNamedTestsFile(tests_csv, actions,
action_base_name_to_default_param))
tests_root_node = ActionNode(Action("root", "root", False))
CreateFullCoverageActionGraph(tests_root_node, tests)
AddPartialPaths(tests_root_node, reversed_partial_paths)
(coverage_file_text, full_coverage,
partial_coverage) = GenerateCoverageFileAndPercents(
required_coverage_tests, tests_root_node)
coverage_table_file = output_dir + "/coverage_" + platform + ".tsv"
coverage_file = open(coverage_table_file, 'w')
coverage_file.write(GENERATED_FILE_HEADER + "\n")
coverage_file.write(
"# Full Coverage: {:.2f}, Partial Coverage: {:.2f}\n".format(
full_coverage, partial_coverage))
coverage_file.write(coverage_file_text)
coverage_file.close()
if options.graphs:
coverage_graph = GenerateGraphvizDotFile(tests_root_node)
coverage_graph_filename = (output_dir + "/coverage_graph_" +
platform + ".dot")
coverage_graph_file = open(coverage_graph_filename, 'w')
coverage_graph_file.write(GENERATED_FILE_HEADER + "\n")
for graph_line in coverage_graph:
coverage_graph_file.write(graph_line + "\n")
coverage_graph_file.close()