def TrimGraphToAllowedActions(root_node: ActionNode, allowed_actions: Dict[str, Action]) -> None: new_children = {} for child in root_node.children.values(): if child.action.name in allowed_actions: new_children[child.action.name] = child root_node.children = new_children new_state_check_actions = {} for action in root_node.state_check_actions.values(): if action.name in allowed_actions: new_state_check_actions[action.name] = action root_node.state_check_actions = new_state_check_actions for child in root_node.children.values(): TrimGraphToAllowedActions(child, allowed_actions)
def PruneNonKeepNodes(node: ActionNode): children = {} for child in node.children.values(): if child.keep: children[child.action.name] = child PruneNonKeepNodes(child) node.children = children return node
def CreateFullCoverageActionGraph(root_node: ActionNode, tests: List[CoverageTest]) -> ActionNode: assert isinstance(root_node, ActionNode) assert isinstance(tests, list) for test in tests: assert isinstance(test, CoverageTest) parent = root_node for action in test.actions: assert isinstance(action, Action) assert parent is not None if action.is_state_check: parent.AddStateCheckAction(action, ActionCoverage.FULL) else: node = None if action.name in parent.children: node = parent.children[action.name] else: node = ActionNode(action) parent.AddChild(node) node.coverage = ActionCoverage.FULL node.full_coverage_tests.add(test) parent = node
def AddActionListToGraph(parent: ActionNode, current: ActionNode, action_list: List[Action], end: ActionNode, partial_tests: Set[CoverageTest]) -> None: nonlocal root_node assert isinstance(parent, ActionNode) assert isinstance(end, ActionNode) assert len(action_list) > 0 only_state_check_actions = True current.AssertValidity() end.AssertValidity() for action in action_list: assert isinstance(action, Action) if action.is_state_check: parent.AddStateCheckAction(action, ActionCoverage.PARTIAL) parent.partial_coverage_tests.update(partial_tests) continue only_state_check_actions = False if action.name in parent.children: current = parent.children[action.name] else: current = ActionNode(action) current.coverage = ActionCoverage.PARTIAL parent.AddChild(current) current.partial_coverage_tests.update(partial_tests) parent = current if current == end: return if only_state_check_actions: # If only state_check actions were added, then no need to add a new # edge. return logging.info("Merging children of\n{" + parent.GetGraphPathStr() + "}\n{" + end.GetGraphPathStr() + "}") if current.action.name is end.action.name: MergeNodes(current, end) else: MergeChildren(current, end) root_node.AssertChildrenValidity()
def main(): parser = argparse.ArgumentParser( description='WebApp Integration Test Analysis CLI Tool') parser.add_argument('-v', dest='v', action='store_true', help='Include info logging.', required=False) script_dir = os.path.dirname(os.path.realpath(__file__)) parser.add_argument( '--test_dir', dest='test_dir', action='store', help=('Specify a directory to find all required files, instead of ' + 'specifying each file individually. Overrides those options.'), required=False) parser.add_argument( '--coverage_required', dest='coverage_required', action='store', default=(script_dir + '/data/coverage_required.csv'), help=( 'Test list csv file, which lists all integration tests that would ' + 'give the required full coverage of the system. The first two ' + 'lines are skipped.'), required=False) parser.add_argument('--coverage_test_row', dest='coverage_test_row', action='append', help='Individually select a coverage test row.', required=False) parser.add_argument( '--partial_coverage_paths', dest='partial_coverage_paths', action='store', default=(script_dir + '/data/partial_coverage_paths.csv'), help=('File with path replacement descriptions to create partial ' + 'coverage paths in the tree.'), required=False) parser.add_argument('--actions', dest='actions', action='store', default=(script_dir + '/data/actions.csv'), help='Actions csv file, defining all actions.', required=False) parser.add_argument('--framework_actions', dest='framework_actions', default=(script_dir + '/data/framework_actions_linux.csv'), help=('Framework actions csv file, enumerating ' + 'all actions supported by the framework'), action='store', required=False) parser.add_argument( '--tests', dest='tests', action='append', help=( 'Test csv files, enumerating all existing tests for coverage ' + 'calculations. First line is skipped, and first column is skipped.' ), required=False) subparsers = parser.add_subparsers(dest="cmd", required=True) subparsers.add_parser('list_actions') subparsers.add_parser('list_coverage_tests') subparsers.add_parser('list_tests') subparsers.add_parser('list_partial_paths') subparsers.add_parser('coverage_required_graph') framework_parse = subparsers.add_parser('generate_framework_tests') framework_parse.add_argument('--graph_framework_tests', dest='graph_framework_tests', default=False, action='store_true') coverage_parse = subparsers.add_parser('generate_test_coverage') coverage_parse.add_argument('--graph_test_coverage', dest='graph_test_coverage', default=False, action='store_true') options = parser.parse_args() actions_file = options.actions coverage_required_file = options.coverage_required partial_paths_file = options.partial_coverage_paths framework_actions_file = options.framework_actions if options.test_dir: actions_file = options.test_dir + "/actions.csv" coverage_required_file = options.test_dir + "/coverage_required.csv" partial_paths_file = options.test_dir + "/partial_coverage_paths.csv" framework_actions_file = (options.test_dir + "/framework_actions_linux.csv") logging.basicConfig(level=logging.INFO if options.v else logging.WARN, format='[%(asctime)s %(levelname)s] %(message)s', datefmt='%H:%M:%S') logging.info('Script directory: ' + script_dir) actions_csv = csv.reader(open(actions_file), delimiter=',') (actions, action_base_name_to_default_param, action_base_name_to_all_params) = ReadActionsFile(actions_csv) if options.cmd == 'list_actions': for action in actions.values(): print(action) return if options.cmd == 'list_tests': tests = [] for tests_file in options.tests: tests_csv = csv.reader(open(tests_file), delimiter=',') tests.extend( ReadNamedTestsFile(tests_csv, actions, action_base_name_to_default_param)) for test in tests: print(test) return if options.cmd == 'list_coverage_tests': coverage_csv = csv.reader(open(coverage_required_file), delimiter=',') required_coverage_tests = ReadCoverageTestsFile( coverage_csv, actions, action_base_name_to_default_param) required_coverage_tests = MaybeFilterCoverageTests( required_coverage_tests, options.coverage_test_row) for test in required_coverage_tests: print(test) return if options.cmd == 'list_partial_paths': partial_csv = csv.reader(open(partial_paths_file), delimiter=',') partial_paths = ReadPartialCoveragePathsFile( partial_csv, actions, action_base_name_to_all_params) for partial_path in partial_paths: print(partial_path) return if options.cmd == 'coverage_required_graph': coverage_csv = csv.reader(open(coverage_required_file), delimiter=',') required_coverage_tests = ReadCoverageTestsFile( coverage_csv, actions, action_base_name_to_default_param) required_coverage_tests = MaybeFilterCoverageTests( required_coverage_tests, options.coverage_test_row) coverage_root_node = ActionNode(Action("root", "root", False)) CreateFullCoverageActionGraph(coverage_root_node, required_coverage_tests) partial_csv = csv.reader(open(partial_paths_file), delimiter=',') partial_paths = ReadPartialCoveragePathsFile( partial_csv, actions, action_base_name_to_all_params) AddPartialPaths(coverage_root_node, partial_paths) graph_file = GenerateGraphvizDotFile(coverage_root_node) print(graph_file) return if options.cmd == 'generate_framework_tests': coverage_csv = csv.reader(open(coverage_required_file), delimiter=',') framework_actions_csv = csv.reader(open(framework_actions_file), delimiter=',') required_coverage_tests = ReadCoverageTestsFile( coverage_csv, actions, action_base_name_to_default_param) required_coverage_tests = MaybeFilterCoverageTests( required_coverage_tests, options.coverage_test_row) framework_actions = ReadFrameworkActions( framework_actions_csv, actions, action_base_name_to_default_param) coverage_root_node = ActionNode(Action("root", "root", False)) CreateFullCoverageActionGraph(coverage_root_node, required_coverage_tests) partial_csv = csv.reader(open(partial_paths_file), delimiter=',') partial_paths = ReadPartialCoveragePathsFile( partial_csv, actions, action_base_name_to_all_params) AddPartialPaths(coverage_root_node, partial_paths) TrimGraphToAllowedActions(coverage_root_node, framework_actions) if options.graph_framework_tests: return GenerateGraphvizDotFile(coverage_root_node) lines = ["# This is a generated file."] paths = GenerateFrameworkTests(coverage_root_node, required_coverage_tests) for path in paths: all_actions_in_path = [] for node in path[1:]: # Skip the root node all_actions_in_path.append(node.action) all_actions_in_path.extend(node.state_check_actions.values()) lines.append( "," + ",".join([action.name for action in all_actions_in_path])) print("\n".join(lines)) return if options.cmd == 'generate_test_coverage': coverage_csv = csv.reader(open(coverage_required_file), delimiter=',') required_coverage_tests = ReadCoverageTestsFile( coverage_csv, actions, action_base_name_to_default_param) required_coverage_tests = MaybeFilterCoverageTests( required_coverage_tests, options.coverage_test_row) partial_csv = csv.reader(open(partial_paths_file), delimiter=',') partial_paths = ReadPartialCoveragePathsFile( partial_csv, actions, action_base_name_to_all_params) for path in partial_paths: path.Reverse() tests = [] for tests_file in options.tests: tests_csv = csv.reader(open(tests_file), delimiter=',') tests.extend( ReadNamedTestsFile(tests_csv, actions, action_base_name_to_default_param)) tests_root_node = ActionNode(Action("root", "root", False)) CreateFullCoverageActionGraph(tests_root_node, tests) AddPartialPaths(tests_root_node, partial_paths) if options.graph_test_coverage: print(GenerateGraphvizDotFile(tests_root_node)) return (coverage_file, _, _) = GenerateCoverageFileAndPercents(required_coverage_tests, tests_root_node) print(coverage_file)
def MergeNodes(node_a: ActionNode, node_b: ActionNode) -> ActionNode: assert not node_a.dead assert not node_b.dead if node_a == node_b: return node_a assert node_a.action.name == node_b.action.name logging.info("Merging nodes with paths:\n{" + node_a.GetGraphPathStr() + "},\n{" + node_b.GetGraphPathStr() + "}") new_node = ActionNode(node_a.action) new_node.coverage = (ActionCoverage.FULL if node_a.coverage == ActionCoverage.FULL or node_b.coverage == ActionCoverage.FULL else ActionCoverage.PARTIAL) new_node.state_check_actions = node_a.state_check_actions new_node.state_check_actions.update(node_b.state_check_actions) new_node.state_check_actions_coverage = MergeStateCheckCoverage( node_a.state_check_actions_coverage, node_b.state_check_actions_coverage) new_node.full_coverage_tests = node_a.full_coverage_tests new_node.full_coverage_tests.update(node_b.full_coverage_tests) new_node.partial_coverage_tests = node_a.partial_coverage_tests new_node.partial_coverage_tests.update(node_b.partial_coverage_tests) children_a = node_a.children.copy() children_b = node_b.children.copy() parents_a = node_a.parents.copy() parents_b = node_b.parents.copy() # Fully remove the merging nodes from the graph for child in list(children_a.values()) + list(children_b.values()): if new_node.action.name in child.parents: del child.parents[new_node.action.name] for parent in list(parents_a.values()) + list(parents_b.values()): if new_node.action.name in parent.children: del parent.children[new_node.action.name] # Merge children. # Start by adding the non-intersecting children to the dictionary. children = ({ k: children_a.get(k, children_b.get(k)).ResolveToAliveNode() for k in (children_a.keys() ^ children_b.keys()) }) # For all children that are the same, they must be merged. children.update({ k: MergeNodes(children_a[k].ResolveToAliveNode(), children_b[k].ResolveToAliveNode()) for k in (children_a.keys() & children_b.keys()) }) # Merge parents. # Start by adding the non-intersecting parents to the dictionary. parents = ({ k: parents_a.get(k, parents_b.get(k)).ResolveToAliveNode() for k in (parents_a.keys() ^ parents_b.keys()) }) # For all parents that are the same, they must be merged. parents.update({ k: MergeNodes(parents_a[k].ResolveToAliveNode(), parents_b[k].ResolveToAliveNode()) for k in (parents_a.keys() & parents_b.keys()) }) # Re-add the node back into the graph. for child in children.values(): child = child.ResolveToAliveNode() new_node.AddChild(child) for parent in parents.values(): parent = parent.ResolveToAliveNode() parent.AddChild(new_node) node_a.dead = new_node node_b.dead = new_node new_node.AssertValidity() return new_node
def GenerateFrameworkTests( root_node: ActionNode, tests: List[CoverageTest]) -> List[List[ActionNode]]: assert isinstance(root_node, ActionNode) def GetAllPaths(node: ActionNode) -> List[List[ActionNode]]: assert node is not None assert isinstance(node, ActionNode) paths = [] for child in node.children.values(): for path in GetAllPaths(child): assert path is not None assert isinstance(path, list) assert bool(path) paths.append([node] + path) if len(paths) == 0: paths = [[node]] return paths def FindLongestFullCoveragePath(paths: List[List[ActionNode]], test: CoverageTest): best = None best_length = 0 for path in paths: assert isinstance(path, list) len = 0 for node in path: if test in node.full_coverage_tests: len += 100 elif test in node.partial_coverage_tests: len += 1 if len > best_length: best_length = len best = path return best def PruneNonKeepNodes(node: ActionNode): children = {} for child in node.children.values(): if child.keep: children[child.action.name] = child PruneNonKeepNodes(child) node.children = children return node if len(tests) == 0: return [] all_paths = GetAllPaths(root_node) root_node.keep = True for test in tests: best = FindLongestFullCoveragePath(all_paths, test) if best is None: continue if (logging.getLogger().isEnabledFor(logging.INFO)): path_str = ", ".join([node.action.name for node in best]) logging.info(f"Best path for test {test.name}:\n{path_str}") for node in best: node.keep = True PruneNonKeepNodes(root_node) return GetAllPaths(root_node)
def MergeChildren(node_a: ActionNode, node_b: ActionNode) -> None: assert not node_a.dead assert not node_b.dead assert node_a.action.name is not node_b.action.name node_a.AssertValidity() node_b.AssertValidity() node_a_name = node_a.action.name node_b_name = node_b.action.name children_a = list(node_a.children.copy().values()) children_b = list(node_b.children.copy().values()) for child in children_a + children_b: child = child.ResolveToAliveNode() child.AssertValidity() child_name = child.action.name if (child_name in node_a.children and child_name in node_b.children): MergeNodes(node_a.children[child_name], node_b.children[child_name]) elif child_name in node_a.children: if (node_b_name in child.parents): node_b = MergeNodes(node_b, child.parents[node_b_name]) else: node_b.AddChild(child) elif child_name in node_b.children: if (node_a_name in child.parents): node_a = MergeNodes(node_a, child.parents[node_a_name]) else: node_a.AddChild(child) else: assert False node_a = node_a.ResolveToAliveNode() node_b = node_b.ResolveToAliveNode() resolved_children = ({ k: node_a.children.get(k).ResolveToAliveNode() for k in node_a.children.keys() }) node_a.children = resolved_children.copy() node_b.children = resolved_children merged_state_check_coverage = MergeStateCheckCoverage( node_a.state_check_actions_coverage, node_b.state_check_actions_coverage) node_a.state_check_actions.update(node_b.state_check_actions) node_a.state_check_actions_coverage = merged_state_check_coverage node_b.state_check_actions.update(node_a.state_check_actions) node_b.state_check_actions_coverage = merged_state_check_coverage.copy( ) node_a.AssertValidity() node_b.AssertValidity()
def main(): parser = argparse.ArgumentParser(description='WebApp Test List Processor') parser.add_argument('-v', dest='v', action='store_true', help='Include info logging.', required=False) parser.add_argument('--graphs', dest='graphs', action='store_true', help='Output dot graphs from all steps.', required=False) parser.add_argument( '--ignore_audited', dest='ignore_audited', action='store_true', help='Ignore the audited manual and automated test data', required=False) options = parser.parse_args() logging.basicConfig(level=logging.INFO if options.v else logging.WARN, format='[%(asctime)s %(levelname)s] %(message)s', datefmt='%H:%M:%S') script_dir = os.path.dirname(os.path.realpath(__file__)) output_dir = script_dir + "/output" actions_file = script_dir + "/data/actions.csv" coverage_required_file = script_dir + "/data/coverage_required.csv" partial_coverage_file = script_dir + "/data/partial_coverage_paths.csv" audited_manual_tests_file = script_dir + "/data/manual_tests.csv" audited_automated_tests_file = script_dir + "/data/automated_tests.csv" partial_coverage_file = script_dir + "/data/partial_coverage_paths.csv" framework_actions_file_base = script_dir + "/data/framework_actions_" actions_csv = csv.reader(open(actions_file), delimiter=',') coverage_csv = csv.reader(open(coverage_required_file), delimiter=',') partial_csv = csv.reader(open(partial_coverage_file), delimiter=',') (actions, action_base_name_to_default_param, action_base_name_to_all_params) = ReadActionsFile(actions_csv) required_coverage_tests = ReadCoverageTestsFile( coverage_csv, actions, action_base_name_to_default_param) partial_paths = ReadPartialCoveragePathsFile( partial_csv, actions, action_base_name_to_all_params) partial_csv = csv.reader(open(partial_coverage_file), delimiter=',') reversed_partial_paths = ReadPartialCoveragePathsFile( partial_csv, actions, action_base_name_to_all_params) for path in reversed_partial_paths: path.Reverse() if options.graphs: coverage_root_node = ActionNode(Action("root", "root", False)) CreateFullCoverageActionGraph(coverage_root_node, required_coverage_tests) AddPartialPaths(coverage_root_node, partial_paths) coverage_graph = GenerateGraphvizDotFile(coverage_root_node) output_coverage_graph_file_name = (output_dir + "/coverage_required_graph.dot") coverage_graph_file = open(output_coverage_graph_file_name, 'w') coverage_graph_file.write(GENERATED_FILE_HEADER + "\n") coverage_graph_file.write(coverage_graph) coverage_graph_file.close() for platform in ['linux', 'mac', 'cros', 'win']: framework_actions_csv = csv.reader(open(framework_actions_file_base + platform + ".csv"), delimiter=',') framework_actions = ReadFrameworkActions( framework_actions_csv, actions, action_base_name_to_default_param) # Create the coverage graph, then prune all unsupported actions for # this platform. coverage_root_node = ActionNode(Action("root", "root", False)) CreateFullCoverageActionGraph(coverage_root_node, required_coverage_tests) AddPartialPaths(coverage_root_node, partial_paths) TrimGraphToAllowedActions(coverage_root_node, framework_actions) if options.graphs: graph = GenerateGraphvizDotFile(coverage_root_node) output_graph_file_name = (output_dir + "/framework_test_graph_" + platform + ".dot") graph_file = open(output_graph_file_name, 'w') graph_file.write(GENERATED_FILE_HEADER + "\n") graph_file.write(graph) graph_file.close() # Write the framework tests. All tests that involve 'sync' actions must # be in a separate file. output_tests_file_name = (output_dir + "/framework_tests_" + platform + ".csv") output_tests_sync_file_name = (output_dir + "/framework_tests_sync_" + platform + ".csv") framework_file = open(output_tests_file_name, 'w') framework_sync_file = open(output_tests_sync_file_name, 'w') framework_file.write(GENERATED_FILE_HEADER + "\n") framework_sync_file.write(GENERATED_FILE_HEADER + "\n") paths = GenerateFrameworkTests(coverage_root_node, required_coverage_tests) for path in paths: all_actions_in_path = [] for node in path[1:]: # Skip the root node all_actions_in_path.append(node.action) all_actions_in_path.extend(node.state_check_actions.values()) action_names = [action.name for action in all_actions_in_path] file = framework_file for action_name in action_names: if action_name in SYNC_ACTIONS: file = framework_sync_file break file.write("," + ",".join(action_names) + "\n") framework_file.close() framework_sync_file.close() # Analyze all tests to generate coverage test_files = [output_tests_file_name, output_tests_sync_file_name] if not options.ignore_audited: test_files.extend( [audited_automated_tests_file, audited_manual_tests_file]) tests = [] for tests_file in test_files: tests_csv = csv.reader(open(tests_file), delimiter=',') tests.extend( ReadNamedTestsFile(tests_csv, actions, action_base_name_to_default_param)) tests_root_node = ActionNode(Action("root", "root", False)) CreateFullCoverageActionGraph(tests_root_node, tests) AddPartialPaths(tests_root_node, reversed_partial_paths) (coverage_file_text, full_coverage, partial_coverage) = GenerateCoverageFileAndPercents( required_coverage_tests, tests_root_node) coverage_table_file = output_dir + "/coverage_" + platform + ".tsv" coverage_file = open(coverage_table_file, 'w') coverage_file.write(GENERATED_FILE_HEADER + "\n") coverage_file.write( "# Full Coverage: {:.2f}, Partial Coverage: {:.2f}\n".format( full_coverage, partial_coverage)) coverage_file.write(coverage_file_text) coverage_file.close() if options.graphs: coverage_graph = GenerateGraphvizDotFile(tests_root_node) coverage_graph_filename = (output_dir + "/coverage_graph_" + platform + ".dot") coverage_graph_file = open(coverage_graph_filename, 'w') coverage_graph_file.write(GENERATED_FILE_HEADER + "\n") for graph_line in coverage_graph: coverage_graph_file.write(graph_line + "\n") coverage_graph_file.close()