def get_test_list(self, sdt_entry): ''' Use a perfdocs_tree entry to find the test list for the framework that was found. :return: A framework info dictionary with fields: { 'yml_path': Path to YAML, 'yml_content': Content of YAML, 'name': Name of framework, 'test_list': Test list found for the framework } ''' # If it was computed before, return it yaml_path = os.path.join(sdt_entry["path"], sdt_entry['yml']) for entry in self._test_list: if entry['yml_path'] == yaml_path: return entry # Set up framework entry with meta data yaml_content = read_yaml(yaml_path) framework = { 'yml_content': yaml_content, 'yml_path': yaml_path, 'name': yaml_content["name"], } # Get and then store the frameworks tests framework_gatherer = frameworks[framework["name"]]( framework["yml_path"], self.workspace_dir) framework["test_list"] = framework_gatherer.get_test_list() self._test_list.append(framework) return framework
def validate_tree(self): """ Validate the `perfdocs` directory that was found. Returns True if it is good, false otherwise. :return bool: True/False => Passed/Failed Validation """ found_good = 0 # For each framework, check their files and validate descriptions for matched in self._gatherer.perfdocs_tree: # Get the paths to the YAML and RST for this framework matched_yml = os.path.join(matched["path"], matched["yml"]) matched_rst = os.path.join(matched["path"], matched["rst"]) _valid_files = { "yml": self.validate_yaml(matched_yml), "rst": True, } if not read_yaml(matched_yml)["static-only"]: _valid_files["rst"] = self.validate_rst_content(matched_rst) # Log independently the errors found for the matched files for file_format, valid in _valid_files.items(): if not valid: logger.log("File validation error: {}".format(file_format)) if not all(_valid_files.values()): continue found_good += 1 self._check_framework_descriptions(matched) if not found_good: raise Exception("No valid perfdocs directories found")
def build_perfdocs_from_tree(self): ''' Builds up a document for each framework that was found. :return dict: A dictionary containing a mapping from each framework to the document that was built for it, i.e: { framework_name: framework_document, ... } ''' def _append_rst_section(title, content, documentation, type=None): ''' Adds a section to the documentation with the title as the type mentioned and paragraph as content mentioned. :param title: title of the section :param content: content of section paragraph :param documentation: documentation object to add section to :param type: type of the title heading ''' heading_map = {'H4': '-', 'H5': '^'} heading_symbol = heading_map.get(type, '-') documentation.extend( [title, heading_symbol * len(title), content, '']) # Using the verified `perfdocs_tree`, build up the documentation. frameworks_info = {} for framework in self._perfdocs_tree: yaml_content = read_yaml( os.path.join(framework['path'], framework['yml'])) rst_content = read_file(os.path.join(framework['path'], framework['rst']), stringify=True) # Gather all tests and descriptions and format them into # documentation content documentation = [] suites = yaml_content['suites'] for suite_name in sorted(suites.keys()): suite_info = suites[suite_name] # Add the suite with an H4 heading _append_rst_section(suite_name.capitalize(), suite_info['description'], documentation, type="H4") tests = suite_info.get('tests', {}) for test_name in sorted(tests.keys()): documentation.extend( self._verifier._gatherer.framework_gatherers[ yaml_content["name"]].build_test_description( test_name, tests[test_name])) # Insert documentation into `.rst` file framework_rst = re.sub(r'{documentation}', os.linesep.join(documentation), rst_content) frameworks_info[yaml_content['name']] = framework_rst return frameworks_info
def test_perfdocs_framework_gatherers_urls(logger, structured_logger, perfdocs_sample): top_dir = perfdocs_sample["top_dir"] setup_sample_logger(logger, structured_logger, top_dir) from perfdocs.gatherer import frameworks from perfdocs.verifier import Verifier from perfdocs.generator import Generator from perfdocs.utils import read_yaml # This test is only for raptor gatherer = frameworks["raptor"] with open(perfdocs_sample["config"], "w") as f: f.write(DYNAMIC_SAMPLE_CONFIG.format("raptor")) fg = gatherer(perfdocs_sample["config_2"], top_dir) fg.get_suite_list = mock.Mock() fg.get_suite_list.return_value = { "suite": [perfdocs_sample["example1_manifest"]], "another_suite": [perfdocs_sample["example2_manifest"]], } v = Verifier(top_dir) gn = Generator(v, generate=True, workspace=top_dir) # Check to make sure that if a test is present under multiple # suties the urls are generated correctly for the test under # every suite for suite, suitetests in fg.get_test_list().items(): url = fg._urls.get(suite) assert url is not None assert url[0]["test_name"] == "Example" assert url[0]["url"] == "Example_url" perfdocs_tree = gn._perfdocs_tree[0] yaml_content = read_yaml( os.path.join(os.path.join(perfdocs_tree["path"], perfdocs_tree["yml"]))) suites = yaml_content["suites"] # Check that the sections for each suite are generated correctly for suite_name, suite_details in suites.items(): gn._verifier._gatherer = mock.Mock( framework_gatherers={"raptor": gatherer}) section = gn._verifier._gatherer.framework_gatherers[ "raptor"].build_suite_section( fg, suite_name, suites.get(suite_name)["description"]) assert suite_name.capitalize() == section[0] assert suite_name in section[2] tests = suites.get(suite_name).get("tests", {}) for test_name in tests.keys(): desc = gn._verifier._gatherer.framework_gatherers[ "raptor"].build_test_description(fg, test_name, tests[test_name], suite_name) assert suite_name in desc[0] assert test_name in desc[0]
def get_manifest_path(self): """ Returns the path to the manifest based on the manifest entry in the frameworks YAML configuration file. :return str: Path to the manifest. """ if self._manifest_path: return self._manifest_path yaml_content = read_yaml(self._yaml_path) self._manifest_path = os.path.join(self.workspace_dir, yaml_content["manifest"]) return self._manifest_path
def get_test_list(self, sdt_entry): """ Use a perfdocs_tree entry to find the test list for the framework that was found. :return: A framework info dictionary with fields: { 'yml_path': Path to YAML, 'yml_content': Content of YAML, 'name': Name of framework, 'test_list': Test list found for the framework } """ # If it was computed before, return it yaml_path = os.path.join(sdt_entry["path"], sdt_entry["yml"]) for entry in self._test_list: if entry["yml_path"] == yaml_path: return entry # Set up framework entry with meta data yaml_content = read_yaml(yaml_path) framework = { "yml_content": yaml_content, "yml_path": yaml_path, "name": yaml_content["name"], "test_list": {}, } if yaml_content["static-only"]: framework_gatherer_cls = StaticGatherer else: framework_gatherer_cls = frameworks[framework["name"]] # Get and then store the frameworks tests framework_gatherer = self.framework_gatherers[ framework["name"]] = framework_gatherer_cls( framework["yml_path"], self.workspace_dir, self.taskgraph) if not yaml_content["static-only"]: framework["test_list"] = framework_gatherer.get_test_list() self._test_list.append(framework) return framework
def validate_yaml(self, yaml_path): ''' Validate that the YAML file has all the fields that are required and parse the descriptions into strings in case some are give as relative file paths. :param str yaml_path: Path to the YAML to validate. :return bool: True/False => Passed/Failed Validation ''' def _get_description(desc): ''' Recompute the description in case it's a file. ''' desc_path = os.path.join(self.workspace_dir, desc) if os.path.exists(desc_path): with open(desc_path, 'r') as f: desc = f.readlines() return desc def _parse_descriptions(content): for suite, sinfo in content.items(): desc = sinfo['description'] sinfo['description'] = _get_description(desc) # It's possible that the suite has no tests and # only a description. If they exist, then parse them. if 'tests' in sinfo: for test, desc in sinfo['tests'].items(): sinfo['tests'][test] = _get_description(desc) valid = False yaml_content = read_yaml(yaml_path) try: jsonschema.validate(instance=yaml_content, schema=CONFIG_SCHEMA) _parse_descriptions(yaml_content['suites']) valid = True except Exception as e: logger.warning( "YAML ValidationError: {}".format(str(e)), yaml_path ) return valid
def build_perfdocs_from_tree(self): """ Builds up a document for each framework that was found. :return dict: A dictionary containing a mapping from each framework to the document that was built for it, i.e: { framework_name: framework_document, ... } """ def _append_rst_section(title, content, documentation, type=None): """ Adds a section to the documentation with the title as the type mentioned and paragraph as content mentioned. :param title: title of the section :param content: content of section paragraph :param documentation: documentation object to add section to :param type: type of the title heading """ heading_map = {"H4": "-", "H5": "^"} heading_symbol = heading_map.get(type, "-") documentation.extend([title, heading_symbol * len(title), content, ""]) # Using the verified `perfdocs_tree`, build up the documentation. frameworks_info = {} for framework in self._perfdocs_tree: yaml_content = read_yaml(os.path.join(framework["path"], framework["yml"])) rst_content = read_file( os.path.join(framework["path"], framework["rst"]), stringify=True ) # Gather all tests and descriptions and format them into # documentation content documentation = [] suites = yaml_content["suites"] for suite_name in sorted(suites.keys()): suite_info = suites[suite_name] # Add the suite with an H4 heading _append_rst_section( suite_name.capitalize(), suite_info["description"], documentation, type="H4", ) tests = suite_info.get("tests", {}) for test_name in sorted(tests.keys()): documentation.extend( self._verifier._gatherer.framework_gatherers[ yaml_content["name"] ].build_test_description( test_name, tests[test_name], suite_name ) ) documentation.append("") # Insert documentation into `.rst` file framework_rst = re.sub( r"{documentation}", os.linesep.join(documentation), rst_content ) frameworks_info[yaml_content["name"]] = { "dynamic": framework_rst, "static": [], } # For static `.rst` file for static_file in framework["static"]: frameworks_info[yaml_content["name"]]["static"].append( { "file": static_file, "content": read_file( os.path.join(framework["path"], static_file), stringify=True ), } ) return frameworks_info
def build_perfdocs_from_tree(self): """ Builds up a document for each framework that was found. :return dict: A dictionary containing a mapping from each framework to the document that was built for it, i.e: { framework_name: framework_document, ... } """ # Using the verified `perfdocs_tree`, build up the documentation. frameworks_info = {} for framework in self._perfdocs_tree: yaml_content = read_yaml( os.path.join(framework["path"], framework["yml"])) rst_content = read_file(os.path.join(framework["path"], framework["rst"]), stringify=True) # Gather all tests and descriptions and format them into # documentation content documentation = [] suites = yaml_content["suites"] for suite_name in sorted(suites.keys()): suite_info = suites[suite_name] # Add the suite section documentation.extend( self._verifier._gatherer.framework_gatherers[ yaml_content["name"]].build_suite_section( suite_name, suite_info["description"])) tests = suite_info.get("tests", {}) for test_name in sorted(tests.keys()): gatherer = self._verifier._gatherer.framework_gatherers[ yaml_content["name"]] test_description = gatherer.build_test_description( test_name, tests[test_name], suite_name) documentation.extend(test_description) documentation.append("") # Insert documentation into `.rst` file framework_rst = re.sub(r"{documentation}", "\n".join(documentation), rst_content) frameworks_info[yaml_content["name"]] = { "dynamic": framework_rst, "static": [], } # For static `.rst` file for static_file in framework["static"]: frameworks_info[yaml_content["name"]]["static"].append({ "file": static_file, "content": read_file(os.path.join(framework["path"], static_file), stringify=True), }) return frameworks_info