def get_test_list(self, sdt_entry):
'''
Use a perfdocs_tree entry to find the test list for
the framework that was found.
:return: A framework info dictionary with fields: {
'yml_path': Path to YAML,
'yml_content': Content of YAML,
'name': Name of framework,
'test_list': Test list found for the framework
}
'''
# If it was computed before, return it
yaml_path = os.path.join(sdt_entry["path"], sdt_entry['yml'])
for entry in self._test_list:
if entry['yml_path'] == yaml_path:
return entry
# Set up framework entry with meta data
yaml_content = read_yaml(yaml_path)
framework = {
'yml_content': yaml_content,
'yml_path': yaml_path,
'name': yaml_content["name"],
}
# Get and then store the frameworks tests
framework_gatherer = frameworks[framework["name"]](
framework["yml_path"], self.workspace_dir)
framework["test_list"] = framework_gatherer.get_test_list()
self._test_list.append(framework)
return framework
def validate_tree(self):
"""
Validate the `perfdocs` directory that was found.
Returns True if it is good, false otherwise.
:return bool: True/False => Passed/Failed Validation
"""
found_good = 0
# For each framework, check their files and validate descriptions
for matched in self._gatherer.perfdocs_tree:
# Get the paths to the YAML and RST for this framework
matched_yml = os.path.join(matched["path"], matched["yml"])
matched_rst = os.path.join(matched["path"], matched["rst"])
_valid_files = {
"yml": self.validate_yaml(matched_yml),
"rst": True,
}
if not read_yaml(matched_yml)["static-only"]:
_valid_files["rst"] = self.validate_rst_content(matched_rst)
# Log independently the errors found for the matched files
for file_format, valid in _valid_files.items():
if not valid:
logger.log("File validation error: {}".format(file_format))
if not all(_valid_files.values()):
continue
found_good += 1
self._check_framework_descriptions(matched)
if not found_good:
raise Exception("No valid perfdocs directories found")
def build_perfdocs_from_tree(self):
'''
Builds up a document for each framework that was found.
:return dict: A dictionary containing a mapping from each framework
to the document that was built for it, i.e:
{
framework_name: framework_document,
...
}
'''
def _append_rst_section(title, content, documentation, type=None):
'''
Adds a section to the documentation with the title as the type mentioned
and paragraph as content mentioned.
:param title: title of the section
:param content: content of section paragraph
:param documentation: documentation object to add section to
:param type: type of the title heading
'''
heading_map = {'H4': '-', 'H5': '^'}
heading_symbol = heading_map.get(type, '-')
documentation.extend(
[title, heading_symbol * len(title), content, ''])
# Using the verified `perfdocs_tree`, build up the documentation.
frameworks_info = {}
for framework in self._perfdocs_tree:
yaml_content = read_yaml(
os.path.join(framework['path'], framework['yml']))
rst_content = read_file(os.path.join(framework['path'],
framework['rst']),
stringify=True)
# Gather all tests and descriptions and format them into
# documentation content
documentation = []
suites = yaml_content['suites']
for suite_name in sorted(suites.keys()):
suite_info = suites[suite_name]
# Add the suite with an H4 heading
_append_rst_section(suite_name.capitalize(),
suite_info['description'],
documentation,
type="H4")
tests = suite_info.get('tests', {})
for test_name in sorted(tests.keys()):
documentation.extend(
self._verifier._gatherer.framework_gatherers[
yaml_content["name"]].build_test_description(
test_name, tests[test_name]))
# Insert documentation into `.rst` file
framework_rst = re.sub(r'{documentation}',
os.linesep.join(documentation), rst_content)
frameworks_info[yaml_content['name']] = framework_rst
return frameworks_info
def test_perfdocs_framework_gatherers_urls(logger, structured_logger,
perfdocs_sample):
top_dir = perfdocs_sample["top_dir"]
setup_sample_logger(logger, structured_logger, top_dir)
from perfdocs.gatherer import frameworks
from perfdocs.verifier import Verifier
from perfdocs.generator import Generator
from perfdocs.utils import read_yaml
# This test is only for raptor
gatherer = frameworks["raptor"]
with open(perfdocs_sample["config"], "w") as f:
f.write(DYNAMIC_SAMPLE_CONFIG.format("raptor"))
fg = gatherer(perfdocs_sample["config_2"], top_dir)
fg.get_suite_list = mock.Mock()
fg.get_suite_list.return_value = {
"suite": [perfdocs_sample["example1_manifest"]],
"another_suite": [perfdocs_sample["example2_manifest"]],
}
v = Verifier(top_dir)
gn = Generator(v, generate=True, workspace=top_dir)
# Check to make sure that if a test is present under multiple
# suties the urls are generated correctly for the test under
# every suite
for suite, suitetests in fg.get_test_list().items():
url = fg._urls.get(suite)
assert url is not None
assert url[0]["test_name"] == "Example"
assert url[0]["url"] == "Example_url"
perfdocs_tree = gn._perfdocs_tree[0]
yaml_content = read_yaml(
os.path.join(os.path.join(perfdocs_tree["path"],
perfdocs_tree["yml"])))
suites = yaml_content["suites"]
# Check that the sections for each suite are generated correctly
for suite_name, suite_details in suites.items():
gn._verifier._gatherer = mock.Mock(
framework_gatherers={"raptor": gatherer})
section = gn._verifier._gatherer.framework_gatherers[
"raptor"].build_suite_section(
fg, suite_name,
suites.get(suite_name)["description"])
assert suite_name.capitalize() == section[0]
assert suite_name in section[2]
tests = suites.get(suite_name).get("tests", {})
for test_name in tests.keys():
desc = gn._verifier._gatherer.framework_gatherers[
"raptor"].build_test_description(fg, test_name,
tests[test_name], suite_name)
assert suite_name in desc[0]
assert test_name in desc[0]
def get_manifest_path(self):
"""
Returns the path to the manifest based on the
manifest entry in the frameworks YAML configuration
file.
:return str: Path to the manifest.
"""
if self._manifest_path:
return self._manifest_path
yaml_content = read_yaml(self._yaml_path)
self._manifest_path = os.path.join(self.workspace_dir, yaml_content["manifest"])
return self._manifest_path
def get_test_list(self, sdt_entry):
"""
Use a perfdocs_tree entry to find the test list for
the framework that was found.
:return: A framework info dictionary with fields: {
'yml_path': Path to YAML,
'yml_content': Content of YAML,
'name': Name of framework,
'test_list': Test list found for the framework
}
"""
# If it was computed before, return it
yaml_path = os.path.join(sdt_entry["path"], sdt_entry["yml"])
for entry in self._test_list:
if entry["yml_path"] == yaml_path:
return entry
# Set up framework entry with meta data
yaml_content = read_yaml(yaml_path)
framework = {
"yml_content": yaml_content,
"yml_path": yaml_path,
"name": yaml_content["name"],
"test_list": {},
}
if yaml_content["static-only"]:
framework_gatherer_cls = StaticGatherer
else:
framework_gatherer_cls = frameworks[framework["name"]]
# Get and then store the frameworks tests
framework_gatherer = self.framework_gatherers[
framework["name"]] = framework_gatherer_cls(
framework["yml_path"], self.workspace_dir, self.taskgraph)
if not yaml_content["static-only"]:
framework["test_list"] = framework_gatherer.get_test_list()
self._test_list.append(framework)
return framework
def validate_yaml(self, yaml_path):
'''
Validate that the YAML file has all the fields that are
required and parse the descriptions into strings in case
some are give as relative file paths.
:param str yaml_path: Path to the YAML to validate.
:return bool: True/False => Passed/Failed Validation
'''
def _get_description(desc):
'''
Recompute the description in case it's a file.
'''
desc_path = os.path.join(self.workspace_dir, desc)
if os.path.exists(desc_path):
with open(desc_path, 'r') as f:
desc = f.readlines()
return desc
def _parse_descriptions(content):
for suite, sinfo in content.items():
desc = sinfo['description']
sinfo['description'] = _get_description(desc)
# It's possible that the suite has no tests and
# only a description. If they exist, then parse them.
if 'tests' in sinfo:
for test, desc in sinfo['tests'].items():
sinfo['tests'][test] = _get_description(desc)
valid = False
yaml_content = read_yaml(yaml_path)
try:
jsonschema.validate(instance=yaml_content, schema=CONFIG_SCHEMA)
_parse_descriptions(yaml_content['suites'])
valid = True
except Exception as e:
logger.warning(
"YAML ValidationError: {}".format(str(e)), yaml_path
)
return valid
def build_perfdocs_from_tree(self):
"""
Builds up a document for each framework that was found.
:return dict: A dictionary containing a mapping from each framework
to the document that was built for it, i.e:
{
framework_name: framework_document,
...
}
"""
def _append_rst_section(title, content, documentation, type=None):
"""
Adds a section to the documentation with the title as the type mentioned
and paragraph as content mentioned.
:param title: title of the section
:param content: content of section paragraph
:param documentation: documentation object to add section to
:param type: type of the title heading
"""
heading_map = {"H4": "-", "H5": "^"}
heading_symbol = heading_map.get(type, "-")
documentation.extend([title, heading_symbol * len(title), content, ""])
# Using the verified `perfdocs_tree`, build up the documentation.
frameworks_info = {}
for framework in self._perfdocs_tree:
yaml_content = read_yaml(os.path.join(framework["path"], framework["yml"]))
rst_content = read_file(
os.path.join(framework["path"], framework["rst"]), stringify=True
)
# Gather all tests and descriptions and format them into
# documentation content
documentation = []
suites = yaml_content["suites"]
for suite_name in sorted(suites.keys()):
suite_info = suites[suite_name]
# Add the suite with an H4 heading
_append_rst_section(
suite_name.capitalize(),
suite_info["description"],
documentation,
type="H4",
)
tests = suite_info.get("tests", {})
for test_name in sorted(tests.keys()):
documentation.extend(
self._verifier._gatherer.framework_gatherers[
yaml_content["name"]
].build_test_description(
test_name, tests[test_name], suite_name
)
)
documentation.append("")
# Insert documentation into `.rst` file
framework_rst = re.sub(
r"{documentation}", os.linesep.join(documentation), rst_content
)
frameworks_info[yaml_content["name"]] = {
"dynamic": framework_rst,
"static": [],
}
# For static `.rst` file
for static_file in framework["static"]:
frameworks_info[yaml_content["name"]]["static"].append(
{
"file": static_file,
"content": read_file(
os.path.join(framework["path"], static_file), stringify=True
),
}
)
return frameworks_info
def build_perfdocs_from_tree(self):
"""
Builds up a document for each framework that was found.
:return dict: A dictionary containing a mapping from each framework
to the document that was built for it, i.e:
{
framework_name: framework_document,
...
}
"""
# Using the verified `perfdocs_tree`, build up the documentation.
frameworks_info = {}
for framework in self._perfdocs_tree:
yaml_content = read_yaml(
os.path.join(framework["path"], framework["yml"]))
rst_content = read_file(os.path.join(framework["path"],
framework["rst"]),
stringify=True)
# Gather all tests and descriptions and format them into
# documentation content
documentation = []
suites = yaml_content["suites"]
for suite_name in sorted(suites.keys()):
suite_info = suites[suite_name]
# Add the suite section
documentation.extend(
self._verifier._gatherer.framework_gatherers[
yaml_content["name"]].build_suite_section(
suite_name, suite_info["description"]))
tests = suite_info.get("tests", {})
for test_name in sorted(tests.keys()):
gatherer = self._verifier._gatherer.framework_gatherers[
yaml_content["name"]]
test_description = gatherer.build_test_description(
test_name, tests[test_name], suite_name)
documentation.extend(test_description)
documentation.append("")
# Insert documentation into `.rst` file
framework_rst = re.sub(r"{documentation}",
"\n".join(documentation), rst_content)
frameworks_info[yaml_content["name"]] = {
"dynamic": framework_rst,
"static": [],
}
# For static `.rst` file
for static_file in framework["static"]:
frameworks_info[yaml_content["name"]]["static"].append({
"file":
static_file,
"content":
read_file(os.path.join(framework["path"], static_file),
stringify=True),
})
return frameworks_info
