def test_perfdocs_generator_save_perfdocs_pass(logger, structured_logger,
perfdocs_sample):
top_dir = perfdocs_sample["top_dir"]
setup_sample_logger(logger, structured_logger, top_dir)
templates_dir = os.path.join(top_dir, "tools", "lint", "perfdocs",
"templates")
os.makedirs(templates_dir)
from perfdocs.generator import Generator
from perfdocs.verifier import Verifier
verifier = Verifier(top_dir)
verifier.validate_tree()
generator = Generator(verifier, generate=True, workspace=top_dir)
assert not os.path.isdir(generator.perfdocs_path)
with temp_file("index.rst",
tempdir=templates_dir,
content="{test_documentation}"):
perfdocs_tmpdir = generator._create_perfdocs()
generator._save_perfdocs(perfdocs_tmpdir)
expected = ["index.rst", "mozperftest.rst"]
files = [f for f in os.listdir(generator.perfdocs_path)]
files.sort()
for i, file in enumerate(files):
assert file == expected[i]
def test_perfdocs_generator_save_perfdocs_fail(logger, shutil,
structured_logger,
perfdocs_sample):
top_dir = perfdocs_sample["top_dir"]
setup_sample_logger(logger, structured_logger, top_dir)
templates_dir = os.path.join(top_dir, "tools", "lint", "perfdocs",
"templates")
os.makedirs(templates_dir)
from perfdocs.generator import Generator
from perfdocs.verifier import Verifier
verifier = Verifier(top_dir)
verifier.validate_tree()
generator = Generator(verifier, generate=True, workspace=top_dir)
with temp_file("index.rst",
tempdir=templates_dir,
content="{test_documentation}"):
perfdocs_tmpdir = generator._create_perfdocs()
shutil.copytree = mock.Mock(side_effect=Exception())
generator._save_perfdocs(perfdocs_tmpdir)
expected = "There was an error while saving the documentation: "
args, _ = logger.critical.call_args
assert logger.critical.call_count == 1
assert args[0] == expected
def test_perfdocs_generator_created_perfdocs(logger, structured_logger,
perfdocs_sample):
top_dir = perfdocs_sample["top_dir"]
setup_sample_logger(logger, structured_logger, top_dir)
templates_dir = os.path.join(top_dir, "tools", "lint", "perfdocs",
"templates")
os.makedirs(templates_dir)
from perfdocs.generator import Generator
from perfdocs.verifier import Verifier
verifier = Verifier(top_dir)
verifier.validate_tree()
generator = Generator(verifier, generate=True, workspace=top_dir)
with temp_file("index.rst",
tempdir=templates_dir,
content="{test_documentation}"):
perfdocs_tmpdir = generator._create_perfdocs()
files = [f for f in os.listdir(perfdocs_tmpdir)]
files.sort()
expected_files = ["index.rst", "mozperftest.rst"]
for i, file in enumerate(files):
assert file == expected_files[i]
with open(os.path.join(perfdocs_tmpdir, expected_files[0])) as f:
filedata = f.readlines()
assert "".join(filedata) == " * :doc:`mozperftest`"
def test_perfdocs_verifier_invalid_dir(logger, structured_logger, perfdocs_sample):
top_dir = perfdocs_sample["top_dir"]
setup_sample_logger(logger, structured_logger, top_dir)
from perfdocs.verifier import Verifier
verifier = Verifier("invalid_path")
with pytest.raises(Exception) as exceinfo:
verifier.validate_tree()
assert str(exceinfo.value) == "No valid perfdocs directories found"
def test_perfdocs_verification(logger, structured_logger, perfdocs_sample):
top_dir = perfdocs_sample["top_dir"]
setup_sample_logger(logger, structured_logger, top_dir)
from perfdocs.verifier import Verifier
verifier = Verifier(top_dir)
verifier.validate_tree()
# Make sure that we had no warnings
assert logger.warning.call_count == 0
assert logger.log.call_count == 1
assert len(logger.mock_calls) == 1
def test_perfdocs_generator_create_temp_dir(logger, structured_logger,
perfdocs_sample):
top_dir = perfdocs_sample["top_dir"]
setup_sample_logger(logger, structured_logger, top_dir)
from perfdocs.generator import Generator
from perfdocs.verifier import Verifier
verifier = Verifier(top_dir)
verifier.validate_tree()
generator = Generator(verifier, generate=True, workspace=top_dir)
tmpdir = generator._create_temp_dir()
assert os.path.isdir(tmpdir)
def test_perfdocs_verifier_file_invalidation(logger, structured_logger,
perfdocs_sample):
top_dir = perfdocs_sample["top_dir"]
setup_sample_logger(logger, structured_logger, top_dir)
from perfdocs.verifier import Verifier
Verifier.validate_yaml = mock.Mock(return_value=False)
verifier = Verifier(top_dir)
with pytest.raises(Exception):
verifier.validate_tree()
# Check if "File validation error" log is called
# and Called with a log inside perfdocs_tree().
assert logger.log.call_count == 2
assert len(logger.mock_calls) == 2
def test_perfdocs_generator_needed_regeneration(logger, structured_logger,
perfdocs_sample):
top_dir = perfdocs_sample["top_dir"]
setup_sample_logger(logger, structured_logger, top_dir)
from perfdocs.generator import Generator
from perfdocs.verifier import Verifier
verifier = Verifier(top_dir)
verifier.validate_tree()
generator = Generator(verifier, generate=False, workspace=top_dir)
generator.generate_perfdocs()
expected = "PerfDocs need to be regenerated."
args, _ = logger.warning.call_args
assert logger.warning.call_count == 1
assert args[0] == expected
def test_perfdocs_generator_build_perfdocs(logger, structured_logger,
perfdocs_sample):
top_dir = perfdocs_sample["top_dir"]
setup_sample_logger(logger, structured_logger, top_dir)
from perfdocs.generator import Generator
from perfdocs.verifier import Verifier
verifier = Verifier(top_dir)
verifier.validate_tree()
generator = Generator(verifier, generate=True, workspace=top_dir)
frameworks_info = generator.build_perfdocs_from_tree()
expected = ["dynamic", "static"]
for framework in sorted(frameworks_info.keys()):
for i, framework_info in enumerate(frameworks_info[framework]):
assert framework_info == expected[i]
def test_perfdocs_generator_generate_perfdocs_pass(logger, structured_logger,
perfdocs_sample):
top_dir = perfdocs_sample["top_dir"]
setup_sample_logger(logger, structured_logger, top_dir)
templates_dir = os.path.join(top_dir, "tools", "lint", "perfdocs",
"templates")
os.makedirs(templates_dir)
from perfdocs.generator import Generator
from perfdocs.verifier import Verifier
verifier = Verifier(top_dir)
verifier.validate_tree()
generator = Generator(verifier, generate=True, workspace=top_dir)
with temp_file("index.rst",
tempdir=templates_dir,
content="{test_documentation}"):
generator.generate_perfdocs()
assert logger.warning.call_count == 0
def test_perfdocs_generator_create_temp_dir_fail(logger, os, structured_logger,
perfdocs_sample):
top_dir = perfdocs_sample["top_dir"]
setup_sample_logger(logger, structured_logger, top_dir)
from perfdocs.generator import Generator
from perfdocs.verifier import Verifier
verifier = Verifier(top_dir)
verifier.validate_tree()
generator = Generator(verifier, generate=True, workspace=top_dir)
os.mkdir = mock.Mock(side_effect=OSError())
os.path.isdir.return_value = False
tmpdir = generator._create_temp_dir()
expected = "Error creating temp file: "
args, _ = logger.critical.call_args
assert not tmpdir
assert logger.critical.call_count == 1
assert args[0] == expected
def test_perfdocs_generator_needed_update(logger, structured_logger,
perfdocs_sample):
top_dir = perfdocs_sample["top_dir"]
setup_sample_logger(logger, structured_logger, top_dir)
templates_dir = os.path.join(top_dir, "tools", "lint", "perfdocs",
"templates")
os.makedirs(templates_dir)
from perfdocs.generator import Generator
from perfdocs.verifier import Verifier
# Initializing perfdocs
verifier = Verifier(top_dir)
verifier.validate_tree()
generator = Generator(verifier, generate=True, workspace=top_dir)
with temp_file("index.rst",
tempdir=templates_dir,
content="{test_documentation}"):
generator.generate_perfdocs()
# Removed file for testing and run again
generator._generate = False
files = [f for f in os.listdir(generator.perfdocs_path)]
for f in files:
os.remove(os.path.join(generator.perfdocs_path, f))
generator.generate_perfdocs()
expected = (
"PerfDocs are outdated, run ./mach lint -l perfdocs --fix` to update them."
)
args, _ = logger.warning.call_args
assert logger.warning.call_count == 1
assert args[0] == expected
def run_perfdocs(config, logger=None, paths=None, generate=True):
"""
Build up performance testing documentation dynamically by combining
text data from YAML files that reside in `perfdoc` folders
across the `testing` directory. Each directory is expected to have
an `index.rst` file along with `config.yml` YAMLs defining what needs
to be added to the documentation.
The YAML must also define the name of the "framework" that should be
used in the main index.rst for the performance testing documentation.
The testing documentation list will be ordered alphabetically once
it's produced (to avoid unwanted shifts because of unordered dicts
and path searching).
Note that the suite name headings will be given the H4 (---) style so it
is suggested that you use H3 (===) style as the heading for your
test section. H5 will be used be used for individual tests within each
suite.
Usage for verification: ./mach lint -l perfdocs
Usage for generation: ./mach lint -l perfdocs --fix
For validation, see the Verifier class for a description of how
it works.
The run will fail if the valid result from validate_tree is not
False, implying some warning/problem was logged.
:param dict config: The configuration given by mozlint.
:param StructuredLogger logger: The StructuredLogger instance to be used to
output the linting warnings/errors.
:param list paths: The paths that are being tested. Used to filter
out errors from files outside of these paths.
:param bool generate: If true, the docs will be (re)generated.
"""
from perfdocs.logger import PerfDocLogger
top_dir = os.environ.get("WORKSPACE", None)
if not top_dir:
floc = os.path.abspath(__file__)
top_dir = floc.split("tools")[0]
top_dir = top_dir.replace("\\", "\\\\")
PerfDocLogger.LOGGER = logger
PerfDocLogger.TOP_DIR = top_dir
# Convert all the paths to relative ones
rel_paths = [re.sub(top_dir, "", path) for path in paths]
PerfDocLogger.PATHS = rel_paths
target_dir = [os.path.join(top_dir, i) for i in rel_paths]
for path in target_dir:
if not os.path.exists(path):
raise Exception("Cannot locate directory at %s" % path)
# Late import because logger isn't defined until later
from perfdocs.generator import Generator
from perfdocs.verifier import Verifier
# Run the verifier first
verifier = Verifier(top_dir)
verifier.validate_tree()
if not PerfDocLogger.FAILED:
# Even if the tree is valid, we need to check if the documentation
# needs to be regenerated, and if it does, we throw a linting error.
# `generate` dictates whether or not the documentation is generated.
generator = Generator(verifier, generate=generate, workspace=top_dir)
generator.generate_perfdocs()
def run_perfdocs(config, logger=None, paths=None, verify=True, generate=False):
'''
Build up performance testing documentation dynamically by combining
text data from YAML files that reside in `perfdoc` folders
across the `testing` directory. Each directory is expected to have
an `index.rst` file along with `config.yml` YAMLs defining what needs
to be added to the documentation.
The YAML must also define the name of the "framework" that should be
used in the main index.rst for the performance testing documentation.
The testing documentation list will be ordered alphabetically once
it's produced (to avoid unwanted shifts because of unordered dicts
and path searching).
Note that the suite name headings will be given the H4 (---) style so it
is suggested that you use H3 (===) style as the heading for your
test section. H5 will be used be used for individual tests within each
suite.
Usage for verification: ./mach lint -l perfdocs
Usage for generation: Not Implemented
Currently, doc generation is not implemented - only validation.
For validation, see the Verifier class for a description of how
it works.
The run will fail if the valid result from validate_tree is not
False, implying some warning/problem was logged.
:param dict config: The configuration given by mozlint.
:param StructuredLogger logger: The StructuredLogger instance to be used to
output the linting warnings/errors.
:param list paths: The paths that are being tested. Used to filter
out errors from files outside of these paths.
:param bool verify: If true, the verification will be performed.
:param bool generate: If true, the docs will be generated.
'''
from perfdocs.logger import PerfDocLogger
top_dir = os.environ.get('WORKSPACE', None)
if not top_dir:
floc = os.path.abspath(__file__)
top_dir = floc.split('tools')[0]
PerfDocLogger.LOGGER = logger
# Convert all the paths to relative ones
rel_paths = [re.sub(".*testing", "testing", path) for path in paths]
PerfDocLogger.PATHS = rel_paths
# TODO: Expand search to entire tree rather than just the testing directory
testing_dir = os.path.join(top_dir, 'testing')
if not os.path.exists(testing_dir):
raise Exception("Cannot locate testing directory at %s" % testing_dir)
# Run either the verifier or generator
if generate:
raise NotImplementedError
if verify:
from perfdocs.verifier import Verifier
verifier = Verifier(testing_dir, top_dir)
verifier.validate_tree()