def test_BuildspecParser(tmp_path):
# Invalid path to buildspec file should exit
with pytest.raises(BuildTestError):
BuildspecParser("")
# Passing 'None' will raise an error
with pytest.raises(BuildTestError):
BuildspecParser(None)
directory = os.path.join(here, "invalid_buildspecs")
builders = []
for buildspec in walk_tree(directory, ".yml"):
buildspecfile = os.path.join(directory, buildspec)
print("Processing buildspec: ", buildspecfile)
with pytest.raises(BuildTestError):
BuildspecParser(buildspecfile)
# Examples folder
valid_buildspecs_directory = os.path.join(here, "valid_buildspecs")
# A directory is not allowed either, this will raise an error.
with pytest.raises(BuildTestError):
BuildspecParser(valid_buildspecs_directory)
# Test loading Buildspec files
for buildspec in walk_tree(valid_buildspecs_directory, ".yml"):
buildspecfile = os.path.join(valid_buildspecs_directory, buildspec)
bp = BuildspecParser(buildspecfile)
assert bp.recipe
assert bp.buildspec
assert bp.executors
filters = {"tags": None, "executors": None}
builders = Builder(bp, filters=filters, testdir=tmp_path)
builders = builders.get_builders()
assert builders
for builder in builders:
# Builders (on init) set up metadata attribute
assert builder.metadata
# Invoking build will build the test script
# and write test
builder.build()
def test_Builder(tmp_path):
directory = os.path.join(here, "invalid_builds")
filters = {"tags": None, "executors": None}
print(os.listdir(directory))
for buildspec in walk_tree(directory, ".yml"):
buildspecfile = os.path.join(directory, buildspec)
print("file:", buildspecfile)
bp = BuildspecParser(buildspecfile)
builder = Builder(bp=bp, testdir=tmp_path, filters=filters)
builders = builder.get_builders()
for builder in builders:
with pytest.raises(BuildTestError):
builder.build()
directory = os.path.join(here, "valid_builds")
filters = {"tags": None, "executors": None}
for buildspec in os.listdir(directory):
buildspec = os.path.join(directory, buildspec)
bp = BuildspecParser(buildspec)
builder = Builder(bp=bp, testdir=tmp_path, filters=filters)
builders = builder.get_builders()
for builder in builders:
builder.build()
def _validate_buildspecs(self, buildspecs):
"""Given a list of buildspec files, validate each buildspec using BuildspecParser
and return a list of valid buildspecs. Any invalid buildspecs are added to
separate list
"""
valid_buildspecs = []
self.count = 0
configuration = BuildtestConfiguration(self.settings)
buildexecutor = BuildExecutor(configuration)
for buildspec in buildspecs:
self.count += 1
try:
parse = BuildspecParser(buildspec, buildexecutor)
# any buildspec that raises SystemExit or ValidationError imply
# buildspec is not valid, we add this to invalid list along with
# error message and skip to next buildspec
except (BuildTestError, ValidationError) as err:
self.invalid_buildspecs[buildspec] = err
continue
valid_buildspecs.append(parse)
if self.count % 5 == 0:
print(f"Validated {self.count}/{len(buildspecs)} buildspecs")
print(f"Validated {self.count}/{len(buildspecs)} buildspecs")
# print invalid buildspecs if found
if len(self.invalid_buildspecs) > 0:
print("\n")
print("Invalid buildspecs")
print("{:_<80}".format(""))
for file in self.invalid_buildspecs:
print(file)
print(f"Found {len(self.invalid_buildspecs)} invalid buildspecs")
print("{:_<80}".format(""))
print("\n")
print(f"Adding {len(valid_buildspecs)} buildspec files to cache")
return valid_buildspecs
def test_build_executor(tmp_path):
bc = BuildtestConfiguration(DEFAULT_SETTINGS_FILE)
# Load BuildExecutor
be = BuildExecutor(bc)
# We should have a total of 5 executors (local.bash, local.sh, local.csh, local.zsh, local.python)
assert len(be.executors) == 5
assert list(be.executors.keys()) == [
"generic.local.bash",
"generic.local.sh",
"generic.local.csh",
"generic.local.zsh",
"generic.local.python",
]
# Making sure all executors are created properly by inspecting their class attribute.
# All executors have a class attribute 'type'
for name, executor in be.executors.items():
assert hasattr(executor, "type")
examples_dir = os.path.join(pytest_root, "buildsystem", "valid_buildspecs")
for buildspec in os.listdir(examples_dir):
buildspec = os.path.join(examples_dir, buildspec)
try:
bp = BuildspecParser(buildspec, be)
except (SystemExit, ValidationError):
continue
bp_filters = {"tags": None}
builders = Builder(
bp=bp, buildexecutor=be, filters=bp_filters, testdir=tmp_path
)
valid_builders = builders.get_builders()
# build each test and then run it
for builder in valid_builders:
builder.build()
be.run(builder)
assert builder.metadata["result"]
def test_build_executor(tmp_path):
settings_schema = load_schema(DEFAULT_SETTINGS_SCHEMA)
example = load_recipe(DEFAULT_SETTINGS_FILE)
custom_validator(recipe=example, schema=settings_schema)
# Load BuildExecutor
be = BuildExecutor(example)
# We should have a total of 5 executors (local.bash, local.sh, local.csh, local.zsh, local.python)
assert len(be.executors) == 5
assert list(be.executors.keys()) == [
"local.bash",
"local.sh",
"local.csh",
"local.zsh",
"local.python",
]
# Each should have
for name, executor in be.executors.items():
assert hasattr(executor, "_settings")
examples_dir = os.path.join(pytest_root, "examples", "buildspecs")
for buildspec in os.listdir(examples_dir):
buildspec = os.path.join(examples_dir, buildspec)
try:
bp = BuildspecParser(buildspec)
except (SystemExit, ValidationError):
continue
bp_filters = {"tags": None}
builders = Builder(bp=bp, filters=bp_filters, testdir=tmp_path)
valid_builders = builders.get_builders()
# build each test and then run it
for builder in valid_builders:
builder.build()
be.run(builder)
assert builder.metadata["result"]
def parse_buildspecs(buildspecs,
executor,
test_directory,
filters,
rebuild,
printTable=False):
"""Parse all buildspecs by invoking class ``BuildspecParser``. If buildspec
fails validation we add it to ``skipped_tests`` list and print all skipped
tests at end. If buildspec passes validation we get all builders by invoking
``get_builders`` method in BuildspecParser class which gets all tests from
buildspec file.
:param buildspecs: A list of input buildspecs to parse
:type buildspecs: list, required
:param executor: An instance of BuildExecutor class
:type executor: BuildExecutor
:param test_directory: Test directory where buildspecs will be written
:type test_directory: str, required
:param filters: A dictionary containing filters on builders based on tags and executors
:type filters: dict, required
:param rebuild: Input argument from command line --rebuild
:type rebuild: int or None
:param printTable: a boolean to control if parse table is printed
:type printTable: bool, optional
:return: A list of builder objects which are instances of ``BuilderBase`` class
:rtype: list
"""
builders = []
table = {"schemafile": [], "validstate": [], "buildspec": []}
invalid_buildspecs = []
# build all the tests
for buildspec in buildspecs:
valid_state = True
try:
# Read in Buildspec file here, loading each will validate the buildspec file
bp = BuildspecParser(buildspec, executor)
except (BuildTestError, ValidationError) as err:
invalid_buildspecs.append(
f"Skipping {buildspec} since it failed to validate")
logger.error(err)
continue
table["schemafile"].append(bp.schema_file)
table["validstate"].append(valid_state)
table["buildspec"].append(buildspec)
builder = Builder(bp=bp,
filters=filters,
testdir=test_directory,
rebuild=rebuild)
builders += builder.get_builders()
# print any skipped buildspecs if they failed to validate during build stage
if len(invalid_buildspecs) > 0:
print("\n\n")
print("Error Messages from Stage: Parse")
print("{:_<80}".format(""))
for test in invalid_buildspecs:
print(test)
if not builders:
print("No buildspecs to process because there are no valid buildspecs")
sys.exit(0)
if printTable:
print("""
+---------------------------+
| Stage: Parsing Buildspecs |
+---------------------------+
""")
print(tabulate(table, headers=table.keys(), tablefmt="presto"))
return builders
def test_BuildspecParser(tmp_path):
config = BuildtestConfiguration(DEFAULT_SETTINGS_FILE)
executors = BuildExecutor(config)
# Invalid path to buildspec file should exit
with pytest.raises(BuildTestError):
BuildspecParser("", executors)
# Passing 'None' will raise an error
with pytest.raises(BuildTestError):
BuildspecParser(None, executors)
directory = os.path.join(here, "invalid_buildspecs")
builders = []
for buildspec in walk_tree(directory, ".yml"):
buildspecfile = os.path.join(directory, buildspec)
print("Processing buildspec: ", buildspecfile)
with pytest.raises(BuildTestError):
BuildspecParser(buildspecfile, executors)
directory = os.path.join(here, "invalid_builds")
# invalid builds for compiler schema tests. These tests will raise BuildTestError exception upon building
# even though they are valid buildspecs.\
for buildspec in walk_tree(directory, ".yml"):
buildspecfile = os.path.join(directory, buildspec)
print("Processing buildspec", buildspecfile)
bp = BuildspecParser(buildspecfile, executors)
with pytest.raises(BuildTestError):
builder = Builder(bp=bp,
buildexecutor=executors,
filters=[],
testdir="/tmp")
builders = builder.get_builders()
for test in builders:
test.build()
# Examples folder
valid_buildspecs_directory = os.path.join(here, "valid_buildspecs")
# A directory is not allowed either, this will raise an error.
with pytest.raises(BuildTestError):
BuildspecParser(valid_buildspecs_directory, executors)
# Test loading Buildspec files
for buildspec in walk_tree(valid_buildspecs_directory, ".yml"):
buildspecfile = os.path.join(valid_buildspecs_directory, buildspec)
bp = BuildspecParser(buildspecfile, executors)
assert hasattr(bp, "recipe")
assert hasattr(bp, "buildspec")
assert hasattr(bp, "buildexecutors")
filters = []
builders = Builder(bp=bp,
buildexecutor=executors,
filters=filters,
testdir=tmp_path)
builders = builders.get_builders()
assert builders
for builder in builders:
# Builders (on init) set up metadata attribute
assert hasattr(builder, "metadata")
# Invoking build will build the test script
# and write test
builder.build()