def create_suites(args): # pylint: disable=W0621
suites = []
# ========================================================================
# Run nrunner interface checks for all available runners
# ========================================================================
config_nrunner_interface = {
'resolver.references':
['selftests/functional/test_nrunner_interface.py'],
'run.dict_variants.variant_id_keys': ['runner'],
'run.dict_variants': [
{
'runner': 'avocado-runner',
'runnable-run-no-args-exit-code': 2,
'runnable-run-uri-only-exit-code': 2
},
{
'runner': 'avocado-runner-dry-run',
'runnable-run-no-args-exit-code': 0,
'runnable-run-uri-only-exit-code': 0
},
{
'runner': 'avocado-runner-noop',
'runnable-run-no-args-exit-code': 0,
'runnable-run-uri-only-exit-code': 0
},
{
'runner': 'avocado-runner-exec-test',
'runnable-run-no-args-exit-code': 0,
'runnable-run-uri-only-exit-code': 0
},
{
'runner': 'avocado-runner-python-unittest',
'runnable-run-no-args-exit-code': 0,
'runnable-run-uri-only-exit-code': 0
},
{
'runner': 'avocado-runner-avocado-instrumented',
'runnable-run-no-args-exit-code': 0,
'runnable-run-uri-only-exit-code': 0
},
{
'runner': 'avocado-runner-tap',
'runnable-run-no-args-exit-code': 0,
'runnable-run-uri-only-exit-code': 0
},
]
}
if (python_module_available('avocado-framework-plugin-golang')
and 'golang' not in args.disable_plugin_checks):
config_nrunner_interface['run.dict_variants'].append({
'runner':
'avocado-runner-golang',
'runnable-run-no-args-exit-code':
0,
'runnable-run-uri-only-exit-code':
0
})
if (python_module_available('avocado-framework-plugin-robot')
and 'robot' not in args.disable_plugin_checks):
config_nrunner_interface['run.dict_variants'].append({
'runner':
'avocado-runner-robot',
'runnable-run-no-args-exit-code':
0,
'runnable-run-uri-only-exit-code':
0
})
if args.dict_tests['nrunner-interface']:
suites.append(
TestSuite.from_config(config_nrunner_interface,
"nrunner-interface"))
# ========================================================================
# Run all static checks, unit and functional tests
# ========================================================================
config_check = {
'run.ignore_missing_references': True,
}
if args.dict_tests['unit']:
config_check_unit = copy.copy(config_check)
config_check_unit['resolver.references'] = ['selftests/unit/']
suites.append(TestSuite.from_config(config_check_unit, "unit"))
if args.dict_tests['jobs']:
config_check_jobs = copy.copy(config_check)
config_check_jobs['resolver.references'] = ['selftests/jobs/']
suites.append(TestSuite.from_config(config_check_jobs, "jobs"))
if args.dict_tests['functional']:
functional_path = os.path.join('selftests', 'functional')
references = glob.glob(os.path.join(functional_path, 'test*.py'))
references.extend([
os.path.join(functional_path, 'utils'),
os.path.join(functional_path, 'plugin')
])
config_check_functional_parallel = copy.copy(config_check)
config_check_functional_parallel['resolver.references'] = references
suites.append(
TestSuite.from_config(config_check_functional_parallel,
"functional-parallel"))
config_check_functional_serial = copy.copy(config_check)
config_check_functional_serial['resolver.references'] = [
'selftests/functional/serial/'
]
config_check_functional_serial['nrunner.max_parallel_tasks'] = 1
suites.append(
TestSuite.from_config(config_check_functional_serial,
"functional-serial"))
if args.dict_tests['static-checks']:
config_check_static = copy.copy(config_check)
config_check_static['resolver.references'] = glob.glob(
'selftests/*.sh')
suites.append(
TestSuite.from_config(config_check_static, "static-checks"))
if args.dict_tests['optional-plugins']:
config_check_optional = copy.copy(config_check)
config_check_optional['resolver.references'] = []
for optional_plugin in glob.glob('optional_plugins/*'):
plugin_name = os.path.basename(optional_plugin)
if plugin_name not in args.disable_plugin_checks:
pattern = '%s/tests/*' % optional_plugin
config_check_optional['resolver.references'] += glob.glob(
pattern)
suites.append(
TestSuite.from_config(config_check_optional, "optional-plugins"))
return suites
def create_suite_job_api(args): # pylint: disable=W0621
suites = []
def get_ref(method_short_name):
return [
'%s:JobAPIFeaturesTest.test_%s' % (__file__, method_short_name)
]
# ========================================================================
# Test if the archive file was created
# ========================================================================
config_check_archive_file_exists = {
'resolver.references':
get_ref('check_archive_file_exists'),
'run.dict_variants.variant_id_keys': ['namespace', 'value'],
'run.dict_variants': [
{
'namespace': 'run.results.archive',
'value': True,
'assert': True
},
]
}
suites.append(
TestSuite.from_config(config_check_archive_file_exists,
"job-api-%s" % (len(suites) + 1)))
# ========================================================================
# Test if the category directory was created
# ========================================================================
config_check_category_directory_exists = {
'resolver.references':
get_ref('check_category_directory_exists'),
'run.dict_variants.variant_id_keys': ['namespace', 'value'],
'run.dict_variants': [
{
'namespace': 'run.job_category',
'value': 'foo',
'assert': True
},
]
}
suites.append(
TestSuite.from_config(config_check_category_directory_exists,
"job-api-%s" % (len(suites) + 1)))
# ========================================================================
# Test if a directory was created
# ========================================================================
config_check_directory_exists = {
'resolver.references':
get_ref('check_directory_exists'),
'run.dict_variants.variant_id_keys': ['namespace', 'value'],
'run.dict_variants': [
{
'namespace': 'sysinfo.collect.enabled',
'value': True,
'directory': 'sysinfo',
'assert': True
},
{
'namespace': 'sysinfo.collect.enabled',
'value': False,
'directory': 'sysinfo',
'assert': False
},
]
}
suites.append(
TestSuite.from_config(config_check_directory_exists,
"job-api-%s" % (len(suites) + 1)))
# ========================================================================
# Test the content of a file
# ========================================================================
config_check_file_content = {
'resolver.references':
get_ref('check_file_content'),
'run.dict_variants.variant_id_keys': ['namespace', 'value', 'file'],
'run.dict_variants': [
# finding the correct 'content' here is trick because any
# simple string is added to the variant file name and is
# found in the log file.
# Using DEBUG| makes the variant name have DEBUG_, working
# fine here.
{
'namespace': 'job.output.loglevel',
'value': 'INFO',
'file': 'job.log',
'content': r'DEBUG\| Test metadata:$',
'assert': False,
'regex': True
},
{
'namespace': 'job.run.result.tap.include_logs',
'value': True,
'file': 'results.tap',
'reference': ['examples/tests/passtest.py:PassTest.test'],
'content': 'PASS 1-examples/tests/passtest.py:PassTest.test',
'assert': True
},
{
'namespace': 'job.run.result.tap.include_logs',
'value': False,
'file': 'results.tap',
'content': "Command '/bin/true' finished with 0",
'assert': False
},
{
'namespace': 'job.run.result.xunit.job_name',
'value': 'foo',
'file': 'results.xml',
'content': 'name="foo"',
'assert': True
},
{
'namespace': 'job.run.result.xunit.max_test_log_chars',
'value': 1,
'file': 'results.xml',
'content': '--[ CUT DUE TO XML PER TEST LIMIT ]--',
'assert': True,
'reference': ['examples/tests/failtest.py:FailTest.test'],
'exit_code': 1
},
{
'namespace': 'run.failfast',
'value': True,
'file': 'results.json',
'content': '"skip": 1',
'assert': True,
'reference': ['/bin/false', '/bin/true'],
'exit_code': 9,
'extra_job_config': {
'nrunner.max_parallel_tasks': 1
}
},
{
'namespace': 'run.ignore_missing_references',
'value': 'on',
'file': 'results.json',
'content': '"pass": 1',
'assert': True,
'reference': ['/bin/true', 'foo']
},
{
'namespace': 'run.unique_job_id',
'value': 'abcdefghi',
'file': 'job.log',
'content': 'Job ID: abcdefghi',
'assert': True
},
{
'namespace': 'job.run.timeout',
'value': 1,
'reference': ['examples/tests/sleeptenmin.py'],
'file': 'job.log',
'content': 'RuntimeError: Test interrupted by SIGTERM',
'assert': True,
'exit_code': 8
},
]
}
suites.append(
TestSuite.from_config(config_check_file_content,
"job-api-%s" % (len(suites) + 1)))
# ========================================================================
# Test if the result file was created
# ========================================================================
config_check_file_exists = {
'resolver.references':
get_ref('check_file_exists'),
'run.dict_variants.variant_id_keys': ['namespace', 'value'],
'run.dict_variants': [
{
'namespace': 'job.run.result.json.enabled',
'value': True,
'file': 'results.json',
'assert': True
},
{
'namespace': 'job.run.result.json.enabled',
'value': False,
'file': 'results.json',
'assert': False
},
{
'namespace': 'job.run.result.tap.enabled',
'value': True,
'file': 'results.tap',
'assert': True
},
{
'namespace': 'job.run.result.tap.enabled',
'value': False,
'file': 'results.tap',
'assert': False
},
{
'namespace': 'job.run.result.xunit.enabled',
'value': True,
'file': 'results.xml',
'assert': True
},
{
'namespace': 'job.run.result.xunit.enabled',
'value': False,
'file': 'results.xml',
'assert': False
},
{
'namespace': 'run.dry_run.enabled',
'value': True,
'file': 'job.log',
'assert': False
},
{
'namespace': 'run.dry_run.no_cleanup',
'value': True,
'file': 'job.log',
'assert': True
},
{
'namespace': 'plugins.disable',
'value': ['result.xunit'],
'file': 'result.xml',
'assert': False
},
# this test needs a huge improvement
{
'namespace': 'run.journal.enabled',
'value': True,
'file': '.journal.sqlite',
'assert': True
},
]
}
if (python_module_available('avocado-framework-plugin-result-html')
and 'html' not in args.disable_plugin_checks):
config_check_file_exists['run.dict_variants'].append({
'namespace': 'job.run.result.html.enabled',
'value': True,
'file': 'results.html',
'assert': True
})
config_check_file_exists['run.dict_variants'].append({
'namespace': 'job.run.result.html.enabled',
'value': False,
'file': 'results.html',
'assert': False
})
suites.append(
TestSuite.from_config(config_check_file_exists,
"job-api-%s" % (len(suites) + 1)))
# ========================================================================
# Test if a file was created
# ========================================================================
config_check_output_file = {
'resolver.references':
get_ref('check_output_file'),
'run.dict_variants.variant_id_keys': ['namespace', 'file'],
'run.dict_variants': [
{
'namespace': 'job.run.result.json.output',
'file': 'custom.json',
'assert': True
},
# https://github.com/avocado-framework/avocado/issues/4034
{
'namespace': 'job.run.result.tap.output',
'file': 'custom.tap',
'assert': True
},
{
'namespace': 'job.run.result.xunit.output',
'file': 'custom.xml',
'assert': True
},
]
}
if (python_module_available('avocado-framework-plugin-result-html')
and 'html' not in args.disable_plugin_checks):
config_check_output_file['run.dict_variants'].append({
'namespace': 'job.run.result.html.output',
'file': 'custom.html',
'assert': True
})
suites.append(
TestSuite.from_config(config_check_output_file,
"job-api-%s" % (len(suites) + 1)))
# ========================================================================
# Test if the temporary directory was created
# ========================================================================
config_check_tmp_directory_exists = {
'resolver.references':
get_ref('check_tmp_directory_exists'),
'run.dict_variants.variant_id_keys': ['namespace', 'value'],
'run.dict_variants': [
{
'namespace': 'run.keep_tmp',
'value': True,
'assert': True
},
]
}
suites.append(
TestSuite.from_config(config_check_tmp_directory_exists,
"job-api-%s" % (len(suites) + 1)))
return suites
def test_plugin_order(self):
"""
Tests plugin order by configuration file
First it checks if html, json, xunit and zip_archive plugins are enabled.
Then it runs a test with zip_archive running first, which means the html,
json and xunit output files do not make into the archive.
Then it runs with zip_archive set to run last, which means the html,
json and xunit output files *do* make into the archive.
"""
def run_config(config_path):
cmd = (f"{AVOCADO} --config {config_path} "
f"run examples/tests/passtest.py --archive "
f"--job-results-dir {self.tmpdir.name} "
f"--disable-sysinfo")
result = process.run(cmd, ignore_status=True)
expected_rc = exit_codes.AVOCADO_ALL_OK
self.assertEqual(
result.exit_status,
expected_rc,
(f"Avocado did not return rc {expected_rc}:"
f"\n{result}"),
)
result_plugins = ["json", "xunit", "zip_archive"]
result_outputs = ["results.json", "results.xml"]
if python_module_available("avocado-framework-plugin-result-html"):
result_plugins.append("html")
result_outputs.append("results.html")
cmd_line = f"{AVOCADO} plugins"
result = process.run(cmd_line, ignore_status=True)
expected_rc = exit_codes.AVOCADO_ALL_OK
self.assertEqual(
result.exit_status,
expected_rc,
f"Avocado did not return rc {expected_rc}:\n{result}",
)
for result_plugin in result_plugins:
self.assertIn(result_plugin, result.stdout_text)
config_content_zip_first = "[plugins.result]\norder=['zip_archive']"
config_zip_first = script.TemporaryScript("zip_first.conf",
config_content_zip_first)
with config_zip_first:
run_config(config_zip_first)
archives = glob.glob(os.path.join(self.tmpdir.name, "*.zip"))
self.assertEqual(len(archives), 1, "ZIP Archive not generated")
zip_file = zipfile.ZipFile(archives[0], "r")
zip_file_list = zip_file.namelist()
for result_output in result_outputs:
self.assertNotIn(result_output, zip_file_list)
os.unlink(archives[0])
config_content_zip_last = ("[plugins.result]\norder=['html', 'json',"
"'xunit', 'non_existing_plugin_is_ignored'"
",'zip_archive']")
config_zip_last = script.TemporaryScript("zip_last.conf",
config_content_zip_last)
with config_zip_last:
run_config(config_zip_last)
archives = glob.glob(os.path.join(self.tmpdir.name, "*.zip"))
self.assertEqual(len(archives), 1, "ZIP Archive not generated")
zip_file = zipfile.ZipFile(archives[0], "r")
zip_file_list = zip_file.namelist()
for result_output in result_outputs:
self.assertIn(result_output, zip_file_list)
def create_suites(args): # pylint: disable=W0621
suites = []
# ========================================================================
# Run nrunner interface checks for all available runners
# ========================================================================
config_nrunner_interface = {
'resolver.references':
['selftests/functional/test_nrunner_interface.py'],
'run.dict_variants.variant_id_keys': ['runner'],
'run.dict_variants': [
{
'runner': 'avocado-runner',
'runnable-run-no-args-exit-code': 2,
'runnable-run-uri-only-exit-code': 2
},
{
'runner': 'avocado-runner-noop',
'runnable-run-no-args-exit-code': 0,
'runnable-run-uri-only-exit-code': 0
},
{
'runner': 'avocado-runner-exec-test',
'runnable-run-no-args-exit-code': 0,
'runnable-run-uri-only-exit-code': 0
},
{
'runner': 'avocado-runner-python-unittest',
'runnable-run-no-args-exit-code': 0,
'runnable-run-uri-only-exit-code': 0
},
{
'runner': 'avocado-runner-avocado-instrumented',
'runnable-run-no-args-exit-code': 0,
'runnable-run-uri-only-exit-code': 0
},
{
'runner': 'avocado-runner-tap',
'runnable-run-no-args-exit-code': 0,
'runnable-run-uri-only-exit-code': 0
},
]
}
if (python_module_available('avocado-framework-plugin-golang')
and 'golang' not in args.disable_plugin_checks):
config_nrunner_interface['run.dict_variants'].append({
'runner':
'avocado-runner-golang',
'runnable-run-no-args-exit-code':
0,
'runnable-run-uri-only-exit-code':
0
})
if (python_module_available('avocado-framework-plugin-robot')
and 'robot' not in args.disable_plugin_checks):
config_nrunner_interface['run.dict_variants'].append({
'runner':
'avocado-runner-robot',
'runnable-run-no-args-exit-code':
0,
'runnable-run-uri-only-exit-code':
0
})
if args.nrunner_interface:
suites.append(
TestSuite.from_config(config_nrunner_interface,
"nrunner-interface"))
# ========================================================================
# Run all static checks, unit and functional tests
# ========================================================================
selftests = []
if args.unit:
selftests.append('selftests/unit/')
if args.jobs:
selftests.append('selftests/jobs/')
if args.functional:
selftests.append('selftests/functional/')
config_check = {
'resolver.references': selftests,
'run.ignore_missing_references': True
}
if args.static_checks:
config_check['resolver.references'] += glob.glob('selftests/*.sh')
if args.optional_plugins:
for optional_plugin in glob.glob('optional_plugins/*'):
plugin_name = os.path.basename(optional_plugin)
if plugin_name not in args.disable_plugin_checks:
pattern = '%s/tests/*' % optional_plugin
config_check['resolver.references'] += glob.glob(pattern)
suites.append(TestSuite.from_config(config_check, "check"))
return suites
def create_suites(args): # pylint: disable=W0621
suites = []
# ========================================================================
# Run nrunner interface checks for all available runners
# ========================================================================
config_nrunner_interface = {
"resolver.references":
["selftests/functional/test_nrunner_interface.py"],
"run.dict_variants.variant_id_keys": ["runner"],
"run.dict_variants": [
{
"runner": "avocado-runner",
"runnable-run-no-args-exit-code": 2,
"runnable-run-uri-only-exit-code": 2,
"task-run-id-only-exit-code": 2,
},
{
"runner": "avocado-runner-dry-run",
"runnable-run-no-args-exit-code": 0,
"runnable-run-uri-only-exit-code": 0,
"task-run-id-only-exit-code": 0,
},
{
"runner": "avocado-runner-noop",
"runnable-run-no-args-exit-code": 0,
"runnable-run-uri-only-exit-code": 0,
"task-run-id-only-exit-code": 0,
},
{
"runner": "avocado-runner-exec-test",
"runnable-run-no-args-exit-code": 0,
"runnable-run-uri-only-exit-code": 0,
"task-run-id-only-exit-code": 0,
},
{
"runner": "avocado-runner-python-unittest",
"runnable-run-no-args-exit-code": 0,
"runnable-run-uri-only-exit-code": 0,
"task-run-id-only-exit-code": 0,
},
{
"runner": "avocado-runner-avocado-instrumented",
"runnable-run-no-args-exit-code": 0,
"runnable-run-uri-only-exit-code": 0,
"task-run-id-only-exit-code": 0,
},
{
"runner": "avocado-runner-tap",
"runnable-run-no-args-exit-code": 0,
"runnable-run-uri-only-exit-code": 0,
"task-run-id-only-exit-code": 0,
},
{
"runner": "avocado-runner-podman-image",
"runnable-run-no-args-exit-code": 0,
"runnable-run-uri-only-exit-code": 0,
"task-run-id-only-exit-code": 0,
},
],
}
if (python_module_available("avocado-framework-plugin-golang")
and "golang" not in args.disable_plugin_checks):
config_nrunner_interface["run.dict_variants"].append({
"runner":
"avocado-runner-golang",
"runnable-run-no-args-exit-code":
0,
"runnable-run-uri-only-exit-code":
0,
"task-run-id-only-exit-code":
0,
})
if (python_module_available("avocado-framework-plugin-robot")
and "robot" not in args.disable_plugin_checks):
config_nrunner_interface["run.dict_variants"].append({
"runner":
"avocado-runner-robot",
"runnable-run-no-args-exit-code":
0,
"runnable-run-uri-only-exit-code":
0,
"task-run-id-only-exit-code":
0,
})
if (python_module_available("avocado-framework-plugin-ansible")
and "ansible" not in args.disable_plugin_checks):
config_nrunner_interface["run.dict_variants"].append({
"runner":
"avocado-runner-ansible-module",
"runnable-run-no-args-exit-code":
0,
"runnable-run-uri-only-exit-code":
0,
"task-run-id-only-exit-code":
0,
})
if args.dict_tests["nrunner-interface"]:
suites.append(
TestSuite.from_config(config_nrunner_interface,
"nrunner-interface"))
# ========================================================================
# Run functional requirement tests
# ========================================================================
config_nrunner_requirement = {
"resolver.references":
["selftests/functional/serial/test_requirements.py"],
"nrunner.max_parallel_tasks": 1,
"run.dict_variants": [
{
"spawner": "process"
},
{
"spawner": "podman"
},
],
}
if args.dict_tests["nrunner-requirement"]:
suites.append(
TestSuite.from_config(config_nrunner_requirement,
"nrunner-requirement"))
# ========================================================================
# Run all static checks, unit and functional tests
# ========================================================================
config_check = {
"run.ignore_missing_references": True,
}
if args.dict_tests["unit"]:
config_check_unit = copy.copy(config_check)
config_check_unit["resolver.references"] = ["selftests/unit/"]
suites.append(TestSuite.from_config(config_check_unit, "unit"))
if args.dict_tests["jobs"]:
config_check_jobs = copy.copy(config_check)
config_check_jobs["resolver.references"] = ["selftests/jobs/"]
suites.append(TestSuite.from_config(config_check_jobs, "jobs"))
if args.dict_tests["functional"]:
functional_path = os.path.join("selftests", "functional")
references = glob.glob(os.path.join(functional_path, "test*.py"))
references.extend([
os.path.join(functional_path, "utils"),
os.path.join(functional_path, "plugin"),
])
config_check_functional_parallel = copy.copy(config_check)
config_check_functional_parallel["resolver.references"] = references
suites.append(
TestSuite.from_config(config_check_functional_parallel,
"functional-parallel"))
config_check_functional_serial = copy.copy(config_check)
config_check_functional_serial["resolver.references"] = [
"selftests/functional/serial/"
]
config_check_functional_serial["nrunner.max_parallel_tasks"] = 1
suites.append(
TestSuite.from_config(config_check_functional_serial,
"functional-serial"))
if args.dict_tests["static-checks"]:
config_check_static = copy.copy(config_check)
config_check_static["resolver.references"] = glob.glob(
"selftests/*.sh")
suites.append(
TestSuite.from_config(config_check_static, "static-checks"))
if args.dict_tests["optional-plugins"]:
config_check_optional = copy.copy(config_check)
config_check_optional["resolver.references"] = []
for optional_plugin in glob.glob("optional_plugins/*"):
plugin_name = os.path.basename(optional_plugin)
if plugin_name not in args.disable_plugin_checks:
pattern = f"{optional_plugin}/tests/*"
config_check_optional["resolver.references"] += glob.glob(
pattern)
suites.append(
TestSuite.from_config(config_check_optional, "optional-plugins"))
return suites
def create_suite_job_api(args): # pylint: disable=W0621
suites = []
def get_ref(method_short_name):
return [f"{__file__}:JobAPIFeaturesTest.test_{method_short_name}"]
# ========================================================================
# Test if the archive file was created
# ========================================================================
config_check_archive_file_exists = {
"resolver.references":
get_ref("check_archive_file_exists"),
"run.dict_variants.variant_id_keys": ["namespace", "value"],
"run.dict_variants": [
{
"namespace": "run.results.archive",
"value": True,
"assert": True
},
],
}
suites.append(
TestSuite.from_config(config_check_archive_file_exists,
f"job-api-{len(suites) + 1}"))
# ========================================================================
# Test if the category directory was created
# ========================================================================
config_check_category_directory_exists = {
"resolver.references":
get_ref("check_category_directory_exists"),
"run.dict_variants.variant_id_keys": ["namespace", "value"],
"run.dict_variants": [
{
"namespace": "run.job_category",
"value": "foo",
"assert": True
},
],
}
suites.append(
TestSuite.from_config(config_check_category_directory_exists,
f"job-api-{len(suites) + 1}"))
# ========================================================================
# Test if a directory was created
# ========================================================================
config_check_directory_exists = {
"resolver.references":
get_ref("check_directory_exists"),
"run.dict_variants.variant_id_keys": ["namespace", "value"],
"run.dict_variants": [
{
"namespace": "sysinfo.collect.enabled",
"value": True,
"directory": "sysinfo",
"assert": True,
},
{
"namespace": "sysinfo.collect.enabled",
"value": False,
"directory": "sysinfo",
"assert": False,
},
],
}
suites.append(
TestSuite.from_config(config_check_directory_exists,
f"job-api-{len(suites) + 1}"))
# ========================================================================
# Test the content of a file
# ========================================================================
config_check_file_content = {
"resolver.references":
get_ref("check_file_content"),
"run.dict_variants.variant_id_keys": ["namespace", "value", "file"],
"run.dict_variants": [
# finding the correct 'content' here is trick because any
# simple string is added to the variant file name and is
# found in the log file.
# Using DEBUG| makes the variant name have DEBUG_, working
# fine here.
{
"namespace": "job.output.loglevel",
"value": "INFO",
"file": "job.log",
"content": r"DEBUG\| Test metadata:$",
"assert": False,
"regex": True,
},
{
"namespace": "job.run.result.tap.include_logs",
"value": True,
"file": "results.tap",
"reference": ["examples/tests/passtest.py:PassTest.test"],
"content": "PASS 1-examples/tests/passtest.py:PassTest.test",
"assert": True,
},
{
"namespace": "job.run.result.tap.include_logs",
"value": False,
"file": "results.tap",
"content": "Command '/bin/true' finished with 0",
"assert": False,
},
{
"namespace": "job.run.result.xunit.job_name",
"value": "foo",
"file": "results.xml",
"content": 'name="foo"',
"assert": True,
},
{
"namespace": "job.run.result.xunit.max_test_log_chars",
"value": 1,
"file": "results.xml",
"content": "--[ CUT DUE TO XML PER TEST LIMIT ]--",
"assert": True,
"reference": ["examples/tests/failtest.py:FailTest.test"],
"exit_code": 1,
},
{
"namespace": "run.failfast",
"value": True,
"file": "results.json",
"content": '"skip": 1',
"assert": True,
"reference": ["/bin/false", "/bin/true"],
"exit_code": 9,
"extra_job_config": {
"nrunner.max_parallel_tasks": 1
},
},
{
"namespace": "run.ignore_missing_references",
"value": "on",
"file": "results.json",
"content": '"pass": 1',
"assert": True,
"reference": ["/bin/true", "foo"],
},
{
"namespace": "run.unique_job_id",
"value": "abcdefghi",
"file": "job.log",
"content": "Job ID: abcdefghi",
"assert": True,
},
{
"namespace": "job.run.timeout",
"value": 1,
"reference": ["examples/tests/sleeptenmin.py"],
"file": "job.log",
"content": "RuntimeError: Test interrupted by SIGTERM",
"assert": True,
"exit_code": 8,
},
],
}
suites.append(
TestSuite.from_config(config_check_file_content,
f"job-api-{len(suites) + 1}"))
# ========================================================================
# Test if the result file was created
# ========================================================================
config_check_file_exists = {
"resolver.references":
get_ref("check_file_exists"),
"run.dict_variants.variant_id_keys": ["namespace", "value"],
"run.dict_variants": [
{
"namespace": "job.run.result.json.enabled",
"value": True,
"file": "results.json",
"assert": True,
},
{
"namespace": "job.run.result.json.enabled",
"value": False,
"file": "results.json",
"assert": False,
},
{
"namespace": "job.run.result.tap.enabled",
"value": True,
"file": "results.tap",
"assert": True,
},
{
"namespace": "job.run.result.tap.enabled",
"value": False,
"file": "results.tap",
"assert": False,
},
{
"namespace": "job.run.result.xunit.enabled",
"value": True,
"file": "results.xml",
"assert": True,
},
{
"namespace": "job.run.result.xunit.enabled",
"value": False,
"file": "results.xml",
"assert": False,
},
{
"namespace": "run.dry_run.enabled",
"value": True,
"file": "job.log",
"assert": False,
},
{
"namespace": "run.dry_run.no_cleanup",
"value": True,
"file": "job.log",
"assert": True,
},
{
"namespace": "plugins.disable",
"value": ["result.xunit"],
"file": "result.xml",
"assert": False,
},
# this test needs a huge improvement
{
"namespace": "run.journal.enabled",
"value": True,
"file": ".journal.sqlite",
"assert": True,
},
],
}
if (python_module_available("avocado-framework-plugin-result-html")
and "html" not in args.disable_plugin_checks):
config_check_file_exists["run.dict_variants"].append({
"namespace": "job.run.result.html.enabled",
"value": True,
"file": "results.html",
"assert": True,
})
config_check_file_exists["run.dict_variants"].append({
"namespace": "job.run.result.html.enabled",
"value": False,
"file": "results.html",
"assert": False,
})
suites.append(
TestSuite.from_config(config_check_file_exists,
f"job-api-{len(suites) + 1}"))
# ========================================================================
# Test if a file was created
# ========================================================================
config_check_output_file = {
"resolver.references":
get_ref("check_output_file"),
"run.dict_variants.variant_id_keys": ["namespace", "file"],
"run.dict_variants": [
{
"namespace": "job.run.result.json.output",
"file": "custom.json",
"assert": True,
},
# https://github.com/avocado-framework/avocado/issues/4034
{
"namespace": "job.run.result.tap.output",
"file": "custom.tap",
"assert": True,
},
{
"namespace": "job.run.result.xunit.output",
"file": "custom.xml",
"assert": True,
},
],
}
if (python_module_available("avocado-framework-plugin-result-html")
and "html" not in args.disable_plugin_checks):
config_check_output_file["run.dict_variants"].append({
"namespace": "job.run.result.html.output",
"file": "custom.html",
"assert": True,
})
suites.append(
TestSuite.from_config(config_check_output_file,
f"job-api-{len(suites) + 1}"))
# ========================================================================
# Test if the temporary directory was created
# ========================================================================
config_check_tmp_directory_exists = {
"resolver.references":
get_ref("check_tmp_directory_exists"),
"run.dict_variants.variant_id_keys": ["namespace", "value"],
"run.dict_variants": [
{
"namespace": "run.keep_tmp",
"value": True,
"assert": True
},
],
}
suites.append(
TestSuite.from_config(config_check_tmp_directory_exists,
f"job-api-{len(suites) + 1}"))
return suites