def go(self):
""" Execute the test step """
super(Execute, self).go()
try:
self.executor.go(self.plan.workdir)
except tmt.utils.GeneralError as error:
self.info('Error occured during test execution.', color='red')
self.plan.provision.execute('cat nohup.out')
self.plan.provision.sync_workdir_from_guest()
output = {}
for logname in ['stdout.log', 'stderr.log']:
logpath = os.path.join(self.workdir, logname)
output[logname] = open(logpath).read()
self.debug(logname, output[logname], 'yellow')
# Process the stdout.log
overview = output['stdout.log'].rstrip('\nD')
self.verbose('overview', overview, color='green', shift=1)
passed = 0
failed = 0
for character in output['stdout.log']:
if character == '.':
passed += 1
if character == 'F':
failed += 1
passed = listed(passed, 'test')
failed = listed(failed, 'test')
message = f"{passed} passed, {failed} failed"
self.info('result', message, color='green', shift=1)
def overview(tree):
""" Show overview of available tests """
tests = [style(str(test), fg='red') for test in tree.tests()]
echo(
style('Found {}{}{}.'.format(listed(tests, 'test'),
': ' if tests else '',
listed(tests, max=12)),
fg='blue'))
def overview(tree):
""" Show overview of available plans """
plans = [style(str(plan), fg='red') for plan in tree.plans()]
echo(
style('Found {}{}{}.'.format(listed(plans, 'plan'),
': ' if plans else '',
listed(plans, max=12)),
fg='blue'))
def overview(tree):
""" Show overview of available stories """
stories = [style(str(story), fg='red') for story in tree.stories()]
echo(
style('Found {}{}{}.'.format(listed(stories, 'story'),
': ' if stories else '',
listed(stories, max=12)),
fg='blue'))
def finito(commands, *args, **kwargs):
""" Process the main callback """
# Show all commands that have been provided
log.info('Detected {0}{1}.'.format(listed(
commands, 'command'), (': ' + listed(commands)) if commands else ''))
# Run test steps if any explicitly requested or no command given at all
if not commands or any([step in commands for step in tmt.steps.STEPS]):
run.go()
def show(self, brief=False, formatting=None, values=[]):
""" Show metadata """
# Show nothing if there's nothing
if not self.data:
return None
# Custom formatting
if formatting is not None:
formatting = re.sub("\\\\n", "\n", formatting)
name = self.name
data = self.data
root = self.root
evaluated = []
for value in values:
evaluated.append(eval(value))
return formatting.format(*evaluated)
# Show the name
output = utils.color(self.name, 'red')
if brief:
return output
# List available attributes
for key, value in sorted(self.data.items()):
output += "\n{0}: ".format(utils.color(key, 'yellow'))
if isinstance(value, type("")):
output += value
elif isinstance(value, list) and all(
[isinstance(item, type("")) for item in value]):
output += utils.listed(value)
else:
output += pretty(value)
output
return output + "\n"
def show(self, brief=False):
""" Show metadata for each path given """
output = []
for path in self.options.paths or ["."]:
if self.options.verbose:
utils.info("Checking {0} for metadata.".format(path))
tree = fmf.Tree(path)
for node in tree.prune(self.options.whole, self.options.keys,
self.options.names, self.options.filters,
self.options.conditions):
if brief:
show = node.show(brief=True)
else:
show = node.show(brief=False,
formatting=self.options.formatting,
values=self.options.values)
# List source files when in debug mode
if self.options.debug:
for source in node.sources:
show += utils.color("{0}\n".format(source), "blue")
if show is not None:
output.append(show)
# Print output and summary
if brief or self.options.formatting:
joined = "".join(output)
else:
joined = "\n".join(output)
print(joined, end="")
if self.options.verbose:
utils.info("Found {0}.".format(utils.listed(len(output),
"object")))
self.output = joined
def main(cmdline=None):
""" Parse options, gather metadata, print requested data """
# Parse command line arguments
options, arguments = Options().parse(cmdline)
if not arguments:
arguments = ["."]
output = ""
# Enable debugging output
if options.debug:
utils.log.setLevel(utils.LOG_DEBUG)
# Show metadata for each path given
counter = 0
for path in arguments:
if options.verbose:
utils.info("Checking {0} for metadata.".format(path))
tree = fmf.Tree(path)
for node in tree.prune(options.whole, options.keys, options.names,
options.filters):
show = node.show(options.brief, options.formatting, options.values)
if show is not None:
print(show, end="")
output += show
counter += 1
# Print summary
if options.verbose:
utils.info("Found {0}.".format(utils.listed(counter, "object")))
return output
def summary(self):
""" Give a concise summary of the discovery """
# Summary of selected tests
text = listed(len(self.tests()), 'test') + ' selected'
self.info('summary', text, 'green', shift=1)
# Test list in verbose mode
for test in self.tests():
self.verbose(test.name, color='red', shift=2)
def go():
""" Go and do test steps for selected plans """
echo(
style('Found {0}.\n'.format(listed(tree.plans(), 'plan')),
fg='magenta'))
for plan in tree.plans():
plan.ls(summary=True)
plan.go()
echo()
def go(self):
""" Go and do test steps for selected plans """
# Show run id / workdir path
self.info(self.workdir, color='magenta')
# Attempt to load run data
self.load()
if self.opt('follow'):
self.follow()
# Enable selected steps
enabled_steps = self._context.obj.steps
all_steps = self.opt('all') or not enabled_steps
since = self.opt('since')
until = self.opt('until')
after = self.opt('after')
before = self.opt('before')
skip = self.opt('skip')
if all_steps or since or until:
# Detect index of the first and last enabled step
if since:
first = tmt.steps.STEPS.index(since)
elif after:
first = tmt.steps.STEPS.index(after) + 1
else:
first = tmt.steps.STEPS.index('discover')
if until:
last = tmt.steps.STEPS.index(until)
elif before:
last = tmt.steps.STEPS.index(before) - 1
else:
last = tmt.steps.STEPS.index('finish')
# Enable all steps between the first and last
for index in range(first, last + 1):
step = tmt.steps.STEPS[index]
if step not in skip:
enabled_steps.add(step)
self.debug(f"Enabled steps: {fmf.utils.listed(enabled_steps)}")
# Show summary, store run data
if not self.plans:
raise tmt.utils.GeneralError("No plans found.")
self.verbose('Found {0}.'.format(listed(self.plans, 'plan')))
self.save()
# Iterate over plans
for plan in self.plans:
plan.go()
# Update the last run id at the very end
# (override possible runs created during execution)
self.config.last_run(self.workdir)
# Give the final summary, remove workdir, handle exit codes
self.finish()
def go(self):
""" Go and do test steps for selected plans """
# Show run id / workdir path
self.info(self.workdir, color='magenta')
# Enable all steps if none selected or --all provided
if self.opt('all_') or not self._context.obj.steps:
self._context.obj.steps = set(tmt.steps.STEPS)
# Show summary and iterate over plans
self.verbose('Found {0}.'.format(listed(self.plans, 'plan')))
for plan in self.plans:
plan.go()
def coverage(
context, code, test, docs,
implemented, verified, documented, covered,
unimplemented, unverified, undocumented, uncovered, **kwargs):
"""
Show code, test and docs coverage for given stories.
Regular expression can be used to filter stories by name.
Use '.' to select stories under the current working directory.
"""
tmt.Story._save_context(context)
def headfoot(text):
""" Format simple header/footer """
echo(style(text.rjust(4) + ' ', fg='blue'), nl=False)
header = False
total = code_coverage = test_coverage = docs_coverage = 0
if not any([code, test, docs]):
code = test = docs = True
for story in context.obj.tree.stories():
# Check conditions
if not story._match(
implemented, verified, documented, covered, unimplemented,
unverified, undocumented, uncovered):
continue
# Show header once
if not header:
if code:
headfoot('code')
if test:
headfoot('test')
if docs:
headfoot('docs')
headfoot('story')
echo()
header = True
# Show individual stats
status = story.coverage(code, test, docs)
total += 1
code_coverage += status[0]
test_coverage += status[1]
docs_coverage += status[2]
# Summary
if not total:
return
if code:
headfoot('{}%'.format(round(100 * code_coverage / total)))
if test:
headfoot('{}%'.format(round(100 * test_coverage / total)))
if docs:
headfoot('{}%'.format(round(100 * docs_coverage / total)))
headfoot('from {}'.format(listed(total, 'story')))
echo()
def go(self):
""" Execute the test step """
super(Execute, self).go()
lognames = ('stdout.log', 'stderr.log', 'nohup.out')
# Remove logs prior to write
for name in lognames:
logpath = os.path.join(self.workdir, name)
if os.path.exists(logpath):
os.remove(logpath)
try:
self.executor.go(self.plan.workdir)
except tmt.utils.GeneralError as error:
self.get_logs(lognames)
raise tmt.utils.GeneralError(f'Test execution failed: {error}')
output = self.get_logs(lognames)
# Process the stdout.log
overview = output['stdout.log'].rstrip('\nD')
self.verbose('overview', overview, color='green', shift=1)
passed = 0
failed = 0
errors = 0
for character in output['stdout.log']:
if character == '.':
passed += 1
if character == 'F':
failed += 1
if character == 'E':
errors += 1
passed = listed(passed, 'test')
failed = listed(failed, 'test')
message = f"{passed} passed, {failed} failed"
self.info('result', message, color='green', shift=1)
if errors > 0:
raise tmt.utils.GeneralError(
f"{errors} errors occured during tests.")
def init(context, path, template, force, **kwargs):
"""
Initialize a new tmt tree.
By default tree is created in the current directory.
Provide a PATH to create it in a different location.
\b
A tree can be optionally populated with example metadata:
* 'mini' template contains a minimal plan and no tests,
* 'base' template contains a plan and a beakerlib test,
* 'full' template contains a 'full' story, an 'full' plan and a shell test.
"""
# Check for existing tree
path = os.path.realpath(path)
try:
tree = tmt.Tree(path)
# Are we creating a new tree under the existing one?
if path == tree.root:
echo("Tree '{}' already exists.".format(tree.root))
else:
tree = None
except tmt.utils.GeneralError:
tree = None
# Create a new tree
if tree is None:
try:
fmf.Tree.init(path)
tree = tmt.Tree(path)
except fmf.utils.GeneralError as error:
raise tmt.utils.GeneralError(
"Failed to initialize tree in '{}': {}".format(
path, error))
echo("Tree '{}' initialized.".format(tree.root))
# Populate the tree with example objects if requested
if template == 'empty':
non_empty_choices = [c for c in _init_template_choices if c != 'empty']
echo("To populate it with example content, use --template with "
"{}.".format(listed(non_empty_choices, join='or')))
else:
echo("Applying template '{}'.".format(template, _init_templates))
if template == 'mini':
tmt.Plan.create('/plans/example', 'mini', tree, force)
elif template == 'base':
tmt.Test.create('/tests/example', 'beakerlib', tree, force)
tmt.Plan.create('/plans/example', 'base', tree, force)
elif template == 'full':
tmt.Test.create('/tests/example', 'shell', tree, force)
tmt.Plan.create('/plans/example', 'full', tree, force)
tmt.Story.create('/stories/example', 'full', tree, force)
def get_command(self, context, cmd_name):
""" Allow command shortening """
# Support both story & stories
cmd_name = cmd_name.replace('story', 'stories')
found = click.Group.get_command(self, context, cmd_name)
if found is not None:
return found
matches = [command for command in self.list_commands(context)
if command.startswith(cmd_name)]
if not matches:
return None
elif len(matches) == 1:
return click.Group.get_command(self, context, matches[0])
context.fail('Did you mean {}?'.format(
listed(sorted(matches), join='or')))
def tests(self):
""" Return all discovered tests """
# Show filters if provided
if self.filters:
for filter_ in self.filters:
self.info('filter', filter_, 'green')
# Nothing to do in dry mode
if self.opt('dry'):
return []
# Prepare test name filter if provided
tests = self.tests_tree.tests(filters=self.filters)
# Summary of selected tests, test list in verbose mode
self.info('tests', listed(len(tests), 'test') + ' selected', 'green')
for test in tests:
self.verbose(test.name, color='red', shift=1)
return tests
def show(self, brief=False):
""" Show metadata """
# Show the name
output = utils.color(self.name, 'red')
if brief:
return output
# List available attributes
for key, value in sorted(self.data.iteritems()):
output += "\n{0}: ".format(utils.color(key, 'yellow'))
if isinstance(value, basestring):
output += value
elif isinstance(value, list) and all(
[isinstance(item, basestring) for item in value]):
output += utils.listed(value)
else:
output += pretty(value)
output
return output
def coverage(
context, code, test, docs,
implemented, tested, documented, covered,
unimplemented, untested, undocumented, uncovered, **kwargs):
""" Show code, test and docs coverage for given stories. """
tmt.Story._context = context
def headfoot(text):
""" Format simple header/footer """
echo(style(text.rjust(4) + ' ', fg='blue'), nl=False)
header = False
total = code_coverage = test_coverage = docs_coverage = 0
if not any([code, test, docs]):
code = test = docs = True
for story in context.obj.tree.stories():
# Header
if not header:
if code:
headfoot('code')
if test:
headfoot('test')
if docs:
headfoot('docs')
headfoot('story')
echo()
header = True
# Coverage
if story._match(implemented, tested, documented, covered,
unimplemented, untested, undocumented, uncovered):
status = story.coverage(code, test, docs)
total += 1
code_coverage += status[0]
test_coverage += status[1]
docs_coverage += status[2]
# Summary
if code:
headfoot('{}%'.format(round(100 * code_coverage / total)))
if test:
headfoot('{}%'.format(round(100 * test_coverage / total)))
if docs:
headfoot('{}%'.format(round(100 * docs_coverage / total)))
headfoot('from {}'.format(listed(total, 'story')))
echo()
def main(cmdline=None):
""" Parse options, gather metadata, print requested data """
# Parse command line arguments
options, arguments = Options().parse(cmdline)
if not arguments:
arguments = ["."]
output = ""
# Enable debugging output
if options.debug:
utils.log.setLevel(utils.LOG_DEBUG)
# Show metadata for each path given
counter = 0
for path in arguments:
if options.verbose:
utils.info("Checking {0} for metadata.".format(path))
tree = fmf.Tree(path)
for node in tree.climb(options.whole):
# Select only nodes with key content
if not all([key in node.data for key in options.keys]):
continue
# Select nodes with name matching regular expression
if options.names and not any(
[re.search(name, node.name) for name in options.names]):
continue
# Apply advanced filters if given
try:
if not all([utils.filter(filter, node.data)
for filter in options.filters]):
continue
# Handle missing attribute as if filter failed
except utils.FilterError:
continue
show = node.show(brief=options.brief)
print(show)
output += show + "\n"
counter += 1
# Print summary
if options.verbose:
utils.info("Found {0}.".format(utils.listed(counter, "object")))
return output
def export(self, format_='rst', title=True):
""" Export story data into requested format """
output = ''
# Title and its anchor
if title:
depth = len(re.findall('/', self.name)) - 1
title = self.title or re.sub('.*/', '', self.name)
output += f'\n.. _{self.name}:\n'
output += '\n{}\n{}\n'.format(title, '=~^:-><'[depth] * len(title))
# Summary, story and description
if self.summary and self.summary != self.node.parent.get('summary'):
output += '\n{}\n'.format(self.summary)
if self.story != self.node.parent.get('story'):
output += '\n*{}*\n'.format(self.story.strip())
if self.description:
output += '\n{}\n'.format(self.description)
# Examples
if self.example:
output += '\nExamples::\n\n'
output += tmt.utils.format('',
self.example,
wrap=False,
indent=4,
key_color=None,
value_color=None) + '\n'
# Status
if not self.node.children:
status = []
for coverage in ['implemented', 'tested', 'documented']:
if getattr(self, coverage):
status.append(coverage)
output += "\nStatus: {}\n".format(
listed(status) if status else 'idea')
return output
def go(self):
""" Discover available tests """
super(DiscoverFmf, self).go()
testdir = os.path.join(self.workdir, 'tests')
# Clone provided git repository
if self.repository:
self.info('repository', self.repository, 'green')
self.debug(f"Clone '{self.repository}' to '{testdir}'.")
self.run(f'git clone {self.repository} {testdir}')
# Copy current directory to workdir
else:
directory = self.step.plan.run.tree.root
self.info('directory', directory, 'green')
self.debug("Copy '{}' to '{}'.".format(directory, testdir))
shutil.copytree(directory, testdir)
# Checkout revision if requested
if self.revision:
self.info('revision', self.revision, 'green')
self.debug(f"Checkout revision '{self.revision}'.")
self.run(f"git checkout -f {self.revision}", cwd=testdir)
# Show filters if provided
if self.filters:
for filter_ in self.filters:
self.info('filter', filter_, 'green')
# Initialize the metadata tree
self.debug(f"Check metadata tree in '{testdir}'.")
# Nothing more to do here when in dry mode
if self.opt('dry'):
return []
tests = tmt.Tree(testdir).tests(filters=self.filters)
# Modify test names and paths to make them unique
for test in tests:
test.name = f"/{self.name}{test.name}"
test.path = f"/{self.name}/tests{test.path}"
# Summary of selected tests, test list in verbose mode
self.info('tests', listed(len(tests), 'test') + ' selected', 'green')
for test in tests:
self.verbose(test.name, color='red', shift=1)
self._tests = tests
def test_text(self):
assert (listed(range(6), 'category') == '6 categories')
assert (listed(7, "leaf", "leaves") == '7 leaves')
assert (listed(0, "item") == "0 items")
Regular expression can be used to filter tests for linting.
Use '.' to select tests under the current working directory.
"""
# FIXME: Workaround https://github.com/pallets/click/pull/1840 for click 7
context.params.update(kwargs)
tmt.Test._save_context(context)
exit_code = 0
for test in context.obj.tree.tests():
if not test.lint():
exit_code = 1
echo()
raise SystemExit(exit_code)
_test_templates = listed(tmt.templates.TEST, join='or')
@tests.command()
@click.pass_context
@click.argument('name')
@click.option('-t',
'--template',
metavar='TEMPLATE',
help='Test template ({}).'.format(_test_templates),
prompt='Template ({})'.format(_test_templates))
@verbose_debug_quiet
@force_dry
def create(context, name, template, force, **kwargs):
"""
Create a new test based on given template.
def test_basic(self):
assert (listed(range(1)) == '0')
assert (listed(range(2)) == '0 and 1')
def test_quoting(self):
assert (listed(range(3), quote='"') == '"0", "1" and "2"')
@name_filter_condition
@verbose_debug_quiet
def lint(context, **kwargs):
"""
Check tests against the L1 metadata specification
Regular expression can be used to filter tests for linting.
Use '.' to select tests under the current working directory.
"""
tmt.Test._save_context(context)
for test in context.obj.tree.tests():
test.lint()
echo()
_test_templates = listed(tmt.templates.TEST, join='or')
@tests.command()
@click.pass_context
@click.argument('name')
@click.option('-t',
'--template',
metavar='TEMPLATE',
help='Test template ({}).'.format(_test_templates),
prompt='Template ({})'.format(_test_templates))
@verbose_debug_quiet
@force_dry
def create(context, name, template, force, **kwargs):
""" Create a new test based on given template. """
tmt.Test._save_context(context)
def test_max(self):
assert (listed(range(4), max=3) == '0, 1, 2 and 1 more')
assert (listed(range(5), 'number', max=2) == '0, 1 and 3 more numbers')
echo()
@tests.command()
@click.pass_context
@name_filter_condition
@verbose_debug_quiet
def lint(context, **kwargs):
""" Check tests against the L1 metadata specification. """
tmt.Test._context = context
for test in context.obj.tree.tests():
test.lint()
echo()
_test_templates = listed(tmt.templates.TEST, join='or')
@tests.command()
@click.pass_context
@click.argument('name')
@click.option(
'-t', '--template', metavar='TEMPLATE',
help='Test template ({}).'.format(_test_templates),
prompt='Template ({})'.format(_test_templates))
@verbose_debug_quiet
@force_dry
def create(context, name, template, **kwargs):
""" Create a new test based on given template. """
tmt.Test._context = context
tmt.Test.create(name, template, context.obj.tree, force)