def xcov_run(self, main):
"""run MAIN through "xcov run" to produce an execution trace."""
# Feed xcov run with full path (absolute dir) of the program so we
# can directly get to the binary from the trace when reading it from
# a different directory, such as in consolidation tests.
ofile = "xcov_run_%s.out" % main
# Some execution engines (e.g. valgrind) do not let us distinguish
# executed program errors from engine errors. Because of them, we
# ignore here any kind of execution error for tests expected to trigger
# failures (such as harness tests), assuming that they will perform
# further checks that are bound to fail if the execution doesn't
# proceed as expected somehow (e.g. not producing a trace).
xrun([
self.abdir_for(main) + exename_for(main),
"--level=%s" % self.xcovlevel
] + self.scoptions,
out=ofile,
register_failure=not self.testcase.expect_failures)
thistest.fail_if(
match(
"(!!! EXCEPTION RAISED !!!"
"|raised [A-Z_]+ : [-._a-zA-Z]+:[0-9]+ \w+)", ofile),
"exception raised while running '%s'." % main)
def check(self):
Rchapter.check(self)
for sec in self.skeys:
thistest.fail_if(
len(sec.start_hits) > 0 and not self.checked[sec],
"summary count check missing for section %s" % sec.name)
def try_one(subdir, extra_covargs, xreports, xwarnings):
"""
Setup a temp ``subdir`` and perform a build/run/coverage sequence
for our example, passing ``extra_covargs`` in addition to gnatcov
coverage. Verify that we obtain the reports stated as expected
in ``xreports``, and that possible warnings on units-of-interest
discrepancies (induced by the extra covargs), stated as expected in
``xwarnings``, are found in the logs.
"""
wd.to_subdir(subdir)
gpr = gprfor(srcdirs="../src", mains="test_t.adb",
extra='\n'.join(
['for Source_Files use',
' ("test_t.adb","flip.ads", "flip.adb");']))
build_run_and_coverage(
gprsw=GPRswitches(root_project=gpr),
covlevel='stmt',
mains=['test_t'],
extra_coverage_args=['--annotate=xcov'] + extra_covargs)
check_xcov_reports('obj/*.xcov', xreports)
wlog = contents_of('coverage.log')
for xw in xwarnings:
thistest.fail_if(
xw not in wlog,
'expected warning "%s" not found in log' % xw)
wd.to_homedir()
def run(self, register_failure=True):
'''
Return if "gnatcov coverage" executed properly.
'''
Wdir('tmp_')
# Compile and run separately each test driver.
for test_driver, switches in self.test_drivers.items():
self._compile(test_driver, switches)
self._run(test_driver)
# Tell to gnatcov which symbols we are interested in (those are symbols
# that appear in the coverage expectations).
self._generate_routines_list()
# Consolidate resulting traces and parse the object coverage results.
# If consolidation fails, return False.
if (not self._consolidate_traces(self.RESULT_FILE, register_failure)
and not register_failure):
return False
# We can parse the result only if the output is an annotated ASM.
if self.annotate == 'asm':
coverage_result = self._parse_coverage_results(self.RESULT_FILE)
# Compare results with expectations...
thistest.fail_if(
coverage_result != self.coverage_expectations,
'Coverage result:\n'
'{}'
'do not match coverage expectations:\n'
'{}'.format(self.format_coverage(coverage_result),
self.format_coverage(self.coverage_expectations)))
return True
def check(options, xunits):
"""
Check that running gnatcov coverage with the provided list of
`options` for project selection yields `xunits` as the set of
units of interest, conveyed as a list. Verify the list of units
reported by --dump-units-to and the set of produced .xcov reports.
"""
# Compute the name of a unique output dir for the reports,
# based on the options.
opt_str = ''.join([
opt.replace('--projects=',
'').replace('root', 'r').replace('../', '').replace(
'.gpr',
'').replace('_X=True',
'').replace('--no-subprojects',
'ns').strip('-')
for opt in options
])
odir = 'tmp_' + opt_str
ofile = os.path.join(odir, 'dump-units')
xcov([
'coverage', trace, '--level=stmt', '--annotate=xcov',
'--output-dir={}'.format(odir), '--dump-units-to={}'.format(ofile),
board_arg
] + options,
out='cov-' + opt_str + '.log')
# Primary check, from the list of units reported by --dump-units-to
xunits = set(xunits)
runits = set(lines_of(ofile))
thistest.fail_if(
runits != xunits,
"for options '{}', reported list of units not as expected:\n"
"expected: {}\n"
"obtained: {}".format(' '.join(options), xunits, runits))
# Secondary check, from the set of generated reports
# We have a list/set of expected unit names on the one hand, and a set of
# .xcov files in `odir` on the other hand. In the latter set, we can have
# multiple files for a unit, for example a spec and a body for a package.
# Map the report files to a set of units by stripping both the leading
# subdir part and the extension.
runits = set(
os.path.basename(r).split('.')[0]
for r in e3.fs.ls(os.path.join(odir, '*.xcov')))
thistest.fail_if(
runits != xunits,
"for options '{}', list of units from reports not as expected:\n"
"expected: {}\n"
"obtained: {}".format(' '.join(options), xunits, runits))
def run_test(self, main):
"""Execute the MAIN program to produce an execution trace, and
trigger a failure if it raises an unhandled exception."""
out_file = self.mode_execute(main=main)
thistest.fail_if(
match(
"(!!! EXCEPTION RAISED !!!"
"|raised [A-Z_]+ : [-._a-zA-Z]+:[0-9]+ \w+)", out_file),
"exception raised while running '%s'." % main)
def xcov(args,
out=None,
err=None,
inp=None,
env=None,
register_failure=True,
auto_config_args=True,
auto_target_args=True):
"""
Run xcov with arguments ARGS, timeout control, valgrind control if
available and enabled, output directed to OUT and failure registration if
register_failure is True. Return the process status descriptor. ARGS may be
a list or a whitespace separated string.
See xcov_suite_args for the meaning of AUTO_*_ARGS arguments.
"""
# Make ARGS a list from whatever it is, to allow unified processing.
# Then fetch the requested command, always first:
args = to_list(args)
covcmd = args[0]
covargs = args[1:]
covargs = xcov_suite_args(covcmd, covargs, auto_config_args,
auto_target_args) + covargs
# Determine which program we are actually going launch. This is
# "gnatcov <cmd>" unless we are to execute some designated program
# for this:
covpgm = thistest.suite_covpgm_for(covcmd)
covpgm = ([covpgm] if covpgm is not None else maybe_valgrind([XCOV]) +
[covcmd])
# Execute, check status, raise on error and return otherwise.
#
# The gprvar options are only needed for the "libsupport" part of our
# projects. They are pointless wrt coverage run or analysis activities
# so we don't include them here.
p = cmdrun(cmd=covpgm + covargs,
inp=inp,
out=out,
err=err,
env=env,
register_failure=register_failure)
if thistest.options.enable_valgrind == 'memcheck':
memcheck_log = contents_of(MEMCHECK_LOG)
thistest.fail_if(
memcheck_log, 'MEMCHECK log not empty'
'\nFROM "%s":'
'\n%s' % (' '.join(covpgm + covargs), memcheck_log))
return p
def check(self):
"""Once we're done reading the entire report, sanity check what we
found for this block. Raise a test failure"""
n_starts = len(self.start_hits)
n_ends = len(self.end_hits)
thistest.fail_if (
n_starts != n_ends,
"(%s report section): %d starts != %d ends" % (
self.name, n_starts, n_ends)
)
def try_match(self, sec, rline):
p = re.match(self.skeys[sec], rline)
if p:
sum_count = self.value(p.group(1))
sec_count = sec.ecount
thistest.fail_if(
sum_count != sec_count,
"summary count %d != section count %d for %s" %
(sum_count, sec_count, sec.name))
thistest.fail_if(
len(sec.start_hits) != 1,
"summary found for section starts != 1 (%s)" % sec.name)
self.checked[sec] = True
def _tryone(sco_args, odir):
"""
Exercise one execution and analysis of the test_ab program
for stmt+mcdc with the provided `sco_args`, switching to a
temporary output dir `odir` that we create.
"""
# Create the output dir and execute the commands from there
wsd = Wdir(subdir=odir)
_run_and_cov(pgm='../test_ab', level='stmt+mcdc', sco_args=sco_args)
# Check report contents
args = 'stmt+mcdc / ' + ' '.join(sco_args)
test_xcov = contents_of('test_ab.adb.xcov')
thistest.fail_if(not re.search(r'\+: Monitor.Diamond', test_xcov),
'failure on check for %s' % args)
mon_xcov = contents_of('monitor.adb.xcov')
thistest.fail_if(not re.search(r'\!:.*and then', mon_xcov),
'failure on Decision monitor check for %s' % args)
thistest.fail_if(not re.search(r'\+:.*Hit := Hit \+ 1;', mon_xcov),
'failure on Hit monitor check for %s' % args)
thistest.fail_if(not re.search(r'\+:.*Miss := Miss \+ 1;', mon_xcov),
'failure on Miss monitor check for %s' % args)
wsd.to_homedir()
def check(explicit_exe):
outbase = explicit_exe if explicit_exe else "noexe"
trace = '%s.trace' % outbase
dump = '%s.dt' % outbase
runcmd = ['-P', gprname, '-o', trace]
if explicit_exe:
runcmd.append(explicit_exe)
xrun(runcmd)
xcov(['dump-trace', trace], out=dump)
thistest.fail_if(
len(re.findall('t block$', contents_of(dump), flags=re.M)) < 1,
'with %s, no block trace entry found in %s' % (outbase, trace))
def check_unexpected_reports(self):
"""Check that we don't have unexpected reports or notes."""
[
thistest.fail_if(
self.covctl.unexpected(s),
"report note found for %s, not in expected list" % s)
for s in self.ernotes
]
[
thistest.fail_if(
self.covctl.unexpected(s.rstrip(".xcov")),
"%s report found, for source not in expected list" % s)
for s in ls("*.xcov")
]
def try_one_gpr(gpr, no_such):
label = os.path.basename(os.getcwd())
dump = 'xcov.out'
build_run_and_coverage(
gprsw=GPRswitches(root_project=gpr),
covlevel='stmt',
mains=['p'],
extra_coverage_args=['-axcov'],
out=dump, register_failure=False)
dump = contents_of(dump)
expected_warning = (
'no unit {} in project gen (coverage.units attribute)'.format(no_such)
if no_such else 'no unit of interest')
thistest.fail_if(
expected_warning not in dump,
'[{}] missing warning on absence of ALI for unit'.format(label))
def try_cov(id, gpr):
"""
gnatcov coverage with common set of options & variations via gpr. Expect
valid options and check for commonly expected outcome
"""
log = id + '.clog'
xcov(['coverage',
'-P%s' % gpr,
'%s.trace' % id, '-o',
rep_for(id)],
out=log,
register_failure=False)
# Check that we get a report with expected contents wrt options
# eventually. The tag & level checks make sure that
#
# * options intended for run do get there and not to coverage
# * options intended for coverage do get there and not to run
thistest.fail_if(not empty(log), 'unexpected contents in %s' % log)
rep = contents_of(rep_for(id))
thistest.fail_if('level: %s' % lev not in rep,
'missing expected level indication in %s' % rep_for(id))
thistest.fail_if(not re.search('tag.*: %s' % tag_for(id), rep),
'missing expected tag indication in %s' % rep_for(id))
def gen_one_xcov_report(self, inputs, format, options=""):
"""Helper for gen_xcov_reports, to produce one specific report for a
particulat FORMAT, from provided INPUTS. The command output is saved
in a file named FORMAT.out."""
# Compute the set of arguments we are to pass to gnatcov coverage.
# When project files are used, force report output in the current
# directory where it would be without a project file, and which the
# project file might arbitrarily redirect otherwise. Doing this
# conditionally prevents the gratuitous addition of command line
# options which might be absent from the tool qualified interface
# descriptions.
covargs = ['--annotate=' + format, inputs
] + (self.covoptions + to_list(options))
if self.gprmode:
covargs.append('--output-dir=.')
# Run, latching standard output in a file so we can check contents on
# return.
ofile = format + ".out"
p = xcov(args=['coverage'] + covargs, out=ofile)
# Standard output might typically contain labeling warnings issued
# by the static analysis phase, or error messages issued when a trace
# indicates that some unlabeled edge was taken. None of this should
# happen so we simply fail as soon as the output file is not empty.
# Note that we do this in qualification mode as well, even though what
# we're looking at is not stricly part of the qualified interface.
thistest.fail_if(
os.path.getsize(ofile) > 0,
"xcov standard output not empty (%s):\n--\n%s" %
(ofile, contents_of(ofile)))
def mode_build(self):
# We first need to instrument, with proper selection of the units of
# interest. Expect we are to provide this through a project file as
# we have no LI file at hand:
assert self.gprmode
# If we have a request for specific options, honor that. Otherwise,
# use the already computed project file for this test:
if self.covctl and self.covctl.gprsw:
instrument_gprsw = self.covctl.gprsw
else:
instrument_gprsw = GPRswitches(root_project=self.gpr)
out = 'xinstr.out'
xcov_instrument(
covlevel=self.xcovlevel,
isi_file=self.ISI_FILE,
extra_args=to_list(self.covctl.covoptions) if self.covctl else [],
gprsw=instrument_gprsw,
gpr_obj_dir=self.gpr_obj_dir,
out=out)
# Standard output might contain warnings indicating instrumentation
# issues. This should not happen, so simply fail as soon as the output
# file is not empty.
thistest.fail_if(
os.path.getsize(out) > 0,
'xcov instrument standard output not empty ({}):'
'\n--'
'\n{}'.format(out, contents_of(out)))
# Now we can build, instructing gprbuild to fetch the instrumented
# sources in their dedicated subdir:
gprbuild(self.gpr,
extracargs=self.extracargs,
gargs='--src-subdirs=gnatcov-instr')
deps=['../App/app'],
extra=gprcov_for(switches=[
Csw('*', ['--level=stmt+decision']),
Csw('coverage', ['--annotate=report'])
]))
build_run_and_coverage(gprsw=GPRswitches(root_project=gpr,
no_subprojects=False),
covlevel=None,
mains=[pgm],
extra_coverage_args=['-o', 'def.rep'])
# Check that we get results corresponding to the root project file despite
# "overrides" (other Switches) in subprojects.
rep = contents_of('def.rep')
thistest.fail_if(not os.path.exists('def.rep'), "couldn't find default report")
thistest.fail_if(not re.search('statement not executed', rep),
'missing expected stmt coverage failure indication')
thistest.fail_if(
not re.search(r'test.*\.adb:.*: decision outcome .* never exercised', rep),
'missing expected decision coverage failure indication')
thistest.fail_if(
not re.search('values.adb:.*: decision outcome .* never exercised', rep),
'missing expected decision coverage failure indication')
thistest.result()
from SCOV.minicheck import build_run_and_coverage
from SUITE.context import thistest
from SUITE.cutils import Wdir
from SUITE.gprutils import GPRswitches
from SUITE.tutils import gprfor
wd = Wdir('wd_', clean=True)
gprname = 'p'
mainunit = 'foo.adb'
subdir = 'gnatcov'
mainunit_xcov = os.path.join('obj', subdir, mainunit + '.xcov')
# Arrange to build, run and perform coverage analysis passing
# --subdirs to all gprbuild and gnatcov commands, then verify that
# we find a report in the designated subdir afterwards.
build_run_and_coverage(gprsw=GPRswitches(root_project=gprfor(prjid=gprname,
mains=[mainunit],
srcdirs=['..']),
subdirs=subdir),
covlevel='stmt',
mains=['foo'],
extra_coverage_args=['-a', 'xcov'])
thistest.fail_if(not os.path.exists(mainunit_xcov),
'The coverage report is missing: {}'.format(mainunit_xcov))
thistest.result()
# Check that command specific args prevail over "*", with "*" placed
# before in the gpr file.
id = 'star_postover'
check_valid_sequence_for(id=id,
gprcov=gprcov_for(switches=[
Csw('*', ['--level=stmt+mcdc']),
Csw('run', tagopt_for(id)),
Csw('coverage',
levopt_for(id) + annopt_for(id))
]))
# Likewise, with "*" placed after in the gpr file.
id = 'star_preover'
check_valid_sequence_for(id=id,
gprcov=gprcov_for(switches=[
Csw('run', tagopt_for(id)),
Csw('coverage',
levopt_for(id) + annopt_for(id)),
Csw('*', ['--level=stmt+mcdc'])
]))
# Check that "*" applies to all. Pass invalid for run, check failure.
id = 'star_invalid'
gpr = gprvariant(id=id, extra=gprcov_for(switches=[Csw('*', annopt_for(id))]))
rlog = try_run(id, gpr)
thistest.fail_if('--annotate is not valid with the "run" command.' not in rlog,
'missing expected failure indication in run log for %s' % id)
thistest.result()
def check(self):
thistest.fail_if(
len(self.start_hits) > 0,
"Unexpected headers caught by %s:\n%s" \
% (self.__class__.__name__, ''.join(self.start_hits))
)
def __validate_ecount(self, count):
self.ecount = len(self.enotes)
thistest.fail_if (
count != self.ecount,
"(%s report section) recognized %d notes != summary (%d)\n" %
(self.name, self.ecount, count))
trace = tracename_for(mainbase)
# Build with the real target as the Target attribute.
instantiate_gpr(target)
gprbuild(os.path.abspath(gpr_basename))
argv = ['-P{}'.format(gprname), '-o', trace, exe]
# Run with a bad target as the Target attribute in order to check that the
# --target argument actually takes precedence.
with_target_arg = mode == 'with_arg'
if with_target_arg:
instantiate_gpr('this_target_does_not_exist')
# Force the passing of --target in the native case, as xrun() does not
# pass it when it is the default.
if not env.is_cross:
argv.append('--target={}'.format(target))
xrun(argv, auto_config_args=False, auto_target_args=with_target_arg)
dump = 'dump.txt'
xcov('dump-trace {}'.format(trace), out=dump)
thistest.fail_if(
len(re.findall('t block$', contents_of(dump), flags=re.M)) < 1,
'with exe, no block execution trace found in {}'.format(trace))
wd.to_homedir()
thistest.result()
def xcov_instrument(gprsw,
covlevel,
extra_args=[],
dump_trigger="auto",
dump_channel="auto",
gpr_obj_dir=None,
runtime_project=None,
out=None,
err=None,
warnings_as_errors=True,
register_failure=True):
"""
Run "gnatcov instrument" on a project.
:param GPRswitches gprsw: Project file command line switches to honor.
:param None|str covlevel: Coverage level for the instrumentation
(--level argument). Not passed if None.
:param list[str] extra_args: Extra arguments to append to the command line.
:param None|str dump_trigger: If None, do not pass the --dump-trigger
argument. If "auto", pass the result of default_dump_trigger().
Otherwise, pass the given value.
:param None|str dump_channel: If None, do not pass the --dump-channel
argument. If "auto", pass the result of default_dump_channel().
Otherwise, pass the given value.
:param None|str gpr_obj_dir: Optional name of the directory where gprbuild
will create build artifacts. If left to None, assume they are produced
in the current directory.
:param None|str runtime_project: If None, use the default name for the
instrumentation runtime project. Otherwise, use the name given for this
option.
:param bool warnings_as_errors: Whether to make the test fail if there are
warnings in gnatcov's output.
See SUITE.tutils.xcov for the other supported options.
"""
# Create the object directory so that gnatcov does not warn that it
# does not exist. This is specific to the source trace mode because
# we run gnatcov before gprbuild.
if gpr_obj_dir:
mkdir(gpr_obj_dir)
covlevel_args = [] if covlevel is None else ['--level', covlevel]
# We want to get the mains to know which dump-trigger should be passed to
# the instrumentation command.
#
# Capture the list of main file names, double quoted and comma separated.
m = re.search(pattern=r"for Main use \((?P<mains>.*)\)",
string=contents_of(gprsw.root_project))
# If found, split then remove whitespaces and double quotes
mains = []
if m:
mains = m.group('mains').split(',')
mains = [main.strip(' "') for main in mains]
args = ['instrument'] + covlevel_args
if dump_trigger:
if dump_trigger == "auto":
dump_trigger = default_dump_trigger(mains)
args += ["--dump-trigger", dump_trigger]
if dump_channel:
if dump_channel == "auto":
dump_channel = default_dump_channel()
args += ["--dump-channel", dump_channel]
if runtime_project:
args += ["--runtime-project", runtime_project]
args += gprsw.cov_switches + extra_args
if thistest.options.pretty_print:
args.append('--pretty-print')
out = out or "instrument.log"
xcov(args, out=out, err=err, register_failure=register_failure)
if warnings_as_errors:
output = contents_of(out)
thistest.fail_if(
"!!!" in output,
f"Warnings detected in the output of 'gnatcov instrument':"
f"\n{indent(output)}")
def __init__(self, testcase, drivers, xfile, xcovlevel, covctl, wdctl):
# The TESTCASE object that delegates the hard work to us :-)
self.testcase = testcase
# The set of DRIVER sources that we are to exercise. We use this
# as a precise approximation of a set of main subprogram or local
# executable names so care about basenames only:
self.drivers = [os.path.basename(d) for d in drivers]
# The "--level" argument we ought to use on gnatcov command lines:
self.xcovlevel = xcovlevel
# The CovControl object that controls aspects of our coverage
# testing operations (project file dependencies, units for which
# we expect reports to be produced, ...)
self.covctl = covctl
# Internal attributes: Directory where the instantiation takes place,
# original expectations file, and base prefix of Working Directory
# names
self.homedir = os.getcwd() + "/"
self.xfile = xfile
# The WdirControl object telling about the Working and Binary
# subdir prefixes we are to use:
self.wdctl = wdctl
# Compute the gnatcov coverage specific extra options that we'll have
# to pass. We need these early for Xnote expansions.
self.covoptions = ['--level=' + self.xcovlevel]
if self.covctl:
self.covoptions += to_list(self.covctl.covoptions)
# Compute the list of test launch options strings that we need for
# expectation CTL lines.
ctl_opts = ['--trace-mode=%s' % thistest.options.trace_mode]
self.extracargs = to_list(self.testcase.extracargs)
# { sourcename -> KnoteDict } dictionaries of emitted/expected
# line/report notes. We'll extract emitted notes from reports when we
# know they have been produced. We extract expected notes from the
# provided expectation file.
# This needs to be done now, to make sure that we can register this
# driver object with maximum details for qualification results before
# run() is called, hence early wrt possible exception occurrences.
self.elnotes = {}
self.ernotes = {}
xnotes = XnotesExpander(
xfile=xfile,
xcov_level=xcovlevel,
ctl_opts=ctl_opts,
ctl_cov=self.covoptions,
ctl_cargs=gprbuild_cargs_with(thiscargs=self.extracargs),
ctl_tags=thistest.options.tags,
ctl_cons=[thistest.options.consolidate])
self.xlnotes = xnotes.xlnotes
self.xrnotes = xnotes.xrnotes
# Even though we remember them here, we won't be looking at the
# xlnotes if we're running for qualification.
# Empty expectation sets here mean we have not a single source on
# which anything will be checked. This can only be a mistake and would
# just pass if we let the processing continue.
thistest.fail_if(not self.xlnotes, "empty xlnotes from %s !!" % xfile)
thistest.fail_if(not self.xrnotes, "empty xrnotes from %s !!" % xfile)
def check_report(label, filename, pattern, check_present=True):
report = contents_of(filename)
matched = re.search(pattern, report)
thistest.fail_if(not matched if check_present else matched, label)
mainunits = [base + '.adb' for base in mainbases]
gprbuild(
gprfor(prjid=gprname,
srcdirs=['../../../../src', '../../src'],
mains=mainunits))
# We expect this to work. The multiple mains in the gpr file are just ignored
# when there is an exe on the command line.
exe = exepath_to('test_tt')
trace = 'tt.trace0'
dump = 'tt.dump0'
xrun(['-P', gprname, '-o', trace, exe])
xcov(['dump-trace', trace], out=dump)
thistest.fail_if(
len(re.findall('t block$', contents_of(dump), flags=re.M)) < 1,
'with exe, no block execution trace found in %s' % trace)
# Again, _not_ providing the executable. Expected to fail
# from missing command line argument.
trace = 'oops.trace0'
dump = 'oops.dump'
xrun(['-P', gprname, '-o', trace], out=dump, register_failure=False)
thistest.fail_if(
not match(': Please specify an executable to run', dump),
'missing expected error diag on main-less invocation of gnatcov run')
thistest.result()
