def _run_and_cov(pgm, level, sco_args):
"""
Execute gnatcov run then gnatcov coverage for the provided
program `pgm`, expected to be the base name of without extension
of an executable file. Pass `level` as --level to both commands
and the provided `sco_args` to gnatcov coverage, and gnatcov run
if we are doing mcdc.
"""
level_arg = '--level=%s' % level
# gnatcov run
run_args = [exepath_to(pgm), level_arg]
if 'mcdc' in level_arg:
run_args.extend(sco_args)
xrun(run_args)
# gnatcov coverage
cov_args = [
'--annotate=xcov',
'--trace=%s' % tracename_for(os.path.basename(pgm)), level_arg
] + sco_args
xcov(['coverage'] + cov_args)
def try_cov(id, gpr):
"""
gnatcov coverage with common set of options & variations via gpr. Expect
valid options and check for commonly expected outcome
"""
log = id + '.clog'
xcov(['coverage',
'-P%s' % gpr,
'%s.trace' % id, '-o',
rep_for(id)],
out=log,
register_failure=False)
# Check that we get a report with expected contents wrt options
# eventually. The tag & level checks make sure that
#
# * options intended for run do get there and not to coverage
# * options intended for coverage do get there and not to run
thistest.fail_if(not empty(log), 'unexpected contents in %s' % log)
rep = contents_of(rep_for(id))
thistest.fail_if('level: %s' % lev not in rep,
'missing expected level indication in %s' % rep_for(id))
thistest.fail_if(not re.search('tag.*: %s' % tag_for(id), rep),
'missing expected tag indication in %s' % rep_for(id))
def xcov_instrument(gprsw,
covlevel,
isi_file,
extra_args=[],
dump_method='atexit',
gpr_obj_dir=None,
out=None,
err=None,
register_failure=True):
"""
Run "gnatcov instrument" on a project.
:param GPRswitches gprsw: Project file command line switches to honor.
:param str covlevel: Coverage level for the instrumentation
(--level argument).
:param str isi_file: Name of the ISI file to create.
:param list[str] extra_args: Extra arguments to append to the command line.
:param bool dump_method: Method to dump coverage buffers (--dump-method)
argument.
:param None|str gpr_obj_dir: Optional name of the directory where gprbuild
will create build artifacts. If left to None, assume they are produced
in the current directory.
See SUITE.tutils.xcov for the other supported options.
"""
# Create the object directory so that gnatcov does not warn that it
# does not exist. This is specific to the source trace mode because
# we run gnatcov before gprbuild.
if gpr_obj_dir:
mkdir(gpr_obj_dir)
args = (['instrument', '--level', covlevel, '--dump-method', dump_method] +
gprsw.as_strings + extra_args + [isi_file])
xcov(args, out=out, err=err, register_failure=register_failure)
def check(options, xunits):
"""
Check that running gnatcov coverage with the provided list of
`options` for project selection yields `xunits` as the set of
units of interest, conveyed as a list. Verify the list of units
reported by --dump-units-to and the set of produced .xcov reports.
"""
# Compute the name of a unique output dir for the reports,
# based on the options.
opt_str = ''.join([
opt.replace('--projects=',
'').replace('root', 'r').replace('../', '').replace(
'.gpr',
'').replace('_X=True',
'').replace('--no-subprojects',
'ns').strip('-')
for opt in options
])
odir = 'tmp_' + opt_str
ofile = os.path.join(odir, 'dump-units')
xcov([
'coverage', trace, '--level=stmt', '--annotate=xcov',
'--output-dir={}'.format(odir), '--dump-units-to={}'.format(ofile),
board_arg
] + options,
out='cov-' + opt_str + '.log')
# Primary check, from the list of units reported by --dump-units-to
xunits = set(xunits)
runits = set(lines_of(ofile))
thistest.fail_if(
runits != xunits,
"for options '{}', reported list of units not as expected:\n"
"expected: {}\n"
"obtained: {}".format(' '.join(options), xunits, runits))
# Secondary check, from the set of generated reports
# We have a list/set of expected unit names on the one hand, and a set of
# .xcov files in `odir` on the other hand. In the latter set, we can have
# multiple files for a unit, for example a spec and a body for a package.
# Map the report files to a set of units by stripping both the leading
# subdir part and the extension.
runits = set(
os.path.basename(r).split('.')[0]
for r in e3.fs.ls(os.path.join(odir, '*.xcov')))
thistest.fail_if(
runits != xunits,
"for options '{}', list of units from reports not as expected:\n"
"expected: {}\n"
"obtained: {}".format(' '.join(options), xunits, runits))
def checked_xcov(args, out_file):
"""
Run "xcov" and make the testcase fail if the output file is not empty.
"""
xcov(args, out_file)
out = contents_of(out_file)
thistest.fail_if(
out, 'gnatcov output not empty ({}):\n'
' {}\n'
'{}'.format(out_file, ' '.join(args), out))
def run(extra_args, covlevel=None):
"""
Build and run the single test program, which volontarily performs stmt and
decision coverage violations.
"""
xcov_args = build_and_run(gprsw=GPRswitches(root_project=gpr),
covlevel=covlevel,
mains=[pgm],
extra_coverage_args=[])
xcov(xcov_args + extra_args)
def xcov_convert_base64(base64_file, output_trace_file, out=None, err=None,
register_failure=True):
"""Extract a trace file out of a Base64 file.
:param str base64_file: Name of the file to read.
:param str output_trace_file: Name of the file to write.
See SUITE.tutils.xcov for the other supported options.
"""
xcov(['extract-base64-trace', base64_file, output_trace_file],
out=out, err=err, register_failure=register_failure)
def build_run_and_coverage(out='coverage.log', err=None, register_failure=True,
**kwargs):
"""
Helper to call build_and_run and then invoke `xcov`.
This invokes `xcov` with the command-line that build_and_run returns to
perform the "gnatcov coverage" step.
`out`, `err` and `register_failure` are forwarded to `xcov`, other
arguments are forwarded to `build_and_run`.
"""
xcov_args = build_and_run(register_failure=register_failure, **kwargs)
xcov(xcov_args, out=out, err=err, register_failure=register_failure)
def check(explicit_exe):
outbase = explicit_exe if explicit_exe else "noexe"
trace = '%s.trace' % outbase
dump = '%s.dt' % outbase
runcmd = ['-P', gprname, '-o', trace]
if explicit_exe:
runcmd.append(explicit_exe)
xrun(runcmd)
xcov(['dump-trace', trace], out=dump)
thistest.fail_if(
len(re.findall('t block$', contents_of(dump), flags=re.M)) < 1,
'with %s, no block trace entry found in %s' % (outbase, trace))
def _consolidate_traces(self, output, register_failure):
xcov_args = [
'coverage',
'--level=' + self.level,
'--annotate=' + self.annotate,
]
if self.level in ('insn', 'branch'):
xcov_args.append('--routines=@' + self.ROUTINES_FILE)
xcov_args.extend(self.extra_xcov_args)
xcov_args.extend(map(tracename_for, self.test_drivers))
p = xcov(xcov_args, out=output, register_failure=register_failure)
return p.status == 0
def xcov_instrument(gprsw, covlevel, extra_args=[], dump_trigger=None,
dump_channel=None, gpr_obj_dir=None, out=None, err=None,
register_failure=True):
"""
Run "gnatcov instrument" on a project.
:param GPRswitches gprsw: Project file command line switches to honor.
:param None|str covlevel: Coverage level for the instrumentation
(--level argument). Not passed if None.
:param list[str] extra_args: Extra arguments to append to the command line.
:param None|str dump_trigger: Trigger to dump coverage buffers
(--dump-trigger argument). If left to None,
use SCOV.instr.default_dump_trigger.
:param None|str dump_channel: Channel to dump coverage buffers
(--dump-channel argument). If left to None,
use SCOV.instr.default_dump_channel.
:param None|str gpr_obj_dir: Optional name of the directory where gprbuild
will create build artifacts. If left to None, assume they are produced
in the current directory.
See SUITE.tutils.xcov for the other supported options.
"""
# Create the object directory so that gnatcov does not warn that it
# does not exist. This is specific to the source trace mode because
# we run gnatcov before gprbuild.
if gpr_obj_dir:
mkdir(gpr_obj_dir)
covlevel_args = [] if covlevel is None else ['--level', covlevel]
args = (['instrument'] + covlevel_args +
['--dump-trigger', dump_trigger or default_dump_trigger(),
'--dump-channel', dump_channel or default_dump_channel()] +
gprsw.cov_switches +
extra_args)
if thistest.options.pretty_print:
args.append('--pretty-print')
xcov(args, out=out, err=err, register_failure=register_failure)
def run(subdir, extra_args, covlevel=None):
"""
Build and run the single test program, which volontarily performs stmt and
decision coverage violations.
"""
dirname = f"tmp_{subdir}"
wd = Wdir(dirname)
gpr = gprfor(mains=[pgm + '.adb'],
srcdirs=['../src'],
extra=gprcov_for(switches=[
Csw('*', ['--level=stmt']),
Csw('coverage', ['--annotate=report'])
]))
xcov_args = build_and_run(
gprsw=GPRswitches(root_project=gpr),
covlevel=covlevel,
mains=[pgm],
extra_coverage_args=[] if covlevel is None else ['--level', covlevel])
xcov(xcov_args + extra_args)
wd.to_homedir()
return dirname
def xcov_instrument(gprsw, covlevel, checkpoint,
auto_dump_buffers=True, out=None,
err=None, register_failure=True):
"""
Run "gnatcov instrument" on a project.
:param GPRswitches gprsw: Project file command line switches to honor.
:param str covlevel: Coverage level for the instrumentation
(--level argument).
:param str checkpoint: Name of the checkpoint file to create.
:param bool auto_dump_buffers: Whether to instrument main sources to add a
dump of coverage buffers at the end (done it by default,
--auto-dump-buffers option).
See SUITE.tutils.xcov for the other supported options.
"""
args = ['instrument', '--level={}'.format(covlevel)] + gprsw.as_strings
if auto_dump_buffers:
args.append('--auto-dump-buffers')
args.append(checkpoint)
xcov(args, out=out, err=err, register_failure=register_failure)
def gen_one_xcov_report(self, inputs, format, options=""):
"""Helper for gen_xcov_reports, to produce one specific report for a
particulat FORMAT, from provided INPUTS. The command output is saved
in a file named FORMAT.out."""
# Compute the set of arguments we are to pass to gnatcov coverage.
# When project files are used, force report output in the current
# directory where it would be without a project file, and which the
# project file might arbitrarily redirect otherwise. Doing this
# conditionally prevents the gratuitous addition of command line
# options which might be absent from the tool qualified interface
# descriptions.
covargs = ['--annotate=' + format, inputs
] + (self.covoptions + to_list(options))
if self.gprmode:
covargs.append('--output-dir=.')
# Run, latching standard output in a file so we can check contents on
# return.
ofile = format + ".out"
p = xcov(args=['coverage'] + covargs, out=ofile)
# Standard output might typically contain labeling warnings issued
# by the static analysis phase, or error messages issued when a trace
# indicates that some unlabeled edge was taken. None of this should
# happen so we simply fail as soon as the output file is not empty.
# Note that we do this in qualification mode as well, even though what
# we're looking at is not stricly part of the qualified interface.
thistest.fail_if(
os.path.getsize(ofile) > 0,
"xcov standard output not empty (%s):\n--\n%s" %
(ofile, contents_of(ofile)))
def xcov_instrument(gprsw,
covlevel,
extra_args=[],
dump_trigger="auto",
dump_channel="auto",
gpr_obj_dir=None,
runtime_project=None,
out=None,
err=None,
warnings_as_errors=True,
register_failure=True):
"""
Run "gnatcov instrument" on a project.
:param GPRswitches gprsw: Project file command line switches to honor.
:param None|str covlevel: Coverage level for the instrumentation
(--level argument). Not passed if None.
:param list[str] extra_args: Extra arguments to append to the command line.
:param None|str dump_trigger: If None, do not pass the --dump-trigger
argument. If "auto", pass the result of default_dump_trigger().
Otherwise, pass the given value.
:param None|str dump_channel: If None, do not pass the --dump-channel
argument. If "auto", pass the result of default_dump_channel().
Otherwise, pass the given value.
:param None|str gpr_obj_dir: Optional name of the directory where gprbuild
will create build artifacts. If left to None, assume they are produced
in the current directory.
:param None|str runtime_project: If None, use the default name for the
instrumentation runtime project. Otherwise, use the name given for this
option.
:param bool warnings_as_errors: Whether to make the test fail if there are
warnings in gnatcov's output.
See SUITE.tutils.xcov for the other supported options.
"""
# Create the object directory so that gnatcov does not warn that it
# does not exist. This is specific to the source trace mode because
# we run gnatcov before gprbuild.
if gpr_obj_dir:
mkdir(gpr_obj_dir)
covlevel_args = [] if covlevel is None else ['--level', covlevel]
# We want to get the mains to know which dump-trigger should be passed to
# the instrumentation command.
#
# Capture the list of main file names, double quoted and comma separated.
m = re.search(pattern=r"for Main use \((?P<mains>.*)\)",
string=contents_of(gprsw.root_project))
# If found, split then remove whitespaces and double quotes
mains = []
if m:
mains = m.group('mains').split(',')
mains = [main.strip(' "') for main in mains]
args = ['instrument'] + covlevel_args
if dump_trigger:
if dump_trigger == "auto":
dump_trigger = default_dump_trigger(mains)
args += ["--dump-trigger", dump_trigger]
if dump_channel:
if dump_channel == "auto":
dump_channel = default_dump_channel()
args += ["--dump-channel", dump_channel]
if runtime_project:
args += ["--runtime-project", runtime_project]
args += gprsw.cov_switches + extra_args
if thistest.options.pretty_print:
args.append('--pretty-print')
out = out or "instrument.log"
xcov(args, out=out, err=err, register_failure=register_failure)
if warnings_as_errors:
output = contents_of(out)
thistest.fail_if(
"!!!" in output,
f"Warnings detected in the output of 'gnatcov instrument':"
f"\n{indent(output)}")
mainbases = ['test_tt', 'test_tf']
mainunits = [base + '.adb' for base in mainbases]
gprbuild(
gprfor(prjid=gprname,
srcdirs=['../../../../src', '../../src'],
mains=mainunits))
# We expect this to work. The multiple mains in the gpr file are just ignored
# when there is an exe on the command line.
exe = exepath_to('test_tt')
trace = 'tt.trace0'
dump = 'tt.dump0'
xrun(['-P', gprname, '-o', trace, exe])
xcov(['dump-trace', trace], out=dump)
thistest.fail_if(
len(re.findall('t block$', contents_of(dump), flags=re.M)) < 1,
'with exe, no block execution trace found in %s' % trace)
# Again, _not_ providing the executable. Expected to fail
# from missing command line argument.
trace = 'oops.trace0'
dump = 'oops.dump'
xrun(['-P', gprname, '-o', trace], out=dump, register_failure=False)
thistest.fail_if(
not match(': Please specify an executable to run', dump),
'missing expected error diag on main-less invocation of gnatcov run')
thistest.result()
trace = tracename_for(mainbase)
# Build with the real target as the Target attribute.
instantiate_gpr(target)
gprbuild(os.path.abspath(gpr_basename))
argv = ['-P{}'.format(gprname), '-o', trace, exe]
# Run with a bad target as the Target attribute in order to check that the
# --target argument actually takes precedence.
with_target_arg = mode == 'with_arg'
if with_target_arg:
instantiate_gpr('this_target_does_not_exist')
# Force the passing of --target in the native case, as xrun() does not
# pass it when it is the default.
if not env.is_cross:
argv.append('--target={}'.format(target))
xrun(argv, auto_config_args=False, auto_target_args=with_target_arg)
dump = 'dump.txt'
xcov('dump-trace {}'.format(trace), out=dump)
thistest.fail_if(
len(re.findall('t block$', contents_of(dump), flags=re.M)) < 1,
'with exe, no block execution trace found in {}'.format(trace))
wd.to_homedir()
thistest.result()
