def xcov_run(self, main): """run MAIN through "xcov run" to produce an execution trace.""" # Feed xcov run with full path (absolute dir) of the program so we # can directly get to the binary from the trace when reading it from # a different directory, such as in consolidation tests. ofile = "xcov_run_%s.out" % main # Some execution engines (e.g. valgrind) do not let us distinguish # executed program errors from engine errors. Because of them, we # ignore here any kind of execution error for tests expected to trigger # failures (such as harness tests), assuming that they will perform # further checks that are bound to fail if the execution doesn't # proceed as expected somehow (e.g. not producing a trace). xrun([ self.abdir_for(main) + exename_for(main), "--level=%s" % self.xcovlevel ] + self.scoptions, out=ofile, register_failure=not self.testcase.expect_failures) thistest.fail_if( match( "(!!! EXCEPTION RAISED !!!" "|raised [A-Z_]+ : [-._a-zA-Z]+:[0-9]+ \w+)", ofile), "exception raised while running '%s'." % main)
def check(self): Rchapter.check(self) for sec in self.skeys: thistest.fail_if( len(sec.start_hits) > 0 and not self.checked[sec], "summary count check missing for section %s" % sec.name)
def try_one(subdir, extra_covargs, xreports, xwarnings): """ Setup a temp ``subdir`` and perform a build/run/coverage sequence for our example, passing ``extra_covargs`` in addition to gnatcov coverage. Verify that we obtain the reports stated as expected in ``xreports``, and that possible warnings on units-of-interest discrepancies (induced by the extra covargs), stated as expected in ``xwarnings``, are found in the logs. """ wd.to_subdir(subdir) gpr = gprfor(srcdirs="../src", mains="test_t.adb", extra='\n'.join( ['for Source_Files use', ' ("test_t.adb","flip.ads", "flip.adb");'])) build_run_and_coverage( gprsw=GPRswitches(root_project=gpr), covlevel='stmt', mains=['test_t'], extra_coverage_args=['--annotate=xcov'] + extra_covargs) check_xcov_reports('obj/*.xcov', xreports) wlog = contents_of('coverage.log') for xw in xwarnings: thistest.fail_if( xw not in wlog, 'expected warning "%s" not found in log' % xw) wd.to_homedir()
def run(self, register_failure=True): ''' Return if "gnatcov coverage" executed properly. ''' Wdir('tmp_') # Compile and run separately each test driver. for test_driver, switches in self.test_drivers.items(): self._compile(test_driver, switches) self._run(test_driver) # Tell to gnatcov which symbols we are interested in (those are symbols # that appear in the coverage expectations). self._generate_routines_list() # Consolidate resulting traces and parse the object coverage results. # If consolidation fails, return False. if (not self._consolidate_traces(self.RESULT_FILE, register_failure) and not register_failure): return False # We can parse the result only if the output is an annotated ASM. if self.annotate == 'asm': coverage_result = self._parse_coverage_results(self.RESULT_FILE) # Compare results with expectations... thistest.fail_if( coverage_result != self.coverage_expectations, 'Coverage result:\n' '{}' 'do not match coverage expectations:\n' '{}'.format(self.format_coverage(coverage_result), self.format_coverage(self.coverage_expectations))) return True
def check(options, xunits): """ Check that running gnatcov coverage with the provided list of `options` for project selection yields `xunits` as the set of units of interest, conveyed as a list. Verify the list of units reported by --dump-units-to and the set of produced .xcov reports. """ # Compute the name of a unique output dir for the reports, # based on the options. opt_str = ''.join([ opt.replace('--projects=', '').replace('root', 'r').replace('../', '').replace( '.gpr', '').replace('_X=True', '').replace('--no-subprojects', 'ns').strip('-') for opt in options ]) odir = 'tmp_' + opt_str ofile = os.path.join(odir, 'dump-units') xcov([ 'coverage', trace, '--level=stmt', '--annotate=xcov', '--output-dir={}'.format(odir), '--dump-units-to={}'.format(ofile), board_arg ] + options, out='cov-' + opt_str + '.log') # Primary check, from the list of units reported by --dump-units-to xunits = set(xunits) runits = set(lines_of(ofile)) thistest.fail_if( runits != xunits, "for options '{}', reported list of units not as expected:\n" "expected: {}\n" "obtained: {}".format(' '.join(options), xunits, runits)) # Secondary check, from the set of generated reports # We have a list/set of expected unit names on the one hand, and a set of # .xcov files in `odir` on the other hand. In the latter set, we can have # multiple files for a unit, for example a spec and a body for a package. # Map the report files to a set of units by stripping both the leading # subdir part and the extension. runits = set( os.path.basename(r).split('.')[0] for r in e3.fs.ls(os.path.join(odir, '*.xcov'))) thistest.fail_if( runits != xunits, "for options '{}', list of units from reports not as expected:\n" "expected: {}\n" "obtained: {}".format(' '.join(options), xunits, runits))
def run_test(self, main): """Execute the MAIN program to produce an execution trace, and trigger a failure if it raises an unhandled exception.""" out_file = self.mode_execute(main=main) thistest.fail_if( match( "(!!! EXCEPTION RAISED !!!" "|raised [A-Z_]+ : [-._a-zA-Z]+:[0-9]+ \w+)", out_file), "exception raised while running '%s'." % main)
def xcov(args, out=None, err=None, inp=None, env=None, register_failure=True, auto_config_args=True, auto_target_args=True): """ Run xcov with arguments ARGS, timeout control, valgrind control if available and enabled, output directed to OUT and failure registration if register_failure is True. Return the process status descriptor. ARGS may be a list or a whitespace separated string. See xcov_suite_args for the meaning of AUTO_*_ARGS arguments. """ # Make ARGS a list from whatever it is, to allow unified processing. # Then fetch the requested command, always first: args = to_list(args) covcmd = args[0] covargs = args[1:] covargs = xcov_suite_args(covcmd, covargs, auto_config_args, auto_target_args) + covargs # Determine which program we are actually going launch. This is # "gnatcov <cmd>" unless we are to execute some designated program # for this: covpgm = thistest.suite_covpgm_for(covcmd) covpgm = ([covpgm] if covpgm is not None else maybe_valgrind([XCOV]) + [covcmd]) # Execute, check status, raise on error and return otherwise. # # The gprvar options are only needed for the "libsupport" part of our # projects. They are pointless wrt coverage run or analysis activities # so we don't include them here. p = cmdrun(cmd=covpgm + covargs, inp=inp, out=out, err=err, env=env, register_failure=register_failure) if thistest.options.enable_valgrind == 'memcheck': memcheck_log = contents_of(MEMCHECK_LOG) thistest.fail_if( memcheck_log, 'MEMCHECK log not empty' '\nFROM "%s":' '\n%s' % (' '.join(covpgm + covargs), memcheck_log)) return p
def check(self): """Once we're done reading the entire report, sanity check what we found for this block. Raise a test failure""" n_starts = len(self.start_hits) n_ends = len(self.end_hits) thistest.fail_if ( n_starts != n_ends, "(%s report section): %d starts != %d ends" % ( self.name, n_starts, n_ends) )
def try_match(self, sec, rline): p = re.match(self.skeys[sec], rline) if p: sum_count = self.value(p.group(1)) sec_count = sec.ecount thistest.fail_if( sum_count != sec_count, "summary count %d != section count %d for %s" % (sum_count, sec_count, sec.name)) thistest.fail_if( len(sec.start_hits) != 1, "summary found for section starts != 1 (%s)" % sec.name) self.checked[sec] = True
def _tryone(sco_args, odir): """ Exercise one execution and analysis of the test_ab program for stmt+mcdc with the provided `sco_args`, switching to a temporary output dir `odir` that we create. """ # Create the output dir and execute the commands from there wsd = Wdir(subdir=odir) _run_and_cov(pgm='../test_ab', level='stmt+mcdc', sco_args=sco_args) # Check report contents args = 'stmt+mcdc / ' + ' '.join(sco_args) test_xcov = contents_of('test_ab.adb.xcov') thistest.fail_if(not re.search(r'\+: Monitor.Diamond', test_xcov), 'failure on check for %s' % args) mon_xcov = contents_of('monitor.adb.xcov') thistest.fail_if(not re.search(r'\!:.*and then', mon_xcov), 'failure on Decision monitor check for %s' % args) thistest.fail_if(not re.search(r'\+:.*Hit := Hit \+ 1;', mon_xcov), 'failure on Hit monitor check for %s' % args) thistest.fail_if(not re.search(r'\+:.*Miss := Miss \+ 1;', mon_xcov), 'failure on Miss monitor check for %s' % args) wsd.to_homedir()
def check(explicit_exe): outbase = explicit_exe if explicit_exe else "noexe" trace = '%s.trace' % outbase dump = '%s.dt' % outbase runcmd = ['-P', gprname, '-o', trace] if explicit_exe: runcmd.append(explicit_exe) xrun(runcmd) xcov(['dump-trace', trace], out=dump) thistest.fail_if( len(re.findall('t block$', contents_of(dump), flags=re.M)) < 1, 'with %s, no block trace entry found in %s' % (outbase, trace))
def check_unexpected_reports(self): """Check that we don't have unexpected reports or notes.""" [ thistest.fail_if( self.covctl.unexpected(s), "report note found for %s, not in expected list" % s) for s in self.ernotes ] [ thistest.fail_if( self.covctl.unexpected(s.rstrip(".xcov")), "%s report found, for source not in expected list" % s) for s in ls("*.xcov") ]
def try_one_gpr(gpr, no_such): label = os.path.basename(os.getcwd()) dump = 'xcov.out' build_run_and_coverage( gprsw=GPRswitches(root_project=gpr), covlevel='stmt', mains=['p'], extra_coverage_args=['-axcov'], out=dump, register_failure=False) dump = contents_of(dump) expected_warning = ( 'no unit {} in project gen (coverage.units attribute)'.format(no_such) if no_such else 'no unit of interest') thistest.fail_if( expected_warning not in dump, '[{}] missing warning on absence of ALI for unit'.format(label))
def try_cov(id, gpr): """ gnatcov coverage with common set of options & variations via gpr. Expect valid options and check for commonly expected outcome """ log = id + '.clog' xcov(['coverage', '-P%s' % gpr, '%s.trace' % id, '-o', rep_for(id)], out=log, register_failure=False) # Check that we get a report with expected contents wrt options # eventually. The tag & level checks make sure that # # * options intended for run do get there and not to coverage # * options intended for coverage do get there and not to run thistest.fail_if(not empty(log), 'unexpected contents in %s' % log) rep = contents_of(rep_for(id)) thistest.fail_if('level: %s' % lev not in rep, 'missing expected level indication in %s' % rep_for(id)) thistest.fail_if(not re.search('tag.*: %s' % tag_for(id), rep), 'missing expected tag indication in %s' % rep_for(id))
def gen_one_xcov_report(self, inputs, format, options=""): """Helper for gen_xcov_reports, to produce one specific report for a particulat FORMAT, from provided INPUTS. The command output is saved in a file named FORMAT.out.""" # Compute the set of arguments we are to pass to gnatcov coverage. # When project files are used, force report output in the current # directory where it would be without a project file, and which the # project file might arbitrarily redirect otherwise. Doing this # conditionally prevents the gratuitous addition of command line # options which might be absent from the tool qualified interface # descriptions. covargs = ['--annotate=' + format, inputs ] + (self.covoptions + to_list(options)) if self.gprmode: covargs.append('--output-dir=.') # Run, latching standard output in a file so we can check contents on # return. ofile = format + ".out" p = xcov(args=['coverage'] + covargs, out=ofile) # Standard output might typically contain labeling warnings issued # by the static analysis phase, or error messages issued when a trace # indicates that some unlabeled edge was taken. None of this should # happen so we simply fail as soon as the output file is not empty. # Note that we do this in qualification mode as well, even though what # we're looking at is not stricly part of the qualified interface. thistest.fail_if( os.path.getsize(ofile) > 0, "xcov standard output not empty (%s):\n--\n%s" % (ofile, contents_of(ofile)))
def mode_build(self): # We first need to instrument, with proper selection of the units of # interest. Expect we are to provide this through a project file as # we have no LI file at hand: assert self.gprmode # If we have a request for specific options, honor that. Otherwise, # use the already computed project file for this test: if self.covctl and self.covctl.gprsw: instrument_gprsw = self.covctl.gprsw else: instrument_gprsw = GPRswitches(root_project=self.gpr) out = 'xinstr.out' xcov_instrument( covlevel=self.xcovlevel, isi_file=self.ISI_FILE, extra_args=to_list(self.covctl.covoptions) if self.covctl else [], gprsw=instrument_gprsw, gpr_obj_dir=self.gpr_obj_dir, out=out) # Standard output might contain warnings indicating instrumentation # issues. This should not happen, so simply fail as soon as the output # file is not empty. thistest.fail_if( os.path.getsize(out) > 0, 'xcov instrument standard output not empty ({}):' '\n--' '\n{}'.format(out, contents_of(out))) # Now we can build, instructing gprbuild to fetch the instrumented # sources in their dedicated subdir: gprbuild(self.gpr, extracargs=self.extracargs, gargs='--src-subdirs=gnatcov-instr')
deps=['../App/app'], extra=gprcov_for(switches=[ Csw('*', ['--level=stmt+decision']), Csw('coverage', ['--annotate=report']) ])) build_run_and_coverage(gprsw=GPRswitches(root_project=gpr, no_subprojects=False), covlevel=None, mains=[pgm], extra_coverage_args=['-o', 'def.rep']) # Check that we get results corresponding to the root project file despite # "overrides" (other Switches) in subprojects. rep = contents_of('def.rep') thistest.fail_if(not os.path.exists('def.rep'), "couldn't find default report") thistest.fail_if(not re.search('statement not executed', rep), 'missing expected stmt coverage failure indication') thistest.fail_if( not re.search(r'test.*\.adb:.*: decision outcome .* never exercised', rep), 'missing expected decision coverage failure indication') thistest.fail_if( not re.search('values.adb:.*: decision outcome .* never exercised', rep), 'missing expected decision coverage failure indication') thistest.result()
from SCOV.minicheck import build_run_and_coverage from SUITE.context import thistest from SUITE.cutils import Wdir from SUITE.gprutils import GPRswitches from SUITE.tutils import gprfor wd = Wdir('wd_', clean=True) gprname = 'p' mainunit = 'foo.adb' subdir = 'gnatcov' mainunit_xcov = os.path.join('obj', subdir, mainunit + '.xcov') # Arrange to build, run and perform coverage analysis passing # --subdirs to all gprbuild and gnatcov commands, then verify that # we find a report in the designated subdir afterwards. build_run_and_coverage(gprsw=GPRswitches(root_project=gprfor(prjid=gprname, mains=[mainunit], srcdirs=['..']), subdirs=subdir), covlevel='stmt', mains=['foo'], extra_coverage_args=['-a', 'xcov']) thistest.fail_if(not os.path.exists(mainunit_xcov), 'The coverage report is missing: {}'.format(mainunit_xcov)) thistest.result()
# Check that command specific args prevail over "*", with "*" placed # before in the gpr file. id = 'star_postover' check_valid_sequence_for(id=id, gprcov=gprcov_for(switches=[ Csw('*', ['--level=stmt+mcdc']), Csw('run', tagopt_for(id)), Csw('coverage', levopt_for(id) + annopt_for(id)) ])) # Likewise, with "*" placed after in the gpr file. id = 'star_preover' check_valid_sequence_for(id=id, gprcov=gprcov_for(switches=[ Csw('run', tagopt_for(id)), Csw('coverage', levopt_for(id) + annopt_for(id)), Csw('*', ['--level=stmt+mcdc']) ])) # Check that "*" applies to all. Pass invalid for run, check failure. id = 'star_invalid' gpr = gprvariant(id=id, extra=gprcov_for(switches=[Csw('*', annopt_for(id))])) rlog = try_run(id, gpr) thistest.fail_if('--annotate is not valid with the "run" command.' not in rlog, 'missing expected failure indication in run log for %s' % id) thistest.result()
def check(self): thistest.fail_if( len(self.start_hits) > 0, "Unexpected headers caught by %s:\n%s" \ % (self.__class__.__name__, ''.join(self.start_hits)) )
def __validate_ecount(self, count): self.ecount = len(self.enotes) thistest.fail_if ( count != self.ecount, "(%s report section) recognized %d notes != summary (%d)\n" % (self.name, self.ecount, count))
trace = tracename_for(mainbase) # Build with the real target as the Target attribute. instantiate_gpr(target) gprbuild(os.path.abspath(gpr_basename)) argv = ['-P{}'.format(gprname), '-o', trace, exe] # Run with a bad target as the Target attribute in order to check that the # --target argument actually takes precedence. with_target_arg = mode == 'with_arg' if with_target_arg: instantiate_gpr('this_target_does_not_exist') # Force the passing of --target in the native case, as xrun() does not # pass it when it is the default. if not env.is_cross: argv.append('--target={}'.format(target)) xrun(argv, auto_config_args=False, auto_target_args=with_target_arg) dump = 'dump.txt' xcov('dump-trace {}'.format(trace), out=dump) thistest.fail_if( len(re.findall('t block$', contents_of(dump), flags=re.M)) < 1, 'with exe, no block execution trace found in {}'.format(trace)) wd.to_homedir() thistest.result()
def xcov_instrument(gprsw, covlevel, extra_args=[], dump_trigger="auto", dump_channel="auto", gpr_obj_dir=None, runtime_project=None, out=None, err=None, warnings_as_errors=True, register_failure=True): """ Run "gnatcov instrument" on a project. :param GPRswitches gprsw: Project file command line switches to honor. :param None|str covlevel: Coverage level for the instrumentation (--level argument). Not passed if None. :param list[str] extra_args: Extra arguments to append to the command line. :param None|str dump_trigger: If None, do not pass the --dump-trigger argument. If "auto", pass the result of default_dump_trigger(). Otherwise, pass the given value. :param None|str dump_channel: If None, do not pass the --dump-channel argument. If "auto", pass the result of default_dump_channel(). Otherwise, pass the given value. :param None|str gpr_obj_dir: Optional name of the directory where gprbuild will create build artifacts. If left to None, assume they are produced in the current directory. :param None|str runtime_project: If None, use the default name for the instrumentation runtime project. Otherwise, use the name given for this option. :param bool warnings_as_errors: Whether to make the test fail if there are warnings in gnatcov's output. See SUITE.tutils.xcov for the other supported options. """ # Create the object directory so that gnatcov does not warn that it # does not exist. This is specific to the source trace mode because # we run gnatcov before gprbuild. if gpr_obj_dir: mkdir(gpr_obj_dir) covlevel_args = [] if covlevel is None else ['--level', covlevel] # We want to get the mains to know which dump-trigger should be passed to # the instrumentation command. # # Capture the list of main file names, double quoted and comma separated. m = re.search(pattern=r"for Main use \((?P<mains>.*)\)", string=contents_of(gprsw.root_project)) # If found, split then remove whitespaces and double quotes mains = [] if m: mains = m.group('mains').split(',') mains = [main.strip(' "') for main in mains] args = ['instrument'] + covlevel_args if dump_trigger: if dump_trigger == "auto": dump_trigger = default_dump_trigger(mains) args += ["--dump-trigger", dump_trigger] if dump_channel: if dump_channel == "auto": dump_channel = default_dump_channel() args += ["--dump-channel", dump_channel] if runtime_project: args += ["--runtime-project", runtime_project] args += gprsw.cov_switches + extra_args if thistest.options.pretty_print: args.append('--pretty-print') out = out or "instrument.log" xcov(args, out=out, err=err, register_failure=register_failure) if warnings_as_errors: output = contents_of(out) thistest.fail_if( "!!!" in output, f"Warnings detected in the output of 'gnatcov instrument':" f"\n{indent(output)}")
def __init__(self, testcase, drivers, xfile, xcovlevel, covctl, wdctl): # The TESTCASE object that delegates the hard work to us :-) self.testcase = testcase # The set of DRIVER sources that we are to exercise. We use this # as a precise approximation of a set of main subprogram or local # executable names so care about basenames only: self.drivers = [os.path.basename(d) for d in drivers] # The "--level" argument we ought to use on gnatcov command lines: self.xcovlevel = xcovlevel # The CovControl object that controls aspects of our coverage # testing operations (project file dependencies, units for which # we expect reports to be produced, ...) self.covctl = covctl # Internal attributes: Directory where the instantiation takes place, # original expectations file, and base prefix of Working Directory # names self.homedir = os.getcwd() + "/" self.xfile = xfile # The WdirControl object telling about the Working and Binary # subdir prefixes we are to use: self.wdctl = wdctl # Compute the gnatcov coverage specific extra options that we'll have # to pass. We need these early for Xnote expansions. self.covoptions = ['--level=' + self.xcovlevel] if self.covctl: self.covoptions += to_list(self.covctl.covoptions) # Compute the list of test launch options strings that we need for # expectation CTL lines. ctl_opts = ['--trace-mode=%s' % thistest.options.trace_mode] self.extracargs = to_list(self.testcase.extracargs) # { sourcename -> KnoteDict } dictionaries of emitted/expected # line/report notes. We'll extract emitted notes from reports when we # know they have been produced. We extract expected notes from the # provided expectation file. # This needs to be done now, to make sure that we can register this # driver object with maximum details for qualification results before # run() is called, hence early wrt possible exception occurrences. self.elnotes = {} self.ernotes = {} xnotes = XnotesExpander( xfile=xfile, xcov_level=xcovlevel, ctl_opts=ctl_opts, ctl_cov=self.covoptions, ctl_cargs=gprbuild_cargs_with(thiscargs=self.extracargs), ctl_tags=thistest.options.tags, ctl_cons=[thistest.options.consolidate]) self.xlnotes = xnotes.xlnotes self.xrnotes = xnotes.xrnotes # Even though we remember them here, we won't be looking at the # xlnotes if we're running for qualification. # Empty expectation sets here mean we have not a single source on # which anything will be checked. This can only be a mistake and would # just pass if we let the processing continue. thistest.fail_if(not self.xlnotes, "empty xlnotes from %s !!" % xfile) thistest.fail_if(not self.xrnotes, "empty xrnotes from %s !!" % xfile)
def check_report(label, filename, pattern, check_present=True): report = contents_of(filename) matched = re.search(pattern, report) thistest.fail_if(not matched if check_present else matched, label)
mainunits = [base + '.adb' for base in mainbases] gprbuild( gprfor(prjid=gprname, srcdirs=['../../../../src', '../../src'], mains=mainunits)) # We expect this to work. The multiple mains in the gpr file are just ignored # when there is an exe on the command line. exe = exepath_to('test_tt') trace = 'tt.trace0' dump = 'tt.dump0' xrun(['-P', gprname, '-o', trace, exe]) xcov(['dump-trace', trace], out=dump) thistest.fail_if( len(re.findall('t block$', contents_of(dump), flags=re.M)) < 1, 'with exe, no block execution trace found in %s' % trace) # Again, _not_ providing the executable. Expected to fail # from missing command line argument. trace = 'oops.trace0' dump = 'oops.dump' xrun(['-P', gprname, '-o', trace], out=dump, register_failure=False) thistest.fail_if( not match(': Please specify an executable to run', dump), 'missing expected error diag on main-less invocation of gnatcov run') thistest.result()