def _execute_testpath(self, testspec, testwriter, testpath, exepath, result,
tester, caseid, case=""):
"""Executes the unit test in the specified testing folder for 'case'."""
if not path.exists(testpath):
mkdir(testpath)
#Copy across all the input files we need to run.
for i in testspec.inputs:
i.copy(self._codefolder, testpath, case, self.compiler)
#Also copy any assignment file dependencies.
testwriter.copy(self._codefolder, testpath, case, testspec.identifier, self.compiler)
#Clean the testing folder to remove any target variable output files
#from any earlier test runs.
testspec.clean(testpath)
testwriter.setup(testpath)
#If the testspec needs auto-class support, write the case to file.
if testspec.autoclass:
with open(path.join(testpath, "fpy_case"), 'w') as f:
f.write('"{}"'.format(case))
#Save the path to the folder for execution in the result.
result.paths[caseid] = testpath
msg.okay("Executing {}.x in folder ./tests{}".format(testspec.identifier,
testpath.split("tests")[1]))
start_time = clock()
from os import waitpid
from subprocess import Popen, PIPE
command = "cd {}; {}".format(testpath, exepath)
prun = Popen(command, shell=True, executable="/bin/bash", stdout=PIPE, stderr=PIPE)
waitpid(prun.pid, 0)
if not self.quiet:
output = prun.stdout.readlines()
if len(output) > 0:
msg.std(''.join(output))
#else: #We don't need to get these lines since we are purposefully redirecting them.
error = prun.stderr.readlines()
if len(error) > 0:
if self.quiet:
msg.info("With Executable at {}".format(exepath), 1)
msg.err('\n '+' '.join(error))
code = len(error)
if case == "":
result.cases[caseid] = ExecutionResult(testpath, code, clock() - start_time, tester)
else:
result.cases[caseid] = ExecutionResult(testpath, code, clock() - start_time, tester, case)
self._write_success(testpath, code)
#See if we need to run the post-execution profiling
if self.profile:
profiling.profile(testpath, testspec.testgroup.method_fullname,
exepath, self.compiler)
def _finalize_testpath(self, result, caseid, testpath, exepath, testspec, code,
start_time, tester, case=""):
"""Cleans up and sets results for the test that ran.
"""
if case == "":
result.cases[caseid] = ExecutionResult(testpath, code, clock() - start_time, tester)
else:
result.cases[caseid] = ExecutionResult(testpath, code, clock() - start_time, tester, case)
self._write_success(testpath, code)
#See if we need to run the post-execution profiling
if self.profile:
profiling.profile(testpath, testspec.testgroup.method_fullname,
exepath, self.compiler)
class UnitTester(object):
"""Performs automatic unit testing of individual subroutines and
functions based on XML doctags found in the code files.
:arg libraryroot: the path to folder in which to stage the tests.
"""
def __init__(self, libraryroot=None, verbose=False, compare_templates=None,
fortpy_templates=None, rerun=None , compiler=None,
debug=False, profile=False, quiet=False):
self.parser = CodeParser()
self.parser.verbose = verbose
from fortpy.utility import set_fortpy_templates
set_fortpy_templates(self, fortpy_templates)
self.tgenerator = TestGenerator(self.parser, libraryroot, self.fortpy_templates, self, rerun)
self.compiler = None
"""The full path to the compiler to use for the unit tests.
"""
self.quiet = quiet
"""Specifies whether the tester should run in quiet mode, which only prints
essential information for the unit tests to stdout.
"""
self.debug = debug == True
self.set_compiler(compiler)
self.profile = self._profiler_exists(profile)
#A flag to track whether the generator has already written
#the executables.
self._written = False
#The user's raw value for the compare_templates directory
self._compare_templates = compare_templates
self._templatev = {}
"""Holds the version number of the fortpy.f90 file."""
def template_version(self, filename):
"""Returns the version number of the latest fortpy.f90 file."""
if filename not in self._templatev:
from os import path
from fortpy.utility import get_fortpy_templates_dir
tempath = path.join(get_fortpy_templates_dir(), filename)
self._templatev[filename] = self.get_fortpy_version(tempath)
return self._templatev[filename]
def get_fortpy_version(self, fortpath, recursed=False):
"""Gets the fortpy version number from the first line of the specified file."""
result = []
#If the file doesn't exist yet, we don't try to find the version information.
from os import path
if not path.isfile(fortpath) or path.splitext(fortpath)[1] in [".o", ".mod"]:
if path.isfile(fortpath + '.v'):
return self.get_fortpy_version(fortpath + '.v', True)
else:
return result
with open(fortpath) as f:
for line in f:
try:
lt = line.index("<")
vxml = "<doc>{}</doc>".format(line[lt::])
except ValueError:
vxml = ""
break
if "<fortpy" in vxml:
import xml.etree.ElementTree as ET
x = list(ET.XML(vxml))
if len(x) > 0:
result = list(map(int, x[0].attrib["version"].split(".")))
if len(result) == 0 and not recursed:
return self.get_fortpy_version(fortpath + '.v', True)
else:
return result
def tests(self, identifier):
"""Returns a dictionary of all the tests that need to be run for the
specified module.executable identifier."""
if identifier in self.tgenerator.xtests:
return self.tgenerator.xtests[identifier]
else:
return {}
def writer(self, identifier):
"""Returns the underlying executable writer that has a list of ordered
method/assignment dependencies."""
if identifier in self.tgenerator.xwriters:
return self.tgenerator.xwriters[identifier]
def libraryroot(self, identifier):
"""Returns the absolute path to the staging directory for the unit tests with
the specified testid.
"""
if identifier in self.tgenerator.xgenerator.folders:
return self.tgenerator.xgenerator.folders[identifier]
def _profiler_exists(self, profile):
"""Tests whether we have gprof available to do the profiling of the methods
to unit test.
:arg profile: the value specified in the UnitTester constructor.
"""
if profile==True:
gprof = self.which("gprof")
if gprof is None:
msg.err("gprof is required to run profiling with fortpy.")
exit(1)
else:
return True
else:
return False
def set_compiler(self, compiler):
"""Sets the compiler to use for the unit testing of this code parser.
"""
if compiler is not None:
self.compiler = compiler
self._compiler_exists()
def _compiler_exists(self):
"""Tests whether the specified compiler is available on the machine. If
it isn't give an error and exit."""
from fortpy.testing.compilers import compilers
if self.compiler in compilers:
#Overwrite the *name* of the compiler with its full path; since
#fortpy assumes that self.compiler is the name of a valid executable
#this will still work correctly.
compiler = compilers[self.compiler].path
else:
compiler = self.compiler
if self.which(compiler) is None:
msg.err("compiler {} not found. Exiting.".format(self.compiler))
exit(1)
def which(self, program):
"""Tests whether the specified program is anywhere in the environment
PATH so that it probably exists."""
import os
def is_exe(fpath):
return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
fpath, fname = os.path.split(program)
if fpath:
if is_exe(program):
return program
else:
for path in os.environ["PATH"].split(os.pathsep):
path = path.strip('"')
exe_file = os.path.join(path, program)
if is_exe(exe_file):
return exe_file
return None
def writeall(self, codefolder):
"""Writes all the unit test executables that are new or modified
as well as the makefiles for all subroutines in all modules."""
#The test generator already loops over all modules in the code
#parser and does all the heavy-lifting. We need to pre-load any
#modules that we are interested in testing. Only those loaded
#when write() is first called will have their tests processed.
self._codefolder = path.abspath(codefolder)
if self._compare_templates is not None:
self.compare_templates = path.abspath(self._compare_templates)
else:
self.compare_templates = path.join(self._codefolder, "templates/")
#We will load all the modules in the code folder specified and
#then run the test generator.
files = {}
self.parser.scan_path(self._codefolder, files)
for f in files:
filepath = files[f]
self.parser.parse(filepath, True, True)
#Now that we have loaded all the codefiles in the path, we can
#generate the unittest executables
self.tgenerator.write(self._codefolder)
self._written = True
def runall(self, compiler=None):
"""Compiles and runs each new or modified unit test executable.
After the run is complete, the outcomes are checked for consistency.
:arg compiler: the name of the compiler in 'compilers.xml' to use.
"""
#We will likely need a file comparer for the outcomes testing
self.comparer = FileComparer(self.fortpy_templates, self.compare_templates)
if self._written:
self.set_compiler(compiler)
#Run them each individually and return a dictionary of all the
#test results
result = {}
for composite_id in self.tgenerator.tests_to_run:
identifier, testid = composite_id.split("|")
oneresult = self._run_single(identifier, testid)
if oneresult is not None:
result[composite_id] = oneresult
return result
else:
msg.warn("you can't run tests until the executables have been written. Exiting.")
return None
def _run_single(self, identifier, testid):
"""Runs all unit test cases for the specified identifier."""
#Just initialize a result object and populate its properties
#using the various _run_* methods.
result = TestResult(identifier, testid)
result.compiled, result.target = self._run_compile(identifier, testid)
if result.compiled:
self._run_exec(identifier, testid, result)
if self.tests(identifier)[testid].runchecks:
#Only return a test result if the checks were actually run.
return result
def _compile_fortpyf90(self, tversion):
"""Compiles a fortpy.mod and fortpy.o for the current compiler.
"""
msg.info("Compiling fortpy.mod and fortpy.f90 for {}".format(self.compiler))
from os import waitpid, path
from subprocess import Popen, PIPE
from fortpy.testing.compilers import executor, replace
command = "cd {0}; {1} fortpy.f90; {1} -c fortpy.f90".format(self.fortpy_templates, executor(self.compiler))
pcompile = Popen(command, shell=True, executable="/bin/bash", stdout=PIPE, stderr=PIPE)
waitpid(pcompile.pid, 0)
opath = path.join(self.fortpy_templates, "fortpy.o")
mpath = path.join(self.fortpy_templates, "fortpy.mod")
if path.isfile(opath) and path.isfile(mpath):
from shutil import move
nopath = path.join(self.fortpy_templates, replace("fortpy.o.[c]", self.compiler))
nmpath = path.join(self.fortpy_templates, replace("fortpy.mod.[c]", self.compiler))
move(opath, nopath)
move(mpath, nmpath)
#Create the version files so we can keep track of the compiled versions.
vpaths = [nopath + ".v", nmpath + ".v"]
for vp in vpaths:
with open(vp, 'w') as f:
f.write('#<fortpy version="{}" />'.format('.'.join(map(str, tversion))))
else:
msg.err("Unable to generate fortpy.o and fortpy.mod.")
def _run_compile(self, identifier, testid):
"""Compiles the executable that was created for the specified identifier,
returns True if the compile was successful."""
#Because we often run the tests for multiple compiler versions, we need
#a copy of the execution directory that was setup for the testing.
from fortpy.testing.compilers import replace, executor, family
from fortpy.utility import copytree
from os import path
source = path.join(self.libraryroot(identifier), identifier)
target = replace(source + ".[c]", self.compiler)
copytree(source, target)
#Before we compile, we need to make sure we have the fortpy.o and fortpy.mod
#files for the specific compiler.
tversion = self.template_version("fortpy.f90")
for sdfile in ["fortpy.o", "fortpy.mod"]:
fdfile = replace(sdfile + ".[c]", self.compiler)
ftarget = path.join(target, sdfile)
dversion = self.get_fortpy_version(ftarget)
if not path.isfile(ftarget) or dversion != tversion:
from shutil import copy
source = path.join(self.fortpy_templates, fdfile)
sversion = self.get_fortpy_version(source)
if not path.isfile(source) or sversion != tversion:
self._compile_fortpyf90(tversion)
msg.info(" COPY: {}".format(source))
copy(source, ftarget)
#If the file is a binary, we need to save a .v with version
#information as well for the next time we want to copy it.
pre, ext = path.splitext(ftarget)
if ext in [".o", ".so", ".mod"]:
with open(ftarget + '.v', 'w') as f:
f.write("# <fortpy version=\"{}\" />".format('.'.join(map(str, tversion))))
#Find the target folder that has the executables etc then run
#make and check the exit code.
msg.blank()
options = ""
if self.debug:
options += " DEBUG=true"
if self.profile:
options += " GPROF=true"
codestr = "cd {}; make -f 'Makefile.{}' F90='{}' FAM='{}'" + options
#If we are running in quiet mode, we don't want the compile information
#to post to stdout; only errors should be redirected. This means we need
#to wrap the execution in a subprocess and redirect the std* to PIPE
from os import waitpid
from subprocess import Popen, PIPE
command = codestr.format(target, testid, executor(self.compiler), family(self.compiler))
pcompile = Popen(command, shell=True, executable="/bin/bash", stdout=PIPE, stderr=PIPE)
waitpid(pcompile.pid, 0)
if not self.quiet:
output = [x.decode('utf8') for x in pcompile.stdout.readlines()]
msg.std(''.join(output))
#else: #We don't need to get these lines since we are purposefully redirecting them.
error = pcompile.stderr.readlines()
code = len(error)
if code != 0:
msg.err(''.join(error))
#It turns out that the compiler still returns a code of zero, even if the compile
#failed because the actual compiler didn't fail; it did its job properly. We need to
#check for the existence of errors in the 'compile.log' file.
lcount = 0
errors = []
log = path.join(target, "compile.log")
with open(log) as f:
for line in f:
lcount += 1
if lcount > 21 and lcount < 32:
errors.append(line)
elif lcount > 21:
break
if len(errors) > 0:
#There are 21 lines in the compile.log file when everything runs correctly
#Overwrite code with a bad exit value since we have some other problems.
code = 1
#We want to write the first couple of errors to the console and give them the
#option to still execute if the compile only generated warnings.
msg.warn("compile generated some errors or warnings:")
msg.blank()
msg.info(''.join(errors))
#If the executable exists, we could still prompt them to run it (in case the
#additional lines were just warnings).
exe = path.join(target, "{}.x".format(testid))
if path.isfile(exe):
choice = input("\nWould you still like to run the executable? ").lower()
code = 0 if "y" in choice else code
if "n" in choice:
msg.err("Unit testing terminated by user.")
exit(0)
else:
msg.err("Could not compile executable {}.x".format(testid))
exit(-1)
return code == 0, target
def _run_exec(self, identifier, testid, result):
"""Runs the executable for unit test for the specified identifier
for each of the outcomes specified in the doctags."""
if not self.tests(identifier)[testid].execute:
#We don't want to carry on with this execution at all. User-specified
#override.
return
#Get the home path of the executable. A sub-folder for tests
#needs to be created. For tests that have input and output files
#a home/tests/testid.case folder gets created and the source files
#get copied.
#Create the folder for staging the tests.
tests = path.join(result.target, "tests")
if not path.exists(tests):
mkdir(tests)
#Now we need determine which tests to run from the outcomes and folder tags.
kmodule, kmethod = identifier.lower().split(".")
module = self.parser.modules[kmodule]
method = module.executables[kmethod]
#Get the absolute path to the executable that we created
exepath = path.join(result.target, "{}.x".format(testid))
#Since we have already split out all the tests that need to be run and
#we have a 'testid' for the current test to run, just run that test.
self._run_folder(self.tests(identifier)[testid], tests, result, exepath,
self.writer(identifier))
if not self.tests(identifier)[testid].runchecks:
return
#Now that we have run all of the executables, we can analyze their
#output to see if it matches.
for case in result.cases:
xres = result.cases[case]
xres.test(case, result)
def _run_folder(self, testspec, testsfolder, result, exepath, testwriter):
"""Runs the executable for the sources in the folder doctag.
:arg testspec: a TestSpecification instance for this unit test.
:arg testsfolder: the path to the unit tests unique staging folder.
:arg result: a TestResult instance that execution results can be
added to as they complete.
:arg expath: the full path to the executable to run in the folder.
:arg testwriter: the MethodWriter instance that generated the test
that will be run.
"""
#We can use the same outcome tester afterwards for all of the cases.
tester = OutcomeTester(testspec, self._codefolder, self.comparer,
self.parser.verbose)
#The execution can either be case-based or once-off.
if testspec.cases is not None:
#We need to run the executable multiple times, once for each case
#Each case has input files specified relative to the code folder.
for case in testspec.cases:
caseid = "{}.{}".format(testspec.identifier, case)
if not caseid in result.cases:
#Make a separate directory for the case and copy all its inputs.
casepath = path.join(testsfolder, caseid)
self._execute_testpath(testspec, testwriter, casepath, exepath,
result, tester, caseid, case)
else:
result.warnings.append("Duplicate CASES specified for unit testing:" +
" {}".format(caseid))
else:
#Create a folder for this test specification to run in.
testpath = path.join(testsfolder, testspec.identifier)
self._execute_testpath(testspec, testwriter, testpath, exepath, result,
tester, testspec.identifier)
def _execute_testpath(self, testspec, testwriter, testpath, exepath, result,
tester, caseid, case=""):
"""Executes the unit test in the specified testing folder for 'case'."""
if not path.exists(testpath):
mkdir(testpath)
#Copy across all the input files we need to run.
for i in testspec.inputs:
i.copy(self._codefolder, testpath, case, self.compiler)
#Also copy any assignment file dependencies.
testwriter.copy(self._codefolder, testpath, case, testspec.identifier, self.compiler)
#Clean the testing folder to remove any target variable output files
#from any earlier test runs.
testspec.clean(testpath)
testwriter.setup(testpath)
#If the testspec needs auto-class support, write the case to file.
if testspec.autoclass:
with open(path.join(testpath, "fpy_case"), 'w') as f:
f.write('"{}"'.format(case))
#Save the path to the folder for execution in the result.
result.paths[caseid] = testpath
msg.okay("Executing {}.x in folder ./tests{}".format(testspec.identifier,
testpath.split("tests")[1]))
start_time = clock()
from os import waitpid
from subprocess import Popen, PIPE
command = "cd {}; {}".format(testpath, exepath)
prun = Popen(command, shell=True, executable="/bin/bash", stdout=PIPE, stderr=PIPE)
waitpid(prun.pid, 0)
if not self.quiet:
output = prun.stdout.readlines()
if len(output) > 0:
msg.std(''.join(output))
#else: #We don't need to get these lines since we are purposefully redirecting them.
error = prun.stderr.readlines()
if len(error) > 0:
if self.quiet:
msg.info("With Executable at {}".format(exepath), 1)
msg.err('\n '+' '.join(error))
code = len(error)
if case == "":
result.cases[caseid] = ExecutionResult(testpath, code, clock() - start_time, tester)
else:
result.cases[caseid] = ExecutionResult(testpath, code, clock() - start_time, tester, case)
self._write_success(testpath, code)
#See if we need to run the post-execution profiling
if self.profile:
profiling.profile(testpath, testspec.testgroup.method_fullname,
exepath, self.compiler)
class UnitTester(object):
"""Performs automatic unit testing of individual subroutines and
functions based on XML doctags found in the code files.
:arg libraryroot: the path to folder in which to stage the tests.
"""
def __init__(self,
libraryroot=None,
verbose=False,
compare_templates=None,
fortpy_templates=None,
rerun=None,
compiler=None,
debug=False,
profile=False,
quiet=False,
strict=False):
self.parser = CodeParser()
self.parser.verbose = verbose
from fortpy.utility import set_fortpy_templates
set_fortpy_templates(self, fortpy_templates)
self.tgenerator = TestGenerator(self.parser, libraryroot,
self.fortpy_templates, self, rerun)
self.compiler = None
"""The full path to the compiler to use for the unit tests.
"""
self.quiet = quiet
"""Specifies whether the tester should run in quiet mode, which only prints
essential information for the unit tests to stdout.
"""
self.strict = strict
"""Specifies whether to compile files with *all* warnings enabled."""
self.debug = debug == True
self.set_compiler(compiler)
self.profile = self._profiler_exists(profile)
#A flag to track whether the generator has already written
#the executables.
self._written = False
#The user's raw value for the compare_templates directory
self._compare_templates = compare_templates
def get_fortpy_version(self, fortpath):
"""Gets the fortpy version number from the first line of the specified file."""
from fortpy.testing.compilers import get_fortpy_version
return get_fortpy_version(self.compiler, fortpath)
def tests(self, identifier):
"""Returns a dictionary of all the tests that need to be run for the
specified module.executable identifier."""
if identifier in self.tgenerator.xtests:
return self.tgenerator.xtests[identifier]
else:
return {}
def writer(self, identifier):
"""Returns the underlying executable writer that has a list of ordered
method/assignment dependencies."""
if identifier in self.tgenerator.xwriters:
return self.tgenerator.xwriters[identifier]
def libraryroot(self, identifier):
"""Returns the absolute path to the staging directory for the unit tests with
the specified testid.
"""
if identifier in self.tgenerator.xgenerator.folders:
return self.tgenerator.xgenerator.folders[identifier]
def _profiler_exists(self, profile):
"""Tests whether we have gprof available to do the profiling of the methods
to unit test.
:arg profile: the value specified in the UnitTester constructor.
"""
if profile == True:
gprof = self.which("gprof")
if gprof is None:
msg.err("gprof is required to run profiling with fortpy.")
exit(1)
else:
return True
else:
return False
def set_compiler(self, compiler):
"""Sets the compiler to use for the unit testing of this code parser.
"""
if compiler is not None:
self.compiler = compiler
self._compiler_exists()
def _compiler_exists(self):
"""Tests whether the specified compiler is available on the machine. If
it isn't give an error and exit."""
from fortpy.testing.compilers import compilers
if self.compiler in compilers:
#Overwrite the *name* of the compiler with its full path; since
#fortpy assumes that self.compiler is the name of a valid executable
#this will still work correctly.
compiler = compilers[self.compiler].path
else:
compiler = self.compiler
if self.which(compiler) is None:
msg.err("compiler {} not found. Exiting.".format(self.compiler))
exit(1)
def which(self, program):
"""Tests whether the specified program is anywhere in the environment
PATH so that it probably exists."""
import os
def is_exe(fpath):
return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
fpath, fname = os.path.split(program)
if fpath:
if is_exe(program):
return program
else:
for path in os.environ["PATH"].split(os.pathsep):
path = path.strip('"')
exe_file = os.path.join(path, program)
if is_exe(exe_file):
return exe_file
return None
def writeall(self, codefolder):
"""Writes all the unit test executables that are new or modified
as well as the makefiles for all subroutines in all modules."""
#The test generator already loops over all modules in the code
#parser and does all the heavy-lifting. We need to pre-load any
#modules that we are interested in testing. Only those loaded
#when write() is first called will have their tests processed.
self._codefolder = path.abspath(codefolder)
if self._compare_templates is not None:
self.compare_templates = path.abspath(self._compare_templates)
else:
self.compare_templates = path.join(self._codefolder, "templates/")
#We will load all the modules in the code folder specified and
#then run the test generator.
files = {}
self.parser.scan_path(self._codefolder, files)
for f in files:
filepath = files[f]
self.parser.parse(filepath, True, True)
#Now that we have loaded all the codefiles in the path, we can
#generate the unittest executables
self.tgenerator.write(self._codefolder)
self._written = True
def runall(self, compiler=None):
"""Compiles and runs each new or modified unit test executable.
After the run is complete, the outcomes are checked for consistency.
:arg compiler: the name of the compiler in 'compilers.xml' to use.
"""
#We will likely need a file comparer for the outcomes testing
self.comparer = FileComparer(self.fortpy_templates,
self.compare_templates)
if self._written:
self.set_compiler(compiler)
from fortpy.testing.compilers import replace
from fortpy.testing.auxiliary import generate
from fortpy.utility import copy
#Run them each individually and return a dictionary of all the
#test results
result = {}
fpyauxs = []
for composite_id in self.tgenerator.tests_to_run:
identifier, testid = composite_id.split("|")
#Compile and copy fpy_auxiliary if it isn't in the identifiers directory yet.
source = path.join(self.libraryroot(identifier), identifier)
target = replace(source + ".[c]", self.compiler)
if self.writer(
identifier).autoclass and identifier not in fpyauxs:
code, success, fpytarget = generate(
self.parser, self._codefolder,
self.libraryroot(identifier), self.compiler,
self.debug, self.profile, self.strict)
opath = path.join(fpytarget, "fpy_auxiliary.o")
mpath = path.join(fpytarget, "fpy_auxiliary.mod")
copy(opath, target)
copy(mpath, target)
fpyauxs.append(identifier)
oneresult = self._run_single(identifier, testid, source)
if oneresult is not None:
result[composite_id] = oneresult
return result
else:
msg.warn(
"you can't run tests until the executables have been written. Exiting."
)
return None
def _run_single(self, identifier, testid, staging):
"""Runs all unit test cases for the specified identifier."""
#Just initialize a result object and populate its properties
#using the various _run_* methods.
result = TestResult(identifier, testid)
result.compiled, result.target = self._run_compile(
identifier, testid, staging)
if result.compiled:
self._run_exec(identifier, testid, result)
if self.tests(identifier)[testid].runchecks:
#Only return a test result if the checks were actually run.
return result
def _run_compile(self, identifier, testid, staging):
"""Compiles the executable that was created for the specified identifier,
returns True if the compile was successful."""
#We need to make sure that the fpy_auxiliary .mod and .o files are available
#if the test uses auto-class functionality.
from os import path
from fortpy.testing.compilers import compile_general
msg.blank()
code, success, target = compile_general(staging,
self.compiler,
testid,
self.debug,
self.profile,
self.quiet,
strict=self.strict)
if not success:
#There are 21 lines in the compile.log file when everything runs correctly
#Overwrite code with a bad exit value since we have some other problems.
code = 1
#If the executable exists, we could still prompt them to run it (in case the
#additional lines were just warnings).
exe = path.join(target, "{}.x".format(testid))
if path.isfile(exe):
choice = raw_input(
"\nWould you still like to run the executable? ").lower()
code = 0 if "y" in choice else code
if "n" in choice:
msg.err("Unit testing terminated by user.")
exit(0)
else:
msg.err("Could not compile executable {}.x".format(testid))
exit(-1)
return code == 0, target
def _run_exec(self, identifier, testid, result):
"""Runs the executable for unit test for the specified identifier
for each of the outcomes specified in the doctags."""
if not self.tests(identifier)[testid].execute:
#We don't want to carry on with this execution at all. User-specified
#override.
return
#Get the home path of the executable. A sub-folder for tests
#needs to be created. For tests that have input and output files
#a home/tests/testid.case folder gets created and the source files
#get copied.
#Create the folder for staging the tests.
tests = path.join(result.target, "tests")
if not path.exists(tests):
mkdir(tests)
#Now we need determine which tests to run from the outcomes and folder tags.
kmodule, kmethod = identifier.lower().split(".")
module = self.parser.modules[kmodule]
method = module.executables[kmethod]
#Get the absolute path to the executable that we created
exepath = path.join(result.target, "{}.x".format(testid))
#Since we have already split out all the tests that need to be run and
#we have a 'testid' for the current test to run, just run that test.
self._run_folder(
self.tests(identifier)[testid], tests, result, exepath,
self.writer(identifier))
if not self.tests(identifier)[testid].runchecks:
return
#Now that we have run all of the executables, we can analyze their
#output to see if it matches.
for case in result.cases:
xres = result.cases[case]
xres.test(case, result)
def _run_folder(self, testspec, testsfolder, result, exepath, testwriter):
"""Runs the executable for the sources in the folder doctag.
:arg testspec: a TestSpecification instance for this unit test.
:arg testsfolder: the path to the unit tests unique staging folder.
:arg result: a TestResult instance that execution results can be
added to as they complete.
:arg expath: the full path to the executable to run in the folder.
:arg testwriter: the MethodWriter instance that generated the test
that will be run.
"""
#We can use the same outcome tester afterwards for all of the cases.
tester = OutcomeTester(testspec, self._codefolder, self.comparer,
self.parser.verbose)
#The execution can either be case-based or once-off.
if testspec.cases is not None:
#We need to run the executable multiple times, once for each case
#Each case has input files specified relative to the code folder.
for case in testspec.cases:
caseid = "{}.{}".format(testspec.identifier, case)
if not caseid in result.cases:
#Make a separate directory for the case and copy all its inputs.
casepath = path.join(testsfolder, caseid)
self._execute_testpath(testspec, testwriter, casepath,
exepath, result, tester, caseid,
case)
else:
result.warnings.append(
"Duplicate CASES specified for unit testing:" +
" {}".format(caseid))
else:
#Create a folder for this test specification to run in.
testpath = path.join(testsfolder, testspec.identifier)
self._execute_testpath(testspec, testwriter, testpath, exepath,
result, tester, testspec.identifier)
def _execute_testpath(self,
testspec,
testwriter,
testpath,
exepath,
result,
tester,
caseid,
case=""):
"""Executes the unit test in the specified testing folder for 'case'."""
if not path.exists(testpath):
mkdir(testpath)
#Copy across all the input files we need to run.
for i in testspec.inputs:
i.copy(self._codefolder, testpath, case, self.compiler)
#Also copy any assignment file dependencies.
testwriter.copy(self._codefolder, testpath, case, testspec.identifier,
self.compiler)
#Clean the testing folder to remove any target variable output files
#from any earlier test runs.
testspec.clean(testpath)
testwriter.setup(testspec.identifier, testpath)
#If the testspec needs auto-class support, write the case to file.
if testspec.autoclass:
with open(path.join(testpath, "fpy_case"), 'w') as f:
f.write('"{}"'.format(case))
#Save the path to the folder for execution in the result.
result.paths[caseid] = testpath
msg.okay("Executing {}.x in folder ./tests{}".format(
testspec.identifier,
testpath.split("tests")[1]))
start_time = clock()
from os import waitpid
from subprocess import Popen, PIPE
command = "cd {}; {}".format(testpath, exepath)
prun = Popen(command,
shell=True,
executable="/bin/bash",
stdout=PIPE,
stderr=PIPE)
waitpid(prun.pid, 0)
if not self.quiet:
output = prun.stdout.readlines()
if len(output) > 0:
msg.std(''.join(output))
#else: #We don't need to get these lines since we are purposefully redirecting them.
error = prun.stderr.readlines()
if len(error) > 0:
if self.quiet:
msg.info("With Executable at {}".format(exepath), 1)
msg.err('\n ' + ' '.join(error))
code = len(error)
if case == "":
result.cases[caseid] = ExecutionResult(testpath, code,
clock() - start_time,
tester)
else:
result.cases[caseid] = ExecutionResult(testpath, code,
clock() - start_time,
tester, case)
self._write_success(testpath, code)
#See if we need to run the post-execution profiling
if self.profile:
profiling.profile(testpath, testspec.testgroup.method_fullname,
exepath, self.compiler)
case=""):
"""Cleans up and sets results for the test that ran.
"""
if case == "":
result.cases[caseid] = ExecutionResult(testpath, code,
clock() - start_time,
tester)
else:
result.cases[caseid] = ExecutionResult(testpath, code,
clock() - start_time,
tester, case)
self._write_success(testpath, code)
#See if we need to run the post-execution profiling
if self.profile:
profiling.profile(testpath, testspec.testgroup.method_fullname,
exepath, self.compiler)
def _write_success(self, testpath, code):
"""Creates a SUCCESS file in the specified testpath if code==0 that has
the time of the last execution. If code != 0, any existing SUCCESS file
is deleted.
"""
sucpath = path.join(testpath, "SUCCESS")
if code == 0:
with open(sucpath, 'w') as f:
f.write(time.strftime("%c"))
else:
if path.isfile(sucpath):
remove(sucpath)
