def copy(src, dst, verbose=False):
"""Copies the specified source file to destination directory if it is newer
or does not yet exist.
"""
from os import waitpid, path
from subprocess import Popen, PIPE
desti = path.join(dst, "")
if not path.isdir(desti):
from os import mkdir
mkdir(desti)
prsync = Popen(
"rsync -t -u {} {}".format(src, desti),
close_fds=True,
shell=True,
executable="/bin/bash",
stdout=PIPE,
stderr=PIPE,
)
waitpid(prsync.pid, 0)
# Redirect the output and errors so that we don't pollute stdout.
# output = prsync.stdout.readlines()
error = prsync.stderr.readlines()
prsync.stderr.close()
prsync.stdout.close()
if len(error) > 0:
from fortpy.msg import warn
warn("Error while copying {} using rsync.\n\n{}".format(src, "\n".join(error)))
def _append_member(member, result):
"""Appends the element to the result list *only* if it can actually be
handled by the auxiliary module.
"""
global warnings
if "private contents" in member.customtype.modifiers:
if member.kind.lower() not in warnings:
msg.warn("User type {} skipped ".format(member.kind) +
"because members are private.", 0)
warnings[member.kind.lower()] = 1
else:
warnings[member.kind.lower()] += 1
return
if member.customtype.name.lower() not in member.customtype.module.publics:
if member.kind.lower() not in warnings:
msg.warn("Skipping user type {} ".format(member.kind) +
"because it is not marked as public in its module.")
warnings[member.kind.lower()] = 1
else:
warnings[member.kind.lower()] += 1
return
result.append(member)
def copy(src, dst):
"""Copies the specified source file to destination directory if it is newer
or does not yet exist.
"""
from os import waitpid, path
from subprocess import Popen, PIPE
desti = path.join(dst, "")
if not path.isdir(desti):
from os import mkdir
mkdir(desti)
prsync = Popen("rsync -t -u {} {}".format(src, desti),
shell=True,
executable="/bin/bash",
stdout=PIPE,
stderr=PIPE)
waitpid(prsync.pid, 0)
#Redirect the output and errors so that we don't pollute stdout.
#output = prsync.stdout.readlines()
error = prsync.stderr.readlines()
if len(error) > 0:
from fortpy.msg import warn
warn("Error while copying {} using rsync.\n\n{}".format(
src, '\n'.join(error)))
def copyfile(src, dst, verbose=False):
"""Copies the specified source file to destination *file* if it is newer
or does not yet exist.
"""
from os import waitpid
from subprocess import Popen, PIPE
prsync = Popen(
"rsync {}-t -u {} {}".format("-v " if verbose else "", src, dst),
shell=True,
executable="/bin/bash",
stdout=PIPE,
stderr=PIPE,
close_fds=True,
)
waitpid(prsync.pid, 0)
# Redirect the output and errors so that we don't pollute stdout.
error = prsync.stderr.readlines()
prsync.stderr.close()
prsync.stdout.close()
if len(error) > 0:
from fortpy.msg import warn
warn("Error while copying {} using rsync.\n\n{}".format(src, "\n".join(error)))
def copytree(src, dst):
"""Recursively copies the source directory to the destination
only if the files are newer or modified by using rsync.
"""
from os import path, waitpid
from subprocess import Popen, PIPE
# Append any trailing / that we need to get rsync to work correctly.
source = path.join(src, "")
desti = path.join(dst, "")
if not path.isdir(desti):
from os import mkdir
mkdir(desti)
prsync = Popen(
"rsync -t -u -r {} {}".format(source, desti),
close_fds=True,
shell=True,
executable="/bin/bash",
stdout=PIPE,
stderr=PIPE,
)
waitpid(prsync.pid, 0)
# Redirect the output and errors so that we don't pollute stdout.
# output = prsync.stdout.readlines()
error = prsync.stderr.readlines()
prsync.stderr.close()
prsync.stdout.close()
if len(error) > 0:
from fortpy.msg import warn
warn("Error while copying {} using rsync.\n\n{}".format(source, "\n".join(error)))
def _expand_autotag(atag, container):
"""Expands the contents of the specified auto tag within its parent container.
"""
if atag.tag != "auto":
return
if "names" in atag.attrib:
i = -1
for name in re.split("[\s,]+", atag.attrib["names"]):
if name[0] == '^':
name = name[1::]
insert = True
i += 1
else:
insert = False
for child in atag:
dupe = child.copy()
for attr, value in dupe.items():
dupe.attrib[attr] = value.replace("$", name)
if insert:
container.insert(i, dupe)
else:
container.append(dupe)
else:
from fortpy.msg import warn
warn("'names' is a required attribute of the <auto> tag.")
def resolve_global(xtag, xmlglobals, xtagname, gtagname, attrib, container, op, gassumed):
"""Adds the child in xmlglobals with the relevant id to the tag.
:arg xtag: the tag with the reference to a global tag.
:arg xmlglobals: the dictionary of global tags extracted from <global>.
:arg tagname: the name of the global tag to look for.
:arg attrib: the name of the attribute in 'xtag' that references the
global tag to process.
:arg container: for "op"=="append", the XMLElement to append the global
tag to as a subelement.
:arg op: one of ["append", "update"]. For "append" the global tag is appended
as a subelement to 'container'. For "update", the attributes of the global
tag are copied to 'xtag'.
:arg gassumed: when True, the 'attrib' being examined is assumed to reference
only tags in <globals>, so no 'global:' prefix is required before attributes.
"""
if attrib not in xtag.attrib:
return False
result = False
if xtag.tag == xtagname and (gassumed or "global:" in xtag.attrib[attrib]):
#We allow a ';'-separated list of global tag names, all of
#which are appended to the child's subelements list.
i = -1
for sattr in xtag.attrib[attrib].split(";"):
gid = sattr.strip().split(":")[1] if ":" in sattr else sattr.strip()
#Since ordering matters, if the id starts with ^, we insert it at
#the beginning of the subelements list instead of appending
if gid[0] == '^':
insert = True
gid = gid[1::]
i += 1
else:
insert = False
if gtagname in xmlglobals and gid.lower() in xmlglobals[gtagname]:
gtag = xmlglobals[gtagname][gid.lower()]
if op == "append":
if insert:
container.insert(i, gtag)
else:
container.append(gtag)
elif op == "update":
update_tag(xtag, gtag)
result = True
else:
from fortpy.msg import warn
wstr = 'The global tag of type <{}> for <{} {}="{}"> does not exist.'
warn(wstr.format(gtagname, xtagname, attrib, xtag.attrib[attrib]))
#Overwrite the name of the tag to not have the 'global:' in it anymore.
xtag.attrib[attrib] = xtag.attrib[attrib].replace("global:", "")
return result
def copyfile(src, dst, verbose=False):
"""Copies the specified source file to destination *file* if it is newer
or does not yet exist.
"""
from os import waitpid
from subprocess import Popen, PIPE
prsync = Popen("rsync {}-t -u {} {}".format("-v " if verbose else "", src, dst),
shell=True, executable="/bin/bash", stdout=PIPE, stderr=PIPE, close_fds=True)
waitpid(prsync.pid, 0)
#Redirect the output and errors so that we don't pollute stdout.
error = prsync.stderr.readlines()
prsync.stderr.close()
prsync.stdout.close()
if len(error) > 0:
from fortpy.msg import warn
warn("Error while copying {} using rsync.\n\n{}".format(src, '\n'.join(error)))
def _prompt_general(header, skeys, sources=None, args=None):
msg.okay(header)
for i, k in enumerate(skeys):
msg.std("{}. {}".format(i, k))
msg.blank(1,0)
choice = input("Selection: ")
if re.match("[\d]+", choice.strip()):
value = int(choice)
if sources is None:
return value
else:
if value in sources:
sources[value](*args)
else:
msg.warn("choice '{}' not in list of possibilities.".format(choice))
else:
msg.warn("invalid (non-numeric) choice: '{}'".format(choice))
def _get_data_series(self,
independent,
dependents,
threshold,
functions=None):
"""Used by plot() and table() to get the x and y data series that will be plotted
or tabulated against each other. See the arguments on those methods.
"""
if (("rowvals" in independent or "colvals" in independent)
or any([("rowvals" in d or "colvals" in d)
for d in dependents])):
#We need to make sure that the test case filter they specified returns only a
#single result so that we get a reasonable plot. This will happen if the data
#is 1D in the other dimension or if the filter only has a single case. We
#issue a warning if the filter doesn't match up, so they can check their data.
for dep in dependents:
tfilter, depvar = dep.split("/")
if tfilter is None or sum(
[(1 if self._case_filter(d, tfilter) else 0)
for d in self.details]) > 1:
msg.warn(
"Plotting aggregated data for more than one test case. Check results \n"
"for consistency and completeness.")
break
ys = []
xs = []
for variable in dependents:
tfilter, depvar = variable.split("/")
fullindvar = "{}/{}".format(tfilter, independent)
x, cases = self._get_data(independent, fullindvar, None, threshold,
tfilter, functions)
if (len(x) == 1 and isinstance(x[0], list) and
("rowvals" in independent or "colvals" in independent)):
x = x[0]
xs.append((x, fullindvar))
ypts, names = self._get_data(depvar, variable, cases, threshold,
tfilter, functions, independent, x)
if (len(ypts) == 1 and isinstance(ypts[0], list)
and ("rowvals" in variable or "colvals" in variable)):
ypts = ypts[0]
ys.append((ypts, variable))
return (xs, ys)
def _ensure_fileversion(compiler, modname, folder, target, trycompile=True):
"""Makes sure that the module's f90, mod and o files are up to date with the template
version. If they aren't compile and copy them.
:arg compiler: the compiler key from compilers.xml
:arg modname: the name of the module to check file versions for and move (e.g. "fortpy").
:arg folder: the folder that contains the up-to-date, "template" version of the module.
:arg target: the folder to copy the compiled files to.
:arg trycompile: if the codefile has not been compiled yet, or if the version is out of
date, should the code try a simple compile?
"""
from os import path
codefile = "{}.f90".format(modname)
compfiles = ["{}.{}".format(modname, ext) for ext in ["o", "mod"]]
tversion = template_version(compiler, codefile)
for sdfile in compfiles:
fdfile = replace(sdfile + ".[c]", compiler)
ftarget = path.join(target, sdfile)
dversion = get_fortpy_version(compiler, ftarget)
if not path.isfile(ftarget) or dversion != tversion:
source = path.join(folder, fdfile)
sversion = get_fortpy_version(compiler, source)
if trycompile and (not path.isfile(source)
or sversion != tversion):
_compile_simple(compiler, [modname], folder)
_vupdate_compiled_module(compiler, modname, folder, tversion)
elif not path.isfile(source):
msg.warn("{} does not exist.".format(source))
continue
elif sversion != tversion:
msg.warn("{} has an old version number.".format(source))
from fortpy.utility import symlink
symlink(source, ftarget)
#If the file is a binary, we need to save a .v with version
#information as well for the next time we want to copy it.
pre, ext = path.splitext(ftarget)
if ext in [".o", ".so", ".mod"]:
with open(ftarget + '.v', 'w') as f:
f.write("# <fortpy version=\"{}\" />".format('.'.join(
map(str, tversion))))
def _format_fit(self, key):
"""Formats the function fit at the specified key for printing/plotting."""
if key in self.fits:
params = self.fits[key]["params"]
model = self.fits[key]["model"]
if any(params < 1e-2):
mdict = {
"exp": "{0:.2e}exp({1:.2e}*x){2:+.2e}",
"lin": "{0:.2e}*x{1:+.2e}"
}
else:
mdict = {
"exp": "{0:.2f}exp({1:.2f}*x){2:+.2f}",
"lin": "{0:.2f}*x{1:+.2f}"
}
return mdict[model].format(*params)
else:
msg.warn("Couldn't format the fit {}; fit not found.".format(key))
return key
def _compile(self, dirpath, makename, compiler, debug, profile):
"""Compiles the makefile at the specified location with 'compiler'.
:arg dirpath: the full path to the directory where the makefile lives.
:arg compiler: one of ['ifort', 'gfortran'].
:arg makename: the name of the make file to compile.
"""
from os import path
options = ""
if debug:
options += " DEBUG=true"
if profile:
options += " GPROF=true"
from os import system
codestr = "cd {}; make -f '{}' F90={} FAM={}" + options
code = system(codestr.format(dirpath, makename, compiler, compiler[0]))
#It turns out that the compiler still returns a code of zero, even if the compile
#failed because the actual compiler didn't fail; it did its job properly. We need to
#check for the existence of errors in the 'compile.log' file.
lcount = 0
errors = []
log = path.join(dirpath, "compile.log")
with open(log) as f:
for line in f:
lcount += 1
if lcount > 21 and lcount < 32:
errors.append(line)
elif lcount > 21:
break
if len(errors) > 0:
#There are 21 lines in the compile.log file when everything runs correctly
#Overwrite code with a bad exit value since we have some other problems.
code = 1
#We want to write the first couple of errors to the console and give them the
#option to still execute if the compile only generated warnings.
msg.warn("compile generated some errors or warnings:")
msg.blank()
msg.info(''.join(errors))
return code
def _compile(self, dirpath, makename, compiler, debug, profile):
"""Compiles the makefile at the specified location with 'compiler'.
:arg dirpath: the full path to the directory where the makefile lives.
:arg compiler: one of ['ifort', 'gfortran'].
:arg makename: the name of the make file to compile.
"""
from os import path
options = ""
if debug:
options += " DEBUG=true"
if profile:
options += " GPROF=true"
from os import system
codestr = "cd {}; make -f '{}' F90={} FAM={}" + options
code = system(codestr.format(dirpath, makename, compiler, compiler[0]))
#It turns out that the compiler still returns a code of zero, even if the compile
#failed because the actual compiler didn't fail; it did its job properly. We need to
#check for the existence of errors in the 'compile.log' file.
lcount = 0
errors = []
log = path.join(dirpath, "compile.log")
with open(log) as f:
for line in f:
lcount += 1
if lcount > 21 and lcount < 32:
errors.append(line)
elif lcount > 21:
break
if len(errors) > 0:
#There are 21 lines in the compile.log file when everything runs correctly
#Overwrite code with a bad exit value since we have some other problems.
code = 1
#We want to write the first couple of errors to the console and give them the
#option to still execute if the compile only generated warnings.
msg.warn("compile generated some errors or warnings:")
msg.blank()
msg.info(''.join(errors))
return code
def _format_fit(self, key):
"""Formats the function fit at the specified key for printing/plotting."""
if key in self.fits:
params = self.fits[key]["params"]
model = self.fits[key]["model"]
if any(params < 1e-2):
mdict = {
"exp": "{0:.2e}exp({1:.2e}*x){2:+.2e}",
"lin": "{0:.2e}*x{1:+.2e}"
}
else:
mdict = {
"exp": "{0:.2f}exp({1:.2f}*x){2:+.2f}",
"lin": "{0:.2f}*x{1:+.2f}"
}
return mdict[model].format(*params)
else:
msg.warn("Couldn't format the fit {}; fit not found.".format(key))
return key
def _ensure_fileversion(compiler, modname, folder, target, trycompile=True):
"""Makes sure that the module's f90, mod and o files are up to date with the template
version. If they aren't compile and copy them.
:arg compiler: the compiler key from compilers.xml
:arg modname: the name of the module to check file versions for and move (e.g. "fortpy").
:arg folder: the folder that contains the up-to-date, "template" version of the module.
:arg target: the folder to copy the compiled files to.
:arg trycompile: if the codefile has not been compiled yet, or if the version is out of
date, should the code try a simple compile?
"""
from os import path
codefile = "{}.f90".format(modname)
compfiles = ["{}.{}".format(modname, ext) for ext in ["o", "mod"]]
tversion = template_version(compiler, codefile)
for sdfile in compfiles:
fdfile = replace(sdfile + ".[c]", compiler)
ftarget = path.join(target, sdfile)
dversion = get_fortpy_version(compiler, ftarget)
if not path.isfile(ftarget) or dversion != tversion:
source = path.join(folder, fdfile)
sversion = get_fortpy_version(compiler, source)
if trycompile and (not path.isfile(source) or sversion != tversion):
_compile_simple(compiler, [modname], folder)
_vupdate_compiled_module(compiler, modname, folder, tversion)
elif not path.isfile(source):
msg.warn("{} does not exist.".format(source))
continue
elif sversion != tversion:
msg.warn("{} has an old version number.".format(source))
from fortpy.utility import copyfile
msg.info(" COPY: {}".format(source))
copyfile(source, ftarget)
#If the file is a binary, we need to save a .v with version
#information as well for the next time we want to copy it.
pre, ext = path.splitext(ftarget)
if ext in [".o", ".so", ".mod"]:
with open(ftarget + '.v', 'w') as f:
f.write("# <fortpy version=\"{}\" />".format('.'.join(map(str, tversion))))
def copytree(src, dst):
"""Recursively copies the source directory to the destination
only if the files are newer or modified by using rsync.
"""
from os import path
from os import waitpid
from subprocess import Popen, PIPE
#Append any trailing / that we need to get rsync to work correctly.
source = path.join(src, "")
prsync = Popen("rsync -t -u -r {} {}".format(source, dst),
shell=True, executable="/bin/bash", stdout=PIPE, stderr=PIPE)
waitpid(prsync.pid, 0)
#Redirect the output and errors so that we don't pollute stdout.
output = prsync.stdout.readlines()
error = prsync.stderr.readlines()
if len(error) > 0:
from fortpy.msg import warn
warn("Error while copying {} using rsync.\n\n{}".format(source, '\n'.join(error)))
def _get_data_series(self, independent, dependents, threshold, functions=None):
"""Used by plot() and table() to get the x and y data series that will be plotted
or tabulated against each other. See the arguments on those methods.
"""
if (("rowvals" in independent or "colvals" in independent) or
any([("rowvals" in d or "colvals" in d) for d in dependents])):
#We need to make sure that the test case filter they specified returns only a
#single result so that we get a reasonable plot. This will happen if the data
#is 1D in the other dimension or if the filter only has a single case. We
#issue a warning if the filter doesn't match up, so they can check their data.
for dep in dependents:
tfilter, depvar = dep.split("/")
if tfilter is None or sum([(1 if self._case_filter(d, tfilter) else 0)
for d in self.details]) > 1:
msg.warn("Plotting aggregated data for more than one test case. Check results \n"
"for consistency and completeness.")
break
ys = []
xs = []
for variable in dependents:
tfilter, depvar = variable.split("/")
fullindvar = "{}/{}".format(tfilter, independent)
x, cases = self._get_data(independent, fullindvar, None, threshold, tfilter, functions)
if (len(x) == 1 and isinstance(x[0], list) and
("rowvals" in independent or "colvals" in independent)):
x = x[0]
xs.append((x, fullindvar))
ypts, names = self._get_data(depvar, variable, cases, threshold, tfilter, functions,
independent, x)
if (len(ypts) == 1 and isinstance(ypts[0], list) and
("rowvals" in variable or "colvals" in variable)):
ypts = ypts[0]
ys.append((ypts, variable))
return (xs, ys)
def _complete_general(prompt="$ ", options=None, completer=None, leader=None,
regex=None, cast=None, current=None):
"""Prompts the user for input from the specified options.
"""
import readline
readline.parse_and_bind("tab: complete")
if completer is not None:
readline.set_completer(SimpleCompleter(None, completer).complete)
elif options is not None:
readline.set_completer(SimpleCompleter(options).complete)
if leader is not None:
msg.okay(leader)
if current is not None:
msg.gen("The current value is set as '{}'.".format(current))
choice = input(prompt)
if choice.strip() == '?':
return choice.strip()
if choice.strip() == '':
return None
if regex is not None:
import re
if not re.match(regex, choice):
msg.warn("'{}' is not a valid choice.".format(choice))
return None
if cast is not None:
try:
result = cast(choice.strip())
except ValueError:
return None
else:
result = choice
return result
def _find_executables(self):
"""Finds the list of executables that pass the requirements necessary to have
a wrapper created for them.
"""
if len(self.needs) > 0:
return
for execname, executable in list(self.module.executables.items()):
skip = False
#At the moment we can't handle executables that use special derived types.
if not execname in self.module.publics or not executable.primitive:
msg.info("Skipping {}.{} because it is not public.".format(self.module.name, execname))
skip = True
#Check that all the parameters have intent specified, otherwise we won't handle them well.ha
if any([p.direction == "" for p in executable.ordered_parameters]):
msg.warn("Some parameters in {}.{} have no intent".format(self.module.name, execname) +
" specified. Can't wrap that executable.")
skip = True
if not skip:
self.uses.append(execname)
for depmod in executable.search_dependencies():
if depmod not in self.needs:
self.needs.append(depmod)
def _get_data_series(self, independent, dependents, threshold, functions=None):
"""Used by plot() and table() to get the x and y data series that will be plotted
or tabulated against each other. See the arguments on those methods.
"""
if (("rowvals" in independent or "colvals" in independent) or
any([("rowvals" in d or "colvals" in d) for d in dependents])):
#We need to make sure that the test case filter they specified returns only a
#single result so that we get a reasonable plot. This will happen if the data
#is 1D in the other dimension or if the filter only has a single case. We
#issue a warning if the filter doesn't match up, so they can check their data.
for dep in dependents:
tfilter, depvar = dep.split("/")
if tfilter is None or sum([(1 if self._case_filter(d, tfilter) else 0)
for d in self.details]) > 1:
msg.warn("Plotting aggregated data for more than one test case. Check results \n"
"for consistency and completeness.")
break
rawvars = {}
oseries = []
icases = {}
def _load_raw(variable, rawvars):
if variable in rawvars:
return
tfilter, depvar = variable.split("/")
fullindvar = "{}/{}".format(tfilter, independent)
cases = None if fullindvar not in icases else icases[fullindvar]
if fullindvar not in rawvars:
x, cases = self._get_data(independent, fullindvar, None, threshold, tfilter)
if (len(x) == 1 and isinstance(x[0], list) and
("rowvals" in independent or "colvals" in independent)):
x = x[0]
rawvars[fullindvar] = x
if cases is not None:
x = rawvars[fullindvar]
ypts, names = self._get_data(depvar, variable, cases, threshold, tfilter, independent, x)
if (len(ypts) == 1 and isinstance(ypts[0], list) and
("rowvals" in variable or "colvals" in variable)):
ypts = ypts[0]
rawvars[variable] = ypts
return (fullindvar, cases)
for variable in dependents:
fullindvar, cases = _load_raw(variable, rawvars)
icases[fullindvar] = cases
oseries.append((fullindvar, variable))
#Next, we need to apply any postfix functions and then check that the data sizes
#are commensurate (i.e. same number of points for dependent and independent series).
ys = []
xs = []
if functions is not None:
for postfix, fixdef in functions.items():
fxn = None
vorder = [v for v in fixdef.keys() if v != "lambda"]
values = {}
if postfix not in rawvars:
msg.err("Variable '{}' to be postfixed has no data series.".format(postfix))
break
N = len(rawvars[postfix])
for lvar, gvar in fixdef.items():
if lvar == "lambda":
#This is the function definition for each item, create a lambda for it.
import numpy
import math
try:
evals = gvar.split(":")[1]
lambstr = "lambda {}: {}".format(", ".join(vorder), evals)
fxn = eval(lambstr)
except:
msg.err("Could not evaluate function '{}'.".format(gvar))
else:
if gvar not in rawvars:
_load_raw(gvar, rawvars)
if gvar in rawvars:
values[lvar] = rawvars[gvar]
else:
emsg = "Variable '{}' in the lambda function postfix for '{}' is missing data."
msg.err(emsg.format(gvar, postfix))
#Now if we have a valid function defined and data to operate on, just
#evaluate the postfix one value at a time.
if fxn is not None and N > 0:
for i in range(N):
args = [values[v][i] for v in vorder]
pval = fxn(*args)
rawvars[postfix][i] = pval
#Now we can use the adjusted/postfixed raw values to construct the data series
for indepvar, depvar in oseries:
xs.append((rawvars[indepvar], indepvar))
#We need the arrays to be the same length for plotting; if we are following
#an existing independent variable, we still need to append zero to the list
#for the values we don't have.
if len(rawvars[depvar]) != len(rawvars[indepvar]):
msg.err("Can't coerce an array to a single value without an aggregation "
"function such as numpy.mean or numpy.sum")
ys.append((rawvars[depvar], depvar))
return (xs, ys)
def _get_data_series(self,
independent,
dependents,
threshold,
functions=None):
"""Used by plot() and table() to get the x and y data series that will be plotted
or tabulated against each other. See the arguments on those methods.
"""
if (("rowvals" in independent or "colvals" in independent)
or any([("rowvals" in d or "colvals" in d)
for d in dependents])):
#We need to make sure that the test case filter they specified returns only a
#single result so that we get a reasonable plot. This will happen if the data
#is 1D in the other dimension or if the filter only has a single case. We
#issue a warning if the filter doesn't match up, so they can check their data.
for dep in dependents:
tfilter, depvar = dep.split("/")
if tfilter is None or sum(
[(1 if self._case_filter(d, tfilter) else 0)
for d in self.details]) > 1:
msg.warn(
"Plotting aggregated data for more than one test case. Check results \n"
"for consistency and completeness.")
break
rawvars = {}
oseries = []
icases = {}
def _load_raw(variable, rawvars):
if variable in rawvars:
return
tfilter, depvar = variable.split("/")
fullindvar = "{}/{}".format(tfilter, independent)
cases = None if fullindvar not in icases else icases[fullindvar]
if fullindvar not in rawvars:
x, cases = self._get_data(independent, fullindvar, None,
threshold, tfilter)
if (len(x) == 1 and isinstance(x[0], list) and
("rowvals" in independent or "colvals" in independent)):
x = x[0]
rawvars[fullindvar] = x
if cases is not None:
x = rawvars[fullindvar]
ypts, names = self._get_data(depvar, variable, cases,
threshold, tfilter, independent,
x)
if (len(ypts) == 1 and isinstance(ypts[0], list)
and ("rowvals" in variable or "colvals" in variable)):
ypts = ypts[0]
rawvars[variable] = ypts
return (fullindvar, cases)
for variable in dependents:
fullindvar, cases = _load_raw(variable, rawvars)
icases[fullindvar] = cases
oseries.append((fullindvar, variable))
#Next, we need to apply any postfix functions and then check that the data sizes
#are commensurate (i.e. same number of points for dependent and independent series).
ys = []
xs = []
if functions is not None:
for postfix, fixdef in functions.items():
fxn = None
vorder = [v for v in fixdef.keys() if v != "lambda"]
values = {}
if postfix not in rawvars:
msg.err(
"Variable '{}' to be postfixed has no data series.".
format(postfix))
break
N = len(rawvars[postfix])
for lvar, gvar in fixdef.items():
if lvar == "lambda":
#This is the function definition for each item, create a lambda for it.
import numpy
import math
try:
evals = gvar.split(":")[1]
lambstr = "lambda {}: {}".format(
", ".join(vorder), evals)
fxn = eval(lambstr)
except:
msg.err("Could not evaluate function '{}'.".format(
gvar))
else:
if gvar not in rawvars:
_load_raw(gvar, rawvars)
if gvar in rawvars:
values[lvar] = rawvars[gvar]
else:
emsg = "Variable '{}' in the lambda function postfix for '{}' is missing data."
msg.err(emsg.format(gvar, postfix))
#Now if we have a valid function defined and data to operate on, just
#evaluate the postfix one value at a time.
if fxn is not None and N > 0:
for i in range(N):
args = [values[v][i] for v in vorder]
pval = fxn(*args)
rawvars[postfix][i] = pval
#Now we can use the adjusted/postfixed raw values to construct the data series
for indepvar, depvar in oseries:
xs.append((rawvars[indepvar], indepvar))
#We need the arrays to be the same length for plotting; if we are following
#an existing independent variable, we still need to append zero to the list
#for the values we don't have.
if len(rawvars[depvar]) != len(rawvars[indepvar]):
msg.err(
"Can't coerce an array to a single value without an aggregation "
"function such as numpy.mean or numpy.sum")
ys.append((rawvars[depvar], depvar))
return (xs, ys)
def compile(folder,
compiler=None,
identifier=None,
debug=False,
profile=False,
quiet=False,
moptions=None,
inclfortpy=True,
vupdates=None,
strict=False,
inclfpyaux=False):
"""Runs the makefile in the specified folder to compile the 'all' rule.
:arg vupdates: a list of module names for which the output .mod and .o files
should have version information attached.
"""
if inclfortpy:
#Before we can compile the library, we need to make sure that we have a fortpy
#.mod and .o compiled with the *same* compiler version specified.
from fortpy.utility import get_fortpy_templates_dir
_ensure_fileversion(compiler, "fortpy", get_fortpy_templates_dir(),
folder)
options = ""
if debug:
options += " DEBUG=true"
if profile:
options += " GPROF=true"
if strict:
options += " STRICT=true"
if moptions is not None:
for opt in moptions:
options += " {}".format(opt)
if identifier is not None:
codestr = "cd {}; make -f 'Makefile.{}' F90='{}' FAM='{}'" + options
command = codestr.format(folder, identifier, executor(compiler),
family(compiler))
else:
codestr = "cd {}; make F90='{}' FAM='{}'" + options
command = codestr.format(folder, executor(compiler), family(compiler))
#If we are running in quiet mode, we don't want the compile information
#to post to stdout; only errors should be redirected. This means we need
#to wrap the execution in a subprocess and redirect the std* to PIPE
from os import waitpid, path
from subprocess import Popen, PIPE
pcompile = Popen(command,
shell=True,
executable="/bin/bash",
stdout=PIPE,
stderr=PIPE,
close_fds=True)
waitpid(pcompile.pid, 0)
if not quiet:
output = [x.decode('utf8') for x in pcompile.stdout.readlines()]
msg.std(''.join(output))
#else: #We don't need to get these lines since we are purposefully redirecting them.
error = [x.decode('utf8') for x in pcompile.stderr.readlines()]
code = len(error)
if code != 0:
msg.err(''.join(error))
pcompile.stdout.close()
pcompile.stderr.close()
#It turns out that the compiler still returns a code of zero, even if the compile
#failed because the actual compiler didn't fail; it did its job properly. We need to
#check for the existence of errors in the 'compile.log' file.
lcount = 0
errors = []
log = path.join(
folder, "compile.{}.log".format(
identifier if identifier is not None else "default"))
with open(log) as f:
for line in f:
lcount += 1
if lcount > 21 and lcount < 32:
errors.append(line)
elif lcount > 21:
break
if len(errors) > 0:
#There are 21 lines in the compile.log file when everything runs correctly
#Overwrite code with a bad exit value since we have some other problems.
code = 1
#We want to write the first couple of errors to the console and give them the
#option to still execute if the compile only generated warnings.
msg.warn("compile generated some errors or warnings:")
msg.blank()
msg.info(''.join(errors))
if vupdates is not None:
for modname in vupdates:
_vupdate_compiled_module(compiler, modname, folder, rename=False)
return (code, len(errors) == 0)
def _prompt_model(wizard, parameter, target):
"""Prompts the user for details about the model output for comparison.
"""
#This is essentially the prompt to setup the <output> tags. Look for the existing
#<output> in the test specification.
if target.compareto is None or target.compareto is not in wizard.tauto.outputs:
from xml.etree.ElementTree import Element
output = Element("output", {"identifier": "{}.auto".format(target.name)})
newoutput = True
else:
output = wizard.tauto.outputs[target.compareto].xml
newoutput = False
skeys = ["Automate.", "Set a constant value.", "Don't compare to model output."]
choice = _prompt_general("Choose the source of the model output:", skeys)
attribs = {
"tolerance": {"leader": "Enter % accuracy required for comparisons: 0.0 to 1.0 (default)?",
"cast": float}
}
#Keep track of whether we actually need to set model output for this parameter.
skip = choice == 2
if choice == 0:
#The biggest problem we have here is that we need to specify the file to use
#for the model output. We could have the user search for one, but really the
#way this works is that we need to compile the test, run it without checks and
#then present the user with the output to verify for each test case. Since the
#automator presents the input parameters first and then allows the targets to
#be established without model outputs first, we should be able to compile and
#run the test (if it hasn't been done already).
rdict = {
1: _examine_output,
2: _print_outpath,
3: _start_debug,
4: _set_correct,
5: _set_existing
}
rkeys = ["Re-run the tests to re-create output(s).",
"Examine the variables' output(s) for correctness.",
"Print the location of the model output(s).",
"Start the debugger for the unit test program.",
"Set the variable output(s) as correct.",
"Specify an existing file as model output.",
"Exit the correction loop."]
varfile = None
correct = True
runonce = False
while correct:
msg.blank()
if has_outputs(wizard, True):
rchoice = _prompt_general("The model output for the active test case exists.\n"
"What would you like to do?", rkeys)
if rchoice in rdict:
varfile = rdict[rchoice](wizard, parameter, target)
if rchoice == 4:
correct = False
elif rchoice == 0:
run(wizard, True, True)
elif rchoice == 5:
correct = False
else:
#First run the tests to generate the output, then present it to the user
#so that they can check it is right.
if not runonce:
msg.info("Running the unit test to generate output for examination.")
run(wizard, False, True)
runonce = True
else:
msg.warn("the model outputs weren't generated by running the unit test.\n"
"Check error messages.")
correct = False
if varfile is not None:
output.set("file", varfile)
if "autoclass" in target.xml.attrib and target.xml.attrib["autoclass"] == "true":
output.set("autoclass", "true")
if "tolerance" in selattrs:
output.set("actolerance", selattrs["tolerance"])
elif choice == 1:
attribs["value"] = {"leader": "Enter the correct value; can be any valid python code."}
if skip:
#We need to remove an output if one already exists; otherwise do nothing.
if target.compareto is not None:
if target.compareto in wizard.tauto.outputs:
del wizard.tauto.outputs[target.compareto]
target.compareto = None
else:
#Prompts for the last, sundry attributes on the <output> tag.
selattrs = _prompt_attributes("output", attribs)
for k, v in selattrs.items():
output.set(k, v)
if newoutput:
target.compareto = output.attrib["identifier"]
wizard.tauto.outputs[target.compareto] = TestOutput(output)
def compile(folder, compiler=None, identifier=None, debug=False, profile=False,
quiet=False, moptions=None, inclfortpy=True, vupdates=None,
strict=False, inclfpyaux=False):
"""Runs the makefile in the specified folder to compile the 'all' rule.
:arg vupdates: a list of module names for which the output .mod and .o files
should have version information attached.
"""
if inclfortpy:
#Before we can compile the library, we need to make sure that we have a fortpy
#.mod and .o compiled with the *same* compiler version specified.
from fortpy.utility import get_fortpy_templates_dir
_ensure_fileversion(compiler, "fortpy", get_fortpy_templates_dir(), folder)
options = ""
if debug:
options += " DEBUG=true"
if profile:
options += " GPROF=true"
if strict:
options += " STRICT=true"
if moptions is not None:
for opt in moptions:
options += " {}".format(opt)
if identifier is not None:
codestr = "cd {}; make -f 'Makefile.{}' F90='{}' FAM='{}'" + options
command = codestr.format(folder, identifier, executor(compiler), family(compiler))
else:
codestr = "cd {}; make F90='{}' FAM='{}'" + options
command = codestr.format(folder, executor(compiler), family(compiler))
#If we are running in quiet mode, we don't want the compile information
#to post to stdout; only errors should be redirected. This means we need
#to wrap the execution in a subprocess and redirect the std* to PIPE
from os import waitpid, path
from subprocess import Popen, PIPE
pcompile = Popen(command, shell=True, executable="/bin/bash", stdout=PIPE, stderr=PIPE)
waitpid(pcompile.pid, 0)
if not quiet:
output = [x.decode('utf8') for x in pcompile.stdout.readlines()]
msg.std(''.join(output))
#else: #We don't need to get these lines since we are purposefully redirecting them.
error = [x.decode('utf8') for x in pcompile.stderr.readlines()]
code = len(error)
if code != 0:
msg.err(''.join(error))
#It turns out that the compiler still returns a code of zero, even if the compile
#failed because the actual compiler didn't fail; it did its job properly. We need to
#check for the existence of errors in the 'compile.log' file.
lcount = 0
errors = []
log = path.join(folder, "compile.log")
with open(log) as f:
for line in f:
lcount += 1
if lcount > 21 and lcount < 32:
errors.append(line)
elif lcount > 21:
break
if len(errors) > 0:
#There are 21 lines in the compile.log file when everything runs correctly
#Overwrite code with a bad exit value since we have some other problems.
code = 1
#We want to write the first couple of errors to the console and give them the
#option to still execute if the compile only generated warnings.
msg.warn("compile generated some errors or warnings:")
msg.blank()
msg.info(''.join(errors))
if vupdates is not None:
for modname in vupdates:
_vupdate_compiled_module(compiler, modname, folder, rename=False)
return (code, len(errors)==0)