def _vupdate_compiled_module(compiler, modname, folder, tversion=None, rename=True):
"""Compiles a .mod and .o for the specified compiler and specified.
"""
from os import path
opath = path.join(folder, "{}.o".format(modname))
mpath = path.join(folder, "{}.mod".format(modname))
if path.isfile(opath) and path.isfile(mpath):
if rename:
from shutil import move
nopath = path.join(folder, replace("{}.o.[c]".format(modname), compiler))
nmpath = path.join(folder, replace("{}.mod.[c]".format(modname), compiler))
move(opath, nopath)
move(mpath, nmpath)
else:
nopath = opath
nmpath = mpath
#Create the version files so we can keep track of the compiled versions.
vpaths = [nopath, nmpath]
for np in vpaths:
if tversion is None:
tversion = get_fortpy_version(compiler, np)
vp = np + ".v"
with open(vp, 'w') as f:
f.write('#<fortpy version="{}" />'.format('.'.join(map(str, tversion))))
else:
msg.err("Unable to find {0}.o and {0}.mod.".format(modname))
def _get_rsubroutine_flat(classer, common):
"""Gets the *non* recursive code subroutine for saving a variable with members
of the same type as itself.
"""
variable = classer.variable
lines = []
for memname, member in variable.customtype.members.items():
if member.is_custom:
if member.customtype is None:
msg.err("Unable to find Type instance for : {}".format(member.definition()))
continue
if "allocatable" in member.modifiers:
d0 = ', '.join(['0' for i in range(member.D)])
lines.append("allocate(variable%{}({}))".format(memname, d0))
xr = "call auxread_{varname}{D}d{suffix}(variable%{memname}, lfolder//'-{memname}', nested_=.true.)"
lines.append(xr.format(**{"varname": member.customtype.name,
"D": member.D,
"memname": member.name,
"suffix": _get_suffix(member)}))
else:
lines.append("call fpy_read{1}(lfolder//'-{0}', '#', variable%{0})".format(member.name, _get_suffix(member)))
common["read"] = '\n '.join(lines)
template = """ subroutine {xname}(variable, folder, multi_stack, rstack, nested_)
character(len=*), intent(in) :: folder
{dtype}{kind}, intent(inout) :: variable{Dx}
{dtype}{kind}, pointer, optional, intent(in) :: multi_stack(:)
type(fpy_address), allocatable, intent(in), optional :: rstack(:)
logical, optional, intent(in) :: nested_
character(len=:), allocatable :: lfolder
logical :: nested
if (fpy_verbose > 0) write (*,*) "Start flat read for {xname} in '"//folder//"'."
if (present(nested_)) then
nested = nested_
else
nested = .false.
end if
if ((present(multi_stack) .and. present(rstack)) .or. nested) then
lfolder = folder
else
lfolder = folder//'_'
end if
{read}
if (fpy_verbose > 0) write (*,*) "End flat read for {xname} in '"//folder//"'."
end subroutine {xname}
"""
return ((common["xname"],), template.format(**common))
def _separate_multiple_def(self, defstring, parent, refstring, refline):
"""Separates the text after '::' in a variable definition to extract all the variables,
their dimensions and default values.
"""
import pyparsing
nester = pyparsing.nestedExpr('(', ')')
try:
parsed = nester.parseString("(" +
re.sub("=(>?)", " =\\1 ", defstring) +
")").asList()[0]
except pyparsing.ParseException as err:
from fortpy import msg
repl = (parent.name, refline[0], refstring, defstring,
''.join(['-'] * (err.loc - 1)) + '^', err.msg)
msg.err(
"parsing variable from '{}:{} >> {}': \n'{}'\n{} {}.".format(
*repl))
raise
i = 0
clean = []
while i < len(parsed):
if (isinstance(parsed[i], str) and not re.match("=>?", parsed[i])
and i + 1 < len(parsed)
and isinstance(parsed[i + 1], list)):
clean.append((parsed[i], parsed[i + 1]))
i += 2
elif isinstance(parsed[i], str) and parsed[i] == ",":
i += 1
else:
clean.append(parsed[i])
i += 1
#Now pass through again to handle the default values.
i = 0
ready = []
while i < len(clean):
if isinstance(clean[i], str) and re.match("=>?", clean[i]):
ready.pop()
if ">" in clean[i]:
ready.append([
clean[i - 1], ("> " + clean[i + 1][0], clean[i + 1][1])
])
else:
ready.append([clean[i - 1], clean[i + 1]])
i += 2
else:
ready.append(clean[i])
i += 1
return ready
def make(self,
remake=False,
dependencies=None,
compiler="gfortran",
debug=False,
profile=False):
"""Generates a makefile to compile the wrapper module and link it with the shared
library that can be compiled with self.makelib().
"""
self._check_lib(remake, compiler, debug, profile)
from os import path
if self.link is None or not path.isfile(self.link):
msg.err(
"Can't create shared library; missing compiled original code: {}"
.format(self.link))
exit(0)
if remake or not path.isfile(self.libpath):
if dependencies is None:
dependencies = ["{}_c".format(self.module.name)]
identifier = self.name
else:
identifier = "ftypes.{}".format(self.library)
#We don't need to reorder these *wrapper* modules before compilation because each of
#them should be independent of all the others.
#dependencies = self._process_module_needs(dependencies)
#append the dependency for the deallocation module to be included in the shared lib.
dependencies.append("ftypes_dealloc")
makename = "Makefile.{}".format(
self.library if self.library else self.name)
makepath = path.join(self.f90path, makename)
extralinks = list(self.dependencies.values())
extralinks.append(self.link)
makefile(identifier, dependencies, makepath,
"ftypes.{}".format(self.name), False, False,
self.module.parent, "so", extralinks)
code = self._compile(self.f90path, makename, compiler, debug,
profile)
if code == 0:
#Copy the shared library to the main directory out of the f90 directory
from shutil import copy
copy(path.join(self.f90path, "{}.so".format(identifier)),
self.dirpath)
return code
else:
return 0
def _separate_multiple_def(self, defstring, parent, refstring, refline):
"""Separates the text after '::' in a variable definition to extract all the variables,
their dimensions and default values.
"""
import pyparsing
nester = pyparsing.nestedExpr("(", ")")
try:
parsed = nester.parseString("(" + re.sub("=(>?)", " =\\1 ", defstring) + ")").asList()[0]
except pyparsing.ParseException as err:
from fortpy import msg
repl = (parent.name, refline[0], refstring, defstring, "".join(["-"] * (err.loc - 1)) + "^", err.msg)
msg.err("parsing variable from '{}:{} >> {}': \n'{}'\n{} {}.".format(*repl))
raise
i = 0
clean = []
while i < len(parsed):
if (
isinstance(parsed[i], str)
and not re.match("=>?", parsed[i])
and i + 1 < len(parsed)
and isinstance(parsed[i + 1], list)
):
clean.append((parsed[i], parsed[i + 1]))
i += 2
elif isinstance(parsed[i], str) and parsed[i] == ",":
i += 1
else:
clean.append(parsed[i])
i += 1
# Now pass through again to handle the default values.
i = 0
ready = []
while i < len(clean):
if isinstance(clean[i], str) and re.match("=>?", clean[i]):
ready.pop()
if ">" in clean[i]:
ready.append([clean[i - 1], ("> " + clean[i + 1][0], clean[i + 1][1])])
else:
ready.append([clean[i - 1], clean[i + 1]])
i += 2
else:
ready.append(clean[i])
i += 1
return ready
def get_fortpy_version(compiler,
fortpath,
recursed=False,
attribute="version"):
"""Gets the fortpy version number from the first line of the specified file."""
result = []
#If the file doesn't exist yet, we don't try to find the version information.
from os import path
ext = path.splitext(fortpath)[1]
cext = "" if compiler is None else replace(".[c]", compiler)
if not path.isfile(fortpath) or ext in [".o", ".mod", cext]:
if path.isfile(fortpath + '.v'):
return get_fortpy_version(compiler, fortpath + '.v', True)
else:
return result
with open(fortpath) as f:
for line in f:
try:
lt = line.index("<")
vxml = "<doc>{}</doc>".format(line[lt::])
except ValueError:
vxml = ""
break
if "<fortpy" in vxml:
import xml.etree.ElementTree as ET
x = list(ET.XML(vxml))
if len(x) > 0:
try:
result = list(map(int, x[0].attrib[attribute].split(".")))
except ValueError:
fstr = "can't extract version information from '{}' in '{}'."
msg.err(fstr.format(x[0].attrib[attribute], fortpath))
except KeyError:
fstr = "fortpy tag does not contain attribute '{}': {} ({})"
msg.err(fstr.format(attribute, x[0].attrib, fortpath))
if len(result) == 0 and not recursed:
return get_fortpy_version(compiler, fortpath + '.v', True)
else:
return result
def fit(self, independent, dependent, model, threshold=1., functions=None):
"""Tries to fit the data selection for the specified dependent variable against the
indepedent one using the specified model. Arguments are similar to self.table(). Returns
a tuple of (fitting parameters, 1 standard deviation errors).
:arg model: a description of the function to fit to. Possible values are ['exp', 'lin'].
"""
key = "{}${}".format(independent, dependent)
xs, ys = self._get_data_series(independent, [dependent], threshold,
functions)
if xs[0][0] is None:
return
if len(xs[0][0]) != len(ys[0][0]):
msg.err(
"Can't fit data unless we have the same number of values for the independent "
"and dependent variables. "
"len({}) == {}; len({}) == {}".format(independent,
len(xs[0][0]), dependent,
len(ys[0][0])))
return
from scipy.optimize import curve_fit
from numpy import sqrt, diag
mdict = {"exp": _fit_exp, "lin": _fit_lin}
if not model in mdict:
msg.err(
"Cannot fit using model '{}'; unknown function.".format(model))
return
func = mdict[model]
popt, pcov = curve_fit(func, xs[0][0], ys[0][0])
perr = sqrt(diag(pcov))
self.fits[key] = {
"params": popt,
"covarmat": pcov,
"function": (lambda x: func(x, *popt)),
"model": model,
"error": perr
}
def fit(self, independent, dependent, model, threshold=1., functions=None):
"""Tries to fit the data selection for the specified dependent variable against the
indepedent one using the specified model. Arguments are similar to self.table(). Returns
a tuple of (fitting parameters, 1 standard deviation errors).
:arg model: a description of the function to fit to. Possible values are ['exp', 'lin'].
"""
key = "{}${}".format(independent, dependent)
xs, ys = self._get_data_series(independent, [dependent], threshold, functions)
if xs[0][0] is None:
return
if len(xs[0][0]) != len(ys[0][0]):
msg.err("Can't fit data unless we have the same number of values for the independent "
"and dependent variables. "
"len({}) == {}; len({}) == {}".format(independent, len(xs[0][0]),
dependent, len(ys[0][0])))
return
from scipy.optimize import curve_fit
from numpy import sqrt, diag
mdict = {
"exp": _fit_exp,
"lin": _fit_lin
}
if not model in mdict:
msg.err("Cannot fit using model '{}'; unknown function.".format(model))
return
func = mdict[model]
popt, pcov = curve_fit(func, xs[0][0], ys[0][0])
perr = sqrt(diag(pcov))
self.fits[key] = {
"params": popt,
"covarmat": pcov,
"function": (lambda x: func(x, *popt)),
"model": model,
"error": perr
}
def get_fortpy_version(compiler, fortpath, recursed=False, attribute="version"):
"""Gets the fortpy version number from the first line of the specified file."""
result = []
#If the file doesn't exist yet, we don't try to find the version information.
from os import path
ext = path.splitext(fortpath)[1]
cext = "" if compiler is None else replace(".[c]", compiler)
if not path.isfile(fortpath) or ext in [".o", ".mod", cext]:
if path.isfile(fortpath + '.v'):
return get_fortpy_version(compiler, fortpath + '.v', True)
else:
return result
with open(fortpath) as f:
for line in f:
try:
lt = line.index("<")
vxml = "<doc>{}</doc>".format(line[lt::])
except ValueError:
vxml = ""
break
if "<fortpy" in vxml:
import xml.etree.ElementTree as ET
x = list(ET.XML(vxml))
if len(x) > 0:
try:
result = list(map(int, x[0].attrib[attribute].split(".")))
except ValueError:
fstr = "can't extract version information from '{}' in '{}'."
msg.err(fstr.format(x[0].attrib[attribute], fortpath))
except KeyError:
fstr = "fortpy tag does not contain attribute '{}': {} ({})"
msg.err(fstr.format(attribute, x[0].attrib, fortpath))
if len(result) == 0 and not recursed:
return get_fortpy_version(compiler, fortpath + '.v', True)
else:
return result
def make(self, remake=False, dependencies=None,
compiler="gfortran", debug=False, profile=False):
"""Generates a makefile to compile the wrapper module and link it with the shared
library that can be compiled with self.makelib().
"""
self._check_lib(remake, compiler, debug, profile)
from os import path
if self.link is None or not path.isfile(self.link):
msg.err("Can't create shared library; missing compiled original code: {}".format(self.link))
exit(0)
if remake or not path.isfile(self.libpath):
if dependencies is None:
dependencies = ["{}_c".format(self.module.name)]
identifier = self.name
else:
identifier = "ftypes.{}".format(self.library)
#We don't need to reorder these *wrapper* modules before compilation because each of
#them should be independent of all the others.
#dependencies = self._process_module_needs(dependencies)
#append the dependency for the deallocation module to be included in the shared lib.
dependencies.append("ftypes_dealloc")
makename = "Makefile.{}".format(self.library if self.library else self.name)
makepath = path.join(self.f90path, makename)
extralinks = list(self.dependencies.values())
extralinks.append(self.link)
makefile(identifier, dependencies, makepath, "ftypes.{}".format(self.name),
False, False, self.module.parent, "so", extralinks)
code = self._compile(self.f90path, makename, compiler, debug, profile)
if code == 0:
#Copy the shared library to the main directory out of the f90 directory
from shutil import copy
copy(path.join(self.f90path, "{}.so".format(identifier)), self.dirpath)
return code
else:
return 0
def _vupdate_compiled_module(compiler,
modname,
folder,
tversion=None,
rename=True):
"""Compiles a .mod and .o for the specified compiler and specified.
"""
from os import path
opath = path.join(folder, "{}.o".format(modname))
mpath = path.join(folder, "{}.mod".format(modname))
if path.isfile(opath) and path.isfile(mpath):
if rename:
from shutil import move
nopath = path.join(folder,
replace("{}.o.[c]".format(modname), compiler))
nmpath = path.join(folder,
replace("{}.mod.[c]".format(modname), compiler))
move(opath, nopath)
move(mpath, nmpath)
else:
nopath = opath
nmpath = mpath
#Create the version files so we can keep track of the compiled versions.
vpaths = [nopath, nmpath]
for np in vpaths:
if tversion is None:
tversion = get_fortpy_version(compiler, np)
if len(tversion) == 0:
#Try to get the template version for the module.
tversion = template_version(compiler, modname + ".f90")
vp = np + ".v"
with open(vp, 'w') as f:
f.write('#<fortpy version="{}" />'.format('.'.join(
map(str, tversion))))
else:
msg.err("Unable to find {0}.o and {0}.mod.".format(modname))
def _start_debug(wizard, parameter, target):
"""Starts a debug session for the unit test executable of the current test
specification and case identifier.
"""
from fortpy.utility import which
if which("gdb"):
#Give the user the option of setting some breakpoints. Basically, list the
#1) main program entry, 2) list of pre-req calls before the executable,
#3) executable itself.
brks = ["Main Unit Test Entry Point.", wizard.xauto.signature]
xnames = ["main", wizard.xauto.name]
from fortpy.testing.elements import Executable
for method in wizard.tauto.methods:
if not isinstance(method, Executable):
#This is an executable that needs to be called before the main one.
brks.append(method.signature)
xnames.append(method.name)
bchoice = _prompt_general("Which methods would you like breakpoints for?", brks)
#Start a gdb session for the compiled unit test executable of the active
#test and test case.
xs = []
for x in bchoice:
if x < len(xnames):
xs.append("-ex '{}'".format(xnames[x]))
from os import system, path
from fortpy.testing.compilers import compile_general
xstage = path.join(wizard.stagedir, wizard.xauto.full_name)
code, success, target = compile_general(xstage, wizard.compiler, wizard.tauto.identifier,
True, quiet=True)
testpath = _get_casepath(wizard)
cmd = "cd {}; gdb {} -ex 'run' ../../{}.x"
system(cmd.format(testpath, ' '.join(xs), wizard.tauto.identifier))
else:
msg.err("The GNU debugger 'gdb' is not available on the system.")
def plot(self, independent=None, dependents=None, threshold=1.,
savefile=None, functions=None, xscale=None, yscale=None,
colors=None, labels=None, fonts=None, markers=None, lines=None, ticks=None,
plottypes=None, limits=None, twinplots=None, legend=None, **kwargs):
"""Plots the specified dependent variables as functions of the independent one.
:arg independent: a string indentifying the independent variable's filename and the
appropriate property of the data in the file. For example: "concs.in|depth" would
use the depth of the data in the file 'concs.in' as its value for each test case.
:arg dependent: a list of strings that use the same format as the string for the
'independent' variable.
:arg threshold: a float value specifying the minimum level of success that the output
file must attain before it can be used in the plot.
:arg xlabel: the label for the x-axis of the plot.
:arg ylabel: the label for the y-axis of the plot.
:arg functions: a dictionary of functions to apply to the values of each variable. E.g.
{"group.in|depth": {lambda dict}}. See the help text help_postfix() for details.
"""
if independent is None or dependents is None:
raise ValueError("Must specify at least the variables to plot.")
import matplotlib.pyplot as plt
from matplotlib import cm, rc
from numpy import linspace, array
from itertools import cycle
from matplotlib.font_manager import FontProperties
xs, ys = self._get_data_series(independent, dependents, threshold, functions)
#Set the default font for the plots, or the custom one.
if fonts is not None and "family" in fonts:
rc('font',family=fonts["family"])
else:
rc('font',family='Times New Roman')
layout = None
if "figsize" in kwargs:
if len(kwargs["figsize"]) > 2:
layout = {"pad": kwargs["figsize"][2]}
fig = plt.figure(figsize=(None if "figsize" not in kwargs else kwargs["figsize"][0:2]), tight_layout=layout)
dfont = FontProperties()
ax = fig.add_subplot(111)
rbcolors = cm.rainbow(linspace(0, 1, len(ys)))
cycols = cycle(rbcolors)
axx = None
axy = None
if twinplots is not None:
if "twinx" in twinplots.values():
axx = ax.twinx()
if "twiny" in twinplots.values():
axy = ax.twiny()
if labels is not None:
if "x" in labels:
ax.set_xlabel(labels["x"], fontproperties=self._get_font(dfont, fonts, "xlabel"))
if "y" in labels:
ax.set_ylabel(labels["y"], fontproperties=self._get_font(dfont, fonts, "ylabel"))
if "plot" in labels:
plt.title(labels["plot"], fontproperties=self._get_font(dfont, fonts, "title"))
if "x-twin" in labels and axx is not None:
axx.set_ylabel(labels["x-twin"], fontproperties=self._get_font(dfont, fonts, "x-twin-label"))
if "y-twin" in labels and axy is not None:
axy.set_xlabel(labels["y-twin"], fontproperties=self._get_font(dfont, fonts, "y-twin-label"))
if xscale is not None:
ax.set_xscale(xscale)
if yscale is not None:
ax.set_yscale(yscale)
for xset, yset in zip(xs, ys):
x, xlabel = xset
y, ylabel = yset
#Set up a dictionary with all the kwargs for the plotting options
#that may be common to both kinds of plots.
dargs = {}
if colors is not None and ylabel in colors:
dargs["color"] = colors[ylabel]
else:
dargs["color"] = next(cycols)
if markers is not None and ylabel in markers:
self._get_markers(dargs, markers[ylabel])
if "size" in markers[ylabel]:
dargs["s"] = float(markers[ylabel]["size"])
if "s" not in dargs:
dargs["s"] = 15
if lines is not None and ylabel in lines:
if "style" in lines[ylabel]:
dargs["linestyle"] = lines[ylabel]["style"]
if "width" in lines[ylabel]:
dargs["linewidth"] = float(lines[ylabel]["width"])
if labels is not None and ylabel in labels:
dargs["label"] = None if labels[ylabel].lower() == "[none]" else labels[ylabel]
else:
dargs["label"] = ylabel
if twinplots is not None and ylabel in twinplots:
if twinplots[ylabel] == "twinx":
liveax = axx
elif twinplots[ylabel] == "twiny":
liveax = axy
else:
liveax = ax
try:
if plottypes is not None and ylabel in plottypes:
lineplot = plottypes[ylabel] == "line"
else:
lineplot = None
if ylabel[len(ylabel)-4:len(ylabel)] in ["|fit", ".fit"]:
varfile = ylabel[0:len(ylabel)-4]
key = "{}${}".format(independent, varfile)
allx = linspace(min(x), max(x), 50)
if labels is not None and ylabel in labels:
dargs["label"] = labels[ylabel].format(self._format_fit(key))
if dargs["label"].lower() == "[none]":
dargs["label"] = None
else:
dargs["label"] = self._format_fit(key)
self._reset_lineplot(dargs)
liveax.plot(allx, self.fits[key]["function"](allx), **dargs)
elif lineplot:
self._reset_lineplot(dargs)
from operator import itemgetter
sdata = array(sorted(zip(x,y),key=itemgetter(0)))
liveax.plot(sdata[:,0], sdata[:,1], **dargs)
else:
size = dargs["s"]
del dargs["s"]
liveax.scatter(x, y, s=size, **dargs)
except ValueError:
msg.err("The values for {} can't be log-plotted.".format(ylabel))
#Set the tick labels font sizes.
def tickfonts(fonts, key, labels):
if fonts is not None and key in fonts:
for label in labels:
tfont = self._get_font(dfont, fonts, key)
label.set_fontproperties(tfont)
tickfonts(fonts, "xticks", ax.get_xticklabels())
tickfonts(fonts, "yticks", ax.get_yticklabels())
if axx is not None:
tickfonts(fonts, "x-twin-ticks", axx.get_yticklabels())
if axy is not None:
tickfonts(fonts, "y-twin-ticks", axy.get_xticklabels())
if ticks is not None and len(ticks) > 0:
for key in ticks:
if "reset" in ticks[key]:
ticks[key]["reset"] = ticks[key]["reset"] == "true"
if "x-twin" in key and axx is not None:
axx.tick_params(**ticks[key])
elif "y-twin" in key and axy is not None:
axy.tick_params(**ticks[key])
else:
ax.tick_params(**ticks[key])
for dim in ["x", "y", "z", "x-twin", "y-twin"]:
if limits is not None and dim in limits:
if dim == "x-twin" and axx is not None:
if isinstance(limits[dim], tuple):
axx.set_ylim(limits[dim])
elif limits[dim] == "auto":
axx.set_ylim(auto=True)
elif dim == "y-twin" and axy is not None:
if isinstance(limits[dim], tuple):
axy.set_ylim(limits[dim])
elif limits[dim] == "auto":
axy.set_ylim(auto=True)
elif "-" not in dim:
if isinstance(limits[dim], tuple):
getattr(ax, "set_{}lim".format(dim))(limits[dim])
elif limits[dim] == "auto":
getattr(ax, "set_{}lim".format(dim))(auto=True)
if len(dependents) > 1:
if legend is not None:
plt.legend(prop=self._get_font(dfont, fonts, "legend"), **legend)
else:
plt.legend(loc='upper right', prop=self._get_font(dfont, fonts, "legend"))
if savefile is None:
plt.show(block=False)
else:
plt.savefig(savefile)
def plot(self, independent, dependents, threshold=1., xlabel=None, ylabel=None,
savefile=None, functions=None, xscale=None, yscale=None,
colors=None, labels=None, fonts=None, markers=None, lines=None, ticks=None,
plottypes=None, limits=None):
"""Plots the specified dependent variables as functions of the independent one.
:arg independent: a string indentifying the independent variable's filename and the
appropriate property of the data in the file. For example: "concs.in|depth" would
use the depth of the data in the file 'concs.in' as its value for each test case.
:arg dependent: a list of strings that use the same format as the string for the
'independent' variable.
:arg threshold: a float value specifying the minimum level of success that the output
file must attain before it can be used in the plot.
:arg xlabel: the label for the x-axis of the plot.
:arg ylabel: the label for the y-axis of the plot.
:arg functions: a dictionary of functions to apply to the values of each variable. E.g.
{"group.in|depth": "numpy.log"} would apply the numpy.log function to each value extracted
from the depth property of the data in 'group.in' before plotting it. If the value
is not a string, it must be callable with some argument.
"""
import matplotlib.pyplot as plt
from matplotlib import cm, rc
from numpy import linspace, array
from itertools import cycle
from matplotlib.font_manager import FontProperties
xs, ys = self._get_data_series(independent, dependents, threshold, functions)
#Set the default font for the plots, or the custom one.
if fonts is not None and "family" in fonts:
rc('font',family=fonts["family"])
else:
rc('font',family='Times New Roman')
fig = plt.figure()
dfont = FontProperties()
ax = fig.add_subplot(111)
if xlabel is not None:
ax.set_xlabel(xlabel, fontproperties=self._get_font(dfont, fonts, "xlabel"))
if ylabel is not None:
ax.set_ylabel(ylabel, fontproperties=self._get_font(dfont, fonts, "ylabel"))
if xscale is not None:
ax.set_xscale(xscale)
if yscale is not None:
ax.set_yscale(yscale)
if labels is not None and "plot" in labels:
plt.title(labels["plot"], fontproperties=self._get_font(dfont, fonts, "title"))
rbcolors = cm.rainbow(linspace(0, 1, len(ys)))
cycols = cycle(rbcolors)
for xset, yset in zip(xs, ys):
x, xlabel = xset
y, ylabel = yset
#Set up a dictionary with all the kwargs for the plotting options
#that may be common to both kinds of plots.
dargs = {}
if colors is not None and ylabel in colors:
dargs["color"] = colors[ylabel]
else:
dargs["color"] = next(cycols)
if markers is not None and ylabel in markers:
self._get_markers(dargs, markers[ylabel])
if "size" in markers[ylabel]:
dargs["s"] = float(markers[ylabel]["size"])
if "s" not in dargs:
dargs["s"] = 15
if lines is not None and ylabel in lines:
if "style" in lines[ylabel]:
dargs["linestyle"] = lines[ylabel]["style"]
if "width" in lines[ylabel]:
dargs["linewidth"] = float(lines[ylabel]["width"])
if labels is not None and ylabel in labels:
dargs["label"] = None if labels[ylabel].lower() == "[none]" else labels[ylabel]
else:
dargs["label"] = ylabel
try:
if plottypes is not None and ylabel in plottypes:
lineplot = plottypes[ylabel] == "line"
else:
lineplot = None
if ylabel[len(ylabel)-4:len(ylabel)] in ["|fit", ".fit"]:
varfile = ylabel[0:len(ylabel)-4]
key = "{}${}".format(independent, varfile)
allx = linspace(min(x), max(x), 50)
if labels is not None and ylabel in labels:
dargs["label"] = labels[ylabel].format(self._format_fit(key))
if dargs["label"].lower() == "[none]":
dargs["label"] = None
else:
dargs["label"] = self._format_fit(key)
self._reset_lineplot(dargs)
ax.plot(allx, self.fits[key]["function"](allx), **dargs)
elif lineplot:
self._reset_lineplot(dargs)
from operator import itemgetter
sdata = array(sorted(zip(x,y),key=itemgetter(0)))
ax.plot(sdata[:,0], sdata[:,1], **dargs)
else:
size = dargs["s"]
del dargs["s"]
ax.scatter(x, y, s=size, **dargs)
except ValueError:
msg.err("The values for {} can't be log-plotted.".format(ylabel))
#Set the tick labels font sizes.
if fonts is not None and "xticks" in fonts:
tfont = self._get_font(dfont, fonts, "xticks")
for label in ax.get_xticklabels():
label.set_fontproperties(tfont)
if fonts is not None and "yticks" in fonts:
for label in ax.get_yticklabels():
tfont = self._get_font(dfont, fonts, "yticks")
label.set_fontproperties(tfont)
if ticks is not None and len(ticks) > 0:
for key in ticks:
if "reset" in ticks[key]:
ticks[key]["reset"] = ticks[key]["reset"] == "true"
ax.tick_params(**ticks[key])
for dim in ["x", "y", "z"]:
if limits is not None and dim in limits:
getattr(plt, "{}lim".format(dim))(limits[dim])
if len(dependents) > 1:
plt.legend(loc='upper left', prop=self._get_font(dfont, fonts, "legend"))
if savefile is None:
plt.show(block=False)
else:
plt.savefig(savefile)
def _get_data(self, variable, fullvar, order=None, threshold=1., tfilter=None, functions=None,
independent=None, x=None):
"""Returns a list of the valid data points for the specified variable.
:arg variable: a string indentifying the variable's filename and the
appropriate property of the data in the file. For example: "concs.in|depth" would
use the depth of the data in the file 'concs.in' as its value for each test case.
:arg fullvar: the full variable name including the filter and property spec.
:arg threshold: a float value specifying the minimum level of success that the output
file must attain before it can be used in the plot.
:arg independent: the name of the independent variable including property.
:arg x: a list of values for the *independent* variable to use when evaluating
a *fitted* curve to a variable.
"""
from numpy import ndarray
target = None
if variable != "timing" or "timing|" in variable:
varfile, attribute = variable.split("|")
else:
target = varfile = "timing"
attribute = None
if target is None:
if varfile in self.infiles:
target = "inputs"
elif varfile in self.outfiles:
target = "outputs"
elif varfile == "timing" and attribute is not None:
target = "timing"
else:
raise ValueError("Cannot locate the values for variable {}".format(variable))
values = []
cases = []
#If we have already ruled out some of the test cases while doing the independent
#variable, we want to only use those cases.
if order is not None:
details = order
else:
details = self.details
#Handle cases where the values need to have functions applied to them before tabulating
#or printing the values.
if functions is not None and fullvar in functions:
if isinstance(functions[fullvar], str) or isinstance(functions[fullvar], str):
from importlib import import_module
module = functions[fullvar].split(".")
fname = module.pop()
numpy = import_module('.'.join(module))
fx = getattr(numpy, fname)
else:
fx = functions[fullvar]
else:
fx = None
for testcase in details:
if not self._case_filter(testcase, tfilter):
#We don't consider test cases that are blocked by the filter.
continue
value = 0
if target == "timing" and self._is_successful(self.details[testcase]) and attribute is None:
value = self.details[testcase][target]
else:
if (self._is_successful(self.details[testcase], varfile, threshold)):
if target != "timing" and varfile in self.details[testcase][target]:
data = self.details[testcase][target][varfile]
else:
data = None
if data is not None and attribute in data:
value = data[attribute]
elif attribute == "fit" and independent is not None and x is not None:
#Return the value of the *fitted* function for the corresponding independent
#variable value; this is one-to-one for tabulating. For plotting we don't use
#these values.
key = "{}${}".format(independent, varfile)
if len(values) < len(x) and key in self.fits:
ival = x[len(values)]
value = self.fits[key]["function"](ival)
#We need the arrays to be the same length for plotting; if we are following
#an existing independent variable, we still need to append zero to the list
#for the values we don't have.
if isinstance(value, ndarray) or value != 0 or order is not None:
if fx is not None:
value = fx(value)
#Handle the case where we are plotting an entire row as a single point
#and an aggregrate function should have been specified
if isinstance(value, list):
if len(value) == 1:
values.append(value[0] if value[0] is not None else 0)
else:
msg.err("Can't coerce an array to a single value without an aggregation "
"function such as numpy.mean or numpy.sum")
return ([0], "Array Aggregation Error")
else:
values.append(value if value is not None else 0)
cases.append(testcase)
return (values, cases)
def _get_data(self,
variable,
fullvar,
order=None,
threshold=1.,
tfilter=None,
functions=None,
independent=None,
x=None):
"""Returns a list of the valid data points for the specified variable.
:arg variable: a string indentifying the variable's filename and the
appropriate property of the data in the file. For example: "concs.in|depth" would
use the depth of the data in the file 'concs.in' as its value for each test case.
:arg fullvar: the full variable name including the filter and property spec.
:arg threshold: a float value specifying the minimum level of success that the output
file must attain before it can be used in the plot.
:arg independent: the name of the independent variable including property.
:arg x: a list of values for the *independent* variable to use when evaluating
a *fitted* curve to a variable.
"""
from numpy import ndarray
target = None
if variable != "timing" or "timing|" in variable:
varfile, attribute = variable.split("|")
else:
target = varfile = "timing"
attribute = None
if target is None:
if varfile in self.infiles:
target = "inputs"
elif varfile in self.outfiles:
target = "outputs"
elif varfile == "timing" and attribute is not None:
target = "timing"
else:
raise ValueError(
"Cannot locate the values for variable {}".format(
variable))
values = []
cases = []
#If we have already ruled out some of the test cases while doing the independent
#variable, we want to only use those cases.
if order is not None:
details = order
else:
details = self.details
#Handle cases where the values need to have functions applied to them before tabulating
#or printing the values.
if functions is not None and fullvar in functions:
if isinstance(functions[fullvar], str) or isinstance(
functions[fullvar], str):
from importlib import import_module
module = functions[fullvar].split(".")
fname = module.pop()
numpy = import_module('.'.join(module))
fx = getattr(numpy, fname)
else:
fx = functions[fullvar]
else:
fx = None
for testcase in details:
if not self._case_filter(testcase, tfilter):
#We don't consider test cases that are blocked by the filter.
continue
value = 0
if target == "timing" and self._is_successful(
self.details[testcase]) and attribute is None:
value = self.details[testcase][target]
else:
if (self._is_successful(self.details[testcase], varfile,
threshold)):
if target != "timing" and varfile in self.details[
testcase][target]:
data = self.details[testcase][target][varfile]
else:
data = None
if data is not None and attribute in data:
value = data[attribute]
elif attribute == "fit" and independent is not None and x is not None:
#Return the value of the *fitted* function for the corresponding independent
#variable value; this is one-to-one for tabulating. For plotting we don't use
#these values.
key = "{}${}".format(independent, varfile)
if len(values) < len(x) and key in self.fits:
ival = x[len(values)]
value = self.fits[key]["function"](ival)
#We need the arrays to be the same length for plotting; if we are following
#an existing independent variable, we still need to append zero to the list
#for the values we don't have.
if isinstance(value, ndarray) or value != 0 or order is not None:
if fx is not None:
value = fx(value)
#Handle the case where we are plotting an entire row as a single point
#and an aggregrate function should have been specified
if isinstance(value, list):
if len(value) == 1:
values.append(value[0] if value[0] is not None else 0)
else:
msg.err(
"Can't coerce an array to a single value without an aggregation "
"function such as numpy.mean or numpy.sum")
return ([0], "Array Aggregation Error")
else:
values.append(value if value is not None else 0)
cases.append(testcase)
return (values, cases)
def plot(self,
independent,
dependents,
threshold=1.,
xlabel=None,
ylabel=None,
savefile=None,
functions=None,
xscale=None,
yscale=None,
colors=None,
labels=None,
fonts=None,
markers=None,
lines=None,
ticks=None,
plottypes=None,
limits=None):
"""Plots the specified dependent variables as functions of the independent one.
:arg independent: a string indentifying the independent variable's filename and the
appropriate property of the data in the file. For example: "concs.in|depth" would
use the depth of the data in the file 'concs.in' as its value for each test case.
:arg dependent: a list of strings that use the same format as the string for the
'independent' variable.
:arg threshold: a float value specifying the minimum level of success that the output
file must attain before it can be used in the plot.
:arg xlabel: the label for the x-axis of the plot.
:arg ylabel: the label for the y-axis of the plot.
:arg functions: a dictionary of functions to apply to the values of each variable. E.g.
{"group.in|depth": "numpy.log"} would apply the numpy.log function to each value extracted
from the depth property of the data in 'group.in' before plotting it. If the value
is not a string, it must be callable with some argument.
"""
import matplotlib.pyplot as plt
from matplotlib import cm, rc
from numpy import linspace, array
from itertools import cycle
from matplotlib.font_manager import FontProperties
xs, ys = self._get_data_series(independent, dependents, threshold,
functions)
#Set the default font for the plots, or the custom one.
if fonts is not None and "family" in fonts:
rc('font', family=fonts["family"])
else:
rc('font', family='Times New Roman')
fig = plt.figure()
dfont = FontProperties()
ax = fig.add_subplot(111)
if xlabel is not None:
ax.set_xlabel(xlabel,
fontproperties=self._get_font(
dfont, fonts, "xlabel"))
if ylabel is not None:
ax.set_ylabel(ylabel,
fontproperties=self._get_font(
dfont, fonts, "ylabel"))
if xscale is not None:
ax.set_xscale(xscale)
if yscale is not None:
ax.set_yscale(yscale)
if labels is not None and "plot" in labels:
plt.title(labels["plot"],
fontproperties=self._get_font(dfont, fonts, "title"))
rbcolors = cm.rainbow(linspace(0, 1, len(ys)))
cycols = cycle(rbcolors)
for xset, yset in zip(xs, ys):
x, xlabel = xset
y, ylabel = yset
#Set up a dictionary with all the kwargs for the plotting options
#that may be common to both kinds of plots.
dargs = {}
if colors is not None and ylabel in colors:
dargs["color"] = colors[ylabel]
else:
dargs["color"] = next(cycols)
if markers is not None and ylabel in markers:
self._get_markers(dargs, markers[ylabel])
if "size" in markers[ylabel]:
dargs["s"] = float(markers[ylabel]["size"])
if "s" not in dargs:
dargs["s"] = 15
if lines is not None and ylabel in lines:
if "style" in lines[ylabel]:
dargs["linestyle"] = lines[ylabel]["style"]
if "width" in lines[ylabel]:
dargs["linewidth"] = float(lines[ylabel]["width"])
if labels is not None and ylabel in labels:
dargs["label"] = None if labels[ylabel].lower(
) == "[none]" else labels[ylabel]
else:
dargs["label"] = ylabel
try:
if plottypes is not None and ylabel in plottypes:
lineplot = plottypes[ylabel] == "line"
else:
lineplot = None
if ylabel[len(ylabel) - 4:len(ylabel)] in ["|fit", ".fit"]:
varfile = ylabel[0:len(ylabel) - 4]
key = "{}${}".format(independent, varfile)
allx = linspace(min(x), max(x), 50)
if labels is not None and ylabel in labels:
dargs["label"] = labels[ylabel].format(
self._format_fit(key))
if dargs["label"].lower() == "[none]":
dargs["label"] = None
else:
dargs["label"] = self._format_fit(key)
self._reset_lineplot(dargs)
ax.plot(allx, self.fits[key]["function"](allx), **dargs)
elif lineplot:
self._reset_lineplot(dargs)
from operator import itemgetter
sdata = array(sorted(zip(x, y), key=itemgetter(0)))
ax.plot(sdata[:, 0], sdata[:, 1], **dargs)
else:
size = dargs["s"]
del dargs["s"]
ax.scatter(x, y, s=size, **dargs)
except ValueError:
msg.err(
"The values for {} can't be log-plotted.".format(ylabel))
#Set the tick labels font sizes.
if fonts is not None and "xticks" in fonts:
tfont = self._get_font(dfont, fonts, "xticks")
for label in ax.get_xticklabels():
label.set_fontproperties(tfont)
if fonts is not None and "yticks" in fonts:
for label in ax.get_yticklabels():
tfont = self._get_font(dfont, fonts, "yticks")
label.set_fontproperties(tfont)
if ticks is not None and len(ticks) > 0:
for key in ticks:
if "reset" in ticks[key]:
ticks[key]["reset"] = ticks[key]["reset"] == "true"
ax.tick_params(**ticks[key])
for dim in ["x", "y", "z"]:
if limits is not None and dim in limits:
getattr(plt, "{}lim".format(dim))(limits[dim])
if len(dependents) > 1:
plt.legend(loc='upper left',
prop=self._get_font(dfont, fonts, "legend"))
if savefile is None:
plt.show(block=False)
else:
plt.savefig(savefile)
def plot(self,
independent=None,
dependents=None,
threshold=1.,
savefile=None,
functions=None,
xscale=None,
yscale=None,
colors=None,
labels=None,
fonts=None,
markers=None,
lines=None,
ticks=None,
plottypes=None,
limits=None,
twinplots=None,
legend=None,
**kwargs):
"""Plots the specified dependent variables as functions of the independent one.
:arg independent: a string indentifying the independent variable's filename and the
appropriate property of the data in the file. For example: "concs.in|depth" would
use the depth of the data in the file 'concs.in' as its value for each test case.
:arg dependent: a list of strings that use the same format as the string for the
'independent' variable.
:arg threshold: a float value specifying the minimum level of success that the output
file must attain before it can be used in the plot.
:arg xlabel: the label for the x-axis of the plot.
:arg ylabel: the label for the y-axis of the plot.
:arg functions: a dictionary of functions to apply to the values of each variable. E.g.
{"group.in|depth": {lambda dict}}. See the help text help_postfix() for details.
"""
if independent is None or dependents is None:
raise ValueError("Must specify at least the variables to plot.")
import matplotlib.pyplot as plt
from matplotlib import cm, rc
from numpy import linspace, array
from itertools import cycle
from matplotlib.font_manager import FontProperties
xs, ys = self._get_data_series(independent, dependents, threshold,
functions)
#Set the default font for the plots, or the custom one.
if fonts is not None and "family" in fonts:
rc('font', family=fonts["family"])
else:
rc('font', family='Times New Roman')
layout = None
if "figsize" in kwargs:
if len(kwargs["figsize"]) > 2:
layout = {"pad": kwargs["figsize"][2]}
fig = plt.figure(figsize=(None if "figsize" not in kwargs else
kwargs["figsize"][0:2]),
tight_layout=layout)
dfont = FontProperties()
ax = fig.add_subplot(111)
rbcolors = cm.rainbow(linspace(0, 1, len(ys)))
cycols = cycle(rbcolors)
axx = None
axy = None
if twinplots is not None:
if "twinx" in twinplots.values():
axx = ax.twinx()
if "twiny" in twinplots.values():
axy = ax.twiny()
if labels is not None:
if "x" in labels:
ax.set_xlabel(labels["x"],
fontproperties=self._get_font(
dfont, fonts, "xlabel"))
if "y" in labels:
ax.set_ylabel(labels["y"],
fontproperties=self._get_font(
dfont, fonts, "ylabel"))
if "plot" in labels:
plt.title(labels["plot"],
fontproperties=self._get_font(dfont, fonts, "title"))
if "x-twin" in labels and axx is not None:
axx.set_ylabel(labels["x-twin"],
fontproperties=self._get_font(
dfont, fonts, "x-twin-label"))
if "y-twin" in labels and axy is not None:
axy.set_xlabel(labels["y-twin"],
fontproperties=self._get_font(
dfont, fonts, "y-twin-label"))
if xscale is not None:
ax.set_xscale(xscale)
if yscale is not None:
ax.set_yscale(yscale)
for xset, yset in zip(xs, ys):
x, xlabel = xset
y, ylabel = yset
#Set up a dictionary with all the kwargs for the plotting options
#that may be common to both kinds of plots.
dargs = {}
if colors is not None and ylabel in colors:
dargs["color"] = colors[ylabel]
else:
dargs["color"] = next(cycols)
if markers is not None and ylabel in markers:
self._get_markers(dargs, markers[ylabel])
if "size" in markers[ylabel]:
dargs["s"] = float(markers[ylabel]["size"])
if "s" not in dargs:
dargs["s"] = 15
if lines is not None and ylabel in lines:
if "style" in lines[ylabel]:
dargs["linestyle"] = lines[ylabel]["style"]
if "width" in lines[ylabel]:
dargs["linewidth"] = float(lines[ylabel]["width"])
if labels is not None and ylabel in labels:
dargs["label"] = None if labels[ylabel].lower(
) == "[none]" else labels[ylabel]
else:
dargs["label"] = ylabel
if twinplots is not None and ylabel in twinplots:
if twinplots[ylabel] == "twinx":
liveax = axx
elif twinplots[ylabel] == "twiny":
liveax = axy
else:
liveax = ax
try:
if plottypes is not None and ylabel in plottypes:
lineplot = plottypes[ylabel] == "line"
else:
lineplot = None
if ylabel[len(ylabel) - 4:len(ylabel)] in ["|fit", ".fit"]:
varfile = ylabel[0:len(ylabel) - 4]
key = "{}${}".format(independent, varfile)
allx = linspace(min(x), max(x), 50)
if labels is not None and ylabel in labels:
dargs["label"] = labels[ylabel].format(
self._format_fit(key))
if dargs["label"].lower() == "[none]":
dargs["label"] = None
else:
dargs["label"] = self._format_fit(key)
self._reset_lineplot(dargs)
liveax.plot(allx, self.fits[key]["function"](allx),
**dargs)
elif lineplot:
self._reset_lineplot(dargs)
from operator import itemgetter
sdata = array(sorted(zip(x, y), key=itemgetter(0)))
liveax.plot(sdata[:, 0], sdata[:, 1], **dargs)
else:
size = dargs["s"]
del dargs["s"]
liveax.scatter(x, y, s=size, **dargs)
except ValueError:
msg.err(
"The values for {} can't be log-plotted.".format(ylabel))
#Set the tick labels font sizes.
def tickfonts(fonts, key, labels):
if fonts is not None and key in fonts:
for label in labels:
tfont = self._get_font(dfont, fonts, key)
label.set_fontproperties(tfont)
tickfonts(fonts, "xticks", ax.get_xticklabels())
tickfonts(fonts, "yticks", ax.get_yticklabels())
if axx is not None:
tickfonts(fonts, "x-twin-ticks", axx.get_yticklabels())
if axy is not None:
tickfonts(fonts, "y-twin-ticks", axy.get_xticklabels())
if ticks is not None and len(ticks) > 0:
for key in ticks:
if "reset" in ticks[key]:
ticks[key]["reset"] = ticks[key]["reset"] == "true"
if "x-twin" in key and axx is not None:
axx.tick_params(**ticks[key])
elif "y-twin" in key and axy is not None:
axy.tick_params(**ticks[key])
else:
ax.tick_params(**ticks[key])
for dim in ["x", "y", "z", "x-twin", "y-twin"]:
if limits is not None and dim in limits:
if dim == "x-twin" and axx is not None:
if isinstance(limits[dim], tuple):
axx.set_ylim(limits[dim])
elif limits[dim] == "auto":
axx.set_ylim(auto=True)
elif dim == "y-twin" and axy is not None:
if isinstance(limits[dim], tuple):
axy.set_ylim(limits[dim])
elif limits[dim] == "auto":
axy.set_ylim(auto=True)
elif "-" not in dim:
if isinstance(limits[dim], tuple):
getattr(ax, "set_{}lim".format(dim))(limits[dim])
elif limits[dim] == "auto":
getattr(ax, "set_{}lim".format(dim))(auto=True)
if len(dependents) > 1:
if legend is not None:
plt.legend(prop=self._get_font(dfont, fonts, "legend"),
**legend)
else:
plt.legend(loc='upper right',
prop=self._get_font(dfont, fonts, "legend"))
if savefile is None:
plt.show(block=False)
else:
plt.savefig(savefile)
def _get_data_series(self,
independent,
dependents,
threshold,
functions=None):
"""Used by plot() and table() to get the x and y data series that will be plotted
or tabulated against each other. See the arguments on those methods.
"""
if (("rowvals" in independent or "colvals" in independent)
or any([("rowvals" in d or "colvals" in d)
for d in dependents])):
#We need to make sure that the test case filter they specified returns only a
#single result so that we get a reasonable plot. This will happen if the data
#is 1D in the other dimension or if the filter only has a single case. We
#issue a warning if the filter doesn't match up, so they can check their data.
for dep in dependents:
tfilter, depvar = dep.split("/")
if tfilter is None or sum(
[(1 if self._case_filter(d, tfilter) else 0)
for d in self.details]) > 1:
msg.warn(
"Plotting aggregated data for more than one test case. Check results \n"
"for consistency and completeness.")
break
rawvars = {}
oseries = []
icases = {}
def _load_raw(variable, rawvars):
if variable in rawvars:
return
tfilter, depvar = variable.split("/")
fullindvar = "{}/{}".format(tfilter, independent)
cases = None if fullindvar not in icases else icases[fullindvar]
if fullindvar not in rawvars:
x, cases = self._get_data(independent, fullindvar, None,
threshold, tfilter)
if (len(x) == 1 and isinstance(x[0], list) and
("rowvals" in independent or "colvals" in independent)):
x = x[0]
rawvars[fullindvar] = x
if cases is not None:
x = rawvars[fullindvar]
ypts, names = self._get_data(depvar, variable, cases,
threshold, tfilter, independent,
x)
if (len(ypts) == 1 and isinstance(ypts[0], list)
and ("rowvals" in variable or "colvals" in variable)):
ypts = ypts[0]
rawvars[variable] = ypts
return (fullindvar, cases)
for variable in dependents:
fullindvar, cases = _load_raw(variable, rawvars)
icases[fullindvar] = cases
oseries.append((fullindvar, variable))
#Next, we need to apply any postfix functions and then check that the data sizes
#are commensurate (i.e. same number of points for dependent and independent series).
ys = []
xs = []
if functions is not None:
for postfix, fixdef in functions.items():
fxn = None
vorder = [v for v in fixdef.keys() if v != "lambda"]
values = {}
if postfix not in rawvars:
msg.err(
"Variable '{}' to be postfixed has no data series.".
format(postfix))
break
N = len(rawvars[postfix])
for lvar, gvar in fixdef.items():
if lvar == "lambda":
#This is the function definition for each item, create a lambda for it.
import numpy
import math
try:
evals = gvar.split(":")[1]
lambstr = "lambda {}: {}".format(
", ".join(vorder), evals)
fxn = eval(lambstr)
except:
msg.err("Could not evaluate function '{}'.".format(
gvar))
else:
if gvar not in rawvars:
_load_raw(gvar, rawvars)
if gvar in rawvars:
values[lvar] = rawvars[gvar]
else:
emsg = "Variable '{}' in the lambda function postfix for '{}' is missing data."
msg.err(emsg.format(gvar, postfix))
#Now if we have a valid function defined and data to operate on, just
#evaluate the postfix one value at a time.
if fxn is not None and N > 0:
for i in range(N):
args = [values[v][i] for v in vorder]
pval = fxn(*args)
rawvars[postfix][i] = pval
#Now we can use the adjusted/postfixed raw values to construct the data series
for indepvar, depvar in oseries:
xs.append((rawvars[indepvar], indepvar))
#We need the arrays to be the same length for plotting; if we are following
#an existing independent variable, we still need to append zero to the list
#for the values we don't have.
if len(rawvars[depvar]) != len(rawvars[indepvar]):
msg.err(
"Can't coerce an array to a single value without an aggregation "
"function such as numpy.mean or numpy.sum")
ys.append((rawvars[depvar], depvar))
return (xs, ys)
def compile(folder,
compiler=None,
identifier=None,
debug=False,
profile=False,
quiet=False,
moptions=None,
inclfortpy=True,
vupdates=None,
strict=False,
inclfpyaux=False):
"""Runs the makefile in the specified folder to compile the 'all' rule.
:arg vupdates: a list of module names for which the output .mod and .o files
should have version information attached.
"""
if inclfortpy:
#Before we can compile the library, we need to make sure that we have a fortpy
#.mod and .o compiled with the *same* compiler version specified.
from fortpy.utility import get_fortpy_templates_dir
_ensure_fileversion(compiler, "fortpy", get_fortpy_templates_dir(),
folder)
options = ""
if debug:
options += " DEBUG=true"
if profile:
options += " GPROF=true"
if strict:
options += " STRICT=true"
if moptions is not None:
for opt in moptions:
options += " {}".format(opt)
if identifier is not None:
codestr = "cd {}; make -f 'Makefile.{}' F90='{}' FAM='{}'" + options
command = codestr.format(folder, identifier, executor(compiler),
family(compiler))
else:
codestr = "cd {}; make F90='{}' FAM='{}'" + options
command = codestr.format(folder, executor(compiler), family(compiler))
#If we are running in quiet mode, we don't want the compile information
#to post to stdout; only errors should be redirected. This means we need
#to wrap the execution in a subprocess and redirect the std* to PIPE
from os import waitpid, path
from subprocess import Popen, PIPE
pcompile = Popen(command,
shell=True,
executable="/bin/bash",
stdout=PIPE,
stderr=PIPE,
close_fds=True)
waitpid(pcompile.pid, 0)
if not quiet:
output = [x.decode('utf8') for x in pcompile.stdout.readlines()]
msg.std(''.join(output))
#else: #We don't need to get these lines since we are purposefully redirecting them.
error = [x.decode('utf8') for x in pcompile.stderr.readlines()]
code = len(error)
if code != 0:
msg.err(''.join(error))
pcompile.stdout.close()
pcompile.stderr.close()
#It turns out that the compiler still returns a code of zero, even if the compile
#failed because the actual compiler didn't fail; it did its job properly. We need to
#check for the existence of errors in the 'compile.log' file.
lcount = 0
errors = []
log = path.join(
folder, "compile.{}.log".format(
identifier if identifier is not None else "default"))
with open(log) as f:
for line in f:
lcount += 1
if lcount > 21 and lcount < 32:
errors.append(line)
elif lcount > 21:
break
if len(errors) > 0:
#There are 21 lines in the compile.log file when everything runs correctly
#Overwrite code with a bad exit value since we have some other problems.
code = 1
#We want to write the first couple of errors to the console and give them the
#option to still execute if the compile only generated warnings.
msg.warn("compile generated some errors or warnings:")
msg.blank()
msg.info(''.join(errors))
if vupdates is not None:
for modname in vupdates:
_vupdate_compiled_module(compiler, modname, folder, rename=False)
return (code, len(errors) == 0)
def _get_data_series(self, independent, dependents, threshold, functions=None):
"""Used by plot() and table() to get the x and y data series that will be plotted
or tabulated against each other. See the arguments on those methods.
"""
if (("rowvals" in independent or "colvals" in independent) or
any([("rowvals" in d or "colvals" in d) for d in dependents])):
#We need to make sure that the test case filter they specified returns only a
#single result so that we get a reasonable plot. This will happen if the data
#is 1D in the other dimension or if the filter only has a single case. We
#issue a warning if the filter doesn't match up, so they can check their data.
for dep in dependents:
tfilter, depvar = dep.split("/")
if tfilter is None or sum([(1 if self._case_filter(d, tfilter) else 0)
for d in self.details]) > 1:
msg.warn("Plotting aggregated data for more than one test case. Check results \n"
"for consistency and completeness.")
break
rawvars = {}
oseries = []
icases = {}
def _load_raw(variable, rawvars):
if variable in rawvars:
return
tfilter, depvar = variable.split("/")
fullindvar = "{}/{}".format(tfilter, independent)
cases = None if fullindvar not in icases else icases[fullindvar]
if fullindvar not in rawvars:
x, cases = self._get_data(independent, fullindvar, None, threshold, tfilter)
if (len(x) == 1 and isinstance(x[0], list) and
("rowvals" in independent or "colvals" in independent)):
x = x[0]
rawvars[fullindvar] = x
if cases is not None:
x = rawvars[fullindvar]
ypts, names = self._get_data(depvar, variable, cases, threshold, tfilter, independent, x)
if (len(ypts) == 1 and isinstance(ypts[0], list) and
("rowvals" in variable or "colvals" in variable)):
ypts = ypts[0]
rawvars[variable] = ypts
return (fullindvar, cases)
for variable in dependents:
fullindvar, cases = _load_raw(variable, rawvars)
icases[fullindvar] = cases
oseries.append((fullindvar, variable))
#Next, we need to apply any postfix functions and then check that the data sizes
#are commensurate (i.e. same number of points for dependent and independent series).
ys = []
xs = []
if functions is not None:
for postfix, fixdef in functions.items():
fxn = None
vorder = [v for v in fixdef.keys() if v != "lambda"]
values = {}
if postfix not in rawvars:
msg.err("Variable '{}' to be postfixed has no data series.".format(postfix))
break
N = len(rawvars[postfix])
for lvar, gvar in fixdef.items():
if lvar == "lambda":
#This is the function definition for each item, create a lambda for it.
import numpy
import math
try:
evals = gvar.split(":")[1]
lambstr = "lambda {}: {}".format(", ".join(vorder), evals)
fxn = eval(lambstr)
except:
msg.err("Could not evaluate function '{}'.".format(gvar))
else:
if gvar not in rawvars:
_load_raw(gvar, rawvars)
if gvar in rawvars:
values[lvar] = rawvars[gvar]
else:
emsg = "Variable '{}' in the lambda function postfix for '{}' is missing data."
msg.err(emsg.format(gvar, postfix))
#Now if we have a valid function defined and data to operate on, just
#evaluate the postfix one value at a time.
if fxn is not None and N > 0:
for i in range(N):
args = [values[v][i] for v in vorder]
pval = fxn(*args)
rawvars[postfix][i] = pval
#Now we can use the adjusted/postfixed raw values to construct the data series
for indepvar, depvar in oseries:
xs.append((rawvars[indepvar], indepvar))
#We need the arrays to be the same length for plotting; if we are following
#an existing independent variable, we still need to append zero to the list
#for the values we don't have.
if len(rawvars[depvar]) != len(rawvars[indepvar]):
msg.err("Can't coerce an array to a single value without an aggregation "
"function such as numpy.mean or numpy.sum")
ys.append((rawvars[depvar], depvar))
return (xs, ys)
def compile(folder, compiler=None, identifier=None, debug=False, profile=False,
quiet=False, moptions=None, inclfortpy=True, vupdates=None,
strict=False, inclfpyaux=False):
"""Runs the makefile in the specified folder to compile the 'all' rule.
:arg vupdates: a list of module names for which the output .mod and .o files
should have version information attached.
"""
if inclfortpy:
#Before we can compile the library, we need to make sure that we have a fortpy
#.mod and .o compiled with the *same* compiler version specified.
from fortpy.utility import get_fortpy_templates_dir
_ensure_fileversion(compiler, "fortpy", get_fortpy_templates_dir(), folder)
options = ""
if debug:
options += " DEBUG=true"
if profile:
options += " GPROF=true"
if strict:
options += " STRICT=true"
if moptions is not None:
for opt in moptions:
options += " {}".format(opt)
if identifier is not None:
codestr = "cd {}; make -f 'Makefile.{}' F90='{}' FAM='{}'" + options
command = codestr.format(folder, identifier, executor(compiler), family(compiler))
else:
codestr = "cd {}; make F90='{}' FAM='{}'" + options
command = codestr.format(folder, executor(compiler), family(compiler))
#If we are running in quiet mode, we don't want the compile information
#to post to stdout; only errors should be redirected. This means we need
#to wrap the execution in a subprocess and redirect the std* to PIPE
from os import waitpid, path
from subprocess import Popen, PIPE
pcompile = Popen(command, shell=True, executable="/bin/bash", stdout=PIPE, stderr=PIPE)
waitpid(pcompile.pid, 0)
if not quiet:
output = [x.decode('utf8') for x in pcompile.stdout.readlines()]
msg.std(''.join(output))
#else: #We don't need to get these lines since we are purposefully redirecting them.
error = [x.decode('utf8') for x in pcompile.stderr.readlines()]
code = len(error)
if code != 0:
msg.err(''.join(error))
#It turns out that the compiler still returns a code of zero, even if the compile
#failed because the actual compiler didn't fail; it did its job properly. We need to
#check for the existence of errors in the 'compile.log' file.
lcount = 0
errors = []
log = path.join(folder, "compile.log")
with open(log) as f:
for line in f:
lcount += 1
if lcount > 21 and lcount < 32:
errors.append(line)
elif lcount > 21:
break
if len(errors) > 0:
#There are 21 lines in the compile.log file when everything runs correctly
#Overwrite code with a bad exit value since we have some other problems.
code = 1
#We want to write the first couple of errors to the console and give them the
#option to still execute if the compile only generated warnings.
msg.warn("compile generated some errors or warnings:")
msg.blank()
msg.info(''.join(errors))
if vupdates is not None:
for modname in vupdates:
_vupdate_compiled_module(compiler, modname, folder, rename=False)
return (code, len(errors)==0)
