def _vupdate_compiled_module(compiler, modname, folder, tversion=None, rename=True): """Compiles a .mod and .o for the specified compiler and specified. """ from os import path opath = path.join(folder, "{}.o".format(modname)) mpath = path.join(folder, "{}.mod".format(modname)) if path.isfile(opath) and path.isfile(mpath): if rename: from shutil import move nopath = path.join(folder, replace("{}.o.[c]".format(modname), compiler)) nmpath = path.join(folder, replace("{}.mod.[c]".format(modname), compiler)) move(opath, nopath) move(mpath, nmpath) else: nopath = opath nmpath = mpath #Create the version files so we can keep track of the compiled versions. vpaths = [nopath, nmpath] for np in vpaths: if tversion is None: tversion = get_fortpy_version(compiler, np) vp = np + ".v" with open(vp, 'w') as f: f.write('#<fortpy version="{}" />'.format('.'.join(map(str, tversion)))) else: msg.err("Unable to find {0}.o and {0}.mod.".format(modname))
def _get_rsubroutine_flat(classer, common): """Gets the *non* recursive code subroutine for saving a variable with members of the same type as itself. """ variable = classer.variable lines = [] for memname, member in variable.customtype.members.items(): if member.is_custom: if member.customtype is None: msg.err("Unable to find Type instance for : {}".format(member.definition())) continue if "allocatable" in member.modifiers: d0 = ', '.join(['0' for i in range(member.D)]) lines.append("allocate(variable%{}({}))".format(memname, d0)) xr = "call auxread_{varname}{D}d{suffix}(variable%{memname}, lfolder//'-{memname}', nested_=.true.)" lines.append(xr.format(**{"varname": member.customtype.name, "D": member.D, "memname": member.name, "suffix": _get_suffix(member)})) else: lines.append("call fpy_read{1}(lfolder//'-{0}', '#', variable%{0})".format(member.name, _get_suffix(member))) common["read"] = '\n '.join(lines) template = """ subroutine {xname}(variable, folder, multi_stack, rstack, nested_) character(len=*), intent(in) :: folder {dtype}{kind}, intent(inout) :: variable{Dx} {dtype}{kind}, pointer, optional, intent(in) :: multi_stack(:) type(fpy_address), allocatable, intent(in), optional :: rstack(:) logical, optional, intent(in) :: nested_ character(len=:), allocatable :: lfolder logical :: nested if (fpy_verbose > 0) write (*,*) "Start flat read for {xname} in '"//folder//"'." if (present(nested_)) then nested = nested_ else nested = .false. end if if ((present(multi_stack) .and. present(rstack)) .or. nested) then lfolder = folder else lfolder = folder//'_' end if {read} if (fpy_verbose > 0) write (*,*) "End flat read for {xname} in '"//folder//"'." end subroutine {xname} """ return ((common["xname"],), template.format(**common))
def _separate_multiple_def(self, defstring, parent, refstring, refline): """Separates the text after '::' in a variable definition to extract all the variables, their dimensions and default values. """ import pyparsing nester = pyparsing.nestedExpr('(', ')') try: parsed = nester.parseString("(" + re.sub("=(>?)", " =\\1 ", defstring) + ")").asList()[0] except pyparsing.ParseException as err: from fortpy import msg repl = (parent.name, refline[0], refstring, defstring, ''.join(['-'] * (err.loc - 1)) + '^', err.msg) msg.err( "parsing variable from '{}:{} >> {}': \n'{}'\n{} {}.".format( *repl)) raise i = 0 clean = [] while i < len(parsed): if (isinstance(parsed[i], str) and not re.match("=>?", parsed[i]) and i + 1 < len(parsed) and isinstance(parsed[i + 1], list)): clean.append((parsed[i], parsed[i + 1])) i += 2 elif isinstance(parsed[i], str) and parsed[i] == ",": i += 1 else: clean.append(parsed[i]) i += 1 #Now pass through again to handle the default values. i = 0 ready = [] while i < len(clean): if isinstance(clean[i], str) and re.match("=>?", clean[i]): ready.pop() if ">" in clean[i]: ready.append([ clean[i - 1], ("> " + clean[i + 1][0], clean[i + 1][1]) ]) else: ready.append([clean[i - 1], clean[i + 1]]) i += 2 else: ready.append(clean[i]) i += 1 return ready
def make(self, remake=False, dependencies=None, compiler="gfortran", debug=False, profile=False): """Generates a makefile to compile the wrapper module and link it with the shared library that can be compiled with self.makelib(). """ self._check_lib(remake, compiler, debug, profile) from os import path if self.link is None or not path.isfile(self.link): msg.err( "Can't create shared library; missing compiled original code: {}" .format(self.link)) exit(0) if remake or not path.isfile(self.libpath): if dependencies is None: dependencies = ["{}_c".format(self.module.name)] identifier = self.name else: identifier = "ftypes.{}".format(self.library) #We don't need to reorder these *wrapper* modules before compilation because each of #them should be independent of all the others. #dependencies = self._process_module_needs(dependencies) #append the dependency for the deallocation module to be included in the shared lib. dependencies.append("ftypes_dealloc") makename = "Makefile.{}".format( self.library if self.library else self.name) makepath = path.join(self.f90path, makename) extralinks = list(self.dependencies.values()) extralinks.append(self.link) makefile(identifier, dependencies, makepath, "ftypes.{}".format(self.name), False, False, self.module.parent, "so", extralinks) code = self._compile(self.f90path, makename, compiler, debug, profile) if code == 0: #Copy the shared library to the main directory out of the f90 directory from shutil import copy copy(path.join(self.f90path, "{}.so".format(identifier)), self.dirpath) return code else: return 0
def _separate_multiple_def(self, defstring, parent, refstring, refline): """Separates the text after '::' in a variable definition to extract all the variables, their dimensions and default values. """ import pyparsing nester = pyparsing.nestedExpr("(", ")") try: parsed = nester.parseString("(" + re.sub("=(>?)", " =\\1 ", defstring) + ")").asList()[0] except pyparsing.ParseException as err: from fortpy import msg repl = (parent.name, refline[0], refstring, defstring, "".join(["-"] * (err.loc - 1)) + "^", err.msg) msg.err("parsing variable from '{}:{} >> {}': \n'{}'\n{} {}.".format(*repl)) raise i = 0 clean = [] while i < len(parsed): if ( isinstance(parsed[i], str) and not re.match("=>?", parsed[i]) and i + 1 < len(parsed) and isinstance(parsed[i + 1], list) ): clean.append((parsed[i], parsed[i + 1])) i += 2 elif isinstance(parsed[i], str) and parsed[i] == ",": i += 1 else: clean.append(parsed[i]) i += 1 # Now pass through again to handle the default values. i = 0 ready = [] while i < len(clean): if isinstance(clean[i], str) and re.match("=>?", clean[i]): ready.pop() if ">" in clean[i]: ready.append([clean[i - 1], ("> " + clean[i + 1][0], clean[i + 1][1])]) else: ready.append([clean[i - 1], clean[i + 1]]) i += 2 else: ready.append(clean[i]) i += 1 return ready
def get_fortpy_version(compiler, fortpath, recursed=False, attribute="version"): """Gets the fortpy version number from the first line of the specified file.""" result = [] #If the file doesn't exist yet, we don't try to find the version information. from os import path ext = path.splitext(fortpath)[1] cext = "" if compiler is None else replace(".[c]", compiler) if not path.isfile(fortpath) or ext in [".o", ".mod", cext]: if path.isfile(fortpath + '.v'): return get_fortpy_version(compiler, fortpath + '.v', True) else: return result with open(fortpath) as f: for line in f: try: lt = line.index("<") vxml = "<doc>{}</doc>".format(line[lt::]) except ValueError: vxml = "" break if "<fortpy" in vxml: import xml.etree.ElementTree as ET x = list(ET.XML(vxml)) if len(x) > 0: try: result = list(map(int, x[0].attrib[attribute].split("."))) except ValueError: fstr = "can't extract version information from '{}' in '{}'." msg.err(fstr.format(x[0].attrib[attribute], fortpath)) except KeyError: fstr = "fortpy tag does not contain attribute '{}': {} ({})" msg.err(fstr.format(attribute, x[0].attrib, fortpath)) if len(result) == 0 and not recursed: return get_fortpy_version(compiler, fortpath + '.v', True) else: return result
def fit(self, independent, dependent, model, threshold=1., functions=None): """Tries to fit the data selection for the specified dependent variable against the indepedent one using the specified model. Arguments are similar to self.table(). Returns a tuple of (fitting parameters, 1 standard deviation errors). :arg model: a description of the function to fit to. Possible values are ['exp', 'lin']. """ key = "{}${}".format(independent, dependent) xs, ys = self._get_data_series(independent, [dependent], threshold, functions) if xs[0][0] is None: return if len(xs[0][0]) != len(ys[0][0]): msg.err( "Can't fit data unless we have the same number of values for the independent " "and dependent variables. " "len({}) == {}; len({}) == {}".format(independent, len(xs[0][0]), dependent, len(ys[0][0]))) return from scipy.optimize import curve_fit from numpy import sqrt, diag mdict = {"exp": _fit_exp, "lin": _fit_lin} if not model in mdict: msg.err( "Cannot fit using model '{}'; unknown function.".format(model)) return func = mdict[model] popt, pcov = curve_fit(func, xs[0][0], ys[0][0]) perr = sqrt(diag(pcov)) self.fits[key] = { "params": popt, "covarmat": pcov, "function": (lambda x: func(x, *popt)), "model": model, "error": perr }
def fit(self, independent, dependent, model, threshold=1., functions=None): """Tries to fit the data selection for the specified dependent variable against the indepedent one using the specified model. Arguments are similar to self.table(). Returns a tuple of (fitting parameters, 1 standard deviation errors). :arg model: a description of the function to fit to. Possible values are ['exp', 'lin']. """ key = "{}${}".format(independent, dependent) xs, ys = self._get_data_series(independent, [dependent], threshold, functions) if xs[0][0] is None: return if len(xs[0][0]) != len(ys[0][0]): msg.err("Can't fit data unless we have the same number of values for the independent " "and dependent variables. " "len({}) == {}; len({}) == {}".format(independent, len(xs[0][0]), dependent, len(ys[0][0]))) return from scipy.optimize import curve_fit from numpy import sqrt, diag mdict = { "exp": _fit_exp, "lin": _fit_lin } if not model in mdict: msg.err("Cannot fit using model '{}'; unknown function.".format(model)) return func = mdict[model] popt, pcov = curve_fit(func, xs[0][0], ys[0][0]) perr = sqrt(diag(pcov)) self.fits[key] = { "params": popt, "covarmat": pcov, "function": (lambda x: func(x, *popt)), "model": model, "error": perr }
def make(self, remake=False, dependencies=None, compiler="gfortran", debug=False, profile=False): """Generates a makefile to compile the wrapper module and link it with the shared library that can be compiled with self.makelib(). """ self._check_lib(remake, compiler, debug, profile) from os import path if self.link is None or not path.isfile(self.link): msg.err("Can't create shared library; missing compiled original code: {}".format(self.link)) exit(0) if remake or not path.isfile(self.libpath): if dependencies is None: dependencies = ["{}_c".format(self.module.name)] identifier = self.name else: identifier = "ftypes.{}".format(self.library) #We don't need to reorder these *wrapper* modules before compilation because each of #them should be independent of all the others. #dependencies = self._process_module_needs(dependencies) #append the dependency for the deallocation module to be included in the shared lib. dependencies.append("ftypes_dealloc") makename = "Makefile.{}".format(self.library if self.library else self.name) makepath = path.join(self.f90path, makename) extralinks = list(self.dependencies.values()) extralinks.append(self.link) makefile(identifier, dependencies, makepath, "ftypes.{}".format(self.name), False, False, self.module.parent, "so", extralinks) code = self._compile(self.f90path, makename, compiler, debug, profile) if code == 0: #Copy the shared library to the main directory out of the f90 directory from shutil import copy copy(path.join(self.f90path, "{}.so".format(identifier)), self.dirpath) return code else: return 0
def _vupdate_compiled_module(compiler, modname, folder, tversion=None, rename=True): """Compiles a .mod and .o for the specified compiler and specified. """ from os import path opath = path.join(folder, "{}.o".format(modname)) mpath = path.join(folder, "{}.mod".format(modname)) if path.isfile(opath) and path.isfile(mpath): if rename: from shutil import move nopath = path.join(folder, replace("{}.o.[c]".format(modname), compiler)) nmpath = path.join(folder, replace("{}.mod.[c]".format(modname), compiler)) move(opath, nopath) move(mpath, nmpath) else: nopath = opath nmpath = mpath #Create the version files so we can keep track of the compiled versions. vpaths = [nopath, nmpath] for np in vpaths: if tversion is None: tversion = get_fortpy_version(compiler, np) if len(tversion) == 0: #Try to get the template version for the module. tversion = template_version(compiler, modname + ".f90") vp = np + ".v" with open(vp, 'w') as f: f.write('#<fortpy version="{}" />'.format('.'.join( map(str, tversion)))) else: msg.err("Unable to find {0}.o and {0}.mod.".format(modname))
def _start_debug(wizard, parameter, target): """Starts a debug session for the unit test executable of the current test specification and case identifier. """ from fortpy.utility import which if which("gdb"): #Give the user the option of setting some breakpoints. Basically, list the #1) main program entry, 2) list of pre-req calls before the executable, #3) executable itself. brks = ["Main Unit Test Entry Point.", wizard.xauto.signature] xnames = ["main", wizard.xauto.name] from fortpy.testing.elements import Executable for method in wizard.tauto.methods: if not isinstance(method, Executable): #This is an executable that needs to be called before the main one. brks.append(method.signature) xnames.append(method.name) bchoice = _prompt_general("Which methods would you like breakpoints for?", brks) #Start a gdb session for the compiled unit test executable of the active #test and test case. xs = [] for x in bchoice: if x < len(xnames): xs.append("-ex '{}'".format(xnames[x])) from os import system, path from fortpy.testing.compilers import compile_general xstage = path.join(wizard.stagedir, wizard.xauto.full_name) code, success, target = compile_general(xstage, wizard.compiler, wizard.tauto.identifier, True, quiet=True) testpath = _get_casepath(wizard) cmd = "cd {}; gdb {} -ex 'run' ../../{}.x" system(cmd.format(testpath, ' '.join(xs), wizard.tauto.identifier)) else: msg.err("The GNU debugger 'gdb' is not available on the system.")
def plot(self, independent=None, dependents=None, threshold=1., savefile=None, functions=None, xscale=None, yscale=None, colors=None, labels=None, fonts=None, markers=None, lines=None, ticks=None, plottypes=None, limits=None, twinplots=None, legend=None, **kwargs): """Plots the specified dependent variables as functions of the independent one. :arg independent: a string indentifying the independent variable's filename and the appropriate property of the data in the file. For example: "concs.in|depth" would use the depth of the data in the file 'concs.in' as its value for each test case. :arg dependent: a list of strings that use the same format as the string for the 'independent' variable. :arg threshold: a float value specifying the minimum level of success that the output file must attain before it can be used in the plot. :arg xlabel: the label for the x-axis of the plot. :arg ylabel: the label for the y-axis of the plot. :arg functions: a dictionary of functions to apply to the values of each variable. E.g. {"group.in|depth": {lambda dict}}. See the help text help_postfix() for details. """ if independent is None or dependents is None: raise ValueError("Must specify at least the variables to plot.") import matplotlib.pyplot as plt from matplotlib import cm, rc from numpy import linspace, array from itertools import cycle from matplotlib.font_manager import FontProperties xs, ys = self._get_data_series(independent, dependents, threshold, functions) #Set the default font for the plots, or the custom one. if fonts is not None and "family" in fonts: rc('font',family=fonts["family"]) else: rc('font',family='Times New Roman') layout = None if "figsize" in kwargs: if len(kwargs["figsize"]) > 2: layout = {"pad": kwargs["figsize"][2]} fig = plt.figure(figsize=(None if "figsize" not in kwargs else kwargs["figsize"][0:2]), tight_layout=layout) dfont = FontProperties() ax = fig.add_subplot(111) rbcolors = cm.rainbow(linspace(0, 1, len(ys))) cycols = cycle(rbcolors) axx = None axy = None if twinplots is not None: if "twinx" in twinplots.values(): axx = ax.twinx() if "twiny" in twinplots.values(): axy = ax.twiny() if labels is not None: if "x" in labels: ax.set_xlabel(labels["x"], fontproperties=self._get_font(dfont, fonts, "xlabel")) if "y" in labels: ax.set_ylabel(labels["y"], fontproperties=self._get_font(dfont, fonts, "ylabel")) if "plot" in labels: plt.title(labels["plot"], fontproperties=self._get_font(dfont, fonts, "title")) if "x-twin" in labels and axx is not None: axx.set_ylabel(labels["x-twin"], fontproperties=self._get_font(dfont, fonts, "x-twin-label")) if "y-twin" in labels and axy is not None: axy.set_xlabel(labels["y-twin"], fontproperties=self._get_font(dfont, fonts, "y-twin-label")) if xscale is not None: ax.set_xscale(xscale) if yscale is not None: ax.set_yscale(yscale) for xset, yset in zip(xs, ys): x, xlabel = xset y, ylabel = yset #Set up a dictionary with all the kwargs for the plotting options #that may be common to both kinds of plots. dargs = {} if colors is not None and ylabel in colors: dargs["color"] = colors[ylabel] else: dargs["color"] = next(cycols) if markers is not None and ylabel in markers: self._get_markers(dargs, markers[ylabel]) if "size" in markers[ylabel]: dargs["s"] = float(markers[ylabel]["size"]) if "s" not in dargs: dargs["s"] = 15 if lines is not None and ylabel in lines: if "style" in lines[ylabel]: dargs["linestyle"] = lines[ylabel]["style"] if "width" in lines[ylabel]: dargs["linewidth"] = float(lines[ylabel]["width"]) if labels is not None and ylabel in labels: dargs["label"] = None if labels[ylabel].lower() == "[none]" else labels[ylabel] else: dargs["label"] = ylabel if twinplots is not None and ylabel in twinplots: if twinplots[ylabel] == "twinx": liveax = axx elif twinplots[ylabel] == "twiny": liveax = axy else: liveax = ax try: if plottypes is not None and ylabel in plottypes: lineplot = plottypes[ylabel] == "line" else: lineplot = None if ylabel[len(ylabel)-4:len(ylabel)] in ["|fit", ".fit"]: varfile = ylabel[0:len(ylabel)-4] key = "{}${}".format(independent, varfile) allx = linspace(min(x), max(x), 50) if labels is not None and ylabel in labels: dargs["label"] = labels[ylabel].format(self._format_fit(key)) if dargs["label"].lower() == "[none]": dargs["label"] = None else: dargs["label"] = self._format_fit(key) self._reset_lineplot(dargs) liveax.plot(allx, self.fits[key]["function"](allx), **dargs) elif lineplot: self._reset_lineplot(dargs) from operator import itemgetter sdata = array(sorted(zip(x,y),key=itemgetter(0))) liveax.plot(sdata[:,0], sdata[:,1], **dargs) else: size = dargs["s"] del dargs["s"] liveax.scatter(x, y, s=size, **dargs) except ValueError: msg.err("The values for {} can't be log-plotted.".format(ylabel)) #Set the tick labels font sizes. def tickfonts(fonts, key, labels): if fonts is not None and key in fonts: for label in labels: tfont = self._get_font(dfont, fonts, key) label.set_fontproperties(tfont) tickfonts(fonts, "xticks", ax.get_xticklabels()) tickfonts(fonts, "yticks", ax.get_yticklabels()) if axx is not None: tickfonts(fonts, "x-twin-ticks", axx.get_yticklabels()) if axy is not None: tickfonts(fonts, "y-twin-ticks", axy.get_xticklabels()) if ticks is not None and len(ticks) > 0: for key in ticks: if "reset" in ticks[key]: ticks[key]["reset"] = ticks[key]["reset"] == "true" if "x-twin" in key and axx is not None: axx.tick_params(**ticks[key]) elif "y-twin" in key and axy is not None: axy.tick_params(**ticks[key]) else: ax.tick_params(**ticks[key]) for dim in ["x", "y", "z", "x-twin", "y-twin"]: if limits is not None and dim in limits: if dim == "x-twin" and axx is not None: if isinstance(limits[dim], tuple): axx.set_ylim(limits[dim]) elif limits[dim] == "auto": axx.set_ylim(auto=True) elif dim == "y-twin" and axy is not None: if isinstance(limits[dim], tuple): axy.set_ylim(limits[dim]) elif limits[dim] == "auto": axy.set_ylim(auto=True) elif "-" not in dim: if isinstance(limits[dim], tuple): getattr(ax, "set_{}lim".format(dim))(limits[dim]) elif limits[dim] == "auto": getattr(ax, "set_{}lim".format(dim))(auto=True) if len(dependents) > 1: if legend is not None: plt.legend(prop=self._get_font(dfont, fonts, "legend"), **legend) else: plt.legend(loc='upper right', prop=self._get_font(dfont, fonts, "legend")) if savefile is None: plt.show(block=False) else: plt.savefig(savefile)
def plot(self, independent, dependents, threshold=1., xlabel=None, ylabel=None, savefile=None, functions=None, xscale=None, yscale=None, colors=None, labels=None, fonts=None, markers=None, lines=None, ticks=None, plottypes=None, limits=None): """Plots the specified dependent variables as functions of the independent one. :arg independent: a string indentifying the independent variable's filename and the appropriate property of the data in the file. For example: "concs.in|depth" would use the depth of the data in the file 'concs.in' as its value for each test case. :arg dependent: a list of strings that use the same format as the string for the 'independent' variable. :arg threshold: a float value specifying the minimum level of success that the output file must attain before it can be used in the plot. :arg xlabel: the label for the x-axis of the plot. :arg ylabel: the label for the y-axis of the plot. :arg functions: a dictionary of functions to apply to the values of each variable. E.g. {"group.in|depth": "numpy.log"} would apply the numpy.log function to each value extracted from the depth property of the data in 'group.in' before plotting it. If the value is not a string, it must be callable with some argument. """ import matplotlib.pyplot as plt from matplotlib import cm, rc from numpy import linspace, array from itertools import cycle from matplotlib.font_manager import FontProperties xs, ys = self._get_data_series(independent, dependents, threshold, functions) #Set the default font for the plots, or the custom one. if fonts is not None and "family" in fonts: rc('font',family=fonts["family"]) else: rc('font',family='Times New Roman') fig = plt.figure() dfont = FontProperties() ax = fig.add_subplot(111) if xlabel is not None: ax.set_xlabel(xlabel, fontproperties=self._get_font(dfont, fonts, "xlabel")) if ylabel is not None: ax.set_ylabel(ylabel, fontproperties=self._get_font(dfont, fonts, "ylabel")) if xscale is not None: ax.set_xscale(xscale) if yscale is not None: ax.set_yscale(yscale) if labels is not None and "plot" in labels: plt.title(labels["plot"], fontproperties=self._get_font(dfont, fonts, "title")) rbcolors = cm.rainbow(linspace(0, 1, len(ys))) cycols = cycle(rbcolors) for xset, yset in zip(xs, ys): x, xlabel = xset y, ylabel = yset #Set up a dictionary with all the kwargs for the plotting options #that may be common to both kinds of plots. dargs = {} if colors is not None and ylabel in colors: dargs["color"] = colors[ylabel] else: dargs["color"] = next(cycols) if markers is not None and ylabel in markers: self._get_markers(dargs, markers[ylabel]) if "size" in markers[ylabel]: dargs["s"] = float(markers[ylabel]["size"]) if "s" not in dargs: dargs["s"] = 15 if lines is not None and ylabel in lines: if "style" in lines[ylabel]: dargs["linestyle"] = lines[ylabel]["style"] if "width" in lines[ylabel]: dargs["linewidth"] = float(lines[ylabel]["width"]) if labels is not None and ylabel in labels: dargs["label"] = None if labels[ylabel].lower() == "[none]" else labels[ylabel] else: dargs["label"] = ylabel try: if plottypes is not None and ylabel in plottypes: lineplot = plottypes[ylabel] == "line" else: lineplot = None if ylabel[len(ylabel)-4:len(ylabel)] in ["|fit", ".fit"]: varfile = ylabel[0:len(ylabel)-4] key = "{}${}".format(independent, varfile) allx = linspace(min(x), max(x), 50) if labels is not None and ylabel in labels: dargs["label"] = labels[ylabel].format(self._format_fit(key)) if dargs["label"].lower() == "[none]": dargs["label"] = None else: dargs["label"] = self._format_fit(key) self._reset_lineplot(dargs) ax.plot(allx, self.fits[key]["function"](allx), **dargs) elif lineplot: self._reset_lineplot(dargs) from operator import itemgetter sdata = array(sorted(zip(x,y),key=itemgetter(0))) ax.plot(sdata[:,0], sdata[:,1], **dargs) else: size = dargs["s"] del dargs["s"] ax.scatter(x, y, s=size, **dargs) except ValueError: msg.err("The values for {} can't be log-plotted.".format(ylabel)) #Set the tick labels font sizes. if fonts is not None and "xticks" in fonts: tfont = self._get_font(dfont, fonts, "xticks") for label in ax.get_xticklabels(): label.set_fontproperties(tfont) if fonts is not None and "yticks" in fonts: for label in ax.get_yticklabels(): tfont = self._get_font(dfont, fonts, "yticks") label.set_fontproperties(tfont) if ticks is not None and len(ticks) > 0: for key in ticks: if "reset" in ticks[key]: ticks[key]["reset"] = ticks[key]["reset"] == "true" ax.tick_params(**ticks[key]) for dim in ["x", "y", "z"]: if limits is not None and dim in limits: getattr(plt, "{}lim".format(dim))(limits[dim]) if len(dependents) > 1: plt.legend(loc='upper left', prop=self._get_font(dfont, fonts, "legend")) if savefile is None: plt.show(block=False) else: plt.savefig(savefile)
def _get_data(self, variable, fullvar, order=None, threshold=1., tfilter=None, functions=None, independent=None, x=None): """Returns a list of the valid data points for the specified variable. :arg variable: a string indentifying the variable's filename and the appropriate property of the data in the file. For example: "concs.in|depth" would use the depth of the data in the file 'concs.in' as its value for each test case. :arg fullvar: the full variable name including the filter and property spec. :arg threshold: a float value specifying the minimum level of success that the output file must attain before it can be used in the plot. :arg independent: the name of the independent variable including property. :arg x: a list of values for the *independent* variable to use when evaluating a *fitted* curve to a variable. """ from numpy import ndarray target = None if variable != "timing" or "timing|" in variable: varfile, attribute = variable.split("|") else: target = varfile = "timing" attribute = None if target is None: if varfile in self.infiles: target = "inputs" elif varfile in self.outfiles: target = "outputs" elif varfile == "timing" and attribute is not None: target = "timing" else: raise ValueError("Cannot locate the values for variable {}".format(variable)) values = [] cases = [] #If we have already ruled out some of the test cases while doing the independent #variable, we want to only use those cases. if order is not None: details = order else: details = self.details #Handle cases where the values need to have functions applied to them before tabulating #or printing the values. if functions is not None and fullvar in functions: if isinstance(functions[fullvar], str) or isinstance(functions[fullvar], str): from importlib import import_module module = functions[fullvar].split(".") fname = module.pop() numpy = import_module('.'.join(module)) fx = getattr(numpy, fname) else: fx = functions[fullvar] else: fx = None for testcase in details: if not self._case_filter(testcase, tfilter): #We don't consider test cases that are blocked by the filter. continue value = 0 if target == "timing" and self._is_successful(self.details[testcase]) and attribute is None: value = self.details[testcase][target] else: if (self._is_successful(self.details[testcase], varfile, threshold)): if target != "timing" and varfile in self.details[testcase][target]: data = self.details[testcase][target][varfile] else: data = None if data is not None and attribute in data: value = data[attribute] elif attribute == "fit" and independent is not None and x is not None: #Return the value of the *fitted* function for the corresponding independent #variable value; this is one-to-one for tabulating. For plotting we don't use #these values. key = "{}${}".format(independent, varfile) if len(values) < len(x) and key in self.fits: ival = x[len(values)] value = self.fits[key]["function"](ival) #We need the arrays to be the same length for plotting; if we are following #an existing independent variable, we still need to append zero to the list #for the values we don't have. if isinstance(value, ndarray) or value != 0 or order is not None: if fx is not None: value = fx(value) #Handle the case where we are plotting an entire row as a single point #and an aggregrate function should have been specified if isinstance(value, list): if len(value) == 1: values.append(value[0] if value[0] is not None else 0) else: msg.err("Can't coerce an array to a single value without an aggregation " "function such as numpy.mean or numpy.sum") return ([0], "Array Aggregation Error") else: values.append(value if value is not None else 0) cases.append(testcase) return (values, cases)
def _get_data(self, variable, fullvar, order=None, threshold=1., tfilter=None, functions=None, independent=None, x=None): """Returns a list of the valid data points for the specified variable. :arg variable: a string indentifying the variable's filename and the appropriate property of the data in the file. For example: "concs.in|depth" would use the depth of the data in the file 'concs.in' as its value for each test case. :arg fullvar: the full variable name including the filter and property spec. :arg threshold: a float value specifying the minimum level of success that the output file must attain before it can be used in the plot. :arg independent: the name of the independent variable including property. :arg x: a list of values for the *independent* variable to use when evaluating a *fitted* curve to a variable. """ from numpy import ndarray target = None if variable != "timing" or "timing|" in variable: varfile, attribute = variable.split("|") else: target = varfile = "timing" attribute = None if target is None: if varfile in self.infiles: target = "inputs" elif varfile in self.outfiles: target = "outputs" elif varfile == "timing" and attribute is not None: target = "timing" else: raise ValueError( "Cannot locate the values for variable {}".format( variable)) values = [] cases = [] #If we have already ruled out some of the test cases while doing the independent #variable, we want to only use those cases. if order is not None: details = order else: details = self.details #Handle cases where the values need to have functions applied to them before tabulating #or printing the values. if functions is not None and fullvar in functions: if isinstance(functions[fullvar], str) or isinstance( functions[fullvar], str): from importlib import import_module module = functions[fullvar].split(".") fname = module.pop() numpy = import_module('.'.join(module)) fx = getattr(numpy, fname) else: fx = functions[fullvar] else: fx = None for testcase in details: if not self._case_filter(testcase, tfilter): #We don't consider test cases that are blocked by the filter. continue value = 0 if target == "timing" and self._is_successful( self.details[testcase]) and attribute is None: value = self.details[testcase][target] else: if (self._is_successful(self.details[testcase], varfile, threshold)): if target != "timing" and varfile in self.details[ testcase][target]: data = self.details[testcase][target][varfile] else: data = None if data is not None and attribute in data: value = data[attribute] elif attribute == "fit" and independent is not None and x is not None: #Return the value of the *fitted* function for the corresponding independent #variable value; this is one-to-one for tabulating. For plotting we don't use #these values. key = "{}${}".format(independent, varfile) if len(values) < len(x) and key in self.fits: ival = x[len(values)] value = self.fits[key]["function"](ival) #We need the arrays to be the same length for plotting; if we are following #an existing independent variable, we still need to append zero to the list #for the values we don't have. if isinstance(value, ndarray) or value != 0 or order is not None: if fx is not None: value = fx(value) #Handle the case where we are plotting an entire row as a single point #and an aggregrate function should have been specified if isinstance(value, list): if len(value) == 1: values.append(value[0] if value[0] is not None else 0) else: msg.err( "Can't coerce an array to a single value without an aggregation " "function such as numpy.mean or numpy.sum") return ([0], "Array Aggregation Error") else: values.append(value if value is not None else 0) cases.append(testcase) return (values, cases)
def plot(self, independent, dependents, threshold=1., xlabel=None, ylabel=None, savefile=None, functions=None, xscale=None, yscale=None, colors=None, labels=None, fonts=None, markers=None, lines=None, ticks=None, plottypes=None, limits=None): """Plots the specified dependent variables as functions of the independent one. :arg independent: a string indentifying the independent variable's filename and the appropriate property of the data in the file. For example: "concs.in|depth" would use the depth of the data in the file 'concs.in' as its value for each test case. :arg dependent: a list of strings that use the same format as the string for the 'independent' variable. :arg threshold: a float value specifying the minimum level of success that the output file must attain before it can be used in the plot. :arg xlabel: the label for the x-axis of the plot. :arg ylabel: the label for the y-axis of the plot. :arg functions: a dictionary of functions to apply to the values of each variable. E.g. {"group.in|depth": "numpy.log"} would apply the numpy.log function to each value extracted from the depth property of the data in 'group.in' before plotting it. If the value is not a string, it must be callable with some argument. """ import matplotlib.pyplot as plt from matplotlib import cm, rc from numpy import linspace, array from itertools import cycle from matplotlib.font_manager import FontProperties xs, ys = self._get_data_series(independent, dependents, threshold, functions) #Set the default font for the plots, or the custom one. if fonts is not None and "family" in fonts: rc('font', family=fonts["family"]) else: rc('font', family='Times New Roman') fig = plt.figure() dfont = FontProperties() ax = fig.add_subplot(111) if xlabel is not None: ax.set_xlabel(xlabel, fontproperties=self._get_font( dfont, fonts, "xlabel")) if ylabel is not None: ax.set_ylabel(ylabel, fontproperties=self._get_font( dfont, fonts, "ylabel")) if xscale is not None: ax.set_xscale(xscale) if yscale is not None: ax.set_yscale(yscale) if labels is not None and "plot" in labels: plt.title(labels["plot"], fontproperties=self._get_font(dfont, fonts, "title")) rbcolors = cm.rainbow(linspace(0, 1, len(ys))) cycols = cycle(rbcolors) for xset, yset in zip(xs, ys): x, xlabel = xset y, ylabel = yset #Set up a dictionary with all the kwargs for the plotting options #that may be common to both kinds of plots. dargs = {} if colors is not None and ylabel in colors: dargs["color"] = colors[ylabel] else: dargs["color"] = next(cycols) if markers is not None and ylabel in markers: self._get_markers(dargs, markers[ylabel]) if "size" in markers[ylabel]: dargs["s"] = float(markers[ylabel]["size"]) if "s" not in dargs: dargs["s"] = 15 if lines is not None and ylabel in lines: if "style" in lines[ylabel]: dargs["linestyle"] = lines[ylabel]["style"] if "width" in lines[ylabel]: dargs["linewidth"] = float(lines[ylabel]["width"]) if labels is not None and ylabel in labels: dargs["label"] = None if labels[ylabel].lower( ) == "[none]" else labels[ylabel] else: dargs["label"] = ylabel try: if plottypes is not None and ylabel in plottypes: lineplot = plottypes[ylabel] == "line" else: lineplot = None if ylabel[len(ylabel) - 4:len(ylabel)] in ["|fit", ".fit"]: varfile = ylabel[0:len(ylabel) - 4] key = "{}${}".format(independent, varfile) allx = linspace(min(x), max(x), 50) if labels is not None and ylabel in labels: dargs["label"] = labels[ylabel].format( self._format_fit(key)) if dargs["label"].lower() == "[none]": dargs["label"] = None else: dargs["label"] = self._format_fit(key) self._reset_lineplot(dargs) ax.plot(allx, self.fits[key]["function"](allx), **dargs) elif lineplot: self._reset_lineplot(dargs) from operator import itemgetter sdata = array(sorted(zip(x, y), key=itemgetter(0))) ax.plot(sdata[:, 0], sdata[:, 1], **dargs) else: size = dargs["s"] del dargs["s"] ax.scatter(x, y, s=size, **dargs) except ValueError: msg.err( "The values for {} can't be log-plotted.".format(ylabel)) #Set the tick labels font sizes. if fonts is not None and "xticks" in fonts: tfont = self._get_font(dfont, fonts, "xticks") for label in ax.get_xticklabels(): label.set_fontproperties(tfont) if fonts is not None and "yticks" in fonts: for label in ax.get_yticklabels(): tfont = self._get_font(dfont, fonts, "yticks") label.set_fontproperties(tfont) if ticks is not None and len(ticks) > 0: for key in ticks: if "reset" in ticks[key]: ticks[key]["reset"] = ticks[key]["reset"] == "true" ax.tick_params(**ticks[key]) for dim in ["x", "y", "z"]: if limits is not None and dim in limits: getattr(plt, "{}lim".format(dim))(limits[dim]) if len(dependents) > 1: plt.legend(loc='upper left', prop=self._get_font(dfont, fonts, "legend")) if savefile is None: plt.show(block=False) else: plt.savefig(savefile)
def plot(self, independent=None, dependents=None, threshold=1., savefile=None, functions=None, xscale=None, yscale=None, colors=None, labels=None, fonts=None, markers=None, lines=None, ticks=None, plottypes=None, limits=None, twinplots=None, legend=None, **kwargs): """Plots the specified dependent variables as functions of the independent one. :arg independent: a string indentifying the independent variable's filename and the appropriate property of the data in the file. For example: "concs.in|depth" would use the depth of the data in the file 'concs.in' as its value for each test case. :arg dependent: a list of strings that use the same format as the string for the 'independent' variable. :arg threshold: a float value specifying the minimum level of success that the output file must attain before it can be used in the plot. :arg xlabel: the label for the x-axis of the plot. :arg ylabel: the label for the y-axis of the plot. :arg functions: a dictionary of functions to apply to the values of each variable. E.g. {"group.in|depth": {lambda dict}}. See the help text help_postfix() for details. """ if independent is None or dependents is None: raise ValueError("Must specify at least the variables to plot.") import matplotlib.pyplot as plt from matplotlib import cm, rc from numpy import linspace, array from itertools import cycle from matplotlib.font_manager import FontProperties xs, ys = self._get_data_series(independent, dependents, threshold, functions) #Set the default font for the plots, or the custom one. if fonts is not None and "family" in fonts: rc('font', family=fonts["family"]) else: rc('font', family='Times New Roman') layout = None if "figsize" in kwargs: if len(kwargs["figsize"]) > 2: layout = {"pad": kwargs["figsize"][2]} fig = plt.figure(figsize=(None if "figsize" not in kwargs else kwargs["figsize"][0:2]), tight_layout=layout) dfont = FontProperties() ax = fig.add_subplot(111) rbcolors = cm.rainbow(linspace(0, 1, len(ys))) cycols = cycle(rbcolors) axx = None axy = None if twinplots is not None: if "twinx" in twinplots.values(): axx = ax.twinx() if "twiny" in twinplots.values(): axy = ax.twiny() if labels is not None: if "x" in labels: ax.set_xlabel(labels["x"], fontproperties=self._get_font( dfont, fonts, "xlabel")) if "y" in labels: ax.set_ylabel(labels["y"], fontproperties=self._get_font( dfont, fonts, "ylabel")) if "plot" in labels: plt.title(labels["plot"], fontproperties=self._get_font(dfont, fonts, "title")) if "x-twin" in labels and axx is not None: axx.set_ylabel(labels["x-twin"], fontproperties=self._get_font( dfont, fonts, "x-twin-label")) if "y-twin" in labels and axy is not None: axy.set_xlabel(labels["y-twin"], fontproperties=self._get_font( dfont, fonts, "y-twin-label")) if xscale is not None: ax.set_xscale(xscale) if yscale is not None: ax.set_yscale(yscale) for xset, yset in zip(xs, ys): x, xlabel = xset y, ylabel = yset #Set up a dictionary with all the kwargs for the plotting options #that may be common to both kinds of plots. dargs = {} if colors is not None and ylabel in colors: dargs["color"] = colors[ylabel] else: dargs["color"] = next(cycols) if markers is not None and ylabel in markers: self._get_markers(dargs, markers[ylabel]) if "size" in markers[ylabel]: dargs["s"] = float(markers[ylabel]["size"]) if "s" not in dargs: dargs["s"] = 15 if lines is not None and ylabel in lines: if "style" in lines[ylabel]: dargs["linestyle"] = lines[ylabel]["style"] if "width" in lines[ylabel]: dargs["linewidth"] = float(lines[ylabel]["width"]) if labels is not None and ylabel in labels: dargs["label"] = None if labels[ylabel].lower( ) == "[none]" else labels[ylabel] else: dargs["label"] = ylabel if twinplots is not None and ylabel in twinplots: if twinplots[ylabel] == "twinx": liveax = axx elif twinplots[ylabel] == "twiny": liveax = axy else: liveax = ax try: if plottypes is not None and ylabel in plottypes: lineplot = plottypes[ylabel] == "line" else: lineplot = None if ylabel[len(ylabel) - 4:len(ylabel)] in ["|fit", ".fit"]: varfile = ylabel[0:len(ylabel) - 4] key = "{}${}".format(independent, varfile) allx = linspace(min(x), max(x), 50) if labels is not None and ylabel in labels: dargs["label"] = labels[ylabel].format( self._format_fit(key)) if dargs["label"].lower() == "[none]": dargs["label"] = None else: dargs["label"] = self._format_fit(key) self._reset_lineplot(dargs) liveax.plot(allx, self.fits[key]["function"](allx), **dargs) elif lineplot: self._reset_lineplot(dargs) from operator import itemgetter sdata = array(sorted(zip(x, y), key=itemgetter(0))) liveax.plot(sdata[:, 0], sdata[:, 1], **dargs) else: size = dargs["s"] del dargs["s"] liveax.scatter(x, y, s=size, **dargs) except ValueError: msg.err( "The values for {} can't be log-plotted.".format(ylabel)) #Set the tick labels font sizes. def tickfonts(fonts, key, labels): if fonts is not None and key in fonts: for label in labels: tfont = self._get_font(dfont, fonts, key) label.set_fontproperties(tfont) tickfonts(fonts, "xticks", ax.get_xticklabels()) tickfonts(fonts, "yticks", ax.get_yticklabels()) if axx is not None: tickfonts(fonts, "x-twin-ticks", axx.get_yticklabels()) if axy is not None: tickfonts(fonts, "y-twin-ticks", axy.get_xticklabels()) if ticks is not None and len(ticks) > 0: for key in ticks: if "reset" in ticks[key]: ticks[key]["reset"] = ticks[key]["reset"] == "true" if "x-twin" in key and axx is not None: axx.tick_params(**ticks[key]) elif "y-twin" in key and axy is not None: axy.tick_params(**ticks[key]) else: ax.tick_params(**ticks[key]) for dim in ["x", "y", "z", "x-twin", "y-twin"]: if limits is not None and dim in limits: if dim == "x-twin" and axx is not None: if isinstance(limits[dim], tuple): axx.set_ylim(limits[dim]) elif limits[dim] == "auto": axx.set_ylim(auto=True) elif dim == "y-twin" and axy is not None: if isinstance(limits[dim], tuple): axy.set_ylim(limits[dim]) elif limits[dim] == "auto": axy.set_ylim(auto=True) elif "-" not in dim: if isinstance(limits[dim], tuple): getattr(ax, "set_{}lim".format(dim))(limits[dim]) elif limits[dim] == "auto": getattr(ax, "set_{}lim".format(dim))(auto=True) if len(dependents) > 1: if legend is not None: plt.legend(prop=self._get_font(dfont, fonts, "legend"), **legend) else: plt.legend(loc='upper right', prop=self._get_font(dfont, fonts, "legend")) if savefile is None: plt.show(block=False) else: plt.savefig(savefile)
def _get_data_series(self, independent, dependents, threshold, functions=None): """Used by plot() and table() to get the x and y data series that will be plotted or tabulated against each other. See the arguments on those methods. """ if (("rowvals" in independent or "colvals" in independent) or any([("rowvals" in d or "colvals" in d) for d in dependents])): #We need to make sure that the test case filter they specified returns only a #single result so that we get a reasonable plot. This will happen if the data #is 1D in the other dimension or if the filter only has a single case. We #issue a warning if the filter doesn't match up, so they can check their data. for dep in dependents: tfilter, depvar = dep.split("/") if tfilter is None or sum( [(1 if self._case_filter(d, tfilter) else 0) for d in self.details]) > 1: msg.warn( "Plotting aggregated data for more than one test case. Check results \n" "for consistency and completeness.") break rawvars = {} oseries = [] icases = {} def _load_raw(variable, rawvars): if variable in rawvars: return tfilter, depvar = variable.split("/") fullindvar = "{}/{}".format(tfilter, independent) cases = None if fullindvar not in icases else icases[fullindvar] if fullindvar not in rawvars: x, cases = self._get_data(independent, fullindvar, None, threshold, tfilter) if (len(x) == 1 and isinstance(x[0], list) and ("rowvals" in independent or "colvals" in independent)): x = x[0] rawvars[fullindvar] = x if cases is not None: x = rawvars[fullindvar] ypts, names = self._get_data(depvar, variable, cases, threshold, tfilter, independent, x) if (len(ypts) == 1 and isinstance(ypts[0], list) and ("rowvals" in variable or "colvals" in variable)): ypts = ypts[0] rawvars[variable] = ypts return (fullindvar, cases) for variable in dependents: fullindvar, cases = _load_raw(variable, rawvars) icases[fullindvar] = cases oseries.append((fullindvar, variable)) #Next, we need to apply any postfix functions and then check that the data sizes #are commensurate (i.e. same number of points for dependent and independent series). ys = [] xs = [] if functions is not None: for postfix, fixdef in functions.items(): fxn = None vorder = [v for v in fixdef.keys() if v != "lambda"] values = {} if postfix not in rawvars: msg.err( "Variable '{}' to be postfixed has no data series.". format(postfix)) break N = len(rawvars[postfix]) for lvar, gvar in fixdef.items(): if lvar == "lambda": #This is the function definition for each item, create a lambda for it. import numpy import math try: evals = gvar.split(":")[1] lambstr = "lambda {}: {}".format( ", ".join(vorder), evals) fxn = eval(lambstr) except: msg.err("Could not evaluate function '{}'.".format( gvar)) else: if gvar not in rawvars: _load_raw(gvar, rawvars) if gvar in rawvars: values[lvar] = rawvars[gvar] else: emsg = "Variable '{}' in the lambda function postfix for '{}' is missing data." msg.err(emsg.format(gvar, postfix)) #Now if we have a valid function defined and data to operate on, just #evaluate the postfix one value at a time. if fxn is not None and N > 0: for i in range(N): args = [values[v][i] for v in vorder] pval = fxn(*args) rawvars[postfix][i] = pval #Now we can use the adjusted/postfixed raw values to construct the data series for indepvar, depvar in oseries: xs.append((rawvars[indepvar], indepvar)) #We need the arrays to be the same length for plotting; if we are following #an existing independent variable, we still need to append zero to the list #for the values we don't have. if len(rawvars[depvar]) != len(rawvars[indepvar]): msg.err( "Can't coerce an array to a single value without an aggregation " "function such as numpy.mean or numpy.sum") ys.append((rawvars[depvar], depvar)) return (xs, ys)
def compile(folder, compiler=None, identifier=None, debug=False, profile=False, quiet=False, moptions=None, inclfortpy=True, vupdates=None, strict=False, inclfpyaux=False): """Runs the makefile in the specified folder to compile the 'all' rule. :arg vupdates: a list of module names for which the output .mod and .o files should have version information attached. """ if inclfortpy: #Before we can compile the library, we need to make sure that we have a fortpy #.mod and .o compiled with the *same* compiler version specified. from fortpy.utility import get_fortpy_templates_dir _ensure_fileversion(compiler, "fortpy", get_fortpy_templates_dir(), folder) options = "" if debug: options += " DEBUG=true" if profile: options += " GPROF=true" if strict: options += " STRICT=true" if moptions is not None: for opt in moptions: options += " {}".format(opt) if identifier is not None: codestr = "cd {}; make -f 'Makefile.{}' F90='{}' FAM='{}'" + options command = codestr.format(folder, identifier, executor(compiler), family(compiler)) else: codestr = "cd {}; make F90='{}' FAM='{}'" + options command = codestr.format(folder, executor(compiler), family(compiler)) #If we are running in quiet mode, we don't want the compile information #to post to stdout; only errors should be redirected. This means we need #to wrap the execution in a subprocess and redirect the std* to PIPE from os import waitpid, path from subprocess import Popen, PIPE pcompile = Popen(command, shell=True, executable="/bin/bash", stdout=PIPE, stderr=PIPE, close_fds=True) waitpid(pcompile.pid, 0) if not quiet: output = [x.decode('utf8') for x in pcompile.stdout.readlines()] msg.std(''.join(output)) #else: #We don't need to get these lines since we are purposefully redirecting them. error = [x.decode('utf8') for x in pcompile.stderr.readlines()] code = len(error) if code != 0: msg.err(''.join(error)) pcompile.stdout.close() pcompile.stderr.close() #It turns out that the compiler still returns a code of zero, even if the compile #failed because the actual compiler didn't fail; it did its job properly. We need to #check for the existence of errors in the 'compile.log' file. lcount = 0 errors = [] log = path.join( folder, "compile.{}.log".format( identifier if identifier is not None else "default")) with open(log) as f: for line in f: lcount += 1 if lcount > 21 and lcount < 32: errors.append(line) elif lcount > 21: break if len(errors) > 0: #There are 21 lines in the compile.log file when everything runs correctly #Overwrite code with a bad exit value since we have some other problems. code = 1 #We want to write the first couple of errors to the console and give them the #option to still execute if the compile only generated warnings. msg.warn("compile generated some errors or warnings:") msg.blank() msg.info(''.join(errors)) if vupdates is not None: for modname in vupdates: _vupdate_compiled_module(compiler, modname, folder, rename=False) return (code, len(errors) == 0)
def _get_data_series(self, independent, dependents, threshold, functions=None): """Used by plot() and table() to get the x and y data series that will be plotted or tabulated against each other. See the arguments on those methods. """ if (("rowvals" in independent or "colvals" in independent) or any([("rowvals" in d or "colvals" in d) for d in dependents])): #We need to make sure that the test case filter they specified returns only a #single result so that we get a reasonable plot. This will happen if the data #is 1D in the other dimension or if the filter only has a single case. We #issue a warning if the filter doesn't match up, so they can check their data. for dep in dependents: tfilter, depvar = dep.split("/") if tfilter is None or sum([(1 if self._case_filter(d, tfilter) else 0) for d in self.details]) > 1: msg.warn("Plotting aggregated data for more than one test case. Check results \n" "for consistency and completeness.") break rawvars = {} oseries = [] icases = {} def _load_raw(variable, rawvars): if variable in rawvars: return tfilter, depvar = variable.split("/") fullindvar = "{}/{}".format(tfilter, independent) cases = None if fullindvar not in icases else icases[fullindvar] if fullindvar not in rawvars: x, cases = self._get_data(independent, fullindvar, None, threshold, tfilter) if (len(x) == 1 and isinstance(x[0], list) and ("rowvals" in independent or "colvals" in independent)): x = x[0] rawvars[fullindvar] = x if cases is not None: x = rawvars[fullindvar] ypts, names = self._get_data(depvar, variable, cases, threshold, tfilter, independent, x) if (len(ypts) == 1 and isinstance(ypts[0], list) and ("rowvals" in variable or "colvals" in variable)): ypts = ypts[0] rawvars[variable] = ypts return (fullindvar, cases) for variable in dependents: fullindvar, cases = _load_raw(variable, rawvars) icases[fullindvar] = cases oseries.append((fullindvar, variable)) #Next, we need to apply any postfix functions and then check that the data sizes #are commensurate (i.e. same number of points for dependent and independent series). ys = [] xs = [] if functions is not None: for postfix, fixdef in functions.items(): fxn = None vorder = [v for v in fixdef.keys() if v != "lambda"] values = {} if postfix not in rawvars: msg.err("Variable '{}' to be postfixed has no data series.".format(postfix)) break N = len(rawvars[postfix]) for lvar, gvar in fixdef.items(): if lvar == "lambda": #This is the function definition for each item, create a lambda for it. import numpy import math try: evals = gvar.split(":")[1] lambstr = "lambda {}: {}".format(", ".join(vorder), evals) fxn = eval(lambstr) except: msg.err("Could not evaluate function '{}'.".format(gvar)) else: if gvar not in rawvars: _load_raw(gvar, rawvars) if gvar in rawvars: values[lvar] = rawvars[gvar] else: emsg = "Variable '{}' in the lambda function postfix for '{}' is missing data." msg.err(emsg.format(gvar, postfix)) #Now if we have a valid function defined and data to operate on, just #evaluate the postfix one value at a time. if fxn is not None and N > 0: for i in range(N): args = [values[v][i] for v in vorder] pval = fxn(*args) rawvars[postfix][i] = pval #Now we can use the adjusted/postfixed raw values to construct the data series for indepvar, depvar in oseries: xs.append((rawvars[indepvar], indepvar)) #We need the arrays to be the same length for plotting; if we are following #an existing independent variable, we still need to append zero to the list #for the values we don't have. if len(rawvars[depvar]) != len(rawvars[indepvar]): msg.err("Can't coerce an array to a single value without an aggregation " "function such as numpy.mean or numpy.sum") ys.append((rawvars[depvar], depvar)) return (xs, ys)
def compile(folder, compiler=None, identifier=None, debug=False, profile=False, quiet=False, moptions=None, inclfortpy=True, vupdates=None, strict=False, inclfpyaux=False): """Runs the makefile in the specified folder to compile the 'all' rule. :arg vupdates: a list of module names for which the output .mod and .o files should have version information attached. """ if inclfortpy: #Before we can compile the library, we need to make sure that we have a fortpy #.mod and .o compiled with the *same* compiler version specified. from fortpy.utility import get_fortpy_templates_dir _ensure_fileversion(compiler, "fortpy", get_fortpy_templates_dir(), folder) options = "" if debug: options += " DEBUG=true" if profile: options += " GPROF=true" if strict: options += " STRICT=true" if moptions is not None: for opt in moptions: options += " {}".format(opt) if identifier is not None: codestr = "cd {}; make -f 'Makefile.{}' F90='{}' FAM='{}'" + options command = codestr.format(folder, identifier, executor(compiler), family(compiler)) else: codestr = "cd {}; make F90='{}' FAM='{}'" + options command = codestr.format(folder, executor(compiler), family(compiler)) #If we are running in quiet mode, we don't want the compile information #to post to stdout; only errors should be redirected. This means we need #to wrap the execution in a subprocess and redirect the std* to PIPE from os import waitpid, path from subprocess import Popen, PIPE pcompile = Popen(command, shell=True, executable="/bin/bash", stdout=PIPE, stderr=PIPE) waitpid(pcompile.pid, 0) if not quiet: output = [x.decode('utf8') for x in pcompile.stdout.readlines()] msg.std(''.join(output)) #else: #We don't need to get these lines since we are purposefully redirecting them. error = [x.decode('utf8') for x in pcompile.stderr.readlines()] code = len(error) if code != 0: msg.err(''.join(error)) #It turns out that the compiler still returns a code of zero, even if the compile #failed because the actual compiler didn't fail; it did its job properly. We need to #check for the existence of errors in the 'compile.log' file. lcount = 0 errors = [] log = path.join(folder, "compile.log") with open(log) as f: for line in f: lcount += 1 if lcount > 21 and lcount < 32: errors.append(line) elif lcount > 21: break if len(errors) > 0: #There are 21 lines in the compile.log file when everything runs correctly #Overwrite code with a bad exit value since we have some other problems. code = 1 #We want to write the first couple of errors to the console and give them the #option to still execute if the compile only generated warnings. msg.warn("compile generated some errors or warnings:") msg.blank() msg.info(''.join(errors)) if vupdates is not None: for modname in vupdates: _vupdate_compiled_module(compiler, modname, folder, rename=False) return (code, len(errors)==0)