def test_parse1(dir):
# A 'normal' ini file that uses all subgrouping mechanisms
parsed = parse_ini_file(dir + "parse1.ini")
assert(len(parsed) == 8)
assert(parsed['x'] == '5')
assert(parsed['y'] == 'str')
assert(parsed['group.y'] == 'str')
assert(parsed['group.x'] == '5')
assert(parsed['group.z'] == '1')
assert(parsed['group.subgroup.y'] == 'str')
assert(parsed['group.subgroup.z'] == '1')
def call(executable, inifile=None):
# If we have an inifile, parse it and look for special keys that modify the execution
command = ["./" + executable]
if inifile:
iniargument = inifile
iniinfo = parse_ini_file(inifile)
if "__inifile_optionkey" in iniinfo:
command.append(iniinfo["__inifile_optionkey"])
command.append(iniargument)
return subprocess.call(command)
def call(executable, inifile=None, *additional_args):
# If we have an inifile, parse it and look for special keys that modify the execution
command = ["./" + executable]
if inifile:
iniargument = inifile
iniinfo = parse_ini_file(inifile)
if "__inifile_optionkey" in iniinfo:
command.append(iniinfo["__inifile_optionkey"])
command.append(iniargument)
for arg in additional_args:
command.append(arg)
return subprocess.call(command)
def call(executable, inifile=None, *additional_args):
# If we have an inifile, parse it and look for special keys that modify the execution
command = ["./" + executable]
if inifile:
iniargument = inifile
iniinfo = parse_ini_file(inifile)
if "__inifile_optionkey" in iniinfo:
command.append(iniinfo["__inifile_optionkey"])
command.append(iniargument)
for arg in additional_args:
command.append(arg)
# forwarding the env ensures child is executed in the dune virtualenv
# since this script itself is always called in the dune virtualenv
return subprocess.call(command, env=os.environ)
def call_parallel(executable, mpi_exec, mpi_numprocflag, mpi_preflags, mpi_postflags, max_processors, inifile=None):
# If we have an inifile, parse it and look for special keys that modify the execution
num_processes = "2" # a default
command = [mpi_exec, mpi_numprocflag, num_processes]
if mpi_preflags:
command += mpi_preflags
command += [executable]
if mpi_postflags:
command += mpi_postflags
if inifile:
iniargument = inifile
iniinfo = parse_ini_file(inifile)
if "__inifile_optionkey" in iniinfo:
command.append(iniinfo["__inifile_optionkey"])
command.append(iniargument)
if "wrapper.execute_parallel.numprocesses" in iniinfo:
command[2] = iniinfo["wrapper.execute_parallel.numprocesses"]
if int(command[2]) <= int(max_processors):
return subprocess.call(command)
else:
return 77
def check_parser(ini):
try:
parsed = parse_ini_file(ini)
except Exception:
try:
file = open(ini, "r")
except Exception:
print("Reading the source file failed. Did you give a correct path?")
sys.exit(1)
print("Parsing the meta ini file failed.")
print("Now attempting to find lines that cannot be parsed.")
parser = MetaIniParser(path=os.path.dirname(ini))
for line in file:
try:
parser.apply(line)
except Exception:
print("ERROR Malformed line: '{}'".format(line))
# TODO give some info about how its malformed: Needs inspection of the exception.
sys.exit(1)
# If we got until here, the file could be parsed correctly
print("Parsing the ini file was successful...")
return parsed
def expand_meta_ini(filename,
assignment="=",
commentChar="#",
whiteFilter=None,
blackFilter=None,
addNameKey=True):
"""
Take a meta ini file and construct the set of ini files it defines
Required Arguments:
:param filename: The filename of the meta ini file
:type filename: string
Optional Arguments:
:type commentChar: string
:param commentChar: A character that defines comments. Everything on a line
after such character is ignored during the parsing process.
:type whiteFilter: tuple
:param whiteFilter: Filter the given keys. The elements of the returned set of
configurations will be unique.
:type blackFilter: tuple
:param blackFilter: The standard assignment operator
:type addNameKey: bool
:param addNameKey: Whether a key ``__name`` should be in the output. Defaults to true, where
a unique name key is generated from the given name key and added to the
file (even when no generation pattern is given). If set to false, no
name key will be in the output, whether a scheme was given or not.
"""
# parse the ini file
parse, cmds = parse_ini_file(filename,
assignment=assignment,
commentChar=commentChar,
returnCommands=True)
# initialize the list of configurations with the parsed configuration
configurations = [parse]
# HOOK: PRE_EXPANSION
apply_commands(configurations,
cmds[CommandType.PRE_EXPANSION],
all_cmds=cmds)
# Preprocessing expansion: Sort and group all expand commands by their argument:
expanddict = {}
expandlist = []
for expcmd in cmds[CommandType.AT_EXPANSION]:
if len(expcmd.args) == 0:
expandlist.append(CommandToApply("expand", [], [expcmd.key]))
else:
if expcmd.args[0] in expanddict:
expanddict[expcmd.args[0]].append(expcmd.key)
else:
expanddict[expcmd.args[0]] = [expcmd.key]
for ident, keylist in expanddict.items():
expandlist.append(CommandToApply("expand", [], keylist))
cmds[CommandType.AT_EXPANSION] = expandlist
# Now apply expansion through the machinery
apply_commands(configurations,
cmds[CommandType.AT_EXPANSION],
all_cmds=cmds)
# HOOK: POST_EXPANSION
apply_commands(configurations,
cmds[CommandType.POST_EXPANSION],
all_cmds=cmds)
def check_for_unique(d, k):
for cta in cmds[CommandType.POST_FILTERING]:
if (cta.key == k and cta.name == "unique") or (k in uniquekeys()):
raise ValueError(
"You cannot have keys depend on keys which are marked unique. This is a chicken-egg situation!"
)
return d[k]
def resolve_key_dependencies(d):
""" replace curly brackets with keys by the appropriate key from the dictionary - recursively """
resolved = False
for key, value in d.items():
if exists_unescaped(value, "}") and exists_unescaped(value, "{"):
# Check whether this key has an AT_RESOLUTION command applied
lookup_key = extract_delimited(value,
leftdelimiter="{",
rightdelimiter="}")
if lookup_key in [
c.key for c in cmds[CommandType.AT_RESOLUTION]
]:
continue
# split the contents form the innermost curly brackets from the rest
d[key] = replace_delimited(value,
d,
access_func=check_for_unique)
resolved = True
return resolved
# HOOK: PRE_RESOLUTION
apply_commands(configurations,
cmds[CommandType.PRE_RESOLUTION],
all_cmds=cmds)
# resolve all key-dependent names present in the configurations
for c in configurations:
# values might depend on keys, whose value also depend on other keys.
# In a worst case scenario concerning the order of resolution,
# a call to resolve_key_dependencies only resolves one such layer.
# That is why we need to do this until all dependencies are resolved.
while resolve_key_dependencies(c):
pass
# If we have AT_RESOLUTION commands present, we need to reiterate resolution
# until all of these are resolved!
at_resolution_commands = cmds[CommandType.AT_RESOLUTION]
while at_resolution_commands:
for cmd in cmds[CommandType.AT_RESOLUTION]:
skip = False
for c in configurations:
value = c[cmd.key]
# If the value still contains curly brackets, we have to skip this!
if exists_unescaped(value, "}") and exists_unescaped(
value, "{"):
skip = True
# If the argument list still contains curly brackets we do the same
for arg in cmd.args:
if exists_unescaped(arg, "}") and exists_unescaped(
arg, "{"):
argval = c[extract_delimited(arg,
leftdelimiter="{",
rightdelimiter="}")]
if exists_unescaped(argval, "}") and exists_unescaped(
argval, "{"):
skip = True
if skip:
continue
apply_commands(configurations, [cmd], all_cmds=cmds)
at_resolution_commands.remove(cmd)
for c in configurations:
while resolve_key_dependencies(c):
pass
# HOOK: POST_RESOLUTION
apply_commands(configurations,
cmds[CommandType.POST_RESOLUTION],
all_cmds=cmds)
# HOOK: PRE_FILTERING
apply_commands(configurations,
cmds[CommandType.PRE_FILTERING],
all_cmds=cmds)
# Apply filtering
if blackFilter:
# check whether a single filter has been given and make a tuple if so
if not hasattr(blackFilter, '__iter__'):
blackFilter = [blackFilter]
else:
blackFilter = []
# always ignore the section called "__local". Its keys by definition do not influence the number of configuration.
blackFilter = [f for f in blackFilter] + ["__local"]
# remove all keys that match the given filtering
configurations = [
c.filter([
k for k in c if True not in [k.startswith(f) for f in blackFilter]
]) for c in configurations
]
if whiteFilter:
# check whether a single filter has been given and make a tuple if so
if not hasattr(whiteFilter, '__iter__'):
whiteFilter = (whiteFilter, )
# remove all keys that do not match the given filtering
configurations = [c.filter(whiteFilter) for c in configurations]
# remove duplicate configurations - we added hashing to the DotDict class just for this purpose.
configurations = [c for c in sorted(set(configurations))]
# Implement the naming scheme through the special key __name
if addNameKey:
# circumvent the fact, that commands on non-existent keys are ignored
if "__name" not in configurations[0]:
configurations[0]["__name"] = ''
cmds[CommandType.POST_FILTERING].append(
CommandToApply(name="unique", args=[], key="__name"))
else:
for c in configurations:
if "__name" in c:
del c["__name"]
# HOOK: POST_FILTERING
apply_commands(configurations, cmds[CommandType.POST_FILTERING])
# Strip escapes TODO: Which charaters should be escaped not to mess with our code?
possibly_escaped_chars = "[]{}="
for c in configurations:
for k, v in list(c.items()):
escaped_value = v
for char in possibly_escaped_chars:
escaped_value = strip_escapes(escaped_value, char)
c[k] = escaped_value
return configurations
from dune.testtools.wrapper.argumentparser import get_args
from dune.testtools.wrapper.call_executable import call
from dune.testtools.wrapper.compareini import compare_ini, fuzzy_compare_ini
from dune.testtools.parser import parse_ini_file
# Parse the given arguments
args = get_args()
# Execute the actual test!
ret = call(args["exec"], args["ini"])
# do the outputtree comparison if execution was succesful
if ret is 0:
# Parse the inifile to learn about where the output file and its reference are located.
ini = parse_ini_file(args["ini"])
try:
# get reference solutions
names = ini["wrapper.outputtreecompare.name"].split(' ')
exts = ini.get("wrapper.outputtreecompare.extension",
"out " * len(names)).split(' ')
references = ini["wrapper.outputtreecompare.reference"].split(' ')
except KeyError:
sys.stdout.write(
"The test wrapper outputtreecompare assumes keys wrapper.outputtreecompare.name \
and wrapper.outputtreecompare.reference to be existent in the inifile"
)
# loop over all outputtree comparisons
for n, e, r in zip(names, exts, references):
# if we have multiple vtks search in the subgroup prefixed with the vtk-name for options
def call(executable, metaini=None):
# check for the meta ini file
if not metaini:
sys.stderr.write("No meta ini file found for this convergence test!")
return 1
# expand the meta ini file
from dune.testtools.metaini import expand_meta_ini
configurations = expand_meta_ini(metaini)
# Find out in which sections the test data is
testsections = configurations[0].get(
"wrapper.convergencetest.testsections", "").split()
if testsections:
testsections = [
"wrapper.convergencetest.{}".format(s) for s in testsections
]
else:
testsections = ["wrapper.convergencetest"]
# execute all runs with temporary ini files and process the temporary output
output = []
for c in configurations:
c.setdefault("__output_extension", "out")
# write a temporary ini file. Prefix them with the name key to be unique
tmp_file = c["__name"] + "_tmp.ini"
write_dict_to_ini(c, tmp_file)
# execute the run
command = [executable]
iniinfo = parse_ini_file(metaini)
if "__inifile_optionkey" in iniinfo:
command.append(iniinfo["__inifile_optionkey"])
command.append(tmp_file)
if subprocess.call(command):
return 1
# collect the information from the output file
output.append([
parse_ini_file(
os.path.basename(c["__name"]) + "." + c["__output_extension"])
][0])
# remove temporary files
os.remove(
os.path.basename(c["__name"]) + "." + c["__output_extension"])
os.remove(tmp_file)
# store return value (because we do not want to return as soon as one section fails)
returnvalue = 0
# calculate the rate according to the outputted data
for section in testsections:
for idx, c in list(enumerate(configurations))[:-1]:
# check if all necessary keys are given
if "expectedrate" not in c[section]:
sys.stderr.write(
"The convergencetest wrapper excepts a key expectedrate \
in section {} of the ini file!".format(
section))
return 1
# specify all default keys if not specified already
c[section].setdefault("absolutedifference", "0.1")
c.setdefault("__output_extension", "out")
norm1 = float(output[idx][section]["norm"])
norm2 = float(output[idx + 1][section]["norm"])
hmax1 = float(output[idx][section]["scale"])
hmax2 = float(output[idx + 1][section]["scale"])
rate = math.log(norm2 / norm1) / math.log(hmax2 / hmax1)
# test passes
if math.fabs(rate - float(c[section]["expectedrate"])) <= float(
c[section]["absolutedifference"]):
sys.stdout.write(
"Test {} passed because the absolute difference "
"between the calculated convergence rate ({}) "
"and the expected convergence rate ({}) was within "
"tolerance ({}). \n".format(
section, rate, c[section]["expectedrate"],
c[section]["absolutedifference"]))
# test fails because rates are off
elif math.fabs(rate - float(c[section]["expectedrate"])) > float(
c[section]["absolutedifference"]):
sys.stderr.write(
"Test {} failed because the absolute difference "
"between the calculated convergence rate ({}) "
"and the expected convergence rate ({}) was greater "
"than tolerance ({}). \n".format(
section, rate, c[section]["expectedrate"],
c[section]["absolutedifference"]))
returnvalue = 1
# test fails because rates are nan or inf
elif math.isnan(rate) or math.isinf(rate):
sys.stderr.write(
"Test {} failed because calculated rate is ({})."
"Expected was ({}) with tolerance ({}). \n".format(
section, rate, c[section]["expectedrate"],
c[section]["absolutedifference"]))
returnvalue = 1
# if we are here, something unexpcted happened
else:
sys.stderr.write(
"Test {} failed for unknown reason with calculated rate ({}), "
"expected rate ({}) and tolerance ({}). \n".format(
section, rate, c[section]["expectedrate"],
c[section]["absolutedifference"]))
returnvalue = 1
return returnvalue
def test_parse3(dir):
# Testing all sorts of escapes
parsed = parse_ini_file(dir + "parse3.ini")
assert(count_unescaped(parsed['a'], '|') == 0)
assert(count_unescaped(parsed['c'], ',') == 3)
assert(count_unescaped(parsed['d'], '"') == 2)
def test_parse2(dir):
# A file that contains non-key-value data
parsed = parse_ini_file(dir + "parse2.ini")['__local.conditionals']
assert(len(parsed) == 2)
assert(parsed['0'] == '{x} == {y}')
assert(parsed['1'] == '{x} == {y}')