Beispiel #1
0
def test_parse1(dir):
    # A 'normal' ini file that uses all subgrouping mechanisms
    parsed = parse_ini_file(dir + "parse1.ini")
    assert(len(parsed) == 8)
    assert(parsed['x'] == '5')
    assert(parsed['y'] == 'str')
    assert(parsed['group.y'] == 'str')
    assert(parsed['group.x'] == '5')
    assert(parsed['group.z'] == '1')
    assert(parsed['group.subgroup.y'] == 'str')
    assert(parsed['group.subgroup.z'] == '1')
Beispiel #2
0
def call(executable, inifile=None):
    # If we have an inifile, parse it and look for special keys that modify the execution
    command = ["./" + executable]
    if inifile:
        iniargument = inifile
        iniinfo = parse_ini_file(inifile)
        if "__inifile_optionkey" in iniinfo:
            command.append(iniinfo["__inifile_optionkey"])
        command.append(iniargument)

    return subprocess.call(command)
def call(executable, inifile=None, *additional_args):
    # If we have an inifile, parse it and look for special keys that modify the execution
    command = ["./" + executable]
    if inifile:
        iniargument = inifile
        iniinfo = parse_ini_file(inifile)
        if "__inifile_optionkey" in iniinfo:
            command.append(iniinfo["__inifile_optionkey"])
        command.append(iniargument)
    for arg in additional_args:
        command.append(arg)
    return subprocess.call(command)
def call(executable, inifile=None, *additional_args):
    # If we have an inifile, parse it and look for special keys that modify the execution
    command = ["./" + executable]
    if inifile:
        iniargument = inifile
        iniinfo = parse_ini_file(inifile)
        if "__inifile_optionkey" in iniinfo:
            command.append(iniinfo["__inifile_optionkey"])
        command.append(iniargument)
    for arg in additional_args:
        command.append(arg)
    # forwarding the env ensures child is executed in the dune virtualenv
    # since this script itself is always called in the dune virtualenv
    return subprocess.call(command, env=os.environ)
Beispiel #5
0
def call_parallel(executable, mpi_exec, mpi_numprocflag, mpi_preflags, mpi_postflags, max_processors, inifile=None):
    # If we have an inifile, parse it and look for special keys that modify the execution
    num_processes = "2"  # a default
    command = [mpi_exec, mpi_numprocflag, num_processes]
    if mpi_preflags:
        command += mpi_preflags
    command += [executable]
    if mpi_postflags:
        command += mpi_postflags
    if inifile:
        iniargument = inifile
        iniinfo = parse_ini_file(inifile)
        if "__inifile_optionkey" in iniinfo:
            command.append(iniinfo["__inifile_optionkey"])
        command.append(iniargument)
        if "wrapper.execute_parallel.numprocesses" in iniinfo:
            command[2] = iniinfo["wrapper.execute_parallel.numprocesses"]

    if int(command[2]) <= int(max_processors):
        return subprocess.call(command)
    else:
        return 77
Beispiel #6
0
    def check_parser(ini):
        try:
            parsed = parse_ini_file(ini)
        except Exception:
            try:
                file = open(ini, "r")
            except Exception:
                print("Reading the source file failed. Did you give a correct path?")
                sys.exit(1)
            print("Parsing the meta ini file failed.")
            print("Now attempting to find lines that cannot be parsed.")
            parser = MetaIniParser(path=os.path.dirname(ini))
            for line in file:
                try:
                    parser.apply(line)
                except Exception:
                    print("ERROR Malformed line: '{}'".format(line))
                    # TODO give some info about how its malformed: Needs inspection of the exception.
            sys.exit(1)

        # If we got until here, the file could be parsed correctly
        print("Parsing the ini file was successful...")
        return parsed
Beispiel #7
0
def expand_meta_ini(filename,
                    assignment="=",
                    commentChar="#",
                    whiteFilter=None,
                    blackFilter=None,
                    addNameKey=True):
    """
    Take a meta ini file and construct the set of ini files it defines

    Required Arguments:

    :param filename: The filename of the meta ini file
    :type filename:  string

    Optional Arguments:

    :type commentChar:  string
    :param commentChar: A  character that defines comments. Everything on a line
                        after such character is ignored during the parsing process.

    :type whiteFilter:  tuple
    :param whiteFilter: Filter the given keys. The elements of the returned set of
                        configurations will be unique.

    :type blackFilter:  tuple
    :param blackFilter: The standard assignment operator

    :type addNameKey:  bool
    :param addNameKey: Whether a key ``__name`` should be in the output. Defaults to true, where
                       a unique name key is generated from the given name key and added to the
                       file (even when no generation pattern is given). If set to false, no
                       name key will be in the output, whether a scheme was given or not.
    """

    # parse the ini file
    parse, cmds = parse_ini_file(filename,
                                 assignment=assignment,
                                 commentChar=commentChar,
                                 returnCommands=True)

    # initialize the list of configurations with the parsed configuration
    configurations = [parse]

    # HOOK: PRE_EXPANSION
    apply_commands(configurations,
                   cmds[CommandType.PRE_EXPANSION],
                   all_cmds=cmds)

    # Preprocessing expansion: Sort and group all expand commands by their argument:
    expanddict = {}
    expandlist = []
    for expcmd in cmds[CommandType.AT_EXPANSION]:
        if len(expcmd.args) == 0:
            expandlist.append(CommandToApply("expand", [], [expcmd.key]))
        else:
            if expcmd.args[0] in expanddict:
                expanddict[expcmd.args[0]].append(expcmd.key)
            else:
                expanddict[expcmd.args[0]] = [expcmd.key]
    for ident, keylist in expanddict.items():
        expandlist.append(CommandToApply("expand", [], keylist))
    cmds[CommandType.AT_EXPANSION] = expandlist

    # Now apply expansion through the machinery
    apply_commands(configurations,
                   cmds[CommandType.AT_EXPANSION],
                   all_cmds=cmds)

    # HOOK: POST_EXPANSION
    apply_commands(configurations,
                   cmds[CommandType.POST_EXPANSION],
                   all_cmds=cmds)

    def check_for_unique(d, k):
        for cta in cmds[CommandType.POST_FILTERING]:
            if (cta.key == k and cta.name == "unique") or (k in uniquekeys()):
                raise ValueError(
                    "You cannot have keys depend on keys which are marked unique. This is a chicken-egg situation!"
                )
        return d[k]

    def resolve_key_dependencies(d):
        """ replace curly brackets with keys by the appropriate key from the dictionary - recursively """
        resolved = False
        for key, value in d.items():
            if exists_unescaped(value, "}") and exists_unescaped(value, "{"):
                # Check whether this key has an AT_RESOLUTION command applied
                lookup_key = extract_delimited(value,
                                               leftdelimiter="{",
                                               rightdelimiter="}")
                if lookup_key in [
                        c.key for c in cmds[CommandType.AT_RESOLUTION]
                ]:
                    continue

                # split the contents form the innermost curly brackets from the rest
                d[key] = replace_delimited(value,
                                           d,
                                           access_func=check_for_unique)
                resolved = True

        return resolved

    # HOOK: PRE_RESOLUTION
    apply_commands(configurations,
                   cmds[CommandType.PRE_RESOLUTION],
                   all_cmds=cmds)

    # resolve all key-dependent names present in the configurations
    for c in configurations:
        # values might depend on keys, whose value also depend on other keys.
        # In a worst case scenario concerning the order of resolution,
        # a call to resolve_key_dependencies only resolves one such layer.
        # That is why we need to do this until all dependencies are resolved.
        while resolve_key_dependencies(c):
            pass

    # If we have AT_RESOLUTION commands present, we need to reiterate resolution
    # until all of these are resolved!
    at_resolution_commands = cmds[CommandType.AT_RESOLUTION]
    while at_resolution_commands:
        for cmd in cmds[CommandType.AT_RESOLUTION]:
            skip = False
            for c in configurations:
                value = c[cmd.key]

                # If the value still contains curly brackets, we have to skip this!
                if exists_unescaped(value, "}") and exists_unescaped(
                        value, "{"):
                    skip = True

                # If the argument list still contains curly brackets we do the same
                for arg in cmd.args:
                    if exists_unescaped(arg, "}") and exists_unescaped(
                            arg, "{"):
                        argval = c[extract_delimited(arg,
                                                     leftdelimiter="{",
                                                     rightdelimiter="}")]
                        if exists_unescaped(argval, "}") and exists_unescaped(
                                argval, "{"):
                            skip = True

            if skip:
                continue

            apply_commands(configurations, [cmd], all_cmds=cmds)
            at_resolution_commands.remove(cmd)

        for c in configurations:
            while resolve_key_dependencies(c):
                pass

    # HOOK: POST_RESOLUTION
    apply_commands(configurations,
                   cmds[CommandType.POST_RESOLUTION],
                   all_cmds=cmds)

    # HOOK: PRE_FILTERING
    apply_commands(configurations,
                   cmds[CommandType.PRE_FILTERING],
                   all_cmds=cmds)

    # Apply filtering
    if blackFilter:
        # check whether a single filter has been given and make a tuple if so
        if not hasattr(blackFilter, '__iter__'):
            blackFilter = [blackFilter]
    else:
        blackFilter = []

    # always ignore the section called "__local". Its keys by definition do not influence the number of configuration.
    blackFilter = [f for f in blackFilter] + ["__local"]
    # remove all keys that match the given filtering
    configurations = [
        c.filter([
            k for k in c if True not in [k.startswith(f) for f in blackFilter]
        ]) for c in configurations
    ]

    if whiteFilter:
        # check whether a single filter has been given and make a tuple if so
        if not hasattr(whiteFilter, '__iter__'):
            whiteFilter = (whiteFilter, )
        # remove all keys that do not match the given filtering
        configurations = [c.filter(whiteFilter) for c in configurations]

    # remove duplicate configurations - we added hashing to the DotDict class just for this purpose.
    configurations = [c for c in sorted(set(configurations))]

    # Implement the naming scheme through the special key __name
    if addNameKey:
        # circumvent the fact, that commands on non-existent keys are ignored
        if "__name" not in configurations[0]:
            configurations[0]["__name"] = ''
        cmds[CommandType.POST_FILTERING].append(
            CommandToApply(name="unique", args=[], key="__name"))
    else:
        for c in configurations:
            if "__name" in c:
                del c["__name"]

    # HOOK: POST_FILTERING
    apply_commands(configurations, cmds[CommandType.POST_FILTERING])

    # Strip escapes TODO: Which charaters should be escaped not to mess with our code?
    possibly_escaped_chars = "[]{}="
    for c in configurations:
        for k, v in list(c.items()):
            escaped_value = v
            for char in possibly_escaped_chars:
                escaped_value = strip_escapes(escaped_value, char)
            c[k] = escaped_value

    return configurations
Beispiel #8
0
    from dune.testtools.wrapper.argumentparser import get_args
    from dune.testtools.wrapper.call_executable import call
    from dune.testtools.wrapper.compareini import compare_ini, fuzzy_compare_ini
    from dune.testtools.parser import parse_ini_file

    # Parse the given arguments
    args = get_args()

    # Execute the actual test!
    ret = call(args["exec"], args["ini"])

    # do the outputtree comparison if execution was succesful
    if ret is 0:
        # Parse the inifile to learn about where the output file and its reference are located.
        ini = parse_ini_file(args["ini"])
        try:
            # get reference solutions
            names = ini["wrapper.outputtreecompare.name"].split(' ')
            exts = ini.get("wrapper.outputtreecompare.extension",
                           "out " * len(names)).split(' ')
            references = ini["wrapper.outputtreecompare.reference"].split(' ')
        except KeyError:
            sys.stdout.write(
                "The test wrapper outputtreecompare assumes keys wrapper.outputtreecompare.name \
                              and wrapper.outputtreecompare.reference to be existent in the inifile"
            )

        # loop over all outputtree comparisons
        for n, e, r in zip(names, exts, references):
            # if we have multiple vtks search in the subgroup prefixed with the vtk-name for options
Beispiel #9
0
def call(executable, metaini=None):
    # check for the meta ini file
    if not metaini:
        sys.stderr.write("No meta ini file found for this convergence test!")
        return 1

    # expand the meta ini file
    from dune.testtools.metaini import expand_meta_ini
    configurations = expand_meta_ini(metaini)

    # Find out in which sections the test data is
    testsections = configurations[0].get(
        "wrapper.convergencetest.testsections", "").split()
    if testsections:
        testsections = [
            "wrapper.convergencetest.{}".format(s) for s in testsections
        ]
    else:
        testsections = ["wrapper.convergencetest"]

    # execute all runs with temporary ini files and process the temporary output
    output = []
    for c in configurations:
        c.setdefault("__output_extension", "out")

        # write a temporary ini file. Prefix them with the name key to be unique
        tmp_file = c["__name"] + "_tmp.ini"
        write_dict_to_ini(c, tmp_file)

        # execute the run
        command = [executable]
        iniinfo = parse_ini_file(metaini)
        if "__inifile_optionkey" in iniinfo:
            command.append(iniinfo["__inifile_optionkey"])
        command.append(tmp_file)

        if subprocess.call(command):
            return 1

        # collect the information from the output file
        output.append([
            parse_ini_file(
                os.path.basename(c["__name"]) + "." + c["__output_extension"])
        ][0])

        # remove temporary files
        os.remove(
            os.path.basename(c["__name"]) + "." + c["__output_extension"])
        os.remove(tmp_file)

    # store return value (because we do not want to return as soon as one section fails)
    returnvalue = 0

    # calculate the rate according to the outputted data
    for section in testsections:
        for idx, c in list(enumerate(configurations))[:-1]:
            # check if all necessary keys are given
            if "expectedrate" not in c[section]:
                sys.stderr.write(
                    "The convergencetest wrapper excepts a key expectedrate \
                                  in section {} of the ini file!".format(
                        section))
                return 1

            # specify all default keys if not specified already
            c[section].setdefault("absolutedifference", "0.1")
            c.setdefault("__output_extension", "out")

            norm1 = float(output[idx][section]["norm"])
            norm2 = float(output[idx + 1][section]["norm"])
            hmax1 = float(output[idx][section]["scale"])
            hmax2 = float(output[idx + 1][section]["scale"])
            rate = math.log(norm2 / norm1) / math.log(hmax2 / hmax1)
            # test passes
            if math.fabs(rate - float(c[section]["expectedrate"])) <= float(
                    c[section]["absolutedifference"]):
                sys.stdout.write(
                    "Test {} passed because the absolute difference "
                    "between the calculated convergence rate ({}) "
                    "and the expected convergence rate ({}) was within "
                    "tolerance ({}). \n".format(
                        section, rate, c[section]["expectedrate"],
                        c[section]["absolutedifference"]))
            # test fails because rates are off
            elif math.fabs(rate - float(c[section]["expectedrate"])) > float(
                    c[section]["absolutedifference"]):
                sys.stderr.write(
                    "Test {} failed because the absolute difference "
                    "between the calculated convergence rate ({}) "
                    "and the expected convergence rate ({}) was greater "
                    "than tolerance ({}). \n".format(
                        section, rate, c[section]["expectedrate"],
                        c[section]["absolutedifference"]))
                returnvalue = 1
            # test fails because rates are nan or inf
            elif math.isnan(rate) or math.isinf(rate):
                sys.stderr.write(
                    "Test {} failed because calculated rate is ({})."
                    "Expected was ({}) with tolerance ({}). \n".format(
                        section, rate, c[section]["expectedrate"],
                        c[section]["absolutedifference"]))
                returnvalue = 1
            # if we are here, something unexpcted happened
            else:
                sys.stderr.write(
                    "Test {} failed for unknown reason with calculated rate ({}), "
                    "expected rate ({}) and tolerance ({}). \n".format(
                        section, rate, c[section]["expectedrate"],
                        c[section]["absolutedifference"]))
                returnvalue = 1

    return returnvalue
Beispiel #10
0
def test_parse3(dir):
    # Testing all sorts of escapes
    parsed = parse_ini_file(dir + "parse3.ini")
    assert(count_unescaped(parsed['a'], '|') == 0)
    assert(count_unescaped(parsed['c'], ',') == 3)
    assert(count_unescaped(parsed['d'], '"') == 2)
Beispiel #11
0
def test_parse2(dir):
    # A file that contains non-key-value data
    parsed = parse_ini_file(dir + "parse2.ini")['__local.conditionals']
    assert(len(parsed) == 2)
    assert(parsed['0'] == '{x} == {y}')
    assert(parsed['1'] == '{x} == {y}')