Beispiel #1
0
def check_meson_commands_work():
    global backend, compile_commands, test_commands, install_commands
    testdir = PurePath('test cases', 'common', '1 trivial').as_posix()
    meson_commands = mesonlib.python_command + [get_meson_script()]
    with AutoDeletedDir(tempfile.mkdtemp(prefix='b ', dir='.')) as build_dir:
        print('Checking that configuring works...')
        gen_cmd = meson_commands + [testdir, build_dir] + backend_flags
        pc, o, e = Popen_safe(gen_cmd)
        if pc.returncode != 0:
            raise RuntimeError('Failed to configure {!r}:\n{}\n{}'.format(
                testdir, e, o))
        print('Checking that building works...')
        dir_args = get_backend_args_for_dir(backend, build_dir)
        pc, o, e = Popen_safe(compile_commands + dir_args, cwd=build_dir)
        if pc.returncode != 0:
            raise RuntimeError('Failed to build {!r}:\n{}\n{}'.format(
                testdir, e, o))
        print('Checking that testing works...')
        pc, o, e = Popen_safe(test_commands, cwd=build_dir)
        if pc.returncode != 0:
            raise RuntimeError('Failed to test {!r}:\n{}\n{}'.format(
                testdir, e, o))
        if install_commands:
            print('Checking that installing works...')
            pc, o, e = Popen_safe(install_commands, cwd=build_dir)
            if pc.returncode != 0:
                raise RuntimeError('Failed to install {!r}:\n{}\n{}'.format(
                    testdir, e, o))
Beispiel #2
0
def check_meson_commands_work():
    global backend, meson_command, compile_commands, test_commands, install_commands
    testdir = 'test cases/common/1 trivial'
    with AutoDeletedDir(tempfile.mkdtemp(prefix='b ', dir='.')) as build_dir:
        print('Checking that configuring works...')
        gen_cmd = [sys.executable, meson_command, testdir, build_dir
                   ] + backend_flags
        pc, o, e = Popen_safe(gen_cmd)
        if pc.returncode != 0:
            raise RuntimeError('Failed to configure {!r}:\n{}\n{}'.format(
                testdir, e, o))
        print('Checking that building works...')
        dir_args = get_backend_args_for_dir(backend, build_dir)
        pc, o, e = Popen_safe(compile_commands + dir_args, cwd=build_dir)
        if pc.returncode != 0:
            raise RuntimeError('Failed to build {!r}:\n{}\n{}'.format(
                testdir, e, o))
        print('Checking that testing works...')
        pc, o, e = Popen_safe(test_commands, cwd=build_dir)
        if pc.returncode != 0:
            raise RuntimeError('Failed to test {!r}:\n{}\n{}'.format(
                testdir, e, o))
        if install_commands:
            print('Checking that installing works...')
            pc, o, e = Popen_safe(install_commands, cwd=build_dir)
            if pc.returncode != 0:
                raise RuntimeError('Failed to install {!r}:\n{}\n{}'.format(
                    testdir, e, o))
Beispiel #3
0
def check_meson_commands_work():
    global backend, meson_command, compile_commands, test_commands, install_commands
    testdir = 'test cases/common/1 trivial'
    with AutoDeletedDir(tempfile.mkdtemp(prefix='b ', dir='.')) as build_dir:
        print('Checking that configuring works...')
        gen_cmd = mesonlib.meson_command + [testdir, build_dir] + backend_flags
        pc, o, e = Popen_safe(gen_cmd)
        if pc.returncode != 0:
            raise RuntimeError('Failed to configure {!r}:\n{}\n{}'.format(testdir, e, o))
        print('Checking that building works...')
        dir_args = get_backend_args_for_dir(backend, build_dir)
        pc, o, e = Popen_safe(compile_commands + dir_args, cwd=build_dir)
        if pc.returncode != 0:
            raise RuntimeError('Failed to build {!r}:\n{}\n{}'.format(testdir, e, o))
        print('Checking that testing works...')
        pc, o, e = Popen_safe(test_commands, cwd=build_dir)
        if pc.returncode != 0:
            raise RuntimeError('Failed to test {!r}:\n{}\n{}'.format(testdir, e, o))
        if install_commands:
            print('Checking that installing works...')
            pc, o, e = Popen_safe(install_commands, cwd=build_dir)
            if pc.returncode != 0:
                raise RuntimeError('Failed to install {!r}:\n{}\n{}'.format(testdir, e, o))
Beispiel #4
0
def _run_test(testdir, test_build_dir, install_dir, extra_args, compiler,
              backend, flags, commands, should_fail):
    compile_commands, clean_commands, install_commands, uninstall_commands = commands
    test_args = parse_test_args(testdir)
    gen_start = time.time()
    # Configure in-process
    if pass_prefix_to_test(testdir):
        gen_args = ['--prefix', '/usr']
    else:
        gen_args = []
    if pass_libdir_to_test(testdir):
        gen_args += ['--libdir', 'lib']
    gen_args += [testdir, test_build_dir] + flags + test_args + extra_args
    nativefile = os.path.join(testdir, 'nativefile.ini')
    if os.path.exists(nativefile):
        gen_args.extend(['--native-file', nativefile])
    crossfile = os.path.join(testdir, 'crossfile.ini')
    if os.path.exists(crossfile):
        gen_args.extend(['--cross-file', crossfile])
    (returncode, stdo, stde) = run_configure(gen_args)
    try:
        logfile = Path(test_build_dir, 'meson-logs', 'meson-log.txt')
        mesonlog = logfile.open(errors='ignore', encoding='utf-8').read()
    except Exception:
        mesonlog = no_meson_log_msg
    gen_time = time.time() - gen_start
    if should_fail == 'meson':
        if returncode == 1:
            return TestResult('', BuildStep.configure, stdo, stde, mesonlog,
                              gen_time)
        elif returncode != 0:
            return TestResult(
                'Test exited with unexpected status {}'.format(returncode),
                BuildStep.configure, stdo, stde, mesonlog, gen_time)
        else:
            return TestResult('Test that should have failed succeeded',
                              BuildStep.configure, stdo, stde, mesonlog,
                              gen_time)
    if returncode != 0:
        return TestResult('Generating the build system failed.',
                          BuildStep.configure, stdo, stde, mesonlog, gen_time)
    builddata = build.load(test_build_dir)
    # Touch the meson.build file to force a regenerate so we can test that
    # regeneration works before a build is run.
    ensure_backend_detects_changes(backend)
    os.utime(os.path.join(testdir, 'meson.build'))
    # Build with subprocess
    dir_args = get_backend_args_for_dir(backend, test_build_dir)
    build_start = time.time()
    pc, o, e = Popen_safe(compile_commands + dir_args, cwd=test_build_dir)
    build_time = time.time() - build_start
    stdo += o
    stde += e
    if should_fail == 'build':
        if pc.returncode != 0:
            return TestResult('', BuildStep.build, stdo, stde, mesonlog,
                              gen_time)
        return TestResult('Test that should have failed to build succeeded',
                          BuildStep.build, stdo, stde, mesonlog, gen_time)
    if pc.returncode != 0:
        return TestResult('Compiling source code failed.', BuildStep.build,
                          stdo, stde, mesonlog, gen_time, build_time)
    # Touch the meson.build file to force a regenerate so we can test that
    # regeneration works after a build is complete.
    ensure_backend_detects_changes(backend)
    os.utime(os.path.join(testdir, 'meson.build'))
    test_start = time.time()
    # Test in-process
    (returncode, tstdo, tstde, test_log) = run_test_inprocess(test_build_dir)
    test_time = time.time() - test_start
    stdo += tstdo
    stde += tstde
    mesonlog += test_log
    if should_fail == 'test':
        if returncode != 0:
            return TestResult('', BuildStep.test, stdo, stde, mesonlog,
                              gen_time)
        return TestResult(
            'Test that should have failed to run unit tests succeeded',
            BuildStep.test, stdo, stde, mesonlog, gen_time)
    if returncode != 0:
        return TestResult('Running unit tests failed.', BuildStep.test, stdo,
                          stde, mesonlog, gen_time, build_time, test_time)
    # Do installation, if the backend supports it
    if install_commands:
        env = os.environ.copy()
        env['DESTDIR'] = install_dir
        # Install with subprocess
        pi, o, e = Popen_safe(install_commands, cwd=test_build_dir, env=env)
        stdo += o
        stde += e
        if pi.returncode != 0:
            return TestResult('Running install failed.', BuildStep.install,
                              stdo, stde, mesonlog, gen_time, build_time,
                              test_time)
    # Clean with subprocess
    env = os.environ.copy()
    pi, o, e = Popen_safe(clean_commands + dir_args,
                          cwd=test_build_dir,
                          env=env)
    stdo += o
    stde += e
    if pi.returncode != 0:
        return TestResult('Running clean failed.', BuildStep.clean, stdo, stde,
                          mesonlog, gen_time, build_time, test_time)
    if not install_commands:
        return TestResult('', BuildStep.install, '', '', mesonlog, gen_time,
                          build_time, test_time)
    return TestResult(
        validate_install(testdir, install_dir, compiler,
                         builddata.environment), BuildStep.validate, stdo,
        stde, mesonlog, gen_time, build_time, test_time)
Beispiel #5
0
def _run_test(testdir, test_build_dir, install_dir, extra_args, compiler, backend, flags, commands, should_fail):
    compile_commands, clean_commands, install_commands, uninstall_commands = commands
    test_args = parse_test_args(testdir)
    gen_start = time.time()
    # Configure in-process
    if pass_prefix_to_test(testdir):
        gen_args = ['--prefix', '/usr']
    else:
        gen_args = []
    if pass_libdir_to_test(testdir):
        gen_args += ['--libdir', 'lib']
    gen_args += [testdir, test_build_dir] + flags + test_args + extra_args
    (returncode, stdo, stde) = run_configure(meson_command, gen_args)
    try:
        logfile = os.path.join(test_build_dir, 'meson-logs/meson-log.txt')
        with open(logfile, errors='ignore') as f:
            mesonlog = f.read()
    except Exception:
        mesonlog = no_meson_log_msg
    gen_time = time.time() - gen_start
    if should_fail == 'meson':
        if returncode == 1:
            return TestResult('', BuildStep.configure, stdo, stde, mesonlog, gen_time)
        elif returncode != 0:
            return TestResult('Test exited with unexpected status {}'.format(returncode), BuildStep.configure, stdo, stde, mesonlog, gen_time)
        else:
            return TestResult('Test that should have failed succeeded', BuildStep.configure, stdo, stde, mesonlog, gen_time)
    if returncode != 0:
        return TestResult('Generating the build system failed.', BuildStep.configure, stdo, stde, mesonlog, gen_time)
    # Touch the meson.build file to force a regenerate so we can test that
    # regeneration works before a build is run.
    ensure_backend_detects_changes(backend)
    os.utime(os.path.join(testdir, 'meson.build'))
    # Build with subprocess
    dir_args = get_backend_args_for_dir(backend, test_build_dir)
    build_start = time.time()
    pc, o, e = Popen_safe(compile_commands + dir_args, cwd=test_build_dir)
    build_time = time.time() - build_start
    stdo += o
    stde += e
    if should_fail == 'build':
        if pc.returncode != 0:
            return TestResult('', BuildStep.build, stdo, stde, mesonlog, gen_time)
        return TestResult('Test that should have failed to build succeeded', BuildStep.build, stdo, stde, mesonlog, gen_time)
    if pc.returncode != 0:
        return TestResult('Compiling source code failed.', BuildStep.build, stdo, stde, mesonlog, gen_time, build_time)
    # Touch the meson.build file to force a regenerate so we can test that
    # regeneration works after a build is complete.
    ensure_backend_detects_changes(backend)
    os.utime(os.path.join(testdir, 'meson.build'))
    test_start = time.time()
    # Test in-process
    (returncode, tstdo, tstde, test_log) = run_test_inprocess(test_build_dir)
    test_time = time.time() - test_start
    stdo += tstdo
    stde += tstde
    mesonlog += test_log
    if should_fail == 'test':
        if returncode != 0:
            return TestResult('', BuildStep.test, stdo, stde, mesonlog, gen_time)
        return TestResult('Test that should have failed to run unit tests succeeded', BuildStep.test, stdo, stde, mesonlog, gen_time)
    if returncode != 0:
        return TestResult('Running unit tests failed.', BuildStep.test, stdo, stde, mesonlog, gen_time, build_time, test_time)
    # Do installation, if the backend supports it
    if install_commands:
        env = os.environ.copy()
        env['DESTDIR'] = install_dir
        # Install with subprocess
        pi, o, e = Popen_safe(install_commands, cwd=test_build_dir, env=env)
        stdo += o
        stde += e
        if pi.returncode != 0:
            return TestResult('Running install failed.', BuildStep.install, stdo, stde, mesonlog, gen_time, build_time, test_time)
    # Clean with subprocess
    env = os.environ.copy()
    pi, o, e = Popen_safe(clean_commands + dir_args, cwd=test_build_dir, env=env)
    stdo += o
    stde += e
    if pi.returncode != 0:
        return TestResult('Running clean failed.', BuildStep.clean, stdo, stde, mesonlog, gen_time, build_time, test_time)
    if not install_commands:
        return TestResult('', BuildStep.install, '', '', mesonlog, gen_time, build_time, test_time)
    return TestResult(validate_install(testdir, install_dir, compiler), BuildStep.validate, stdo, stde, mesonlog, gen_time, build_time, test_time)
Beispiel #6
0
def _run_test(testdir, test_build_dir, install_dir, extra_args, compiler, backend, flags, commands, should_fail):
    compile_commands, clean_commands, install_commands, uninstall_commands = commands
    test_args = parse_test_args(testdir)
    gen_start = time.time()
    # Configure in-process
    gen_command = [meson_command, '--prefix', '/usr', '--libdir', 'lib', testdir, test_build_dir]\
        + flags + test_args + extra_args
    (returncode, stdo, stde) = run_configure_inprocess(gen_command)
    try:
        logfile = os.path.join(test_build_dir, 'meson-logs/meson-log.txt')
        with open(logfile, errors='ignore') as f:
            mesonlog = f.read()
    except Exception:
        mesonlog = no_meson_log_msg
    gen_time = time.time() - gen_start
    if should_fail == 'meson':
        if returncode != 0:
            return TestResult('', BuildStep.configure, stdo, stde, mesonlog, gen_time)
        return TestResult('Test that should have failed succeeded', BuildStep.configure, stdo, stde, mesonlog, gen_time)
    if returncode != 0:
        return TestResult('Generating the build system failed.', BuildStep.configure, stdo, stde, mesonlog, gen_time)
    # Touch the meson.build file to force a regenerate so we can test that
    # regeneration works before a build is run.
    ensure_backend_detects_changes(backend)
    os.utime(os.path.join(testdir, 'meson.build'))
    # Build with subprocess
    dir_args = get_backend_args_for_dir(backend, test_build_dir)
    build_start = time.time()
    pc, o, e = Popen_safe(compile_commands + dir_args, cwd=test_build_dir)
    build_time = time.time() - build_start
    stdo += o
    stde += e
    if should_fail == 'build':
        if pc.returncode != 0:
            return TestResult('', BuildStep.build, stdo, stde, mesonlog, gen_time)
        return TestResult('Test that should have failed to build succeeded', BuildStep.build, stdo, stde, mesonlog, gen_time)
    if pc.returncode != 0:
        return TestResult('Compiling source code failed.', BuildStep.build, stdo, stde, mesonlog, gen_time, build_time)
    # Touch the meson.build file to force a regenerate so we can test that
    # regeneration works after a build is complete.
    ensure_backend_detects_changes(backend)
    os.utime(os.path.join(testdir, 'meson.build'))
    test_start = time.time()
    # Test in-process
    (returncode, tstdo, tstde, test_log) = run_test_inprocess(test_build_dir)
    test_time = time.time() - test_start
    stdo += tstdo
    stde += tstde
    mesonlog += test_log
    if should_fail == 'test':
        if returncode != 0:
            return TestResult('', BuildStep.test, stdo, stde, mesonlog, gen_time)
        return TestResult('Test that should have failed to run unit tests succeeded', BuildStep.test, stdo, stde, mesonlog, gen_time)
    if returncode != 0:
        return TestResult('Running unit tests failed.', BuildStep.test, stdo, stde, mesonlog, gen_time, build_time, test_time)
    # Do installation, if the backend supports it
    if install_commands:
        env = os.environ.copy()
        env['DESTDIR'] = install_dir
        # Install with subprocess
        pi, o, e = Popen_safe(install_commands, cwd=test_build_dir, env=env)
        stdo += o
        stde += e
        if pi.returncode != 0:
            return TestResult('Running install failed.', BuildStep.install, stdo, stde, mesonlog, gen_time, build_time, test_time)
    # Clean with subprocess
    env = os.environ.copy()
    pi, o, e = Popen_safe(clean_commands + dir_args, cwd=test_build_dir, env=env)
    stdo += o
    stde += e
    if pi.returncode != 0:
        return TestResult('Running clean failed.', BuildStep.clean, stdo, stde, mesonlog, gen_time, build_time, test_time)
    if not install_commands:
        return TestResult('', BuildStep.install, '', '', mesonlog, gen_time, build_time, test_time)
    return TestResult(validate_install(testdir, install_dir, compiler), BuildStep.validate, stdo, stde, mesonlog, gen_time, build_time, test_time)
Beispiel #7
0
def _run_test(test: TestDef, test_build_dir: str, install_dir: str, extra_args,
              compiler, backend, flags, commands, should_fail):
    compile_commands, clean_commands, install_commands, uninstall_commands = commands
    gen_start = time.time()
    # Configure in-process
    gen_args = []  # type: T.List[str]
    if 'prefix' not in test.do_not_set_opts:
        gen_args += ['--prefix', 'x:/usr'
                     ] if mesonlib.is_windows() else ['--prefix', '/usr']
    if 'libdir' not in test.do_not_set_opts:
        gen_args += ['--libdir', 'lib']
    gen_args += [test.path.as_posix(), test_build_dir] + flags + extra_args
    nativefile = test.path / 'nativefile.ini'
    crossfile = test.path / 'crossfile.ini'
    if nativefile.exists():
        gen_args.extend(['--native-file', nativefile.as_posix()])
    if crossfile.exists():
        gen_args.extend(['--cross-file', crossfile.as_posix()])
    (returncode, stdo, stde) = run_configure(gen_args, env=test.env)
    try:
        logfile = Path(test_build_dir, 'meson-logs', 'meson-log.txt')
        mesonlog = logfile.open(errors='ignore', encoding='utf-8').read()
    except Exception:
        mesonlog = no_meson_log_msg
    cicmds = run_ci_commands(mesonlog)
    testresult = TestResult(cicmds)
    testresult.add_step(BuildStep.configure, stdo, stde, mesonlog,
                        time.time() - gen_start)
    if should_fail == 'meson':
        if returncode == 1:
            return testresult
        elif returncode != 0:
            testresult.fail(
                'Test exited with unexpected status {}.'.format(returncode))
            return testresult
        else:
            testresult.fail('Test that should have failed succeeded.')
            return testresult
    if returncode != 0:
        testresult.fail('Generating the build system failed.')
        return testresult
    builddata = build.load(test_build_dir)
    dir_args = get_backend_args_for_dir(backend, test_build_dir)

    # Build with subprocess
    def build_step():
        build_start = time.time()
        pc, o, e = Popen_safe(compile_commands + dir_args, cwd=test_build_dir)
        testresult.add_step(BuildStep.build, o, e, '',
                            time.time() - build_start)
        if should_fail == 'build':
            if pc.returncode != 0:
                raise testresult
            testresult.fail('Test that should have failed to build succeeded.')
            raise testresult
        if pc.returncode != 0:
            testresult.fail('Compiling source code failed.')
            raise testresult

    # Touch the meson.build file to force a regenerate
    def force_regenerate():
        ensure_backend_detects_changes(backend)
        os.utime(str(test.path / 'meson.build'))

    # just test building
    build_step()

    # test that regeneration works for build step
    force_regenerate()
    build_step()  # TBD: assert nothing gets built after the regenerate?

    # test that regeneration works for test step
    force_regenerate()

    # Test in-process
    test_start = time.time()
    (returncode, tstdo, tstde, test_log) = run_test_inprocess(test_build_dir)
    testresult.add_step(BuildStep.test, tstdo, tstde, test_log,
                        time.time() - test_start)
    if should_fail == 'test':
        if returncode != 0:
            return testresult
        testresult.fail(
            'Test that should have failed to run unit tests succeeded.')
        return testresult
    if returncode != 0:
        testresult.fail('Running unit tests failed.')
        return testresult

    # Do installation, if the backend supports it
    if install_commands:
        env = os.environ.copy()
        env['DESTDIR'] = install_dir
        # Install with subprocess
        pi, o, e = Popen_safe(install_commands, cwd=test_build_dir, env=env)
        testresult.add_step(BuildStep.install, o, e)
        if pi.returncode != 0:
            testresult.fail('Running install failed.')
            return testresult

    # Clean with subprocess
    env = os.environ.copy()
    pi, o, e = Popen_safe(clean_commands + dir_args,
                          cwd=test_build_dir,
                          env=env)
    testresult.add_step(BuildStep.clean, o, e)
    if pi.returncode != 0:
        testresult.fail('Running clean failed.')
        return testresult

    # Validate installed files
    testresult.add_step(BuildStep.install, '', '')
    if not install_commands:
        return testresult
    install_msg = validate_install(test, Path(install_dir), compiler,
                                   builddata.environment)
    if install_msg:
        testresult.fail('\n' + install_msg)
        return testresult

    return testresult
Beispiel #8
0
def _run_test(testdir, test_build_dir, install_dir, extra_args, compiler,
              backend, flags, commands, should_fail):
    compile_commands, clean_commands, install_commands, uninstall_commands = commands
    test_args = parse_test_args(testdir)
    gen_start = time.time()
    setup_env = None
    # Configure in-process
    if pass_prefix_to_test(testdir):
        gen_args = ['--prefix', 'x:/usr'
                    ] if mesonlib.is_windows() else ['--prefix', '/usr']
    else:
        gen_args = []
    if pass_libdir_to_test(testdir):
        gen_args += ['--libdir', 'lib']
    gen_args += [testdir, test_build_dir] + flags + test_args + extra_args
    nativefile = os.path.join(testdir, 'nativefile.ini')
    if os.path.exists(nativefile):
        gen_args.extend(['--native-file', nativefile])
    crossfile = os.path.join(testdir, 'crossfile.ini')
    if os.path.exists(crossfile):
        gen_args.extend(['--cross-file', crossfile])
    setup_env_file = os.path.join(testdir, 'setup_env.json')
    if os.path.exists(setup_env_file):
        setup_env = os.environ.copy()
        with open(setup_env_file, 'r') as fp:
            data = json.load(fp)
            for key, val in data.items():
                val = val.replace('@ROOT@', os.path.abspath(testdir))
                setup_env[key] = val
    (returncode, stdo, stde) = run_configure(gen_args, env=setup_env)
    try:
        logfile = Path(test_build_dir, 'meson-logs', 'meson-log.txt')
        mesonlog = logfile.open(errors='ignore', encoding='utf-8').read()
    except Exception:
        mesonlog = no_meson_log_msg
    cicmds = run_ci_commands(mesonlog)
    testresult = TestResult(cicmds)
    testresult.add_step(BuildStep.configure, stdo, stde, mesonlog,
                        time.time() - gen_start)
    if should_fail == 'meson':
        if returncode == 1:
            return testresult
        elif returncode != 0:
            testresult.fail(
                'Test exited with unexpected status {}.'.format(returncode))
            return testresult
        else:
            testresult.fail('Test that should have failed succeeded.')
            return testresult
    if returncode != 0:
        testresult.fail('Generating the build system failed.')
        return testresult
    builddata = build.load(test_build_dir)
    # Touch the meson.build file to force a regenerate so we can test that
    # regeneration works before a build is run.
    ensure_backend_detects_changes(backend)
    os.utime(os.path.join(testdir, 'meson.build'))
    # Build with subprocess
    dir_args = get_backend_args_for_dir(backend, test_build_dir)
    build_start = time.time()
    pc, o, e = Popen_safe(compile_commands + dir_args, cwd=test_build_dir)
    testresult.add_step(BuildStep.build, o, e, '', time.time() - build_start)
    if should_fail == 'build':
        if pc.returncode != 0:
            return testresult
        testresult.fail('Test that should have failed to build succeeded.')
        return testresult
    if pc.returncode != 0:
        testresult.fail('Compiling source code failed.')
        return testresult
    # Touch the meson.build file to force a regenerate so we can test that
    # regeneration works after a build is complete.
    ensure_backend_detects_changes(backend)
    os.utime(os.path.join(testdir, 'meson.build'))
    test_start = time.time()
    # Test in-process
    (returncode, tstdo, tstde, test_log) = run_test_inprocess(test_build_dir)
    testresult.add_step(BuildStep.test, tstdo, tstde, test_log,
                        time.time() - test_start)
    if should_fail == 'test':
        if returncode != 0:
            return testresult
        testresult.fail(
            'Test that should have failed to run unit tests succeeded.')
        return testresult
    if returncode != 0:
        testresult.fail('Running unit tests failed.')
        return testresult
    # Do installation, if the backend supports it
    if install_commands:
        env = os.environ.copy()
        env['DESTDIR'] = install_dir
        # Install with subprocess
        pi, o, e = Popen_safe(install_commands, cwd=test_build_dir, env=env)
        testresult.add_step(BuildStep.install, o, e)
        if pi.returncode != 0:
            testresult.fail('Running install failed.')
            return testresult

    # Clean with subprocess
    env = os.environ.copy()
    pi, o, e = Popen_safe(clean_commands + dir_args,
                          cwd=test_build_dir,
                          env=env)
    testresult.add_step(BuildStep.clean, o, e)
    if pi.returncode != 0:
        testresult.fail('Running clean failed.')
        return testresult

    # Validate installed files
    testresult.add_step(BuildStep.install, '', '')
    if not install_commands:
        return testresult
    install_msg = validate_install(testdir, install_dir, compiler,
                                   builddata.environment)
    if install_msg:
        testresult.fail(install_msg)
        return testresult

    return testresult