Exemplo n.º 1
0
def compile_test_runner(package):
    binary_name = os.path.basename(package)
    binary_name += '.test'
    runner = os.path.join('/tmp', package, '_test', binary_name)

    if os.path.exists(runner):
        os.remove(runner)

    run('nice go test -c -i -o {} {}'.format(runner, package), fail_exits=True, quiet=True)

    return runner
Exemplo n.º 2
0
def compile_and_run(package):
    logName = os.path.join('/tmp', package, 'out.txt')
    print('Testing {}'.format(package))

    with open(logName, 'w') as f:
        runner = compile_test_runner(package)
        out, rc = run('nice ' + runner, write_to=f, fail_ok=True, quiet=True)
        if rc:

            print('  FAIL: {}\n{}\n'.format(package, out[:800]))
    os.remove(runner)
    return rc
Exemplo n.º 3
0
def main(args):
    long_tests = {
        'state': 270,
        'mongo': 19,
    }
    start_time = datetime.now()
    try:
        install_out = run('go install  -v github.com/juju/juju/...')
    except subprocess.CalledProcessError as e:
        exit(e.returncode)
    output_filename = os.path.join(os.path.expanduser('~'),
                                   '.jujutestoutput.txt')
    rerun_filename = os.path.join(os.path.expanduser('~'),
                                  '.jujutestoutput_rerun.txt')

    print('Build duration:', datetime.now() - start_time)
    start_time = datetime.now()

    filename = None
    packages = []
    if args.changed:
        git_status = run('git status', quiet=True)
        # TODO: support new, deleted, modified, moved etc.
        for line in git_status.splitlines():
            mod = re.search('modified:\s+(.*)/.*?\.go$', line)
            if mod and mod.group(1) not in packages:
                packages.append(mod.group(1))
        filename = output_filename

        # Sort packages to do long tests last
        # TODO: long tests first, in parallel...
        packages = sorted(packages, key=lambda p: long_tests.get(p, 0))
        print(packages)

        with open(output_filename, 'w') as f:
            for package in packages:
                test_out, rc = run('GOMAXPROCS=32 go test github.com/juju/juju/' + package, write_to=f, fail_ok=True)
                if rc:
                    exit(rc)
        print('Test duration:', datetime.now() - start_time)
        start_time = datetime.now()
        exit(0)

    if (len(install_out) or args.force) and not args.rerun:
        if os.path.isfile(rerun_filename):
            os.remove(rerun_filename)

        filename = output_filename
        with open(output_filename, 'w') as f:
            test_out, rc = run('go test ./...', write_to=f, fail_ok=True)
            print('Test duration:', datetime.now() - start_time)
            start_time = datetime.now()

    else:
        # Don't have a new binary, so just re-run tests
        if os.path.isfile(rerun_filename):
            filename = rerun_filename
        elif os.path.isfile(output_filename):
            filename = output_filename

    if filename is None:
        return

    with open(filename) as f:
        test_out = f.readlines()

    unrecoverable = False
    re_run = []
    for line in test_out:
        if(not re.search('FAIL\s+github.*\n', line) and
           not line.startswith('# github.com')):
            continue

        if re.search('failed\]\s*$', line):
            # Build failed or setup failed. No point re-running, but need
            # to report
            unrecoverable = True
        elif line.startswith('# github.com'):
            unrecoverable = True
            print(line)
        else:
            s = re.search('FAIL\s+(github.com.*)\s[\d\.]+s\s*$', line)
            re_run.append(s.group(1))

    if len(re_run) == 0:
        return

    print('-' * 20, 'Failing packages', '-' * 20)
    for package in re_run:
        print(' ', package)

    if not unrecoverable:
        print('Re-running failed tests...')
        with open(rerun_filename, 'w') as f:
            for package in re_run:
                out, rc = run('go test ' + package, write_to=f, fail_ok=True)
        print('Re-run duration:', datetime.now() - start_time)
        start_time = datetime.now()
    else:
        print('Some failures are unrecoverable... not re-running.')
Exemplo n.º 4
0
def maas(cmd):
    print('maas maas ' + cmd)
    out = run('maas maas ' + cmd, quiet=True)
    return json.loads(out)
Exemplo n.º 5
0
def test(args, db):
    long_tests = {
        'api': 9.926,
        'api/firewaller': 8.735,
        'api/provisioner': 10.475,
        'api/uniter': 34.885,
        'api/upgrader': 5.926,
        'apiserver': 141.927,
        'apiserver/client': 59.812,
        'apiserver/common': 10.367,
        'apiserver/firewaller': 5.664,
        'apiserver/metricsender': 5.502,
        'apiserver/provisioner': 20.643,
        'apiserver/storageprovisioner': 8.943,
        'apiserver/uniter': 54.406,
        'apiserver/upgrader': 7.856,
        'bzr': 6.002,
        'cmd/juju/action': 18.114,
        'cmd/juju/commands': 70.726,
        'cmd/juju/status': 17.68,
        'cmd/juju/system': 10.319,
        'cmd/jujud/reboot': 9.227,
        'container/lxc': 5.231,
        'downloader': 5.081,
        'environs/bootstrap': 47.446,
        'environs/configstore': 5.096,
        'featuretests': 53.464,
        'mongo': 19.99,
        'provider/ec2': 6.181,
        'provider/maas': 8.041,
        'provider/openstack': 36.273,
        'state': 230.395,
        'state/backups': 12.262,
        'state/presence': 5.551,
        'upgrades': 8.62,
        'utils/ssh': 5.196,
        'worker/addresser': 41.869,
        'worker/firewaller': 7.677,
        'worker/machiner': 11.263,
        'worker/peergrouper': 10.333,
        'worker/provisioner': 55.694,
        'worker/reboot': 10.65,
        'worker/uniter': 227.597,
        'worker/uniter/charm': 5.773,
        'worker/uniter/filter': 9.154,
        'worker/uniter/runner': 18.348,
    }

    output_filename = os.path.join(os.path.expanduser('~'),
                                   '.jujutestoutput.txt')
    rerun_filename = os.path.join(os.path.expanduser('~'),
                                  '.jujutestoutput_rerun.txt')

    start_time = datetime.now()
    try:
        install_out = run('go install  -v github.com/juju/juju/...')
    except subprocess.CalledProcessError as e:
        for line in e.output.splitlines():
            if not line.startswith(JUJU_VCS):
                print('!', line)
        return e.returncode

    print('Build duration:', datetime.now() - start_time)

    start_time = datetime.now()

    filename = None
    packages = []
    if args.fast:
        packages = package_list()
        # Start long tests first
        packages = sorted(packages, key=lambda p: long_tests.get(p, 0), reverse=True)
        packages.remove('state')

        rc, failures = test_packages(packages)

        # Always run state last because it just doesn't like the company
        state_rc, state_failures = test_packages(['state'])

        print('Test duration:', datetime.now() - start_time)
        if rc or state_rc:
            filename = '/tmp/all_failures.txt'
            with open(filename, 'w') as f:
                for p in failures + state_failures:
                    logName = os.path.join('/tmp', p, 'out.txt')
                    with open(logName) as i:
                        f.write(i.read())
                        f.writelines([
                            '\nFAIL {} 0.0123456s\n\n'.format(p),
                            '# End of {}\n'.format(p),
                            '<>' * 40 + '\n\n',
                        ])

    elif args.changed:
        print('Testing changed packages')
        git_status = run('git status', quiet=True)
        # TODO: support new, deleted, modified, moved etc.
        packages_with_tests = package_list()
        for line in git_status.splitlines():
            mod = re.search('modified:\s+(.*)/.*?\.go$', line)
            if mod and mod.group(1) not in packages and mod.group(1) in packages_with_tests:
                packages.append(mod.group(1))

        # Sort packages to do long tests last
        # TODO: long tests first, in parallel...
        packages = sorted(packages, key=lambda p: long_tests.get(p, 0))
        print(packages)

        with open(output_filename, 'w') as f:
            for package in packages:
                package = 'github.com/juju/juju/' + package

                runner = compile_test_runner(package)
                run(runner, write_to=f, fail_ok=True)
                os.remove(runner)
                f.write(">>end of package>>" + package + "\n")

        print('Test duration:', datetime.now() - start_time)
        filename = output_filename

    elif (len(install_out) or args.force) and not args.rerun:
        print('Testing everything')
        if os.path.isfile(rerun_filename):
            os.remove(rerun_filename)

        filename = output_filename
        with open(output_filename, 'w') as f:
            run('go test -i ./...', write_to=f, fail_exits=True)
            run('go test ./...', write_to=f, fail_ok=True)
            print('Test duration:', datetime.now() - start_time)
            start_time = datetime.now()

    else:
        print('Re-running failures from last test')
        # Don't have a new binary, so just re-run tests

        if db.get('empty'):
            del(db['empty'])
            # No old test DB, so parse the last output
            if os.path.isfile(rerun_filename):
                filename = rerun_filename
            elif os.path.isfile(output_filename):
                filename = output_filename

        filename = output_filename

    unrecoverable = False
    if filename is not None:
        db['failures'] = {}
        with open(filename) as f:
            test_out = f.readlines()

        re_run = []
        re_run_tests = []
        for line in test_out:
            s = re.search('>>end of package>>(.*)$', line)
            if s and len(re_run_tests):
                db['failures'][s.group(1)] = re_run_tests
                re_run.append({
                    'package': s.group(1),
                    'tests': re_run_tests,
                })
                re_run_tests = []

            if line.startswith('FAIL:'):
                s = re.search('^FAIL:\s+.*:\d+:\s+(.*)\s*$', line)
                if s:
                    print(s.group(1))
                    re_run_tests.append(s.group(1))
                else:
                    print("--- Error: don't know how to handle this line:")
                    print(line)
                    print("Aborting...")
                    return 1
            if not re.search('FAIL\s+github.*', line):
                continue

            s = re.search('FAIL\s+(.+?)\s+\[.*? failed\]', line)
            if s:
                unrecoverable = True
                db['failures'][s.group(1)] = ['.*']
                re_run_tests = []
                print(line)
            else:
                s = re.search('FAIL\s+(github.com.*)\s[\d\.]+s\s*$', line)
                db['failures'][s.group(1)] = re_run_tests
                re_run.append({
                    'package': s.group(1),
                    'tests': re_run_tests,
                })
                re_run_tests = []
    print('-' * 40 + 'failures:' + '-' * 40)
    pprint(db)

    if len(db['failures'].keys()) == 0:
        return 0

    if not unrecoverable:
        print('Re-running failed tests...')
        failed = False
        remove_packages = []
        with open(rerun_filename, 'w') as f:
            for package, tests in db['failures'].items():
                failures = []
                name_search = re.search(r'github.com/juju/juju/(.*)', package)

                if name_search:
                    runner = compile_test_runner(package)
                    if len(tests):
                        for test in tests:
                            print(package, test)
                            out, rc = run('{} -check.f ^{}$'.format(runner, test), write_to=f, fail_ok=True, quiet=True)

                            if rc:
                                print(out[:400])
                                print('\n' + '-' * 80)

                            if args.stop_on_failure and rc:
                                os.remove(runner)
                                return rc

                            failed = failed or rc != 0
                            if rc:
                                failures.append(test)
                    else:
                        print(package)
                        out, rc = run(runner, write_to=f, fail_ok=True, quiet=True)

                        if rc:
                            print(out[:400])

                        if args.stop_on_failure and rc:
                            os.remove(runner)
                            return rc

                        failed = failed or rc != 0

                    os.remove(runner)

                if len(failures):
                    db['failures'][package] = failures
                elif failed:
                    db['failures'][package] = []
                elif package in db['failures']:
                    remove_packages.append(package)

        # Remove empty lists from database
        for package in remove_packages:
            del(db['failures'][package])

        print('Re-run duration:', datetime.now() - start_time)

        return failed
    else:
        print('Some failures are unrecoverable... not re-running.')

    return 0