def compile_test_runner(package):
binary_name = os.path.basename(package)
binary_name += '.test'
runner = os.path.join('/tmp', package, '_test', binary_name)
if os.path.exists(runner):
os.remove(runner)
run('nice go test -c -i -o {} {}'.format(runner, package), fail_exits=True, quiet=True)
return runner
def compile_and_run(package):
logName = os.path.join('/tmp', package, 'out.txt')
print('Testing {}'.format(package))
with open(logName, 'w') as f:
runner = compile_test_runner(package)
out, rc = run('nice ' + runner, write_to=f, fail_ok=True, quiet=True)
if rc:
print(' FAIL: {}\n{}\n'.format(package, out[:800]))
os.remove(runner)
return rc
def main(args):
long_tests = {
'state': 270,
'mongo': 19,
}
start_time = datetime.now()
try:
install_out = run('go install -v github.com/juju/juju/...')
except subprocess.CalledProcessError as e:
exit(e.returncode)
output_filename = os.path.join(os.path.expanduser('~'),
'.jujutestoutput.txt')
rerun_filename = os.path.join(os.path.expanduser('~'),
'.jujutestoutput_rerun.txt')
print('Build duration:', datetime.now() - start_time)
start_time = datetime.now()
filename = None
packages = []
if args.changed:
git_status = run('git status', quiet=True)
# TODO: support new, deleted, modified, moved etc.
for line in git_status.splitlines():
mod = re.search('modified:\s+(.*)/.*?\.go$', line)
if mod and mod.group(1) not in packages:
packages.append(mod.group(1))
filename = output_filename
# Sort packages to do long tests last
# TODO: long tests first, in parallel...
packages = sorted(packages, key=lambda p: long_tests.get(p, 0))
print(packages)
with open(output_filename, 'w') as f:
for package in packages:
test_out, rc = run('GOMAXPROCS=32 go test github.com/juju/juju/' + package, write_to=f, fail_ok=True)
if rc:
exit(rc)
print('Test duration:', datetime.now() - start_time)
start_time = datetime.now()
exit(0)
if (len(install_out) or args.force) and not args.rerun:
if os.path.isfile(rerun_filename):
os.remove(rerun_filename)
filename = output_filename
with open(output_filename, 'w') as f:
test_out, rc = run('go test ./...', write_to=f, fail_ok=True)
print('Test duration:', datetime.now() - start_time)
start_time = datetime.now()
else:
# Don't have a new binary, so just re-run tests
if os.path.isfile(rerun_filename):
filename = rerun_filename
elif os.path.isfile(output_filename):
filename = output_filename
if filename is None:
return
with open(filename) as f:
test_out = f.readlines()
unrecoverable = False
re_run = []
for line in test_out:
if(not re.search('FAIL\s+github.*\n', line) and
not line.startswith('# github.com')):
continue
if re.search('failed\]\s*$', line):
# Build failed or setup failed. No point re-running, but need
# to report
unrecoverable = True
elif line.startswith('# github.com'):
unrecoverable = True
print(line)
else:
s = re.search('FAIL\s+(github.com.*)\s[\d\.]+s\s*$', line)
re_run.append(s.group(1))
if len(re_run) == 0:
return
print('-' * 20, 'Failing packages', '-' * 20)
for package in re_run:
print(' ', package)
if not unrecoverable:
print('Re-running failed tests...')
with open(rerun_filename, 'w') as f:
for package in re_run:
out, rc = run('go test ' + package, write_to=f, fail_ok=True)
print('Re-run duration:', datetime.now() - start_time)
start_time = datetime.now()
else:
print('Some failures are unrecoverable... not re-running.')
def maas(cmd):
print('maas maas ' + cmd)
out = run('maas maas ' + cmd, quiet=True)
return json.loads(out)
def test(args, db):
long_tests = {
'api': 9.926,
'api/firewaller': 8.735,
'api/provisioner': 10.475,
'api/uniter': 34.885,
'api/upgrader': 5.926,
'apiserver': 141.927,
'apiserver/client': 59.812,
'apiserver/common': 10.367,
'apiserver/firewaller': 5.664,
'apiserver/metricsender': 5.502,
'apiserver/provisioner': 20.643,
'apiserver/storageprovisioner': 8.943,
'apiserver/uniter': 54.406,
'apiserver/upgrader': 7.856,
'bzr': 6.002,
'cmd/juju/action': 18.114,
'cmd/juju/commands': 70.726,
'cmd/juju/status': 17.68,
'cmd/juju/system': 10.319,
'cmd/jujud/reboot': 9.227,
'container/lxc': 5.231,
'downloader': 5.081,
'environs/bootstrap': 47.446,
'environs/configstore': 5.096,
'featuretests': 53.464,
'mongo': 19.99,
'provider/ec2': 6.181,
'provider/maas': 8.041,
'provider/openstack': 36.273,
'state': 230.395,
'state/backups': 12.262,
'state/presence': 5.551,
'upgrades': 8.62,
'utils/ssh': 5.196,
'worker/addresser': 41.869,
'worker/firewaller': 7.677,
'worker/machiner': 11.263,
'worker/peergrouper': 10.333,
'worker/provisioner': 55.694,
'worker/reboot': 10.65,
'worker/uniter': 227.597,
'worker/uniter/charm': 5.773,
'worker/uniter/filter': 9.154,
'worker/uniter/runner': 18.348,
}
output_filename = os.path.join(os.path.expanduser('~'),
'.jujutestoutput.txt')
rerun_filename = os.path.join(os.path.expanduser('~'),
'.jujutestoutput_rerun.txt')
start_time = datetime.now()
try:
install_out = run('go install -v github.com/juju/juju/...')
except subprocess.CalledProcessError as e:
for line in e.output.splitlines():
if not line.startswith(JUJU_VCS):
print('!', line)
return e.returncode
print('Build duration:', datetime.now() - start_time)
start_time = datetime.now()
filename = None
packages = []
if args.fast:
packages = package_list()
# Start long tests first
packages = sorted(packages, key=lambda p: long_tests.get(p, 0), reverse=True)
packages.remove('state')
rc, failures = test_packages(packages)
# Always run state last because it just doesn't like the company
state_rc, state_failures = test_packages(['state'])
print('Test duration:', datetime.now() - start_time)
if rc or state_rc:
filename = '/tmp/all_failures.txt'
with open(filename, 'w') as f:
for p in failures + state_failures:
logName = os.path.join('/tmp', p, 'out.txt')
with open(logName) as i:
f.write(i.read())
f.writelines([
'\nFAIL {} 0.0123456s\n\n'.format(p),
'# End of {}\n'.format(p),
'<>' * 40 + '\n\n',
])
elif args.changed:
print('Testing changed packages')
git_status = run('git status', quiet=True)
# TODO: support new, deleted, modified, moved etc.
packages_with_tests = package_list()
for line in git_status.splitlines():
mod = re.search('modified:\s+(.*)/.*?\.go$', line)
if mod and mod.group(1) not in packages and mod.group(1) in packages_with_tests:
packages.append(mod.group(1))
# Sort packages to do long tests last
# TODO: long tests first, in parallel...
packages = sorted(packages, key=lambda p: long_tests.get(p, 0))
print(packages)
with open(output_filename, 'w') as f:
for package in packages:
package = 'github.com/juju/juju/' + package
runner = compile_test_runner(package)
run(runner, write_to=f, fail_ok=True)
os.remove(runner)
f.write(">>end of package>>" + package + "\n")
print('Test duration:', datetime.now() - start_time)
filename = output_filename
elif (len(install_out) or args.force) and not args.rerun:
print('Testing everything')
if os.path.isfile(rerun_filename):
os.remove(rerun_filename)
filename = output_filename
with open(output_filename, 'w') as f:
run('go test -i ./...', write_to=f, fail_exits=True)
run('go test ./...', write_to=f, fail_ok=True)
print('Test duration:', datetime.now() - start_time)
start_time = datetime.now()
else:
print('Re-running failures from last test')
# Don't have a new binary, so just re-run tests
if db.get('empty'):
del(db['empty'])
# No old test DB, so parse the last output
if os.path.isfile(rerun_filename):
filename = rerun_filename
elif os.path.isfile(output_filename):
filename = output_filename
filename = output_filename
unrecoverable = False
if filename is not None:
db['failures'] = {}
with open(filename) as f:
test_out = f.readlines()
re_run = []
re_run_tests = []
for line in test_out:
s = re.search('>>end of package>>(.*)$', line)
if s and len(re_run_tests):
db['failures'][s.group(1)] = re_run_tests
re_run.append({
'package': s.group(1),
'tests': re_run_tests,
})
re_run_tests = []
if line.startswith('FAIL:'):
s = re.search('^FAIL:\s+.*:\d+:\s+(.*)\s*$', line)
if s:
print(s.group(1))
re_run_tests.append(s.group(1))
else:
print("--- Error: don't know how to handle this line:")
print(line)
print("Aborting...")
return 1
if not re.search('FAIL\s+github.*', line):
continue
s = re.search('FAIL\s+(.+?)\s+\[.*? failed\]', line)
if s:
unrecoverable = True
db['failures'][s.group(1)] = ['.*']
re_run_tests = []
print(line)
else:
s = re.search('FAIL\s+(github.com.*)\s[\d\.]+s\s*$', line)
db['failures'][s.group(1)] = re_run_tests
re_run.append({
'package': s.group(1),
'tests': re_run_tests,
})
re_run_tests = []
print('-' * 40 + 'failures:' + '-' * 40)
pprint(db)
if len(db['failures'].keys()) == 0:
return 0
if not unrecoverable:
print('Re-running failed tests...')
failed = False
remove_packages = []
with open(rerun_filename, 'w') as f:
for package, tests in db['failures'].items():
failures = []
name_search = re.search(r'github.com/juju/juju/(.*)', package)
if name_search:
runner = compile_test_runner(package)
if len(tests):
for test in tests:
print(package, test)
out, rc = run('{} -check.f ^{}$'.format(runner, test), write_to=f, fail_ok=True, quiet=True)
if rc:
print(out[:400])
print('\n' + '-' * 80)
if args.stop_on_failure and rc:
os.remove(runner)
return rc
failed = failed or rc != 0
if rc:
failures.append(test)
else:
print(package)
out, rc = run(runner, write_to=f, fail_ok=True, quiet=True)
if rc:
print(out[:400])
if args.stop_on_failure and rc:
os.remove(runner)
return rc
failed = failed or rc != 0
os.remove(runner)
if len(failures):
db['failures'][package] = failures
elif failed:
db['failures'][package] = []
elif package in db['failures']:
remove_packages.append(package)
# Remove empty lists from database
for package in remove_packages:
del(db['failures'][package])
print('Re-run duration:', datetime.now() - start_time)
return failed
else:
print('Some failures are unrecoverable... not re-running.')
return 0