def run_tests(testnames,
version=None,
fast=False,
changed=False,
coverage=True,
parallel=False,
failed=False,
cores=None,
coverdir='../output/coverage',
html=False,
threshold=90,
outputfile=None,
package='pygsti',
scriptfile=None,
timer=False):
with directory('test_packages'):
# Don't run report or drivers
if fast:
for slowtest in slowtests:
testnames.remove(slowtest)
# Specify the versions of your test :)
if version is None:
# The version this file was run/imported with
pythoncommands = [
'python%s.%s' % (sys.version_info[0], sys.version_info[1])
]
else:
# The version specified
pythoncommands = ['python%s' % version]
# Always use nose
pythoncommands += ['-m', 'nose', '-v']
# Since last commit to current branch
if changed:
testnames = [
name for name in testnames if name in get_changed_packages()
]
if len(testnames) == 0:
print('No tests to run')
sys.exit(0)
# testnames should be final at this point
print('Running tests:\n%s' % ('\n'.join(testnames)))
# Version-specific coverage cmds only sometimes have a dash
# e.g.: coverage, coverage2, coverage3, coverage-2.7, coverage-3.5
if version is None: coverage_cmd = "coverage"
elif "." in version: coverage_cmd = 'coverage-%s' % version
else: coverage_cmd = 'coverage%s' % version
# Run mpi coverage tests differently
covermpi = ('mpi' in testnames) and coverage
if covermpi:
testnames.remove('mpi')
postcommands = []
# Use parallelism native to nose
if parallel:
if cores is None:
pythoncommands.append('--processes=-1')
# (-1) will use all cores
else:
pythoncommands.append('--processes=%s' % cores)
# Some tests take up to an hour
pythoncommands.append('--process-timeout=14400') # Four hours
else:
# Use the failure monitoring native to nose
postcommands = ['--with-id']
if failed:
postcommands = ['--failed'] # ~implies --with-id
if coverage:
# html coverage is prettiest
pythoncommands += [
'--with-coverage', '--cover-erase',
'--cover-package={}'.format(package),
'--cover-min-percentage={}'.format(threshold)
]
if timer:
pythoncommands.append('--with-timer')
returned = 0
if len(testnames) > 0:
commands = pythoncommands + testnames + postcommands
commandStr = ' '.join(commands)
if scriptfile:
#Script file runs command directly from shell so output works normally
# (using subprocess on TravisCI gives incomplete output sometimes). It
# uses a sleep loop to ensure some output is printed every 9 minutes,
# as TravisCI terminates a process when it goes 10m without output.
with open(scriptfile, 'w') as script:
print("#!/usr/bin/bash", file=script)
print('echo "%s"' % commandStr, file=script)
print(
'while sleep 540; do echo "=====[ $SECONDS seconds ]====="; done &',
file=script)
print(commandStr, file=script)
print('kill %1', file=script) # Kill background sleep loop
print("Wrote script file %s" % os.path.join(
'test_packages', scriptfile)) # cwd == 'test_packages'
sys.exit(0)
else:
print(commandStr)
if outputfile is None:
returned = subprocess.call(commands)
else:
with open(outputfile, 'w') as testoutput:
returned = subprocess.call(commands,
stdout=testoutput,
stderr=testoutput)
with open(outputfile, 'r') as testoutput:
print(testoutput.read())
if parallel:
#Only combine when run in parallel mode, since this
# causes nose tests to create .coverage.<processid>
# files instead of just a single .coverage file, which
# "coverage combine" will overwrite with no-data (eek!).
subprocess.call([coverage_cmd, 'combine'])
if covermpi:
print('Running mpi with coverage')
# Combine serial/parallel coverage
serial_coverage_exists = bool(len(testnames) > 0)
if serial_coverage_exists:
#In this case, nose tests have erased old coverage files
shutil.copy2('.coverage', '../output/temp_coverage')
else:
#If no serial tests have run, then we need to erase old files
subprocess.call([coverage_cmd, 'erase'])
run_mpi_coverage_tests(coverage_cmd) #creates .coverage.xxx files
if serial_coverage_exists:
shutil.copy2('../output/temp_coverage', '.coverage.serial')
subprocess.call([coverage_cmd, 'combine']) #combine everything
if html:
create_html(coverdir, coverage_cmd)
webbrowser.open(coverdir + '/index.html')
sys.exit(returned)
#!/usr/bin/env python3
import os
import subprocess
from helpers.automation_tools import directory, get_packages
exclude = ['__pycache__']
print('Generating html reports of duplicated code')
with directory('../pygsti'):
for package in get_packages(os.getcwd()):
if package not in exclude:
print('Finding duplicated code in %s' % package)
subprocess.call([
'clonedigger', package, '-o',
'../../test/output/dup/dup_%s.html' % package
])
print('Done')
def run_tests(testnames,
version=None,
fast=False,
changed=False,
coverage=True,
parallel=False,
failed=False,
cores=None,
coverdir='../output/coverage',
html=False,
threshold=90,
outputfile=None):
with directory('test_packages'):
# Don't run report or drivers
if fast:
for slowtest in slowtests:
testnames.remove(slowtest)
# Specify the versions of your test :)
if version is None:
# The version this file was run/imported with
pythoncommands = [
'python%s.%s' % (sys.version_info[0], sys.version_info[1])
]
else:
# The version specified
pythoncommands = ['python%s' % version]
# Always use nose
pythoncommands += ['-m', 'nose', '-v']
# Since last commit to current branch
if changed:
testnames = [
name for name in testnames if name in get_changed_packages()
]
if len(testnames) == 0:
print('No tests to run')
sys.exit(0)
# testnames should be final at this point
print('Running tests:\n%s' % ('\n'.join(testnames)))
# Version-specific coverage cmds only sometimes have a dash
# e.g.: coverage, coverage2, coverage3, coverage-2.7, coverage-3.5
if version is None: coverage_cmd = "coverage"
elif "." in version: coverage_cmd = 'coverage-%s' % version
else: coverage_cmd = 'coverage%s' % version
# Run mpi coverage tests differently
covermpi = ('mpi' in testnames) and coverage
if covermpi:
testnames.remove('mpi')
postcommands = []
# Use parallelism native to nose
if parallel:
if cores is None:
pythoncommands.append('--processes=-1')
# (-1) will use all cores
else:
pythoncommands.append('--processes=%s' % cores)
# Some tests take up to an hour
pythoncommands.append('--process-timeout=14400') # Four hours
else:
# Use the failure monitoring native to nose
postcommands = ['--with-id']
if failed:
postcommands = ['--failed'] # ~implies --with-id
if coverage:
# html coverage is prettiest
pythoncommands += [
'--with-coverage', '--cover-erase', '--cover-package=pygsti',
'--cover-min-percentage=%s' % threshold
]
returned = 0
if len(testnames) > 0:
commands = pythoncommands + testnames + postcommands
print(commands)
if outputfile is None:
returned = subprocess.call(commands)
else:
with open(outputfile, 'w') as testoutput:
returned = subprocess.call(commands,
stdout=testoutput,
stderr=testoutput)
with open(outputfile, 'r') as testoutput:
print(testoutput.read())
if parallel:
#Only combine when run in parallel mode, since this
# causes nose tests to create .coverage.<processid>
# files instead of just a single .coverage file, which
# "coverage combine" will overwrite with no-data (eek!).
subprocess.call([coverage_cmd, 'combine'])
if covermpi:
print('Running mpi with coverage')
# Combine serial/parallel coverage
serial_coverage_exists = bool(len(testnames) > 0)
if serial_coverage_exists:
#In this case, nose tests have erased old coverage files
shutil.copy2('.coverage', '../output/temp_coverage')
else:
#If no serial tests have run, then we need to erase old files
subprocess.call([coverage_cmd, 'erase'])
run_mpi_coverage_tests(coverage_cmd) #creates .coverage.xxx files
if serial_coverage_exists:
shutil.copy2('../output/temp_coverage', '.coverage.serial')
subprocess.call([coverage_cmd, 'combine']) #combine everything
if html:
create_html(coverdir, coverage_cmd)
sys.exit(returned)
#!/usr/bin/env python3
from helpers.automation_tools import directory, get_packages
import subprocess, os
exclude = ['__pycache__']
print('Generating html reports of duplicated code')
with directory('../packages/pygsti'):
for package in get_packages(os.getcwd()):
if package not in exclude:
print('Finding duplicated code in %s' % package)
subprocess.call(['clonedigger', package, '-o', '../../test/output/dup/dup_%s.html' % package])
print('Done')
def run_tests(testnames, version=None, fast=False, changed=False, coverage=True,
parallel=False, failed=False, cores=None, coverdir='../output/coverage', html=False,
threshold=90, outputfile=None):
with directory('test_packages'):
# Don't run report or drivers
if fast:
for slowtest in slowtests:
testnames.remove(slowtest)
# Specify the versions of your test :)
if version is None:
# The version this file was run/imported with
pythoncommands = ['python%s.%s' % (sys.version_info[0], sys.version_info[1])]
else:
# The version specified
pythoncommands = ['python%s' % version]
# Always use nose
pythoncommands += ['-m', 'nose', '-v']
# Since last commit to current branch
if changed:
testnames = [name for name in testnames if name in get_changed_packages()]
if len(testnames) == 0:
print('No tests to run')
sys.exit(0)
# testnames should be final at this point
print('Running tests:\n%s' % ('\n'.join(testnames)))
# Version-specific coverage cmds only sometimes have a dash
# e.g.: coverage, coverage2, coverage3, coverage-2.7, coverage-3.5
if version is None: coverage_cmd = "coverage"
elif "." in version: coverage_cmd = 'coverage-%s' % version
else: coverage_cmd = 'coverage%s' % version
# Run mpi coverage tests differently
covermpi = ('mpi' in testnames) and coverage
if covermpi:
testnames.remove('mpi')
postcommands = []
# Use parallelism native to nose
if parallel:
if cores is None:
pythoncommands.append('--processes=-1')
# (-1) will use all cores
else:
pythoncommands.append('--processes=%s' % cores)
# Some tests take up to an hour
pythoncommands.append('--process-timeout=14400') # Four hours
else:
# Use the failure monitoring native to nose
postcommands = ['--with-id']
if failed:
postcommands = ['--failed']# ~implies --with-id
if coverage:
# html coverage is prettiest
pythoncommands += ['--with-coverage',
'--cover-erase',
'--cover-package=pygsti',
'--cover-min-percentage=%s' % threshold]
returned = 0
if len(testnames) > 0:
commands = pythoncommands + testnames + postcommands
print(commands)
if outputfile is None:
returned = subprocess.call(commands)
else:
with open(outputfile, 'w') as testoutput:
returned = subprocess.call(commands, stdout=testoutput, stderr=testoutput)
with open(outputfile, 'r') as testoutput:
print(testoutput.read())
if parallel:
#Only combine when run in parallel mode, since this
# causes nose tests to create .coverage.<processid>
# files instead of just a single .coverage file, which
# "coverage combine" will overwrite with no-data (eek!).
subprocess.call([coverage_cmd, 'combine'])
if covermpi:
print('Running mpi with coverage')
# Combine serial/parallel coverage
serial_coverage_exists = bool(len(testnames) > 0)
if serial_coverage_exists:
#In this case, nose tests have erased old coverage files
shutil.copy2('.coverage', '../output/temp_coverage')
else:
#If no serial tests have run, then we need to erase old files
subprocess.call([coverage_cmd, 'erase'])
run_mpi_coverage_tests(coverage_cmd) #creates .coverage.xxx files
if serial_coverage_exists:
shutil.copy2('../output/temp_coverage', '.coverage.serial')
subprocess.call([coverage_cmd, 'combine']) #combine everything
if html:
create_html(coverdir, coverage_cmd)
sys.exit(returned)
#!/usr/bin/env python3
from helpers.automation_tools import directory, get_packages
import subprocess, os
exclude = ['__pycache__']
print('Generating html reports of duplicated code')
with directory('../packages/pygsti'):
for package in get_packages(os.getcwd()):
if package not in exclude:
print('Finding duplicated code in %s' % package)
subprocess.call([
'clonedigger', package, '-o',
'../../test/output/dup/dup_%s.html' % package
])
print('Done')
