def run_tests(testnames, version=None, fast=False, changed=False, coverage=True, parallel=False, failed=False, cores=None, coverdir='../output/coverage', html=False, threshold=90, outputfile=None): with directory('test_packages'): # Don't run report or drivers if fast: for slowtest in slowtests: testnames.remove(slowtest) # Specify the versions of your test :) if version is None: # The version this file was run/imported with pythoncommands = [ 'python%s.%s' % (sys.version_info[0], sys.version_info[1]) ] else: # The version specified pythoncommands = ['python%s' % version] # Always use nose pythoncommands += ['-m', 'nose', '-v'] # Since last commit to current branch if changed: testnames = [ name for name in testnames if name in get_changed_packages() ] if len(testnames) == 0: print('No tests to run') sys.exit(0) # testnames should be final at this point print('Running tests:\n%s' % ('\n'.join(testnames))) # Version-specific coverage cmds only sometimes have a dash # e.g.: coverage, coverage2, coverage3, coverage-2.7, coverage-3.5 if version is None: coverage_cmd = "coverage" elif "." in version: coverage_cmd = 'coverage-%s' % version else: coverage_cmd = 'coverage%s' % version # Run mpi coverage tests differently covermpi = ('mpi' in testnames) and coverage if covermpi: testnames.remove('mpi') postcommands = [] # Use parallelism native to nose if parallel: if cores is None: pythoncommands.append('--processes=-1') # (-1) will use all cores else: pythoncommands.append('--processes=%s' % cores) # Some tests take up to an hour pythoncommands.append('--process-timeout=14400') # Four hours else: # Use the failure monitoring native to nose postcommands = ['--with-id'] if failed: postcommands = ['--failed'] # ~implies --with-id if coverage: # html coverage is prettiest pythoncommands += [ '--with-coverage', '--cover-erase', '--cover-package=pygsti', '--cover-min-percentage=%s' % threshold ] returned = 0 if len(testnames) > 0: commands = pythoncommands + testnames + postcommands print(commands) if outputfile is None: returned = subprocess.call(commands) else: with open(outputfile, 'w') as testoutput: returned = subprocess.call(commands, stdout=testoutput, stderr=testoutput) with open(outputfile, 'r') as testoutput: print(testoutput.read()) if parallel: #Only combine when run in parallel mode, since this # causes nose tests to create .coverage.<processid> # files instead of just a single .coverage file, which # "coverage combine" will overwrite with no-data (eek!). subprocess.call([coverage_cmd, 'combine']) if covermpi: print('Running mpi with coverage') # Combine serial/parallel coverage serial_coverage_exists = bool(len(testnames) > 0) if serial_coverage_exists: #In this case, nose tests have erased old coverage files shutil.copy2('.coverage', '../output/temp_coverage') else: #If no serial tests have run, then we need to erase old files subprocess.call([coverage_cmd, 'erase']) run_mpi_coverage_tests(coverage_cmd) #creates .coverage.xxx files if serial_coverage_exists: shutil.copy2('../output/temp_coverage', '.coverage.serial') subprocess.call([coverage_cmd, 'combine']) #combine everything if html: create_html(coverdir, coverage_cmd) sys.exit(returned)
def run_tests(testnames, version=None, fast=False, changed=False, coverage=True, parallel=False, failed=False, cores=None, coverdir='../output/coverage', html=False, threshold=90, outputfile=None, package='pygsti', scriptfile=None, timer=False): with directory('test_packages'): # Don't run report or drivers if fast: for slowtest in slowtests: testnames.remove(slowtest) # Specify the versions of your test :) if version is None: # The version this file was run/imported with pythoncommands = [ 'python%s.%s' % (sys.version_info[0], sys.version_info[1]) ] else: # The version specified pythoncommands = ['python%s' % version] # Always use nose pythoncommands += ['-m', 'nose', '-v'] # Since last commit to current branch if changed: testnames = [ name for name in testnames if name in get_changed_packages() ] if len(testnames) == 0: print('No tests to run') sys.exit(0) # testnames should be final at this point print('Running tests:\n%s' % ('\n'.join(testnames))) # Version-specific coverage cmds only sometimes have a dash # e.g.: coverage, coverage2, coverage3, coverage-2.7, coverage-3.5 if version is None: coverage_cmd = "coverage" elif "." in version: coverage_cmd = 'coverage-%s' % version else: coverage_cmd = 'coverage%s' % version # Run mpi coverage tests differently covermpi = ('mpi' in testnames) and coverage if covermpi: testnames.remove('mpi') postcommands = [] # Use parallelism native to nose if parallel: if cores is None: pythoncommands.append('--processes=-1') # (-1) will use all cores else: pythoncommands.append('--processes=%s' % cores) # Some tests take up to an hour pythoncommands.append('--process-timeout=14400') # Four hours else: # Use the failure monitoring native to nose postcommands = ['--with-id'] if failed: postcommands = ['--failed'] # ~implies --with-id if coverage: # html coverage is prettiest pythoncommands += [ '--with-coverage', '--cover-erase', '--cover-package={}'.format(package), '--cover-min-percentage={}'.format(threshold) ] if timer: pythoncommands.append('--with-timer') returned = 0 if len(testnames) > 0: commands = pythoncommands + testnames + postcommands commandStr = ' '.join(commands) if scriptfile: #Script file runs command directly from shell so output works normally # (using subprocess on TravisCI gives incomplete output sometimes). It # uses a sleep loop to ensure some output is printed every 9 minutes, # as TravisCI terminates a process when it goes 10m without output. with open(scriptfile, 'w') as script: print("#!/usr/bin/bash", file=script) print('echo "%s"' % commandStr, file=script) print( 'while sleep 540; do echo "=====[ $SECONDS seconds ]====="; done &', file=script) print(commandStr, file=script) print('kill %1', file=script) # Kill background sleep loop print("Wrote script file %s" % os.path.join( 'test_packages', scriptfile)) # cwd == 'test_packages' sys.exit(0) else: print(commandStr) if outputfile is None: returned = subprocess.call(commands) else: with open(outputfile, 'w') as testoutput: returned = subprocess.call(commands, stdout=testoutput, stderr=testoutput) with open(outputfile, 'r') as testoutput: print(testoutput.read()) if parallel: #Only combine when run in parallel mode, since this # causes nose tests to create .coverage.<processid> # files instead of just a single .coverage file, which # "coverage combine" will overwrite with no-data (eek!). subprocess.call([coverage_cmd, 'combine']) if covermpi: print('Running mpi with coverage') # Combine serial/parallel coverage serial_coverage_exists = bool(len(testnames) > 0) if serial_coverage_exists: #In this case, nose tests have erased old coverage files shutil.copy2('.coverage', '../output/temp_coverage') else: #If no serial tests have run, then we need to erase old files subprocess.call([coverage_cmd, 'erase']) run_mpi_coverage_tests(coverage_cmd) #creates .coverage.xxx files if serial_coverage_exists: shutil.copy2('../output/temp_coverage', '.coverage.serial') subprocess.call([coverage_cmd, 'combine']) #combine everything if html: create_html(coverdir, coverage_cmd) webbrowser.open(coverdir + '/index.html') sys.exit(returned)
def run_tests(testnames, version=None, fast=False, changed=False, coverage=True, parallel=False, failed=False, cores=None, coverdir='../output/coverage', html=False, threshold=90, outputfile=None): with directory('test_packages'): # Don't run report or drivers if fast: for slowtest in slowtests: testnames.remove(slowtest) # Specify the versions of your test :) if version is None: # The version this file was run/imported with pythoncommands = ['python%s.%s' % (sys.version_info[0], sys.version_info[1])] else: # The version specified pythoncommands = ['python%s' % version] # Always use nose pythoncommands += ['-m', 'nose', '-v'] # Since last commit to current branch if changed: testnames = [name for name in testnames if name in get_changed_packages()] if len(testnames) == 0: print('No tests to run') sys.exit(0) # testnames should be final at this point print('Running tests:\n%s' % ('\n'.join(testnames))) # Version-specific coverage cmds only sometimes have a dash # e.g.: coverage, coverage2, coverage3, coverage-2.7, coverage-3.5 if version is None: coverage_cmd = "coverage" elif "." in version: coverage_cmd = 'coverage-%s' % version else: coverage_cmd = 'coverage%s' % version # Run mpi coverage tests differently covermpi = ('mpi' in testnames) and coverage if covermpi: testnames.remove('mpi') postcommands = [] # Use parallelism native to nose if parallel: if cores is None: pythoncommands.append('--processes=-1') # (-1) will use all cores else: pythoncommands.append('--processes=%s' % cores) # Some tests take up to an hour pythoncommands.append('--process-timeout=14400') # Four hours else: # Use the failure monitoring native to nose postcommands = ['--with-id'] if failed: postcommands = ['--failed']# ~implies --with-id if coverage: # html coverage is prettiest pythoncommands += ['--with-coverage', '--cover-erase', '--cover-package=pygsti', '--cover-min-percentage=%s' % threshold] returned = 0 if len(testnames) > 0: commands = pythoncommands + testnames + postcommands print(commands) if outputfile is None: returned = subprocess.call(commands) else: with open(outputfile, 'w') as testoutput: returned = subprocess.call(commands, stdout=testoutput, stderr=testoutput) with open(outputfile, 'r') as testoutput: print(testoutput.read()) if parallel: #Only combine when run in parallel mode, since this # causes nose tests to create .coverage.<processid> # files instead of just a single .coverage file, which # "coverage combine" will overwrite with no-data (eek!). subprocess.call([coverage_cmd, 'combine']) if covermpi: print('Running mpi with coverage') # Combine serial/parallel coverage serial_coverage_exists = bool(len(testnames) > 0) if serial_coverage_exists: #In this case, nose tests have erased old coverage files shutil.copy2('.coverage', '../output/temp_coverage') else: #If no serial tests have run, then we need to erase old files subprocess.call([coverage_cmd, 'erase']) run_mpi_coverage_tests(coverage_cmd) #creates .coverage.xxx files if serial_coverage_exists: shutil.copy2('../output/temp_coverage', '.coverage.serial') subprocess.call([coverage_cmd, 'combine']) #combine everything if html: create_html(coverdir, coverage_cmd) sys.exit(returned)