def run_tests(self, suite):
runner = unittest.TextTestRunner(stream=StringIO())
# Run tests across 2 processes
concurrent_suite = ConcurrentTestSuite(suite, fork_for_tests(2))
result = runner.run(concurrent_suite)
self.assertEqual(result.testsRun, suite.countTestCases())
return result
def run_test_suites(sanity_tests, single_tests, parallel_tests):
all_successful = False
sanity_runner = unittest.TextTestRunner(verbosity=255, failfast=True)
sanity_result = sanity_runner.run(sanity_tests)
if sanity_result.wasSuccessful():
print('running %u tests in parallel and %u tests serial' %
(parallel_tests.countTestCases(), single_tests.countTestCases()))
results = []
if parallel_tests.countTestCases():
max_parallel_tests = min(parallel_tests.countTestCases(),
MAX_PARALLEL_TESTS)
parallel_runner = unittest.TextTestRunner(verbosity=255)
parallel_suite = ConcurrentTestSuite(
parallel_tests, fork_for_tests(max_parallel_tests))
results.append(parallel_runner.run(parallel_suite))
# TODO: Tests that are serialized generally depend on hardcoded ports.
# Make them use dynamic ports.
if single_tests.countTestCases():
single_runner = unittest.TextTestRunner(verbosity=255)
results.append(single_runner.run(single_tests))
all_successful = True
for result in results:
if not result.wasSuccessful():
all_successful = False
print(result.printErrors())
else:
print('sanity tests failed - test environment not correct')
return all_successful
def setup():
suite = unittest.TestSuite((
unittest.TestLoader().loadTestsFromTestCase(ForkingWorkersTestCase),
unittest.TestLoader().loadTestsFromTestCase(PartitionTestCase),
))
concurrent_suite = ConcurrentTestSuite(suite, fork_for_tests(2))
return concurrent_suite
def main(argv):
parser = argparse.ArgumentParser(prog=argv[0],
description='run CAmkES tests')
parser.add_argument('--jobs', '-j', nargs='?', type=int,
help='parallelise test execution')
parser.add_argument('--verbosity', '-v', default=1, type=int, help="Verbosity to run tests. 0 = quiet. 1 = default. 2 = verbose")
parser.add_argument('test', nargs='*', choices=TESTS+['all'], default='all', help='run a specific category of tests')
parser.add_argument('--capdl-python', help='Deprecated. Using this argument has no effect.')
options = parser.parse_args(argv[1:])
if options.jobs is None:
# Maximum parallelism.
options.jobs = multiprocessing.cpu_count()
# work out which tests to run
if options.test == 'all' or 'all' in options.test:
test_packages = TESTS
else:
test_packages = options.test
# load the tests we want to run
loader = unittest.TestLoader()
test_suite = unittest.TestSuite()
for v in test_packages:
test_suite.addTests(loader.discover('camkes.' + v, top_level_dir=os.path.dirname(ME)))
concurrent_suite = ConcurrentTestSuite(test_suite, fork_for_tests(options.jobs))
runner = unittest.TextTestRunner(verbosity=options.verbosity)
result = runner.run(concurrent_suite)
if result.wasSuccessful():
return 0
return 1
def run_tests(self, suite):
runner = unittest.TextTestRunner(stream=StringIO())
# Run tests across 2 processes
concurrent_suite = ConcurrentTestSuite(suite, fork_for_tests(2))
result = runner.run(concurrent_suite)
self.assertEqual(result.testsRun, suite.countTestCases())
return result
def runtests(processes=8):
"""
Run the crypto library's unit tests. Will run tests in parallel if the `concurrencytest`
module is installed. Defaults to four processes.
Example:
>>> from crypto.tests import runtests
>>> runtests()
...
"""
import unittest
loader = unittest.TestLoader()
# Discover all tests in the current directory that are prefixed with `test`. Also discovers
# the doctests loaded by defining a load_tests(...) function in each submodule's __init__.py
suite = loader.discover('.', pattern='test*.py')
runner = unittest.runner.TextTestRunner()
try:
from concurrencytest import ConcurrentTestSuite, fork_for_tests
concurrent_suite = ConcurrentTestSuite(suite, fork_for_tests(processes))
runner.run(concurrent_suite)
except ImportError:
runner.run(suite)
# Prevent calling sys.exit() just in case the user is running the tests from an interpreter.
unittest.main(exit=False)
def main(args):
"""
Implementation of main method starts here....
"""
process = 0
iter_num = 0
if len(args) > 1:
iter_num = int(args[0])
process = int(args[1])
else:
iter_num = int(args[0])
# load a TestSuite with 50x TestCases for demo
loader = unittest.TestLoader()
suite = unittest.TestSuite()
for _ in range(iter_num):
suite.addTests(loader.loadTestsFromTestCase(MemcacheConcurrencyTestCase))
print('Loaded %d test cases...' % suite.countTestCases())
runner = unittest.TextTestRunner()
# Run parallely with process and parallel hits at once
if process:
print('\nRun same tests with %s processes: ' % process)
concurrent_suite = ConcurrentTestSuite(suite, fork_for_tests(process))
runner.run(concurrent_suite)
else:
# Sequential run (one by one)
print('\nRun tests sequentially:')
runner.run(suite)
def execute_test_class(test_class): loader = unittest.TestLoader() suite = unittest.TestSuite() suite.addTests(loader.loadTestsFromTestCase(test_class)) runner = unittest.TextTestRunner() concurrent_suite = ConcurrentTestSuite(suite, fork_for_tests(50)) runner.run(concurrent_suite)
def multi_processing(self):
print("multi_processing")
runner = unittest.TextTestRunner()
suite = unittest.TestLoader().loadTestsFromTestCase(test_name)
concurrent_suite = ConcurrentTestSuite(suite,
fork_for_tests(self.processing))
runner.run(concurrent_suite)
def RunTests(debug, processes, args):
"""Run the functional tests and any embedded doctests
Args:
debug: True to enable debugging, which shows a full stack trace on error
args: List of positional args provided to binman. This can hold a test
name to execute (as in 'binman -t testSections', for example)
processes: Number of processes to use to run tests (None=same as #CPUs)
"""
import elf_test
import entry_test
import fdt_test
import ftest
import image_test
import test
import doctest
result = unittest.TestResult()
for module in []:
suite = doctest.DocTestSuite(module)
suite.run(result)
sys.argv = [sys.argv[0]]
if debug:
sys.argv.append('-D')
if debug:
sys.argv.append('-D')
# Run the entry tests first ,since these need to be the first to import the
# 'entry' module.
test_name = args and args[0] or None
suite = unittest.TestSuite()
loader = unittest.TestLoader()
for module in (entry_test.TestEntry, ftest.TestFunctional, fdt_test.TestFdt,
elf_test.TestElf, image_test.TestImage):
if test_name:
try:
suite.addTests(loader.loadTestsFromName(test_name, module))
except AttributeError:
continue
else:
suite.addTests(loader.loadTestsFromTestCase(module))
if use_concurrent and processes != 1:
concurrent_suite = ConcurrentTestSuite(suite,
fork_for_tests(processes or multiprocessing.cpu_count()))
concurrent_suite.run(result)
else:
suite.run(result)
print result
for test, err in result.errors:
print test.id(), err
for test, err in result.failures:
print err, result.failures
if result.errors or result.failures:
print 'binman tests FAILED'
return 1
return 0
def main():
runner = unittest.TextTestRunner()
suite = unittest.TestSuite((
unittest.TestLoader().loadTestsFromTestCase(ForkingWorkersTestCase),
unittest.TestLoader().loadTestsFromTestCase(PartitionTestCase),
))
concurrent_suite = ConcurrentTestSuite(suite, fork_for_tests())
result = runner.run(concurrent_suite)
return len(result.errors) + len(result.failures)
def main():
runner = unittest.TextTestRunner()
suite = unittest.TestSuite((
unittest.TestLoader().loadTestsFromTestCase(ForkingWorkersTestCase),
unittest.TestLoader().loadTestsFromTestCase(PartitionTestCase),
))
concurrent_suite = ConcurrentTestSuite(suite, fork_for_tests())
result = runner.run(concurrent_suite)
return len(result.errors) + len(result.failures)
def run_parallel_test_suites(root_tmpdir, resultclass, parallel_tests):
results = []
if parallel_tests.countTestCases():
max_parallel_tests = min(parallel_tests.countTestCases(), max_loadavg())
print('running maximum of %u parallel tests' % max_parallel_tests)
parallel_runner = test_runner(root_tmpdir, resultclass)
parallel_suite = ConcurrentTestSuite(
parallel_tests, fork_for_tests(max_parallel_tests))
results.append(parallel_runner.run(parallel_suite))
return results
def run_parallel_test_suites(root_tmpdir, resultclass, parallel_tests):
results = []
if parallel_tests.countTestCases():
max_parallel_tests = min(parallel_tests.countTestCases(), max_loadavg())
print('running maximum of %u parallel tests' % max_parallel_tests)
parallel_runner = test_runner(root_tmpdir, resultclass)
parallel_suite = ConcurrentTestSuite(
parallel_tests, fork_for_tests(max_parallel_tests))
results.append(parallel_runner.run(parallel_suite))
return results
def main():
print("\n###################### running tests ##########################\n")
s = unittest.defaultTestLoader.discover('', pattern="Test*")
concurrent_suite = ConcurrentTestSuite(s, fork_for_tests(16))
r = unittest.TextTestRunner()
res = r.run(concurrent_suite)
if len(res.errors) + len(res.failures) > 0:
sys.exit(1)
def RunTestSuites(result, debug, verbosity, test_preserve_dirs, processes,
test_name, toolpath, test_class_list):
"""Run a series of test suites and collect the results
Args:
result: A unittest.TestResult object to add the results to
debug: True to enable debugging, which shows a full stack trace on error
verbosity: Verbosity level to use (0-4)
test_preserve_dirs: True to preserve the input directory used by tests
so that it can be examined afterwards (only useful for debugging
tests). If a single test is selected (in args[0]) it also preserves
the output directory for this test. Both directories are displayed
on the command line.
processes: Number of processes to use to run tests (None=same as #CPUs)
test_name: Name of test to run, or None for all
toolpath: List of paths to use for tools
test_class_list: List of test classes to run
"""
for module in []:
suite = doctest.DocTestSuite(module)
suite.run(result)
sys.argv = [sys.argv[0]]
if debug:
sys.argv.append('-D')
if verbosity:
sys.argv.append('-v%d' % verbosity)
if toolpath:
for path in toolpath:
sys.argv += ['--toolpath', path]
suite = unittest.TestSuite()
loader = unittest.TestLoader()
for module in test_class_list:
# Test the test module about our arguments, if it is interested
if hasattr(module, 'setup_test_args'):
setup_test_args = getattr(module, 'setup_test_args')
setup_test_args(preserve_indir=test_preserve_dirs,
preserve_outdirs=test_preserve_dirs
and test_name is not None,
toolpath=toolpath,
verbosity=verbosity)
if test_name:
try:
suite.addTests(loader.loadTestsFromName(test_name, module))
except AttributeError:
continue
else:
suite.addTests(loader.loadTestsFromTestCase(module))
if use_concurrent and processes != 1:
concurrent_suite = ConcurrentTestSuite(
suite, fork_for_tests(processes or multiprocessing.cpu_count()))
concurrent_suite.run(result)
else:
suite.run(result)
def test_run_default_concurrency(self):
runner = unittest.TextTestRunner(stream=StringIO())
suite = unittest.TestLoader().loadTestsFromTestCase(BothPass)
# Run across all cpu/processesors in machine
concurrent_suite = ConcurrentTestSuite(suite, fork_for_tests())
result = runner.run(concurrent_suite)
self.assertTrue(result.wasSuccessful())
self.assertEqual(result.testsRun, suite.countTestCases())
self.assertEqual(result.errors, [])
self.assertEqual(result.failures, [])
self.assertEqual(result.skipped, [])
def test_run_default_concurrency(self):
runner = unittest.TextTestRunner(stream=StringIO())
suite = unittest.TestLoader().loadTestsFromTestCase(BothPass)
# Run across all cpu/processesors in machine
concurrent_suite = ConcurrentTestSuite(suite, fork_for_tests())
result = runner.run(concurrent_suite)
self.assertTrue(result.wasSuccessful())
self.assertEqual(result.testsRun, suite.countTestCases())
self.assertEqual(result.errors, [])
self.assertEqual(result.failures, [])
self.assertEqual(result.skipped, [])
def _parallelize(suite):
def fdopen(fd, mode, *kwds):
stream = orig_fdopen(fd, mode)
atexit.register(stream.close)
return stream
# Monkey patch concurrencytest lib bug (fdopen() stream not closed).
# https://github.com/cgoldberg/concurrencytest/issues/11
orig_fdopen = os.fdopen
concurrencytest.os.fdopen = fdopen
forker = concurrencytest.fork_for_tests(NWORKERS)
return concurrencytest.ConcurrentTestSuite(suite, forker)
def test_Trigger():
loader = unittest.TestLoader()
suite = unittest.TestSuite()
suite.addTests(loader.loadTestsFromTestCase(TestAuthWithGoogleAuth))
suite.addTests(loader.loadTestsFromTestCase(TestAuthWithOAuth2Client))
suite.addTests(loader.loadTestsFromTestCase(TestAuthWithoutAuth))
suite.addTests(loader.loadTestsFromTestCase(TestGoogleAuthWithoutHttplib2))
suite.addTests(loader.loadTestsFromTestCase(TestMediaUpload))
logger.debug('Loaded %d test cases...' % suite.countTestCases())
runner = unittest.TextTestRunner()
logger.debug('\nRun same tests with 8 processes:')
concurrent_suite = ConcurrentTestSuite(suite, fork_for_tests(8))
runner.run(concurrent_suite)
def runsuite(suite, parallel):
suite_to_run = suite
if parallel > 1:
from concurrencytest import ConcurrentTestSuite, fork_for_tests
if not WiredTigerTestCase._globalSetup:
WiredTigerTestCase.globalSetup()
WiredTigerTestCase._concurrent = True
suite_to_run = ConcurrentTestSuite(suite, fork_for_tests(parallel))
try:
return unittest.TextTestRunner(
verbosity=WiredTigerTestCase._verbose).run(suite_to_run)
except BaseException as e:
# This should not happen for regular test errors, unittest should catch everything
print('ERROR: running test: ', e)
raise e
def runsuite(suite, parallel):
suite_to_run = suite
if parallel > 1:
from concurrencytest import ConcurrentTestSuite, fork_for_tests
if not WiredTigerTestCase._globalSetup:
WiredTigerTestCase.globalSetup()
WiredTigerTestCase._concurrent = True
suite_to_run = ConcurrentTestSuite(suite, fork_for_tests(parallel))
try:
return unittest.TextTestRunner(
verbosity=WiredTigerTestCase._verbose).run(suite_to_run)
except BaseException as e:
# This should not happen for regular test errors, unittest should catch everything
print('ERROR: running test: ', e)
raise e
def main(argv):
parser = argparse.ArgumentParser(prog=argv[0],
description='run CAmkES tests')
parser.add_argument('--jobs',
'-j',
nargs='?',
type=int,
help='parallelise test execution')
parser.add_argument(
'--verbosity',
'-v',
default=1,
type=int,
help="Verbosity to run tests. 0 = quiet. 1 = default. 2 = verbose")
parser.add_argument('test',
nargs='*',
choices=TESTS + ['all'],
default='all',
help='run a specific category of tests')
parser.add_argument('--capdl-python',
help='Deprecated. Using this argument has no effect.')
options = parser.parse_args(argv[1:])
if options.jobs is None:
# Maximum parallelism.
options.jobs = multiprocessing.cpu_count()
# work out which tests to run
if options.test == 'all' or 'all' in options.test:
test_packages = TESTS
else:
test_packages = options.test
# load the tests we want to run
loader = unittest.TestLoader()
test_suite = unittest.TestSuite()
for v in test_packages:
test_suite.addTests(
loader.discover('camkes.' + v, top_level_dir=os.path.dirname(ME)))
concurrent_suite = ConcurrentTestSuite(test_suite,
fork_for_tests(options.jobs))
runner = unittest.TextTestRunner(verbosity=options.verbosity)
result = runner.run(concurrent_suite)
if result.wasSuccessful():
return 0
return 1
def runsuite(suite, parallel):
suite_to_run = suite
if parallel > 1:
from concurrencytest import ConcurrentTestSuite, fork_for_tests
if not WiredTigerTestCase._globalSetup:
WiredTigerTestCase.globalSetup()
WiredTigerTestCase._concurrent = True
suite_to_run = ConcurrentTestSuite(suite, fork_for_tests(parallel))
try:
if WiredTigerTestCase._randomseed:
WiredTigerTestCase.prout("Starting test suite with seedw={0} and seedz={1}. Rerun this test with -seed {0}.{1} to get the same randomness"
.format(str(WiredTigerTestCase._seeds[0]), str(WiredTigerTestCase._seeds[1])))
return unittest.TextTestRunner(
verbosity=WiredTigerTestCase._verbose).run(suite_to_run)
except BaseException as e:
# This should not happen for regular test errors, unittest should catch everything
print('ERROR: running test: ', e)
raise e
def main(argv):
parser = argparse.ArgumentParser(prog=argv[0],
description='Run capdl tests')
parser.add_argument('--verbosity', '-v', default=1, type=int,
help="Verbosity to run tests. 0 = quiet. 1 = default. 2 = verbose")
options = parser.parse_args(argv[1:])
# load the tests we want to run
loader = unittest.TestLoader()
test_suite = unittest.TestSuite()
print("Looking for tests in {0}".format(os.path.dirname(ME)))
test_suite.addTests(loader.discover(os.path.dirname(ME), pattern="*.py"))
concurrent_suite = ConcurrentTestSuite(test_suite, fork_for_tests(multiprocessing.cpu_count()))
runner = unittest.TextTestRunner(verbosity=options.verbosity)
result = runner.run(concurrent_suite)
if result.wasSuccessful():
return 0
return 1
def main():
# runner = TextTestRunner(verbosity=2, failfast=True)
runner = TextTestRunner(verbosity=2)
try:
from concurrencytest import ConcurrentTestSuite, fork_for_tests
useParallerlTest = True
except ImportError:
# concurrencytest is not installed, use regular test runner
useParallerlTest = False
# useParallerlTest = False
if useParallerlTest:
concurrent_suite = ConcurrentTestSuite(suite, fork_for_tests())
res = runner.run(concurrent_suite)
else:
res = runner.run(suite)
if not res.wasSuccessful():
sys.exit(1)
def runtests(processes=4):
"""
Run Bitfield's unit tests. Will run the tests in parallel if the `concurrencytest` library
is installed. Will run serially otherwise.
"""
# Discover all tests in the current directory that are prefixed with `test`. Also discovers
# the doctests loaded by defining a load_tests(...) function in the module __init__.py
loader = unittest.TestLoader()
doctest_suite = loader.discover('.', pattern='test*.py')
runner = unittest.runner.TextTestRunner()
try:
from concurrencytest import ConcurrentTestSuite, fork_for_tests
concurrent_doctest_suite = ConcurrentTestSuite(
doctest_suite, fork_for_tests(processes))
runner.run(concurrent_doctest_suite)
except ImportError:
runner.run(doctest_suite)
# Prevent calling sys.exit() just in case the user is running the tests from an interpreter.
unittest.main(exit=False)
def parallel_run(verbose):
if verbose:
print("[runCompleteTestSuite] verbose run not implemented "
"for parallel version")
return
try:
from concurrencytest import ConcurrentTestSuite, fork_for_tests
except ImportError as e:
print("Need to install the module concurrencytest.")
print("pip install --user concurrencytest")
return
from smodels.tools import runtime
suite = unittest.TestLoader().discover("./")
ncpus = runtime.nCPUs()
## "shuffle" the tests, so that the heavy tests get distributed
## more evenly among threads (didnt help, so I commented it out)
#suite._tests = [ item for sublist in [ suite._tests[x::ncpus] \
# for x in range(ncpus) ] for item in sublist ]
concurrent_suite = ConcurrentTestSuite(suite, fork_for_tests(ncpus))
runner = unittest.TextTestRunner()
runner.run(concurrent_suite)
def parallel_run ( verbose ):
if verbose:
print ("[runCompleteTestSuite] verbose run not implemented "
"for parallel version" )
return
try:
from concurrencytest import ConcurrentTestSuite, fork_for_tests
except ImportError as e:
print ( "Need to install the module concurrencytest." )
print ( "pip install --user concurrencytest" )
return
from smodels.tools import runtime
suite = unittest.TestLoader().discover("./")
ncpus = runtime.nCPUs()
## "shuffle" the tests, so that the heavy tests get distributed
## more evenly among threads (didnt help, so I commented it out)
#suite._tests = [ item for sublist in [ suite._tests[x::ncpus] \
# for x in range(ncpus) ] for item in sublist ]
concurrent_suite = ConcurrentTestSuite(suite, fork_for_tests( ncpus ))
runner = unittest.TextTestRunner()
runner.run(concurrent_suite)
def execute():
test_list = []
setup(test_list)
for t in test_list:
test_name = 'test_%s' % t[0]
test = test_generator(t[1], t[2])
setattr(TestSequence, test_name, test)
fast = unittest.makeSuite(TestSequence, 'test')
print('\nRun mutant tests across 3 processes:')
with open("output.log", 'w') as f:
runner = unittest.TextTestRunner(f)
concurrent_suite = ConcurrentTestSuite(fast, fork_for_tests(3))
output = runner.run(concurrent_suite)
with open("output.log", 'r') as f:
lines = f.readlines()
i = 1
with open("results.txt", 'w') as w:
for c in lines[0].strip():
if c == '.':
w.write(f"\nMutant #{i} was not killed")
print(f"Mutant #{i} was not killed")
else:
w.write(f"\nMutant #{i} was killed")
print(f"Mutant #{i} was killed")
w.write(f"\nvector is {l[i - 1]}\n")
print(f"vector is {l[i - 1]}\n")
i = i + 1
tot_tests = output.testsRun
tot_fail = len(output.failures)
mut_cov = (tot_fail / tot_tests) * 100
with open("results.txt", "a") as w:
w.write("\nMutant Coverage: %{0:.2f}".format(mut_cov))
print("Mutant Coverage: %{0:.2f}".format(mut_cov))
def runsuite(suite, parallel):
suite_to_run = suite
if parallel > 1:
try:
from concurrencytest import ConcurrentTestSuite, fork_for_tests
except ImportError:
print('ERROR: additional python modules must be installed\n' +
' to use the "--parallel N" option. Consult\n' +
' the WiredTiger HOWTO:RunTheTestSuite wiki page.\n')
raise
if not WiredTigerTestCase._globalSetup:
WiredTigerTestCase.globalSetup()
WiredTigerTestCase._concurrent = True
suite_to_run = ConcurrentTestSuite(suite, fork_for_tests(parallel))
try:
return unittest.TextTestRunner(
verbosity=WiredTigerTestCase._verbose).run(suite_to_run)
except BaseException as e:
# This should not happen for regular test errors, unittest should catch everything
print('ERROR: running test: ', e)
raise e
def runsuite(suite, parallel):
suite_to_run = suite
if parallel > 1:
try:
from concurrencytest import ConcurrentTestSuite, fork_for_tests
except ImportError:
print ('ERROR: additional python modules must be installed\n' +
' to use the "--parallel N" option. Consult\n' +
' the WiredTiger HOWTO:RunTheTestSuite wiki page.\n')
raise
if not WiredTigerTestCase._globalSetup:
WiredTigerTestCase.globalSetup()
WiredTigerTestCase._concurrent = True
suite_to_run = ConcurrentTestSuite(suite, fork_for_tests(parallel))
try:
return unittest.TextTestRunner(
verbosity=WiredTigerTestCase._verbose).run(suite_to_run)
except BaseException as e:
# This should not happen for regular test errors, unittest should catch everything
print('ERROR: running test: ', e)
raise e
def run_test_suites(sanity_tests, single_tests, parallel_tests):
sanity = False
all_successful = False
failures = []
sanity_runner = unittest.TextTestRunner(verbosity=255, failfast=True)
sanity_result = sanity_runner.run(sanity_tests)
max_parallel_tests = multiprocessing.cpu_count() * 3
if sanity_result.wasSuccessful():
sanity = True
print('running %u tests in parallel and %u tests serial' % (
parallel_tests.countTestCases(), single_tests.countTestCases()))
results = []
if parallel_tests.countTestCases():
print('running maximum of %u of parallel tests' % max_parallel_tests)
max_parallel_tests = min(parallel_tests.countTestCases(), max_parallel_tests)
parallel_runner = unittest.TextTestRunner(verbosity=255)
parallel_suite = ConcurrentTestSuite(
parallel_tests, fork_for_tests(max_parallel_tests))
results.append(parallel_runner.run(parallel_suite))
# TODO: Tests that are serialized generally depend on hardcoded ports.
# Make them use dynamic ports.
if single_tests.countTestCases():
single_runner = unittest.TextTestRunner(verbosity=255)
results.append(single_runner.run(single_tests))
all_successful = True
for result in results:
if not result.wasSuccessful():
for failure in result.failures + result.errors:
description = failure[0].shortDescription()
if description:
failed_name = '-'.join(description.split('.')[1:])
failures.append(failed_name)
all_successful = False
print(result.printErrors())
else:
print('sanity tests failed - test environment not correct')
return (sanity, all_successful, failures)
def main():
# the following monkeypatching of the path would
# make the execution in subdirectories impossible
#print("\n###################### running single tests ##########################\n")
#s1=unittest.TestSuite()
#from Test_smooth_model_run import TestSmoothModelRun
#s1.addTest(TestSmoothModelRun('test_linearize'))
#r = unittest.TextTestRunner()
#res = r.run(s1)
#if len(res.errors) + len(res.failures) > 0:
# sys.exit(1)
print("\n###################### running tests ##########################\n")
s = unittest.defaultTestLoader.discover('', pattern='Test*')
#p = unittest.defaultTestLoader.discover('', pattern='Pinned_Test*')
#s.addTests(p)
#concurrent_suite = s
concurrent_suite = ConcurrentTestSuite(s, fork_for_tests(64))
r = unittest.TextTestRunner()
res = r.run(concurrent_suite)
if len(res.errors) + len(res.failures) > 0:
sys.exit(1)
Smat = get_shift_matrix(K_ortho_Geier, ex_Geier, ey_Geier)
f = io.StringIO()
with redirect_stdout(f):
print_as_vector(Shift_ortho_Geier, 's')
out = f.getvalue()
f2 = io.StringIO()
with redirect_stdout(f2):
print_as_vector(Smat[3:, 3:], 's')
out2 = f2.getvalue()
assert out == out2
# Pycharm runs them sequentially
# python -m unittest tests/test_example_unit_tests_parallel_run.py # sequential as well
# python tests/test_example_unit_tests_parallel_run.py # concurrent run :)
if __name__ == '__main__':
loader = unittest.TestLoader()
runner = unittest.TextTestRunner()
# Run same tests across 4 processes
cores = multiprocessing.cpu_count()
print(f'\nRunning tests on {cores} cores:')
suite = unittest.TestLoader().loadTestsFromTestCase(TestMatrixGenerator)
concurrent_suite = ConcurrentTestSuite(suite, fork_for_tests(cores))
# concurrent_suite.addTests(loader.loadTestsFromTestCase(TestMatrixGenerator)) # add more :>
runner.run(concurrent_suite)
def main(arguments):
"""
Begins running unit tests.
argv[1] if used must be the source filename else os_env.yaml will be
leveraged instead
argv[2] if used must be the proxy server <host>:<port>
"""
logger.info('Starting test suite')
log_level = LOG_LEVELS.get(arguments.log_level, logging.DEBUG)
flavor_metadata = None
if arguments.flavor_metadata:
flavor_metadata = {
'metadata': {
'hw:mem_page_size': arguments.flavor_metadata
}
}
image_metadata = None
if arguments.image_metadata_file:
image_metadata = file_utils.read_yaml(arguments.image_metadata_file)
concurrent_suite = None
sequential_suite = None
if arguments.env and arguments.ext_net:
unit = arguments.include_unit != ARG_NOT_SET
connection = arguments.include_connection != ARG_NOT_SET
api = arguments.include_api != ARG_NOT_SET
integration = arguments.include_integration != ARG_NOT_SET
ci = arguments.continuous_integration != ARG_NOT_SET
staging = arguments.include_staging != ARG_NOT_SET
if (not unit and not connection and not api and not integration
and not staging and not ci):
unit = True
connection = True
api = True
integration = True
concurrent_suite = __create_concurrent_test_suite(
arguments.env, arguments.ext_net, arguments.proxy,
arguments.ssh_proxy_cmd, unit, connection, api, integration,
staging, flavor_metadata, image_metadata,
arguments.floating_ips != ARG_NOT_SET, ci, log_level)
if (arguments.include_integration != ARG_NOT_SET
and arguments.floating_ips != ARG_NOT_SET):
sequential_suite = __create_sequential_test_suite(
arguments.env, arguments.ext_net, arguments.proxy,
arguments.ssh_proxy_cmd, integration, flavor_metadata,
image_metadata, arguments.floating_ips != ARG_NOT_SET,
log_level)
else:
logger.error('Environment file or external network not defined')
exit(1)
i = 0
while i < int(arguments.num_runs):
i += 1
if concurrent_suite:
logger.info('Running Concurrent Tests')
concurrent_runner = unittest.TextTestRunner(verbosity=2)
concurrent_suite = ConcurrentTestSuite(
concurrent_suite, fork_for_tests(int(arguments.threads)))
concurrent_results = concurrent_runner.run(concurrent_suite)
__output_results(concurrent_results)
if ((concurrent_results.errors
and len(concurrent_results.errors) > 0)
or (concurrent_results.failures
and len(concurrent_results.failures) > 0)):
logger.error('See above for test failures')
exit(1)
else:
logger.info(
'Concurrent tests completed successfully in run #%s', i)
if sequential_suite:
logger.info('Running Sequential Tests')
sequential_runner = unittest.TextTestRunner(verbosity=2)
sequential_results = sequential_runner.run(sequential_suite)
__output_results(sequential_results)
if ((sequential_results.errors
and len(sequential_results.errors) > 0)
or (sequential_results.failures
and len(sequential_results.failures) > 0)):
logger.error('See above for test failures')
exit(1)
else:
logger.info(
'Sequential tests completed successfully in run #%s', i)
logger.info('Successful completion of %s test runs', i)
exit(0)
def main():
# pylint: disable=import-outside-toplevel
if "BUNDLED_LIB_LOCATION" in os.environ:
sys.path.insert(0, os.environ["BUNDLED_LIB_LOCATION"])
if "--installed" in sys.argv:
sys.path.append(PACKAGE_DIR)
from pcs import settings
if settings.pcs_bundled_packages_dir not in sys.path:
sys.path.insert(0, settings.pcs_bundled_packages_dir)
from pcs_test.tools import pcs_runner
pcs_runner.test_installed = True
else:
sys.path.insert(0, PACKAGE_DIR)
from pcs import settings
settings.pcs_data_dir = os.path.join(PACKAGE_DIR, "data")
run_concurrently = can_concurrency and "--no-parallel" not in sys.argv
explicitly_enumerated_tests = [
prepare_test_name(arg) for arg in sys.argv[1:] if arg not in (
"-v",
"--all-but",
"--fast-info", # show a traceback immediatelly after the test fails
"--last-slash",
"--list",
"--no-parallel",
"--traceback-highlight",
"--traditional-verbose",
"--vanilla",
"--installed",
"--tier0",
"--tier1",
)
]
tier = None
if "--tier0" in sys.argv:
tier = 0
elif "--tier1" in sys.argv:
tier = 1
discovered_tests = discover_tests(explicitly_enumerated_tests,
"--all-but" in sys.argv,
tier=tier)
if "--list" in sys.argv:
test_list = tests_from_suite(discovered_tests)
print("\n".join(sorted(test_list)))
print("{0} tests found".format(len(test_list)))
sys.exit()
tests_to_run = discovered_tests
tier1_fixtures_cleanup = None
if tier1_fixtures_needed(tests_to_run):
tier1_fixtures_cleanup = run_tier1_fixtures(
run_concurrently=run_concurrently)
if run_concurrently:
tests_to_run = ConcurrentTestSuite(
discovered_tests,
concurrencytest.fork_for_tests(),
)
use_improved_result_class = (sys.stdout.isatty() and sys.stderr.isatty()
and "--vanilla" not in sys.argv)
ResultClass = unittest.TextTestResult
if use_improved_result_class:
from pcs_test.tools.color_text_runner import get_text_test_result_class
ResultClass = get_text_test_result_class(
slash_last_fail_in_overview=("--last-slash" in sys.argv),
traditional_verbose=(
"--traditional-verbose" in sys.argv or
# temporary workaround - our verbose writer is not compatible with
# running tests in parallel, use our traditional writer
(run_concurrently and "-v" in sys.argv)),
traceback_highlight=("--traceback-highlight" in sys.argv),
fast_info=("--fast-info" in sys.argv),
)
test_runner = unittest.TextTestRunner(
verbosity=2 if "-v" in sys.argv else 1, resultclass=ResultClass)
test_result = test_runner.run(tests_to_run)
if tier1_fixtures_cleanup:
tier1_fixtures_cleanup()
if not test_result.wasSuccessful():
sys.exit(1)
for scoring_algorithm in scoring_algorithms:
file_name = "{} {} {}.csv".format(
scoring_algorithm.value, pairing_algorithm.value, correct_rate_str
)
report_path = "{}/{}".format(REPORT_FOLDER, file_name)
with open(report_path, "w+") as csvfile:
out = csv.writer(csvfile)
heading = []
for index in range(1, 101):
heading.append("Round {}".format(index))
out.writerow(heading)
AlgorithmValidityTests.SCORING_ALGORITHM = scoring_algorithm
AlgorithmValidityTests.PAIRING_ALGORITHM = pairing_algorithm
AlgorithmValidityTests.REPORT_PATH = report_path
AlgorithmValidityTests.WINNER_SELECTOR = winner_selector
AlgorithmValidityTests.CORRECT_RATE = correct_rate
loader = unittest.TestLoader()
test = loader.loadTestsFromName('test_random_students_perform_comparisons', AlgorithmValidityTests)
suite = unittest.TestSuite()
for _ in range(REPETITIONS):
suite.addTest(test)
runner = unittest.TextTestRunner()
concurrent_suite = concurrencytest.ConcurrentTestSuite(suite, concurrencytest.fork_for_tests(CONCURRENCY))
print("Starting Test for {}".format(file_name))
runner.run(concurrent_suite)
print("Finished Test for {}".format(file_name))
print("")
def RunTests(debug, verbosity, processes, test_preserve_dirs, args, toolpath):
"""Run the functional tests and any embedded doctests
Args:
debug: True to enable debugging, which shows a full stack trace on error
verbosity: Verbosity level to use
test_preserve_dirs: True to preserve the input directory used by tests
so that it can be examined afterwards (only useful for debugging
tests). If a single test is selected (in args[0]) it also preserves
the output directory for this test. Both directories are displayed
on the command line.
processes: Number of processes to use to run tests (None=same as #CPUs)
args: List of positional args provided to binman. This can hold a test
name to execute (as in 'binman test testSections', for example)
toolpath: List of paths to use for tools
"""
import cbfs_util_test
import elf_test
import entry_test
import fdt_test
import ftest
import image_test
import test
import doctest
result = unittest.TestResult()
for module in []:
suite = doctest.DocTestSuite(module)
suite.run(result)
sys.argv = [sys.argv[0]]
if debug:
sys.argv.append('-D')
if verbosity:
sys.argv.append('-v%d' % verbosity)
if toolpath:
for path in toolpath:
sys.argv += ['--toolpath', path]
# Run the entry tests first ,since these need to be the first to import the
# 'entry' module.
test_name = args and args[0] or None
suite = unittest.TestSuite()
loader = unittest.TestLoader()
for module in (entry_test.TestEntry, ftest.TestFunctional,
fdt_test.TestFdt, elf_test.TestElf, image_test.TestImage,
cbfs_util_test.TestCbfs):
# Test the test module about our arguments, if it is interested
if hasattr(module, 'setup_test_args'):
setup_test_args = getattr(module, 'setup_test_args')
setup_test_args(preserve_indir=test_preserve_dirs,
preserve_outdirs=test_preserve_dirs
and test_name is not None,
toolpath=toolpath,
verbosity=verbosity)
if test_name:
try:
suite.addTests(loader.loadTestsFromName(test_name, module))
except AttributeError:
continue
else:
suite.addTests(loader.loadTestsFromTestCase(module))
if use_concurrent and processes != 1:
concurrent_suite = ConcurrentTestSuite(
suite, fork_for_tests(processes or multiprocessing.cpu_count()))
concurrent_suite.run(result)
else:
suite.run(result)
# Remove errors which just indicate a missing test. Since Python v3.5 If an
# ImportError or AttributeError occurs while traversing name then a
# synthetic test that raises that error when run will be returned. These
# errors are included in the errors accumulated by result.errors.
if test_name:
errors = []
for test, err in result.errors:
if ("has no attribute '%s'" % test_name) not in err:
errors.append((test, err))
result.testsRun -= 1
result.errors = errors
print(result)
for test, err in result.errors:
print(test.id(), err)
for test, err in result.failures:
print(err, result.failures)
if result.skipped:
print('%d binman test%s SKIPPED:' %
(len(result.skipped), 's' if len(result.skipped) > 1 else ''))
for skip_info in result.skipped:
print('%s: %s' % (skip_info[0], skip_info[1]))
if result.errors or result.failures:
print('binman tests FAILED')
return 1
return 0
files_patched.append(l[len("patching file '"):-1])
elif l.startswith("patching file "):
files_patched.append(l[len("patching file "):])
print(result[0])
if args.add:
subprocess.call(["git", "add"] + files_patched)
print("Done.\nAdded to git:\n" + '\n'.join(files_patched))
else:
print("Done.\nYou now need to run:\n" +
'\n'.join('git add ' + x for x in files_patched))
else:
test_loader = unittest2.TestLoader()
all_test_names = ["test_" + t.name for t in TEST_CASES]
test_loader.sortTestMethodsUsing = lambda x, y: cmp(
all_test_names.index(x), all_test_names.index(y))
suite = test_loader.loadTestsFromTestCase(IntegrationTests)
if args.jobs > 1:
suite = ConcurrentTestSuite(suite, fork_for_tests(args.jobs))
if hasattr(sys.stderr, "isatty") and sys.stderr.isatty():
test_result = colour_runner.runner.ColourTextTestRunner(
verbosity=2).run(suite)
else:
test_result = unittest2.TextTestRunner(verbosity=2).run(suite)
if not test_result.wasSuccessful():
print('\n***\nRun one of these:\n' +
'./integration_tests.py --fix\n' +
'./integration_tests.py --fix --add\n' + '***\n')
exit(1)
else:
exit(0)
sql.upload_dataset()
# Create and run suite
suite = unittest.TestSuite()
for test in to_run:
suite.addTest(SQLShare(test))
result = unittest.TestResult(verbosity=2)
runner = unittest.TextTestRunner(stream=sys.stdout)
#runner.run(suite)
# Concurrency
if settings['concurrent_tests']:
from concurrencytest import ConcurrentTestSuite, fork_for_tests
concurrent_suite = ConcurrentTestSuite(suite, fork_for_tests(settings['concurrent_tests']))
runner.run(concurrent_suite)
else:
runner.run(suite)
for dataset in to_upload:
dataset_name = dataset['dataset_name']
if dataset_name is not sql.to_delete_dataset:
sql.open_dataset(dataset_name)
sql.delete_dataset()
sql.tearDown()
del sql
#!/usr/bin/python
# vim: set expandtab ts=4
import unittest
from concurrencytest import ConcurrentTestSuite, fork_for_tests
import IndexedTest
import ChangeOfBaseTests
if __name__ == '__main__':
s= unittest.TestLoader().loadTestsFromTestCase(IndexedTest.IndexedTest)
suites=[ChangeOfBaseTests.suite()]
for suite in suites:
s.addTests(suite)
# Run same tests across 16 processes
concurrent_suite = ConcurrentTestSuite(s, fork_for_tests(16))
runner = unittest.TextTestRunner()
runner.run(concurrent_suite)
'''
import unittest, sys
import testcases.regression_test_1
import testcases.regression_test_2
from concurrencytest import ConcurrentTestSuite, fork_for_tests
if __name__ == '__main__':
loader = unittest.TestLoader()
suite = unittest.BaseTestSuite()
suite.addTests(loader.loadTestsFromModule(testcases.regression_test_1))
suite.addTests(loader.loadTestsFromModule(testcases.regression_test_2))
runner = unittest.TextTestRunner()
isSuccessful = True
testsRan = False
concurrent_suite = ConcurrentTestSuite(suite, fork_for_tests(10))
isSuccessful = runner.run(concurrent_suite)
if isSuccessful:
sys.exit(0)
sys.exit(1)
def assert_equal(self, iteration, a, b):
if iteration == self.iteration_max: return True #self.assertEqual(a, b)
return a == b
if __name__ == '__main__':
TestCodeSnippets.dump = True
os.system('rm -r paper_statistics/pyct_run_pyct*')
os.system('mkdir -p paper_statistics/pyct_run_pyct')
# load the TestSuite
loader = unittest.TestLoader()
suite = unittest.TestSuite()
suite.addTests(loader.loadTestsFromTestCase(TestCodeSnippets))
# start preparation
def job():
while len(next(os.walk('paper_statistics/pyct_run_pyct'))[2]) != suite.countTestCases(): pass
os.system('echo "ID|Function|Line Coverage|Time (sec.)|# of SMT files|# of SAT|Time of SAT|# of UNSAT|Time of UNSAT|# of OTHERWISE|Time of OTHERWISE" > paper_statistics/pyct_run_pyct.csv')
os.system('cat paper_statistics/pyct_run_pyct/*.csv >> paper_statistics/pyct_run_pyct.csv')
os.system('rm -r paper_statistics/pyct_run_pyct')
pid = os.getpid()
os.kill(pid, signal.SIGTERM) #or signal.SIGKILL
t = threading.Thread(target = job)
t.start()
# run the TestSuite
concurrent_suite = ConcurrentTestSuite(suite, fork_for_tests(args.num))
result = unittest.TextTestRunner().run(concurrent_suite)
result.stop()
print('Finish the integration test!!!')
explicitly_enumerated_tests, "--all-but" in sys.argv
)
if "--list" in sys.argv:
test_list = tests_from_suite(discovered_tests)
print("\n".join(sorted(test_list)))
print("{0} tests found".format(len(test_list)))
sys.exit()
tests_to_run = discovered_tests
if run_concurrently:
# replace the partitioning function with our own
concurrencytest.partition_tests = partition_tests
tests_to_run = ConcurrentTestSuite(
discovered_tests,
# the number doesn't matter, we do our own partitioning which ignores it
concurrencytest.fork_for_tests(1)
)
use_improved_result_class = (
sys.stdout.isatty()
and
sys.stderr.isatty()
and
"--vanilla" not in sys.argv
)
resultclass = unittest.TextTestResult
if use_improved_result_class:
from pcs_test.tools.color_text_runner import get_text_test_result_class
resultclass = get_text_test_result_class(
explicitly_enumerated_tests, "--all-but" in sys.argv
)
if "--list" in sys.argv:
test_list = tests_from_suite(discovered_tests)
print("\n".join(sorted(test_list)))
print("{0} tests found".format(len(test_list)))
sys.exit()
tests_to_run = discovered_tests
if run_concurrently:
# replace the partitioning function with our own
concurrencytest.partition_tests = partition_tests
tests_to_run = ConcurrentTestSuite(
discovered_tests,
# the number doesn't matter, we do our own partitioning which ignores it
concurrencytest.fork_for_tests(1)
)
use_improved_result_class = (
sys.stdout.isatty()
and
sys.stderr.isatty()
and
"--vanilla" not in sys.argv
)
resultclass = unittest.TextTestResult
if use_improved_result_class:
from pcs.test.tools.color_text_runner import get_text_test_result_class
resultclass = get_text_test_result_class(
def testSuiteFromTCs(*tcs):
loader = TestLoader()
loadedTcs = [loader.loadTestsFromTestCase(tc) for tc in tcs]
suite = TestSuite(loadedTcs)
return suite
suite = testSuiteFromTCs(
JsonWriterTC,
VcdParserTC,
VcdWriterTC,
)
if __name__ == '__main__':
runner = TextTestRunner(verbosity=2)
try:
from concurrencytest import ConcurrentTestSuite, fork_for_tests
useParallerlTest = True
except ImportError:
# concurrencytest is not installed, use regular test runner
useParallerlTest = False
if useParallerlTest:
# Run same tests across 4 processes
concurrent_suite = ConcurrentTestSuite(suite, fork_for_tests())
runner.run(concurrent_suite)
else:
sys.exit(not runner.run(suite).wasSuccessful())
sql.upload_dataset()
# Create and run suite
suite = unittest.TestSuite()
for test in to_run:
suite.addTest(SQLShare(test))
result = unittest.TestResult(verbosity=2)
runner = unittest.TextTestRunner(stream=sys.stdout)
#runner.run(suite)
# Concurrency
if settings['concurrent_tests']:
from concurrencytest import ConcurrentTestSuite, fork_for_tests
concurrent_suite = ConcurrentTestSuite(suite, fork_for_tests(settings['concurrent_tests']))
runner.run(concurrent_suite)
else:
runner.run(suite)
for dataset in to_upload:
dataset_name = dataset['dataset_name']
if dataset_name is not sql.to_delete_dataset:
sql.open_dataset(dataset_name)
sql.delete_dataset()
sql.tearDown()
del sql
