def run_tests(complete: bool = True,
strict: bool = True,
dry_run: bool = False) -> (unittest.TestResult, Coverage, unittest.TestSuite):
"""
Run integration tests
:param complete: When true ibllib unit tests are run in addition to the integration tests.
:param strict: When true asserts that all gathered tests were successfully imported. This
means that a module not found error in any test module will raise an exception.
:param dry_run: When true the tests are gathered but not run.
:return Test results and coverage objects, and test suite.
"""
# Coverage recorded for all code within the source directory; otherwise just omit some
# common pyCharm files
options = {'omit': ['*pydevd_file_utils.py', 'test_*'], 'source': []}
# Gather tests
test_dir = str(Path(ci.tests.__file__).parent)
logger.info(f'Loading integration tests from {test_dir}')
ci_tests = unittest.TestLoader().discover(test_dir, pattern='test_*')
if complete: # include ibllib and brainbox unit tests
root = Path(ibllib.__file__).parents[1] # Search relative to our imported ibllib package
test_dirs = [root.joinpath(x) for x in ('brainbox', 'oneibl', 'ibllib', 'alf')]
for tdir in test_dirs:
logger.info(f'Loading unit tests from folders: {tdir}')
assert tdir.exists(), f'Failed to find unit test folders in {tdir}'
unit_tests = unittest.TestLoader().discover(str(tdir), pattern='test_*', top_level_dir=root)
logger.info(f"Found {unit_tests.countTestCases()}, appending to the test suite")
ci_tests = unittest.TestSuite((ci_tests, *unit_tests))
# for coverage, append the path of the test modules to the source key
options['source'].append(str(tdir))
logger.info(f'Complete suite contains {ci_tests.countTestCases()} tests')
# Check all tests loaded successfully
not_loaded = [x[12:] for x in list_tests(ci_tests) if x.startswith('_Failed')]
if len(not_loaded) != 0:
err_msg = 'Failed to import the following tests:\n\t' + '\n\t'.join(not_loaded)
assert not strict, err_msg
logger.warning(err_msg)
if dry_run:
return unittest.TestResult(), Coverage(**options), ci_tests
# Run tests with coverage
cov = Coverage(**options)
cov.start()
result = unittest.TextTestRunner(verbosity=2).run(ci_tests)
cov.stop()
cov.save()
return result, cov, ci_tests
def main(argv=None):
"""Application entry-point.
This will read the given coverage data file and prints a JSON object with
the following fields to stdout:
``covered``
An array of line numbers that are covered.
``excluded``
An array of line numbers that have been excluded from being subject
to coverage checks.
``missing``
An array of line numbers that have not been covered.
"""
args = parse_args(argv)
coverage = Coverage(data_file=str(args.coverage[0]))
coverage.load()
_, covered, excluded, missing, _ = coverage.analysis2(str(args.file[0]))
print(
json.dumps({
"covered": covered,
"excluded": excluded,
"missing": missing,
}))
return 0
def run(self) -> int:
coverage = Coverage()
coverage.start()
exit_code = main(['-v', 'tests'])
coverage.stop()
# Early exit if pytest failed
if exit_code != 0:
return 0
# Generate xml report in StringIO file
coverage.get_data()
coverage.config.from_args(
ignore_errors=None,
report_omit=None,
report_include=None,
xml_output=None,
)
data = StringIO()
reporter = XmlReporter(coverage, coverage.config)
reporter.report(None, data)
data.seek(0)
# Check diff cover compared to origin/master using xml
score = generate_coverage_report(coverage_xml=[data],
compare_branch='origin/master',
exclude=self.exclude_paths)
return score
def run_tests(context, app=None, module=None, doctype=None, test=(),
driver=None, profile=False, coverage=False, junit_xml_output=False, ui_tests = False,
doctype_list_path=None, skip_test_records=False, skip_before_tests=False, failfast=False):
"Run tests"
import frappe.test_runner
tests = test
site = get_site(context)
frappe.init(site=site)
frappe.flags.skip_before_tests = skip_before_tests
frappe.flags.skip_test_records = skip_test_records
if coverage:
# Generate coverage report only for app that is being tested
source_path = os.path.join(get_bench_path(), 'apps', app or 'frappe')
cov = Coverage(source=[source_path], omit=['*.html', '*.js', '*.xml', '*.css', '*/doctype/*/*_dashboard.py', '*/patches/*'])
cov.start()
ret = frappe.test_runner.main(app, module, doctype, context.verbose, tests=tests,
force=context.force, profile=profile, junit_xml_output=junit_xml_output,
ui_tests = ui_tests, doctype_list_path = doctype_list_path, failfast=failfast)
if coverage:
cov.stop()
cov.save()
if len(ret.failures) == 0 and len(ret.errors) == 0:
ret = 0
if os.environ.get('CI'):
sys.exit(ret)
def run_tests():
config = utils.get_config()
cov_conf_file = config.get('file_locations',
'coverage_in_conf',
fallback='config/.coveragerc')
cov_out_file = config.get('file_locations',
'coverage_out_dir',
fallback='coverage')
cov = Coverage(config_file=cov_conf_file)
cov.start()
result = grab_test_results()
if not result.wasSuccessful():
print("Error running unit tests ...")
exit(1)
cov.stop()
cov.save()
print('Coverage Summary:')
cov.report()
cov.html_report(directory=cov_out_file)
print('HTML version: file://{0}/{1}/index.html'.format(
os.getcwd(), cov_out_file))
cov.erase()
exit(0)
def main():
"""Run administrative tasks."""
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'project.settings')
# MyMoney Customization: run coverage.py around tests automatically
try:
command = sys.argv[1]
except IndexError:
command = "help"
running_tests = (command == 'test')
if running_tests:
from coverage import Coverage
cov = Coverage()
cov.erase()
cov.start()
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?") from exc
execute_from_command_line(sys.argv)
if running_tests:
cov.stop()
cov.save()
covered = cov.report()
if covered < 100:
sys.exit(0)
def combinedCov(self, alpha, cluster, tweet):
covInstance = Coverage(cluster, tweet)
covScore = covInstance.getCoverage()
for key in self.result:
origin = self.result[key]
self.result[key] = alpha * origin + (1 - alpha) * covScore[key]
return self.result
def report(testSuite):
cov = Coverage(branch=True)
cov.start()
for test_case in testSuite:
TriangleApp.TriangleTester(test_case[1][0], test_case[1][1], test_case[1][2])
cov.stop()
cov.report(file=open('./tmp/results.txt', 'w'), show_missing=True)
f = open('./tmp/results.txt', 'r')
lines = f.readlines()
imp_elm = []
br_not_cover = 0
elements = lines[2].split(' ')
for e in elements:
if e != '' and e[0].isdigit():
e = e.strip(',')
e = e.strip('\n')
e = e.strip('%')
imp_elm.append(e)
for i in range(5, len(imp_elm)):
if imp_elm[i].find('->') != -1:
br_not_cover = br_not_cover + 1
coverage_report = {'stmt': int(imp_elm[0]), 'miss': int(imp_elm[1]), 'branch': int(imp_elm[2]),
'br_par': int(imp_elm[3]), 'cover': int(imp_elm[4]), 'br_not_cover': br_not_cover}
return coverage_report
def test(package, coverage=True, tests=(), verbose=False, fail_fast=False):
cwd = os.getcwd()
where = os.path.join(cwd, package.replace('.', os.sep))
top_level_dir = cwd
coverage = coverage and not tests
verbosity = 2 if verbose else 1
if coverage:
from coverage import Coverage
cover = Coverage(branch=True, source=[where])
cover.start()
loader = unittest.TestLoader()
if tests:
suite = loader.loadTestsFromNames(tests)
else:
suite = loader.discover(where, top_level_dir=top_level_dir)
runner = unittest.TextTestRunner(verbosity=verbosity, failfast=fail_fast)
runner.run(suite)
if coverage:
cover.stop()
cover.report()
def main():
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'app.settings')
try:
from django.core.management import execute_from_command_line
from django.conf import settings
# MyProject Customization: run coverage.py around tests automatically
running_tests = (sys.argv[1] == 'test')
if running_tests:
from coverage import Coverage
cov = Coverage()
cov.erase()
cov.start()
if settings.DEBUG:
if os.environ.get('RUN_MAIN') or os.environ.get('WERKZEUG_RUN_MAIN'):
import ptvsd
ptvsd.enable_attach(address = ('0.0.0.0', 3500))
print("Attached remote debugger")
except ImportError as exc:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?"
) from exc
execute_from_command_line(sys.argv)
if running_tests:
cov.stop()
cov.save()
covered = cov.report()
if covered < 100:
sys.exit(1)
def generate_coverage_reports(reports_dir=COVERAGE_DIR, report="all"):
'''
This task combines all the coverage data files generated during all the tests and generates the coverage reports.
:param report: type of the coverage report to be generated. Options: text|html|xml. Defaults to 'html'
:param reports_dir: directory under which all coverage files to be found and reports to be generated. Defaults to reports/coverage
'''
print("\nInitiating code coverage analysis ...")
coverage_obj = Coverage(data_file="%s/coverage" % reports_dir)
coverage_obj.combine()
coverage_obj.save()
if report == "all":
with open("%s/coverage.txt" % reports_dir, "w") as cov_rep:
coverage_obj.report(file=cov_rep, show_missing=True)
print("Generated Code-Coverage text report at %s/coverage.txt\n" %
reports_dir)
coverage_obj.html_report(directory=reports_dir)
print("Generated Code-Coverage HTML report at %s\n" % reports_dir)
coverage_obj.xml_report(outfile='%s/coverage.xml' % reports_dir)
print("Generated Code-Coverage XML report at %s/coverage.xml\n" %
reports_dir)
elif report == "text":
with open("%s/coverage.txt" % reports_dir, "w") as cov_rep:
coverage_obj.report(file=cov_rep, show_missing=True)
print("Generated Code-Coverage text report at %s/coverage.txt\n" %
reports_dir)
elif report == "html":
coverage_obj.html_report(directory=reports_dir)
print("Generated Code-Coverage HTML report at %s\n" % reports_dir)
elif report == "xml":
coverage_obj.xml_report(outfile='%s/coverage.xml' % reports_dir)
print("Generated Code-Coverage XML report at %s/coverage.xml\n" %
reports_dir)
def __init__(self, vw):
self.cov = Coverage()
self.comments = Comments()
self.running = True
self.cmt_changes = {}
self.cmt_lock = Lock()
self.stackvar_changes = {}
self.stackvar_lock = Lock()
self.syms = get_syms(vw)
self.syms_lock = Lock()
self.structs = {} # get_structs(vw)
self.structs_lock = Lock()
self.filedata_by_sha = {}
self.filedata_by_fname = {}
for fname in vw.getFiles():
sha256 = vw.getFileMeta(fname, 'sha256')
mdict = dict(vw.getFileMetaDict(fname))
mdict['name'] = fname
mdict['maps'] = [
mmap for mmap in vw.getMemoryMaps() if mmap[MAP_FNAME] == fname
]
mdict['sha256'] = sha256
self.filedata_by_sha[sha256] = mdict
self.filedata_by_fname[fname] = mdict
def main():
options = parse_args()
test_directory = os.path.dirname(os.path.abspath(__file__))
selected_tests = get_selected_tests(options)
if options.verbose:
print_to_stderr('Selected tests: {}'.format(', '.join(selected_tests)))
if options.coverage and not PYTORCH_COLLECT_COVERAGE:
shell(['coverage', 'erase'])
if options.jit:
selected_tests = filter(lambda test_name: "jit" in test_name, TESTS)
if options.determine_from is not None and os.path.exists(options.determine_from):
with open(options.determine_from, 'r') as fh:
touched_files = [
os.path.normpath(name.strip()) for name in fh.read().split('\n')
if len(name.strip()) > 0
]
# HACK: Ensure the 'test' paths can be traversed by Modulefinder
sys.path.append('test')
selected_tests = [
test for test in selected_tests
if determine_target(test, touched_files, options)
]
sys.path.remove('test')
has_failed = False
failure_messages = []
try:
for test in selected_tests:
options_clone = copy.deepcopy(options)
if test in USE_PYTEST_LIST:
options_clone.pytest = True
err_message = run_test_module(test, test_directory, options_clone)
if err_message is None:
continue
has_failed = True
failure_messages.append(err_message)
if not options_clone.continue_through_error:
raise RuntimeError(err_message)
print_to_stderr(err_message)
finally:
if options.coverage:
from coverage import Coverage
test_dir = os.path.dirname(os.path.abspath(__file__))
with set_cwd(test_dir):
cov = Coverage()
if PYTORCH_COLLECT_COVERAGE:
cov.load()
cov.combine(strict=False)
cov.save()
if not PYTORCH_COLLECT_COVERAGE:
cov.html_report()
if options.continue_through_error and has_failed:
for err in failure_messages:
print_to_stderr(err)
sys.exit(1)
def test(config,
tests=(),
fail_fast=False,
with_coverage=True,
with_lint=True):
if tests:
num_tests = len(tests)
s = '' if num_tests == 1 else 's'
printer.header('Running {num_tests} test{s}...'.format_map(locals()))
else:
coverage_message = ' with coverage' if with_coverage else ''
printer.header('Running tests{coverage_message}...'.format_map(
locals()))
runner = unittest.TextTestRunner(failfast=fail_fast)
loader = unittest.TestLoader()
if with_coverage:
from coverage import Coverage
coverage = Coverage(source=['runcommands'])
coverage.start()
if tests:
for name in tests:
runner.run(loader.loadTestsFromName(name))
else:
tests = loader.discover('.')
result = runner.run(tests)
if not result.errors:
if with_coverage:
coverage.stop()
coverage.report()
if with_lint:
printer.header('Checking for lint...')
lint(config)
def test(*tests, coverage=True, verbose=False, fail_fast=False):
top_level_dir = Path.cwd()
where = top_level_dir
for segment in top_level_dir.name.split('.'):
where = where / segment
coverage = coverage and not tests
verbosity = 2 if verbose else 1
if coverage:
from coverage import Coverage
cover = Coverage(branch=True, source=[where])
cover.start()
loader = unittest.TestLoader()
if tests:
suite = loader.loadTestsFromNames(tests)
else:
suite = loader.discover(where, top_level_dir=top_level_dir)
runner = unittest.TextTestRunner(verbosity=verbosity, failfast=fail_fast)
runner.run(suite)
if coverage:
cover.stop()
cover.report()
def coverage_start(self, features, marker):
"""
Start the coverage measurement
"""
# if no explicit modules are specified we just
# use the ones loaded from radish's basedir.
# During the plugin init the basedir modules are
# not loaded yet, but they are during the start method.
# Thus, we are safe to consider the difference between the
# two for coverage measurement.
if not self.cover_packages:
source = list(set(sys.modules.keys()).difference(self.modules_on_init))
else:
source = self.cover_packages
self.coverage = Coverage(source=source,
config_file=world.config.cover_config_file,
branch=world.config.cover_branches)
if world.config.cover_erase:
self.coverage.combine()
self.coverage.erase()
if world.config.cover_append:
self.coverage.load()
self.coverage.start()
def main():
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'django-graphql.settings')
try:
command = sys.argv[1]
except IndexError:
command = "help"
running_tests = (command == 'test')
if running_tests:
from coverage import Coverage
cov = Coverage()
cov.erase()
cov.start()
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?") from exc
execute_from_command_line(sys.argv)
if running_tests:
cov.stop()
cov.save()
covered = cov.report()
if covered < 100:
sys.exit(1)
def cal_coverage(collection_yaml, model_yaml, output_root):
index_path = get_index_path(collection_yaml)
this_output_root = os.path.join(output_root, collection_yaml['name'])
Coverage(index_path).cal_coverage(
model_yaml,
os.path.join(collection_yaml['anserini_root'], collection_yaml['qrels_root'], collection_yaml['qrel']),
this_output_root
)
def _run_avocado(runnable, queue):
try:
# This assumes that a proper resolution (see resolver module)
# was performed, and that a URI contains:
# 1) path to python module
# 2) class
# 3) method
#
# To be defined: if the resolution uri should be composed like
# this, or broken down and stored into other data fields
module_path, klass_method = runnable.uri.split(":", 1)
klass, method = klass_method.split(".", 1)
params = AvocadoInstrumentedTestRunner._create_params(runnable)
result_dir = runnable.output_dir or tempfile.mkdtemp(prefix=".avocado-task")
test_factory = [
klass,
{
"name": TestID(1, runnable.uri, runnable.variant),
"methodName": method,
"config": runnable.config,
"modulePath": module_path,
"params": params,
"tags": runnable.tags,
"run.results_dir": result_dir,
},
]
messages.start_logging(runnable.config, queue)
if "COVERAGE_RUN" in os.environ:
from coverage import Coverage
coverage = Coverage()
coverage.start()
instance = loader.load_test(test_factory)
early_state = instance.get_state()
early_state["type"] = "early_state"
queue.put(early_state)
instance.run_avocado()
if "COVERAGE_RUN" in os.environ:
coverage.stop()
coverage.save()
state = instance.get_state()
fail_reason = state.get("fail_reason")
queue.put(messages.WhiteboardMessage.get(state["whiteboard"]))
queue.put(
messages.FinishedMessage.get(
state["status"].lower(), fail_reason=fail_reason
)
)
except Exception as e:
queue.put(messages.StderrMessage.get(traceback.format_exc()))
queue.put(messages.FinishedMessage.get("error", fail_reason=str(e)))
def _bootstrap(self):
from coverage import Coverage
cov = Coverage(data_suffix=True)
cov.start()
try:
return original_bootstrap(self)
finally:
cov.stop()
cov.save()
def coverage():
from coverage import Coverage
cover = Coverage()
cover.start()
failures = run()
cover.stop()
cover.save()
cover.html_report()
return failures
def __init__(self):
before.all(self.coverage_start)
after.all(self.coverage_stop)
if world.config.cover_packages:
cover_packages = world.config.cover_packages.split(",")
else:
cover_packages = []
self.coverage = Coverage(source=cover_packages)
def setUp(self):
if not os.path.exists(self._testMethodName):
os.system('mkdir %s' % self._testMethodName)
# 这里依赖于命名规范，include指定要测试的文件
if self._testMethodName.startswith("test_"):
include_name = self._testMethodName[5:]
self.cov = Coverage(include=['comm.py', '%s.py' % include_name])
self.cov.start()
return super().setUp()
def _run_avocado(runnable, queue):
try:
# This assumes that a proper resolution (see resolver module)
# was performed, and that a URI contains:
# 1) path to python module
# 2) class
# 3) method
#
# To be defined: if the resolution uri should be composed like
# this, or broken down and stored into other data fields
module_path, klass_method = runnable.uri.split(':', 1)
klass, method = klass_method.split('.', 1)
params = AvocadoInstrumentedTestRunner._create_params(runnable)
result_dir = (runnable.output_dir
or tempfile.mkdtemp(prefix=".avocado-task"))
test_factory = [
klass, {
'name': TestID(1, runnable.uri),
'methodName': method,
'config': runnable.config,
'modulePath': module_path,
'params': params,
'tags': runnable.tags,
'run.results_dir': result_dir,
}
]
messages.start_logging(runnable.config, queue)
if 'COVERAGE_RUN' in os.environ:
from coverage import Coverage
coverage = Coverage()
coverage.start()
instance = loader.load_test(test_factory)
early_state = instance.get_state()
early_state['type'] = "early_state"
queue.put(early_state)
instance.run_avocado()
if 'COVERAGE_RUN' in os.environ:
coverage.stop()
coverage.save()
state = instance.get_state()
fail_reason = state.get('fail_reason')
queue.put(messages.WhiteboardMessage.get(state['whiteboard']))
queue.put(
messages.FinishedMessage.get(state['status'].lower(),
fail_reason=fail_reason))
except Exception as e:
queue.put(messages.StderrMessage.get(traceback.format_exc()))
queue.put(messages.FinishedMessage.get('error',
fail_reason=str(e)))
def setUp(self):
self.regress_dir = None
self.regress_basedir = os.path.abspath('%s/../regression' %
os.path.dirname(__file__))
self.test_file_name = None
self.gold_file_name = None
self.diff_file_name = None
# Setup all the log file paths
self.set_regress_dir_paths()
print("Test: %s is running under PID: %s of parent PID: %s" %
(self.test_name, os.getpid(), os.getppid()))
# test dir is common to all tests in a test class. When the tests are
# running in parallel, sometime makedirs() may see conflicts. So lock
# the operation and ignore the exceptions
try:
lock = threading.Lock()
lock.acquire()
if not os.path.isdir(self.get_test_dir()):
print("Creating the test dir: %s" % self.get_test_dir())
os.makedirs(self.get_test_dir(), 0o755)
except Exception as exc:
print(str(exc))
finally:
lock.release()
# clean up the previous test logs and test dir if exists.
try:
if os.path.isfile(self.test_file_name):
print("Cleaning up the previous test log: %s" %
self.test_file_name)
os.remove(self.test_file_name)
except Exception as exc:
print(str(exc))
# clean up the previous temp and test dir if exists.
try:
if os.path.isdir(self.get_temp_dir()):
print("Cleaning up the previous temp dir: %s" %
self.get_temp_dir())
shutil.rmtree(self.get_temp_dir())
print("Creating the temp dir: %s" % self.get_temp_dir())
os.makedirs(self.get_temp_dir(), 0o755)
except Exception as exc:
print(str(exc))
# Start coverage metrics
self.coverage_obj = Coverage(data_file="%s/coverage" % COVERAGE_DIR,
branch=True,
data_suffix=True,
cover_pylib=False,
omit=COVERAGE_EXCLUDE_RULES,
concurrency=["multiprocessing", "thread"])
self.coverage_obj.start()
def read_coverage_data():
"""
:rtype: CoverageData or None
"""
print('Using coverage data from .coverage file')
# noinspection PyPackageRequirements,PyUnresolvedReferences
from coverage import Coverage
cov = Coverage('.coverage')
cov.load()
return cov.get_data()
def coverage_analysis():
cov = Coverage()
cov.start()
try:
yield
finally:
cov.stop()
cov.report()
print("------ SAVING COVERAGE REPORTS ------ ")
cov.xml_report(outfile=os.path.join(".", 'cobertura.xml'))
def run(self):
cov = Coverage(data_suffix=True)
cov._warn_no_data = True
cov._warn_unimported_source = True
cov.start()
try:
super().run()
finally:
cov.stop()
cov.save()
def setup_coverage(self, subprocess, cov_plugin=None):
params = {
"include": [os.path.join(self.rootdir, "*")],
"omit": _get_python_lib_paths(),
}
self.cov = Coverage(data_file=getattr(self, "sub_cov_file", None),
config_file=False,
**params)
self.cov._warn_no_data = False
def _bootstrap(self):
"""Wrapper around _bootstrap to start coverage."""
from coverage import Coverage
cov = Coverage(data_suffix=True)
cov.start()
try:
return original_bootstrap(self)
finally:
cov.stop()
cov.save()
