class CoverageDBTest(TestCase):
def setUp(self):
self.db = CoverageDB('sqlite:///test.db', logger=logging.getLogger(__name__))
self.db.upgrade()
def tearDown(self):
os.unlink('test.db')
def test_has_seen(self):
self.assertFalse(self.db.has_test('foo.bar'))
self.db.set_test_has_seen_test('foo.bar', '1')
self.assertTrue(self.db.has_test('foo.bar'))
class TestCoveragePlugin(Plugin):
"""
Monitors tests to discover which tests cover which
lines of code.
We find the diff with the parent revision for diff-tests with::
git diff origin/master
If you run with the discover flag, it will attempt to discovery
any tests that are required to run to test the changes in your current
branch, against those of origin/master.
"""
score = 0
name = 'kleenex'
def _get_name_from_test(self, test):
test_method_name = test._testMethodName
# We need to determine the *actual* test path (as thats what nose gives us in wantMethod)
# for example, maybe a test was imported in foo.bar.tests, but originated as foo.bar.something.MyTest
# in this case, we'd need to identify that its *actually* foo.bar.something.MyTest to record the
# proper coverage
test_ = getattr(sys.modules[test.__module__], test.__class__.__name__)
test_name = '%s:%s.%s' % (test_.__module__, test_.__name__,
test_method_name)
return test_name
def _setup_coverage(self):
instance = coverage(include=os.path.join(os.getcwd(), '*'))
instance.collector._trace_class = ExtendedTracer
instance.use_cache(False)
return instance
def options(self, parser, env):
Plugin.options(self, parser, env)
parser.add_option("--kleenex-config", dest="kleenex_config", default="setup.cfg")
parser.add_option("--kleenex-config-section", dest="kleenex_config_section", default="kleenex")
def configure(self, options, config):
Plugin.configure(self, options, config)
if not self.enabled:
return
config = read_config(options.kleenex_config, options.kleenex_config_section)
self.config = config
assert not (self.config.discover and self.config.record), "You cannot use both `record` and `discover` options."
self.logger = logging.getLogger(__name__)
self.pending_funcs = set()
# diff is a mapping of filename->set(linenos)
self.diff_data = defaultdict(set)
# cov is a mapping of filename->set(linenos)
self.cov_data = defaultdict(set)
# test test_name->dict(filename->set(linenos))
self.test_data = defaultdict(dict)
report_output = config.report_output
if not report_output or report_output == '-':
self.report_file = None
elif report_output.startswith('sys://'):
pipe = report_output[6:]
assert pipe in ('stdout', 'stderr')
self.report_file = getattr(sys, pipe)
else:
self.report_file = open(report_output, 'w')
def begin(self):
# XXX: this is pretty hacky
self.db = CoverageDB(self.config.db, self.logger)
if self.config.record:
self.db.upgrade()
if self.config.report or self.config.record:
# If we're recording coverage we need to ensure it gets reset
self.coverage = self._setup_coverage()
if not (self.config.discover or self.config.record):
return
proc = Popen(['git', 'merge-base', 'HEAD', self.config.parent], stdout=PIPE, stderr=STDOUT)
self.parent_revision = proc.stdout.read().strip()
if self.config.discover:
# We need to determine our merge base
self.logger.info("Checking coverage for revision %s", self.parent_revision)
self.revision = self.parent_revision
try:
self.revision_id = self.db.get_revision_id(self.revision)
except ValueError:
raise ValueError('Revision not recorded in coverage database (do you need to rebase?)')
if not (self.config.discover or self.config.report):
return
s = time.time()
self.logger.info("Parsing diff from parent %s", self.parent_revision)
# pull in our diff
# git diff `git merge-base HEAD master`
proc = Popen(['git', 'diff', self.parent_revision], stdout=PIPE, stderr=STDOUT)
diff = proc.stdout.read().strip()
pending_funcs = self.pending_funcs
parser = DiffParser(diff)
files = list(parser.parse())
diff = self.diff_data
s = time.time()
for file in files:
# we dont care about headers
if file['is_header']:
continue
# file was removed
if file['new_filename'] == '/dev/null':
continue
is_new_file = (file['old_filename'] == '/dev/null')
if is_new_file:
filename = file['new_filename']
if not filename.startswith('b/'):
continue
else:
filename = file['old_filename']
if not filename.startswith('a/'):
continue # ??
filename = filename[2:]
# Ignore non python files
if not is_py_script(filename):
continue
# file is new, only record diff state
for chunk in file['chunks']:
linenos = filter(bool, (l['new_lineno'] for l in chunk if l['line']))
diff[file['new_filename'][2:]].update(linenos)
# we dont care about missing coverage for new code, and there
# wont be any "existing coverage" to check for
if is_new_file:
continue
if not self.config.discover:
continue
self.logger.info("Parsed diff in %.2fs as %d file(s)", time.time() - s, len(diff))
if self.config.discover:
self.logger.info("Finding coverage for %d file(s)", len(files))
for filename, linenos in diff.iteritems():
test_coverage = self.db.get_coverage(self.revision_id, filename, linenos)
if not test_coverage:
# check if we have any coverage recorded
if not self.db.has_coverage(self.revision_id, filename):
if self.config.skip_missing:
self.logger.warning('%s has no test coverage recorded', filename)
continue
raise AssertionError("Missing test coverage for %s" % filename)
else:
for test in test_coverage:
pending_funcs.add(test)
self.logger.info("Determined available coverage in %.2fs with %d test(s)", time.time() - s, len(pending_funcs))
def report(self, stream):
if self.config.record:
self._record_test_coverage()
if self.config.report:
self._report_test_coverage(stream)
def _record_test_coverage(self):
trans = self.db.begin()
# Trim the all revisions outside of bounds
if self.config.max_revisions:
self.logger.info("Trimming revision tail (past %s)", self.config.max_revisions)
s = time.time()
num_revisions = self.db.trim_revisions(self.config.max_revisions)
self.logger.info("%d revision(s) were trimmed in %.2fs", num_revisions, time.time() - s)
# Use our current revision
self.logger.info("Recording current revision")
proc = Popen(['git', 'log', '-n 1', '--format=%H %ct'], stdout=PIPE, stderr=STDOUT)
self.revision, commit_date = proc.stdout.read().strip().split(' ')
commit_date = datetime.datetime.fromtimestamp(int(commit_date))
revision_id = self.db.add_revision(self.revision, commit_date)
self.logger.info("Current revision recorded as %s (commit date of %s)", self.revision, commit_date)
# Finally record tests and their coverage
for test_name, files in self.test_data.iteritems():
self.db.remove_test(revision_id, test_name)
test_id = self.db.add_test(revision_id, test_name)
for filename, linenos in files.iteritems():
self.db.add_coverage(revision_id, test_id, filename, linenos)
trans.commit()
def _report_test_coverage(self, stream):
cov_data = self.cov_data
diff_data = self.diff_data
covered = 0
total = 0
missing = defaultdict(set)
for filename, linenos in diff_data.iteritems():
covered_linenos = cov_data[filename]
total += len(linenos)
covered += len(covered_linenos)
missing[filename] = linenos.difference(covered_linenos)
if self.report_file:
self.report_file.write(simplejson.dumps({
'stats': {
'covered': covered,
'total': total,
},
'missing': dict((k, tuple(v)) for k, v in missing.iteritems() if v),
}))
self.report_file.close()
elif total:
stream.writeln('Coverage Report')
stream.writeln('-' * 70)
stream.writeln('Coverage against diff is %.2f%% (%d / %d lines)' % (covered / float(total) * 100, covered, total))
if missing:
stream.writeln()
stream.writeln('%-35s %s' % ('Filename', 'Missing Lines'))
stream.writeln('-' * 70)
for filename, linenos in missing.iteritems():
if not linenos:
continue
stream.writeln('%-35s %s' % (filename, ', '.join(map(str, sorted(linenos)))))
def wantMethod(self, method):
if not self.config.discover:
return
# only works with unittest compatible functions currently
method_name = method.__name__
method = getattr(sys.modules[method.im_class.__module__], method.im_class.__name__)
test_name = '%s:%s.%s' % (method.__module__, method.__name__, method_name)
# test has coverage for diff
if test_name in self.pending_funcs:
return True
# check if this test was modified (e.g. added/changed)
diff_data = self.diff_data[inspect.getfile(method)]
if diff_data:
lines, startlineno = inspect.getsourcelines(method)
for lineno in xrange(startlineno, len(lines) + startlineno):
if lineno in diff_data:
self.pending_funcs.add(test_name)
self.logger.info("Adding test due to new or changed code: %s", test_name)
return True
# test has no coverage recorded, defer to other plugins
if self.config.test_missing and not self.db.has_test(self.revision_id, test_name):
self.pending_funcs.add(test_name)
self.logger.info("Allowing test due to missing coverage report: %s", test_name)
return None
return False
def startTest(self, test):
if not (self.config.record or self.config.report):
return
self.coverage.start()
def stopTest(self, test):
if not (self.config.record or self.config.report):
return
cov = self.coverage
cov.stop()
# this must have been imported under a different name
# if self.discover and test_name not in self.pending_funcs:
# self.logger.warning("Unable to determine origin for test: %s", test_name)
# return
# initialize reporter
rep = Reporter(cov)
# process all files
rep.find_code_units(None, cov.config)
# Compute the standard deviation for all code executed from this test
linenos = []
for filename in cov.data.measured_files():
linenos.extend(cov.data.executed_lines(filename).values())
# We're recording so fetch the test data
if self.config.record:
test_ = test.test
test_name = self._get_name_from_test(test_)
test_data = self.test_data[test_name]
for cu in rep.code_units:
# if sys.modules[test_.__module__].__file__ == cu.filename:
# continue
filename = cu.name + '.py'
linenos = cov.data.executed_lines(cu.filename)
if self.config.record:
linenos_in_prox = dict((k, v) for k, v in linenos.iteritems() if v < self.config.max_distance)
if linenos_in_prox:
test_data[filename] = linenos_in_prox
if self.config.report:
diff = self.diff_data[filename]
cov_linenos = [l for l in linenos if l in diff]
if cov_linenos:
self.cov_data[filename].update(cov_linenos)
cov.erase()
def begin(self):
# XXX: this is pretty hacky
self.db = CoverageDB(self.config.db, self.logger)
if self.config.record:
self.db.upgrade()
if self.config.report or self.config.record:
# If we're recording coverage we need to ensure it gets reset
self.coverage = self._setup_coverage()
if not (self.config.discover or self.config.record):
return
proc = Popen(['git', 'merge-base', 'HEAD', self.config.parent], stdout=PIPE, stderr=STDOUT)
self.parent_revision = proc.stdout.read().strip()
if self.config.discover:
# We need to determine our merge base
self.logger.info("Checking coverage for revision %s", self.parent_revision)
self.revision = self.parent_revision
try:
self.revision_id = self.db.get_revision_id(self.revision)
except ValueError:
raise ValueError('Revision not recorded in coverage database (do you need to rebase?)')
if not (self.config.discover or self.config.report):
return
s = time.time()
self.logger.info("Parsing diff from parent %s", self.parent_revision)
# pull in our diff
# git diff `git merge-base HEAD master`
proc = Popen(['git', 'diff', self.parent_revision], stdout=PIPE, stderr=STDOUT)
diff = proc.stdout.read().strip()
pending_funcs = self.pending_funcs
parser = DiffParser(diff)
files = list(parser.parse())
diff = self.diff_data
s = time.time()
for file in files:
# we dont care about headers
if file['is_header']:
continue
# file was removed
if file['new_filename'] == '/dev/null':
continue
is_new_file = (file['old_filename'] == '/dev/null')
if is_new_file:
filename = file['new_filename']
if not filename.startswith('b/'):
continue
else:
filename = file['old_filename']
if not filename.startswith('a/'):
continue # ??
filename = filename[2:]
# Ignore non python files
if not is_py_script(filename):
continue
# file is new, only record diff state
for chunk in file['chunks']:
linenos = filter(bool, (l['new_lineno'] for l in chunk if l['line']))
diff[file['new_filename'][2:]].update(linenos)
# we dont care about missing coverage for new code, and there
# wont be any "existing coverage" to check for
if is_new_file:
continue
if not self.config.discover:
continue
self.logger.info("Parsed diff in %.2fs as %d file(s)", time.time() - s, len(diff))
if self.config.discover:
self.logger.info("Finding coverage for %d file(s)", len(files))
for filename, linenos in diff.iteritems():
test_coverage = self.db.get_coverage(self.revision_id, filename, linenos)
if not test_coverage:
# check if we have any coverage recorded
if not self.db.has_coverage(self.revision_id, filename):
if self.config.skip_missing:
self.logger.warning('%s has no test coverage recorded', filename)
continue
raise AssertionError("Missing test coverage for %s" % filename)
else:
for test in test_coverage:
pending_funcs.add(test)
self.logger.info("Determined available coverage in %.2fs with %d test(s)", time.time() - s, len(pending_funcs))
def setUp(self):
self.db = CoverageDB('sqlite:///test.db', logger=logging.getLogger(__name__))
self.db.upgrade()
