def preprocess_coverage_data(coverage): ''' Preprocess the given coverage data. Preprocessing includes structuring the coverage data by directory for better performance as well as computing coverage summaries per directory. @type coverage: dict @param coverage: Coverage Data @rtype dict @return Preprocessed Coverage Data ''' ret = {"children": {}} if "source_files" in coverage: # Coveralls format source_files = coverage["source_files"] # Process every source file and store the coverage data in our tree structure for source_file in source_files: # Split the filename into path parts and file part name = source_file["name"] name_parts = name.split(os.sep) path_parts = name_parts[:-1] file_part = name_parts[-1] # Start at the top of the tree for the path walking ptr = ret["children"] # Walk the tree down, one path part at a time and create parts # on the fly if they don't exist yet in our tree. for path_part in path_parts: if path_part not in ptr: ptr[path_part] = {"children": {}} ptr = ptr[path_part]["children"] ptr[file_part] = { "coverage": [-1 if x is None else x for x in source_file["coverage"]] } else: raise RuntimeError("Unknown coverage format") # Now we need to calculate the coverage summaries (lines total and covered) # for each subtree in the tree. We can do this easily by using a recursive # definition. CoverageHelper.calculate_summary_fields(ret) return ret
def preprocess_coverage_data(coverage): ''' Preprocess the given coverage data. Preprocessing includes structuring the coverage data by directory for better performance as well as computing coverage summaries per directory. @type coverage: dict @param coverage: Coverage Data @rtype dict @return Preprocessed Coverage Data ''' ret = {"children": {}} if "source_files" in coverage: # Coveralls format source_files = coverage["source_files"] # Process every source file and store the coverage data in our tree structure for source_file in source_files: # Split the filename into path parts and file part name = source_file["name"] name_parts = name.split(os.sep) path_parts = name_parts[:-1] file_part = name_parts[-1] # Start at the top of the tree for the path walking ptr = ret["children"] # Walk the tree down, one path part at a time and create parts # on the fly if they don't exist yet in our tree. for path_part in path_parts: if path_part not in ptr: ptr[path_part] = {"children": {}} ptr = ptr[path_part]["children"] ptr[file_part] = { "coverage": [-1 if x is None else x for x in source_file["coverage"]] } else: raise RuntimeError("Unknown coverage format") # Now we need to calculate the coverage summaries (lines total and covered) # for each subtree in the tree. We can do this easily by using a recursive # definition. CoverageHelper.calculate_summary_fields(ret) return ret
def test_CoverageHelperApplyDirectivesMakeEmpty(): node = json.loads(covdata) # Check that making the set entirely empty doesn't crash things (tsmith mode) directives = ["-:**"] CoverageHelper.apply_include_exclude_directives(node, directives) result = CoverageHelper.get_flattened_names(node, prefix="") expected_names = [] assert result == set(expected_names)
def runTest(self): node = json.loads(covdata) # Check that making the set entirely empty doesn't crash things (tsmith mode) directives = ["-:**"] CoverageHelper.apply_include_exclude_directives(node, directives) result = CoverageHelper.get_flattened_names(node, prefix="") expected_names = [] self.assertEqual(result, set(expected_names))
def runTest(self): node = json.loads(covdata) # Check that excluding all paths works (specialized case) directives = ["-:**", "+:topdir2/subdir1/**"] CoverageHelper.apply_include_exclude_directives(node, directives) result = CoverageHelper.get_flattened_names(node, prefix="") expected_names = [ 'topdir2', 'topdir2/subdir1', 'topdir2/subdir1/file1.c' ] self.assertEqual(result, set(expected_names))
def runTest(self): node = json.loads(covdata) # Check that any empty childs are pruned (empty childs are not useful) directives = ["-:topdir1/subdir1/**", "-:topdir1/subdir2/**"] CoverageHelper.apply_include_exclude_directives(node, directives) result = CoverageHelper.get_flattened_names(node, prefix="") expected_names = [ 'topdir2', 'topdir2/subdir1', 'topdir2/subdir1/file1.c' ] self.assertEqual(result, set(expected_names))
def test_CoverageHelperApplyDirectivesExcludeAll(): node = json.loads(covdata) # Check that excluding all paths works (specialized case) directives = ["-:**", "+:topdir2/subdir1/**"] CoverageHelper.apply_include_exclude_directives(node, directives) result = CoverageHelper.get_flattened_names(node, prefix="") expected_names = [ 'topdir2', 'topdir2/subdir1', 'topdir2/subdir1/file1.c' ] assert result == set(expected_names)
def runTest(self): node = json.loads(covdata) # Check that mixed directives work properly (exclude multiple paths, include some back) directives = [ "-:topdir1/subdir1/**", "+:topdir1/subdir?/file1.c", "+:topdir1/subdir?/file3.c", "-:topdir1/subdir2/**" ] CoverageHelper.apply_include_exclude_directives(node, directives) result = CoverageHelper.get_flattened_names(node, prefix="") expected_names = [ 'topdir1', 'topdir1/subdir1/file1.c', 'topdir1/subdir1', 'topdir2', 'topdir2/subdir1', 'topdir2/subdir1/file1.c' ] self.assertEqual(result, set(expected_names))
def runTest(self): node = json.loads(covdata) result = CoverageHelper.get_flattened_names(node, prefix="") expected_names = [ 'topdir1', 'topdir1/subdir2', 'topdir1/subdir2/file3.c', 'topdir1/subdir1/file2.c', 'topdir1/subdir1', 'topdir1/subdir1/file1.c', 'topdir2', 'topdir2/subdir1', 'topdir2/subdir1/file1.c' ] self.assertEqual(result, set(expected_names))
def aggregate_coverage_data(pk, pks): from covmanager.models import Collection, CollectionFile # noqa from FTB import CoverageHelper # noqa # Fetch our existing, but incomplete destination collection mergedCollection = Collection.objects.get(pk=pk) # Fetch all source collections collections = Collection.objects.filter(pk__in=pks) # Merge the coverage of all other collections into the first one first_collection = collections[0] first_collection.loadCoverage() newCoverage = first_collection.content total_stats = None for collection in collections[1:]: # Load coverage, perform the merge, then release reference to the JSON blob again collection.loadCoverage() stats = CoverageHelper.merge_coverage_data(newCoverage, collection.content) collection.content = None # Merge stats appropriately if total_stats is None: total_stats = stats else: for x in total_stats: total_stats[x] += stats[x] # Save the new coverage blob to disk and database newCoverage = json.dumps(newCoverage, separators=(',', ':')) h = hashlib.new('sha1') h.update(newCoverage.encode('utf-8')) dbobj = CollectionFile() dbobj.file.save("%s.coverage" % h.hexdigest(), ContentFile(newCoverage)) dbobj.save() mergedCollection.description += " (NC %s, LM %s, CM %s)" % ( stats['null_coverable_count'], stats['length_mismatch_count'], stats['coverable_mismatch_count']) # Save the collection mergedCollection.coverage = dbobj mergedCollection.save() return
def test_CoverageHelperFlattenNames(): node = json.loads(covdata) result = CoverageHelper.get_flattened_names(node, prefix="") expected_names = [ 'topdir1', 'topdir1/subdir2', 'topdir1/subdir2/file3.c', 'topdir1/subdir1/file2.c', 'topdir1/subdir1', 'topdir1/subdir1/file1.c', 'topdir2', 'topdir2/subdir1', 'topdir2/subdir1/file1.c' ] assert result == set(expected_names)
def aggregate_coverage_data(pk, pks): from covmanager.models import Collection, CollectionFile # noqa from FTB import CoverageHelper # noqa # Fetch our existing, but incomplete destination collection mergedCollection = Collection.objects.get(pk=pk) # Fetch all source collections collections = Collection.objects.filter(pk__in=pks) # Merge the coverage of all other collections into the first one first_collection = collections[0] first_collection.loadCoverage() newCoverage = first_collection.content total_stats = None for collection in collections[1:]: # Load coverage, perform the merge, then release reference to the JSON blob again collection.loadCoverage() stats = CoverageHelper.merge_coverage_data(newCoverage, collection.content) collection.content = None # Merge stats appropriately if total_stats is None: total_stats = stats else: for x in total_stats: total_stats[x] += stats[x] # Save the new coverage blob to disk and database newCoverage = json.dumps(newCoverage, separators=(',', ':')) h = hashlib.new('sha1') h.update(newCoverage.encode('utf-8')) dbobj = CollectionFile() dbobj.file.save("%s.coverage" % h.hexdigest(), ContentFile(newCoverage)) dbobj.save() mergedCollection.description += " (NC %s, LM %s, CM %s)" % (stats['null_coverable_count'], stats['length_mismatch_count'], stats['coverable_mismatch_count']) # Save the collection mergedCollection.coverage = dbobj mergedCollection.save() return
def create_combined_coverage(coverage_files): ''' Read coverage data from multiple files and return a single dictionary containing the merged data (already preprocessed). @type coverage_files: list @param coverage_files: List of filenames containing coverage data @return Dictionary with combined coverage data, version information and debug statistics @rtype tuple(dict,dict,dict) ''' ret = None version = None stats = None for coverage_file in coverage_files: with open(coverage_file) as f: coverage = json.load(f) if version is None: version = CovReporter.version_info_from_coverage_data( coverage) coverage = CovReporter.preprocess_coverage_data(coverage) if ret is None: ret = coverage else: merge_stats = CoverageHelper.merge_coverage_data( ret, coverage) if stats is None: stats = merge_stats else: for k in merge_stats: if k in stats: stats[k] += merge_stats[k] return (ret, version, stats)
def create_combined_coverage(coverage_files): ''' Read coverage data from multiple files and return a single dictionary containing the merged data (already preprocessed). @type coverage_files: list @param coverage_files: List of filenames containing coverage data @return Dictionary with combined coverage data, version information and debug statistics @rtype tuple(dict,dict,dict) ''' ret = None version = None stats = None for coverage_file in coverage_files: with open(coverage_file) as f: coverage = json.load(f) if version is None: version = CovReporter.version_info_from_coverage_data(coverage) coverage = CovReporter.preprocess_coverage_data(coverage) if ret is None: ret = coverage else: merge_stats = CoverageHelper.merge_coverage_data(ret, coverage) if stats is None: stats = merge_stats else: for k in merge_stats: if k in stats: stats[k] += merge_stats[k] return (ret, version, stats)
def apply(self, collection): CoverageHelper.apply_include_exclude_directives( collection, self.directives.splitlines()) CoverageHelper.calculate_summary_fields(collection)
def apply(self, collection): CoverageHelper.apply_include_exclude_directives(collection, self.directives.splitlines()) CoverageHelper.calculate_summary_fields(collection)