def coverage_combine(data_files, output_path, source, process=None, absolute_path=True): """ Merges multiples reports. @param data_files report files (``.coverage``) @param output_path output path @param source source directory @param process function which processes the coverage report @param absolute_path relocate sources with absolute paths @return coverage report The function *process* should have the signature: :: def process(content): # ... return content """ def raise_exc(exc, content, ex, ex2, outfile, destcov, source, dests, inter, cov): from coverage.data import CoverageData def shorten(t): if len(t) > 2000: return t[:2000] + "\n..." else: return t if len(content) > 2000: content = content[:2000] + '\n...' ex = "\n-\n".join(shorten(_) for _ in ex) ex2 = "\n-\n".join(shorten(_) for _ in ex2) rows = ["destcov='{0}'".format(destcov), "outfile='{0}'".format(outfile), "source='{0}'".format(source), "cov.source={0}".format(cov.source), "dests='{0}'".format(';'.join(dests)), "inter={0}".format(inter)] if cov is not None and cov.data is not None and cov.data._lines is not None: rows.append("----- LINES") end = min(5, len(cov.data._lines)) for k, v in list(sorted(cov.data._lines.items()))[:end]: rows.append(' {0}:{1}'.format(k, v)) rows.append("----- RUNS") end = min(5, len(cov.data._runs)) for k in cov.data._runs[:end]: rows.append(' {0}'.format(k)) rows.append("----- END") for d in dests: dd = CoverageData() dd.read_file(d + "~") rows.append("------- LINES - '{0}'".format(d)) end = min(5, len(dd._lines)) for k, v in list(sorted(dd._lines.items()))[:end]: rows.append(' {0}:{1}'.format(k, v)) rows.append("------- RUNS - '{0}'".format(d)) end = min(5, len(dd._runs)) for k in dd._runs[:end]: rows.append(' {0}'.format(k)) rows.append("------- END") mes = "{5}. In '{0}'.\n{1}\n{2}\n---AFTER---\n{3}\n---BEGIN---\n{4}" raise RuntimeError(mes.format(output_path, "\n".join( rows), content, ex, ex2, exc, cov)) from exc # We copy the origin coverage if the report is produced # in a folder part of the merge. destcov = os.path.join(output_path, '.coverage') if os.path.exists(destcov): destcov2 = destcov + '_old' shutil.copy(destcov, destcov2) # Starts merging coverage. from coverage import Coverage cov = Coverage(data_file=destcov, source=[source]) # Module coverage may modify the folder, # we take the one it considers. # On Windows, it has to have the right case. # If not, coverage reports an empty coverage and # raises an exception. cov._init() cov.get_data() if cov.source is None or len(cov.source) == 0: raise_exc(FileNotFoundError("Probably unable to find '{0}'".format(source)), "", [], [], "", destcov, source, [], [], cov) source = cov.source[0] inter = [] reg = re.compile(',\\"(.*?[.]py)\\"') def copy_replace(source, dest, root_source): with open(source, "r") as f: content = f.read() if process is not None: content = process(content) cf = reg.findall(content) co = Counter([_.split('src')[0] for _ in cf]) mx = max((v, k) for k, v in co.items()) root = mx[1].rstrip('\\/') if absolute_path: if '\\\\' in root: s2 = root_source.replace('\\', '\\\\').replace('/', '\\\\') s2 += "\\\\" root += "\\\\" elif '\\' in root: s2 = root_source s2 += "\\\\" root += "\\" else: s2 = root_source s2 += "/" root += "/" else: s2 = "" if '\\\\' in root: root += "\\\\" elif '\\' in root: root += "\\" else: root += "/" inter.append((root, root_source, s2)) content = content.replace(root, s2) with open(dest, "w") as f: f.write(content) # We modify the root in every coverage file. dests = [os.path.join(output_path, '.coverage{0}'.format( i)) for i in range(len(data_files))] for fi, de in zip(data_files, dests): copy_replace(fi, de, source) shutil.copy(de, de + "~") # Keeping information (for exception). ex = [] for d in dests: with open(d, "r") as f: ex.append(f.read()) ex2 = [] for d in data_files: with open(d, "r") as f: ex2.append(f.read()) # We replace destcov by destcov2 if found in dests. if destcov in dests: ind = dests.index(destcov) dests[ind] = destcov2 # Let's combine. cov.combine(dests) from coverage.misc import NoSource try: cov.html_report(directory=output_path) except NoSource as e: raise_exc(e, "", ex, ex2, "", destcov, source, dests, inter, cov) outfile = os.path.join(output_path, "coverage_report.xml") cov.xml_report(outfile=outfile) cov.save() # Verifications with open(outfile, "r", encoding="utf-8") as f: content = f.read() if 'line hits="1"' not in content: raise_exc(Exception("Coverage is empty"), content, ex, ex2, outfile, destcov, source, dests, inter, cov) return cov
def coverage_combine(data_files, output_path, source, process=None): """ Merges multiples reports. @param data_files report files (``.coverage``) @param output_path output path @param source source directory @param process function which processes the coverage report @return coverage report The function *process* should have the signature: :: def process(content): # ... return content On :epkg:`Windows`, file name have to have the right case. If not, coverage reports an empty coverage and raises an exception. """ def raise_exc(exc, content, ex, ex2, outfile, destcov, source, dests, inter, cov, infos): # pragma: no cover def shorten(t): if len(t) > 2000: return t[:2000] + "\n..." else: return t if len(content) > 2000: content = content[:2000] + '\n...' ex = "\n-\n".join(shorten(_) for _ in ex) ex2 = "\n-\n".join(shorten(_) for _ in ex2) rows = [ '-----------------', "destcov='{0}'".format(destcov), "outfile='{0}'".format(outfile), "source='{0}'".format(source), "cov.source={0}".format(get_source(cov)), "dests='{0}'".format(';'.join(dests)), "inter={0}".format(inter) ] for ii, info in enumerate(infos): rows.append('----------------- {}/{}'.format(ii, len(infos))) for k, v in sorted(info.items()): rows.append("{}='{}'".format(k, v)) rows.append('-----------------') if cov is not None and _attr_(cov, '_data', 'data')._lines is not None: rows.append("##### LINES") end = min(5, len(_attr_(cov, '_data', 'data')._lines)) for k, v in list( sorted(_attr_(cov, '_data', 'data')._lines.items()))[:end]: rows.append(' {0}:{1}'.format(k, v)) rows.append("----- RUNS") end = min(5, len(_attr_(cov, '_data', 'data')._runs)) for k in _attr_(cov, '_data', 'data')._runs[:end]: rows.append(' {0}'.format(k)) rows.append("----- END") mes = "{5}. In '{0}'.\n{1}\n{2}\n---AFTER---\n{3}\n---BEGIN---\n{4}" raise RuntimeError( mes.format(output_path, "\n".join(rows), content, ex, ex2, exc, cov)) from exc # We copy the origin coverage if the report is produced # in a folder part of the merge. destcov = os.path.join(output_path, '.coverage') if os.path.exists(destcov): destcov2 = destcov + '_old' shutil.copy(destcov, destcov2) # Starts merging coverage. from coverage import Coverage cov = Coverage(data_file=destcov, source=[source]) cov._init() cov.get_data() if get_source(cov) is None or len(get_source(cov)) == 0: raise_exc( FileNotFoundError("Probably unable to find '{0}'".format(source)), "", [], [], "", destcov, source, [], [], cov, []) inter = [] def find_longest_common_root(names, begin): counts = {} for name in names: spl = name.split(begin) for i in range(1, len(spl) + 1): if spl[i - 1] == 'src': break sub = begin.join(spl[:i]) if sub in counts: counts[sub] += 1 else: counts[sub] = 1 item = max((v, k) for k, v in counts.items()) return item[1] def copy_replace(source, dest, root_source, keep_infos): shutil.copy(source, dest) co = Counter(root_source) slash = co.get('/', 0) >= co.get('\\', 0) if slash: begin = "/" root_source_dup = root_source.replace('\\', '/').replace('//', '/') else: begin = "\\" root_source_dup = root_source.replace("\\", "\\\\") keep_infos["slash"] = slash keep_infos["begin"] = begin keep_infos["root_source_dup"] = root_source_dup keep_infos["root_source"] = root_source keep_infos["source"] = source keep_infos["dest"] = dest conn = sqlite3.connect(dest) sql = [] names = [] for row in conn.execute("select * from file"): names.append(row[1]) name = row[1].replace('/', begin) if not name.startswith(root_source): name = root_source + begin + name s = "UPDATE file SET path='{}' WHERE id={};".format(name, row[0]) sql.append(s) keep_infos['root_common'] = find_longest_common_root(names, begin) c = conn.cursor() for s in sql: c.execute(s) conn.commit() conn.close() # We modify the root in every coverage file. dests = [ os.path.join(output_path, '.coverage{0}'.format(i)) for i in range(len(data_files)) ] infos = [] for fi, de in zip(data_files, dests): keep_infos = {} copy_replace(fi, de, source, keep_infos) infos.append(keep_infos) shutil.copy(de, de + "~") # Keeping information (for exception). ex = [] for d in dests: with open(d, "rb") as f: ex.append(f.read()) ex2 = [] for d in data_files: with open(d, "rb") as f: ex2.append(f.read()) # We replace destcov by destcov2 if found in dests. if destcov in dests: ind = dests.index(destcov) dests[ind] = destcov2 # Let's combine. cov.combine(dests) # dest cov.save() report = True try: from coverage.exceptions import CoverageException except ImportError: # older version of coverage from coverage.misc import CoverageException try: from coverage.exceptions import NoSource except ImportError: # older version of coverage from coverage.misc import NoSource try: cov.html_report(directory=output_path, ignore_errors=True) except NoSource as e: raise_exc(e, "", ex, ex2, "", destcov, source, dests, inter, cov, infos) except CoverageException as e: if "No data to report" in str(e): # issue with path report = False else: msg = pprint.pformat(infos) raise RuntimeError( # pragma: no cover "Unable to process report in '{0}'.\n----\n{1}".format( output_path, msg)) from e if report: outfile = os.path.join(output_path, "coverage_report.xml") cov.xml_report(outfile=outfile) cov.save() # Verifications with open(outfile, "r", encoding="utf-8") as f: content = f.read() if len(content) == 0: raise RuntimeError("No report was generated.") return cov