def hg_log(hg: hglib.client, revs: List[bytes]) -> List[Commit]:
if len(revs) == 0:
return []
template = "{node}\\0{author}\\0{desc}\\0{bug}\\0{backedoutby}\\0{author|email}\\0{pushdate|hgdate}\\0{reviewers}\\0{backsoutnodes}\\0"
args = hglib.util.cmdbuilder(
b"log",
template=template,
no_merges=True,
rev=revs,
branch="tip",
)
x = hg.rawcommand(args)
out = x.split(b"\x00")[:-1]
commits = []
for rev in hglib.util.grouper(template.count("\\0"), out):
assert b" " in rev[6]
pushdate_timestamp = rev[6].split(b" ", 1)[0]
if pushdate_timestamp != b"0":
pushdate = datetime.utcfromtimestamp(float(pushdate_timestamp))
else:
pushdate = datetime.utcnow()
bug_id = int(rev[3].decode("ascii")) if rev[3] else None
reviewers = (
list(set(sys.intern(r) for r in rev[7].decode("utf-8").split(" ")))
if rev[7] != b""
else []
)
backsout = (
list(set(sys.intern(r) for r in rev[8].decode("utf-8").split(" ")))
if rev[8] != b""
else []
)
commits.append(
Commit(
node=sys.intern(rev[0].decode("ascii")),
author=sys.intern(rev[1].decode("utf-8")),
desc=rev[2].decode("utf-8"),
pushdate=pushdate,
bug_id=bug_id,
backsout=backsout,
backedoutby=rev[4].decode("ascii"),
author_email=rev[5].decode("utf-8"),
reviewers=reviewers,
)
)
return commits
def set_commits_to_ignore(hg: hglib.client, repo_dir: str,
commits: List[Commit]):
# Skip commits which are in .hg-annotate-ignore-revs or which have
# 'ignore-this-changeset' in their description (mostly consisting of very
# large and not meaningful formatting changes).
ignore_revs_content = hg.cat(
[os.path.join(repo_dir, ".hg-annotate-ignore-revs").encode("ascii")],
rev=b"-1").decode("utf-8")
ignore_revs = set(line[:40] for line in ignore_revs_content.splitlines())
for commit in commits:
commit.ignored = (commit.node in ignore_revs
or "ignore-this-changeset" in commit.desc)
def run_annotate(self, hg: hglib.client, rev: str,
path: str) -> Optional[Tuple[Tuple[str, int], ...]]:
args = hglib.util.cmdbuilder(
b"annotate",
os.path.join(self.repo_dir, path).encode("ascii"),
r=rev,
line=True,
changeset=True,
)
try:
out = hg.rawcommand(args)
except hglib.error.CommandError as e:
if b"no such file in rev" not in e.err:
raise
# The file was removed.
return None
def _collect() -> Iterator[Tuple[str, int]]:
for line in out.splitlines():
orig_changeset, orig_line, _ = line.split(b":", 2)
yield orig_changeset.decode("ascii"), int(orig_line)
return tuple(_collect())
def transform(hg: hglib.client, repo_dir: str, commit: Commit):
hg_modified_files(hg, commit)
if commit.ignored or len(commit.backsout) > 0 or commit.bug_id is None:
return commit
assert code_analysis_server is not None
source_code_sizes = []
other_sizes = []
test_sizes = []
metrics_file_count = 0
patch = hg.export(revs=[commit.node.encode("ascii")], git=True)
try:
patch_data = rs_parsepatch.get_lines(patch)
except Exception:
logger.error(f"Exception while analyzing {commit.node}")
raise
for stats in patch_data:
path = stats["filename"]
if stats["binary"]:
if not is_test(path):
commit.types.add("binary")
continue
size = None
after = None
if not stats["deleted"]:
try:
after = hg.cat(
[os.path.join(repo_dir, path).encode("utf-8")],
rev=commit.node.encode("ascii"),
)
size = after.count(b"\n")
except hglib.error.CommandError as e:
if b"no such file in rev" not in e.err:
raise
type_ = get_type(path)
if is_test(path):
commit.test_files_modified_num += 1
commit.test_added += len(stats["added_lines"])
commit.test_deleted += len(stats["deleted_lines"])
if size is not None:
test_sizes.append(size)
# We don't have a 'test' equivalent of types, as most tests are JS,
# so this wouldn't add useful information.
elif type_ in SOURCE_CODE_TYPES_TO_EXT:
commit.source_code_files_modified_num += 1
commit.source_code_added += len(stats["added_lines"])
commit.source_code_deleted += len(stats["deleted_lines"])
if size is not None:
source_code_sizes.append(size)
if type_ != "IDL/IPDL/WebIDL":
metrics = code_analysis_server.metrics(path,
after,
unit=False)
if metrics.get("spaces"):
metrics_file_count += 1
error = get_metrics(commit, metrics["spaces"])
if error:
logger.debug(
f"rust-code-analysis error on commit {commit.node}, path {path}"
)
touched_functions = get_touched_functions(
metrics["spaces"],
stats["deleted_lines"],
stats["added_lines"],
)
if len(touched_functions) > 0:
commit.functions[path] = list(touched_functions)
# Replace type with "Objective-C/C++" if rust-code-analysis detected this is an Objective-C/C++ file.
if type_ == "C/C++" and metrics.get(
"language") == "obj-c/c++":
type_ = "Objective-C/C++"
commit.types.add(type_)
else:
commit.other_files_modified_num += 1
commit.other_added += len(stats["added_lines"])
commit.other_deleted += len(stats["deleted_lines"])
if size is not None:
other_sizes.append(size)
if type_:
commit.types.add(type_)
commit.total_source_code_file_size = sum(source_code_sizes)
commit.average_source_code_file_size = (
commit.total_source_code_file_size /
len(source_code_sizes) if len(source_code_sizes) > 0 else 0)
commit.maximum_source_code_file_size = max(source_code_sizes, default=0)
commit.minimum_source_code_file_size = min(source_code_sizes, default=0)
commit.total_other_file_size = sum(other_sizes)
commit.average_other_file_size = (commit.total_other_file_size /
len(other_sizes)
if len(other_sizes) > 0 else 0)
commit.maximum_other_file_size = max(other_sizes, default=0)
commit.minimum_other_file_size = min(other_sizes, default=0)
commit.total_test_file_size = sum(test_sizes)
commit.average_test_file_size = (commit.total_test_file_size /
len(test_sizes)
if len(test_sizes) > 0 else 0)
commit.maximum_test_file_size = max(test_sizes, default=0)
commit.minimum_test_file_size = min(test_sizes, default=0)
if metrics_file_count:
commit.average_cyclomatic = commit.total_cyclomatic / metrics_file_count
commit.average_halstead_n2 = commit.total_halstead_n2 / metrics_file_count
commit.average_halstead_N2 = commit.total_halstead_N2 / metrics_file_count
commit.average_halstead_n1 = commit.total_halstead_n1 / metrics_file_count
commit.average_halstead_N1 = commit.total_halstead_N1 / metrics_file_count
commit.average_source_loc = commit.total_source_loc / metrics_file_count
commit.average_logical_loc = commit.total_logical_loc / metrics_file_count
else:
# these values are initialized with sys.maxsize (because we take the min)
# if no files, then reset them to 0 (it'd be stupid to have min > max)
commit.minimum_cyclomatic = 0
commit.minimum_halstead_N2 = 0
commit.minimum_halstead_n2 = 0
commit.minimum_halstead_N1 = 0
commit.minimum_halstead_n1 = 0
commit.minimum_source_loc = 0
commit.minimum_logical_loc = 0
return commit
def hg_log(hg: hglib.client, revs: list[bytes]) -> tuple[Commit, ...]:
if len(revs) == 0:
return tuple()
template = "{node}\\0{author}\\0{desc}\\0{bug}\\0{backedoutby}\\0{author|email}\\0{pushdate|hgdate}\\0{reviewers}\\0{backsoutnodes}\\0"
args = hglib.util.cmdbuilder(
b"log",
template=template,
no_merges=True,
rev=revs,
branch="tip",
)
x = hg.rawcommand(args)
out = x.split(b"\x00")[:-1]
commits = []
for rev in hglib.util.grouper(template.count("\\0"), out):
assert b" " in rev[6]
pushdate_timestamp = rev[6].split(b" ", 1)[0]
if pushdate_timestamp != b"0":
pushdate = datetime.utcfromtimestamp(float(pushdate_timestamp))
else:
pushdate = datetime.utcnow()
bug_id = int(rev[3].decode("ascii")) if rev[3] else None
reviewers = (list(
set(
sys.intern(r) for r in rev[7].decode("utf-8").split(" ")
if r not in (
"",
"testonly",
"gaia-bump",
"me",
"fix",
"wpt-fix",
"testing",
"bustage",
"test-only",
"blocking",
"blocking-fennec",
"blocking1.9",
"backout",
"trivial",
"DONTBUILD",
"blocking-final",
"blocking-firefox3",
"test",
"bustage-fix",
"release",
"tests",
"lint-fix",
))) if rev[7] != b"" else [])
backsout = (list(
set(sys.intern(r) for r in rev[8].decode("utf-8").split(" ")))
if rev[8] != b"" else [])
commits.append(
Commit(
node=sys.intern(rev[0].decode("ascii")),
author=sys.intern(rev[1].decode("utf-8")),
desc=rev[2].decode("utf-8"),
pushdate=pushdate,
bug_id=bug_id,
backsout=backsout,
backedoutby=rev[4].decode("ascii"),
author_email=rev[5].decode("utf-8"),
reviewers=reviewers,
))
return tuple(commits)