def apply_phab(self, hg, diff_id):
phabricator_api = PhabricatorAPI(
api_key=get_secret("PHABRICATOR_TOKEN"),
url=get_secret("PHABRICATOR_URL"))
diffs = phabricator_api.search_diffs(diff_id=diff_id)
assert len(diffs) == 1, "No diff available for {}".format(diff_id)
diff = diffs[0]
# Get the stack of patches
base, patches = phabricator_api.load_patches_stack(hg, diff)
assert len(patches) > 0, "No patches to apply"
# Load all the diffs details with commits messages
diffs = phabricator_api.search_diffs(diff_phid=[p[0] for p in patches],
attachments={"commits": True})
commits = {
diff["phid"]: diff["attachments"]["commits"].get("commits", [])
for diff in diffs
}
# First apply patches on local repo
for diff_phid, patch in patches:
commit = commits.get(diff_phid)
message = ""
if commit:
message += "{}\n".format(commit[0]["message"])
logger.info(f"Applying {diff_phid}")
hg.import_(
patches=io.BytesIO(patch.encode("utf-8")),
message=message,
user="******",
)
def apply_phab(self, hg, diff_id):
phabricator_api = PhabricatorAPI(
api_key=get_secret("PHABRICATOR_TOKEN"),
url=get_secret("PHABRICATOR_URL"))
diffs = phabricator_api.search_diffs(diff_id=diff_id)
assert len(diffs) == 1, f"No diff available for {diff_id}"
diff = diffs[0]
# Get the stack of patches
base, patches = phabricator_api.load_patches_stack(hg, diff)
assert len(patches) > 0, "No patches to apply"
# Load all the diffs details with commits messages
diffs = phabricator_api.search_diffs(diff_phid=[p[0] for p in patches],
attachments={"commits": True})
diffs_data = {}
for diff in diffs:
revision = phabricator_api.load_revision(
rev_phid=diff["revisionPHID"])
logger.info("Diff {} linked to Revision {}".format(
diff["id"], revision["id"]))
diffs_data[diff["phid"]] = {
"commits": diff["attachments"]["commits"].get("commits", []),
"revision": revision,
}
# First apply patches on local repo
for diff_phid, patch in patches:
diff_data = diffs_data.get(diff_phid)
commits = diff_data["commits"]
revision = diff_data["revision"]
if commits and commits[0]["message"]:
message = commits[0]["message"]
else:
message = revision["fields"]["title"]
logger.info(f"Applying {diff_phid}")
hg.import_(
patches=io.BytesIO(patch.encode("utf-8")),
message=message,
user="******",
)
def from_autoland(autoland_task: dict, phabricator: PhabricatorAPI):
"""
Build a revision from a Mozilla autoland decision task
"""
assert (
autoland_task["payload"]["env"]["GECKO_HEAD_REPOSITORY"] == REPO_AUTOLAND
), "Not an autoland decision task"
# Load mercurial revision
mercurial_revision = autoland_task["payload"]["env"]["GECKO_HEAD_REV"]
# Search phabricator revision from commit message
commit_url = (
f"https://hg.mozilla.org/integration/autoland/json-rev/{mercurial_revision}"
)
response = requests.get(commit_url)
response.raise_for_status()
description = response.json()["desc"]
match = REGEX_PHABRICATOR_COMMIT.search(description)
if match is not None:
url, revision_id = match.groups()
revision_id = int(revision_id)
logger.info("Found phabricator revision", id=revision_id, url=url)
else:
raise Exception(f"No phabricator revision found in commit {commit_url}")
# Lookup the Phabricator revision to get details (phid, title, bugzilla_id, ...)
revision = phabricator.load_revision(rev_id=revision_id)
# Search the Phabricator diff with same commit identifier
diffs = phabricator.search_diffs(
revision_phid=revision["phid"], attachments={"commits": True}
)
diff = next(
iter(
d
for d in diffs
if d["attachments"]["commits"]["commits"][0]["identifier"]
== mercurial_revision
),
None,
)
assert (
diff is not None
), f"No Phabricator diff found for D{revision_id} and mercurial revision {mercurial_revision}"
logger.info("Found phabricator diff", id=diff["id"])
return Revision(
id=revision_id,
phid=revision["phid"],
diff_id=diff["id"],
diff_phid=diff["phid"],
mercurial_revision=mercurial_revision,
repository=REPO_AUTOLAND,
target_repository=REPO_MOZILLA_CENTRAL,
revision=revision,
diff=diff,
url=url,
)
def from_try(try_task: dict, phabricator: PhabricatorAPI):
"""
Load identifiers from Phabricator, using the remote task description
"""
# Load build target phid from the task env
code_review = try_task["extra"]["code-review"]
build_target_phid = code_review.get("phabricator-diff") or code_review.get(
"phabricator-build-target"
)
assert (
build_target_phid is not None
), "Missing phabricator-build-target or phabricator-diff declaration"
assert build_target_phid.startswith("PHID-HMBT-")
# And get the diff from the phabricator api
buildable = phabricator.find_target_buildable(build_target_phid)
diff_phid = buildable["fields"]["objectPHID"]
assert diff_phid.startswith("PHID-DIFF-")
# Load diff details to get the diff revision
# We also load the commits list in order to get the email of the author of the
# patch for sending email if builds are failing.
diffs = phabricator.search_diffs(
diff_phid=diff_phid, attachments={"commits": True}
)
assert len(diffs) == 1, "No diff available for {}".format(diff_phid)
diff = diffs[0]
diff_id = diff["id"]
phid = diff["revisionPHID"]
revision = phabricator.load_revision(phid)
# Load repository detailed information
repos = phabricator.request(
"diffusion.repository.search",
constraints={"phids": [revision["fields"]["repositoryPHID"]]},
)
assert len(repos["data"]) == 1, "Repository not found on Phabricator"
# Load target patch from Phabricator for Try mode
patch = phabricator.load_raw_diff(diff_id)
# Build a revision without repositories as they are retrieved later
# when analyzing the full task group
return Revision(
id=revision["id"],
phid=phid,
diff_id=diff_id,
diff_phid=diff_phid,
build_target_phid=build_target_phid,
revision=revision,
phabricator_repository=repos["data"][0],
diff=diff,
url="https://{}/D{}".format(phabricator.hostname, revision["id"]),
patch=patch,
)
def apply_phab(self, hg, phabricator_deployment, diff_id):
if phabricator_deployment == PHAB_PROD:
api_key = get_secret("PHABRICATOR_TOKEN")
url = get_secret("PHABRICATOR_URL")
else:
api_key = get_secret("PHABRICATOR_DEV_TOKEN")
url = get_secret("PHABRICATOR_DEV_URL")
phabricator_api = PhabricatorAPI(api_key=api_key, url=url)
# Get the stack of patches
stack = phabricator_api.load_patches_stack(diff_id)
assert len(stack) > 0, "No patches to apply"
# Find the first unknown base revision
needed_stack = []
revisions = {}
for patch in reversed(stack):
needed_stack.insert(0, patch)
# Stop as soon as a base revision is available
if self.has_revision(hg, patch.base_revision):
logger.info(
f"Stopping at diff {patch.id} and revision {patch.base_revision}"
)
break
if not needed_stack:
logger.info("All the patches are already applied")
return
# Load all the diff revisions
diffs = phabricator_api.search_diffs(diff_phid=[p.phid for p in stack])
revisions = {
diff["phid"]:
phabricator_api.load_revision(rev_phid=diff["revisionPHID"],
attachments={"reviewers": True})
for diff in diffs
}
# Update repo to base revision
hg_base = needed_stack[0].base_revision
if not self.has_revision(hg, hg_base):
logger.warning(
"Missing base revision {} from Phabricator".format(hg_base))
hg_base = "tip"
if hg_base:
hg.update(rev=hg_base, clean=True)
logger.info(f"Updated repo to {hg_base}")
if self.git_repo_dir and hg_base != "tip":
try:
self.git_base = tuple(
vcs_map.mercurial_to_git(self.git_repo_dir,
[hg_base]))[0]
subprocess.run(
[
"git", "checkout", "-b", "analysis_branch",
self.git_base
],
check=True,
cwd=self.git_repo_dir,
)
logger.info(f"Updated git repo to {self.git_base}")
except Exception as e:
logger.info(
f"Updating git repo to Mercurial {hg_base} failed: {e}"
)
def load_user(phid):
if phid.startswith("PHID-USER"):
return phabricator_api.load_user(user_phid=phid)
elif phid.startswith("PHID-PROJ"):
# TODO: Support group reviewers somehow.
logger.info(f"Skipping group reviewer {phid}")
else:
raise Exception(f"Unsupported reviewer {phid}")
for patch in needed_stack:
revision = revisions[patch.phid]
message = "{}\n\n{}".format(revision["fields"]["title"],
revision["fields"]["summary"])
author_name = None
author_email = None
if patch.commits:
author_name = patch.commits[0]["author"]["name"]
author_email = patch.commits[0]["author"]["email"]
if author_name is None:
author = load_user(revision["fields"]["authorPHID"])
author_name = author["fields"]["realName"]
# XXX: Figure out a way to know the email address of the author.
author_email = author["fields"]["username"]
reviewers = list(
filter(
None,
(load_user(reviewer["reviewerPHID"]) for reviewer in
revision["attachments"]["reviewers"]["reviewers"]),
))
reviewers = set(reviewer["fields"]["username"]
for reviewer in reviewers)
if len(reviewers):
message = replace_reviewers(message, reviewers)
logger.info(
f"Applying {patch.phid} from revision {revision['id']}: {message}"
)
hg.import_(
patches=io.BytesIO(patch.patch.encode("utf-8")),
message=message.encode("utf-8"),
user=f"{author_name} <{author_email}>".encode("utf-8"),
)
if self.git_repo_dir:
patch_proc = subprocess.Popen(
["patch", "-p1", "--no-backup-if-mismatch", "--force"],
stdin=subprocess.PIPE,
cwd=self.git_repo_dir,
)
patch_proc.communicate(patch.patch.encode("utf-8"))
assert patch_proc.returncode == 0, "Failed to apply patch"
subprocess.run(
[
"git",
"-c",
f"user.name={author_name}",
"-c",
f"user.email={author_email}",
"commit",
"-am",
message,
],
check=True,
cwd=self.git_repo_dir,
)
def apply_phab(self, hg, diff_id):
def has_revision(revision):
if not revision:
return False
try:
hg.identify(revision)
return True
except hglib.error.CommandError:
return False
phabricator_api = PhabricatorAPI(
api_key=get_secret("PHABRICATOR_TOKEN"),
url=get_secret("PHABRICATOR_URL"))
# Get the stack of patches
stack = phabricator_api.load_patches_stack(diff_id)
assert len(stack) > 0, "No patches to apply"
# Find the first unknown base revision
needed_stack = []
revisions = {}
for patch in reversed(stack):
needed_stack.insert(0, patch)
# Stop as soon as a base revision is available
if has_revision(patch.base_revision):
logger.info(
f"Stopping at diff {patch.id} and revision {patch.base_revision}"
)
break
if not needed_stack:
logger.info("All the patches are already applied")
return
# Load all the diff revisions
diffs = phabricator_api.search_diffs(diff_phid=[p.phid for p in stack])
revisions = {
diff["phid"]:
phabricator_api.load_revision(rev_phid=diff["revisionPHID"],
attachments={"reviewers": True})
for diff in diffs
}
# Update repo to base revision
hg_base = needed_stack[0].base_revision
if not has_revision(hg_base):
logger.warning(
"Missing base revision {} from Phabricator".format(hg_base))
hg_base = "tip"
if hg_base:
hg.update(rev=hg_base, clean=True)
logger.info(f"Updated repo to {hg_base}")
try:
self.git_base = vcs_map.mercurial_to_git(hg_base)
subprocess.run(
[
"git", "checkout", "-b", "analysis_branch",
self.git_base
],
check=True,
cwd=self.git_repo_dir,
)
logger.info(f"Updated git repo to {self.git_base}")
except Exception as e:
logger.info(
f"Updating git repo to Mercurial {hg_base} failed: {e}")
def load_user(phid):
if phid.startswith("PHID-USER"):
return phabricator_api.load_user(user_phid=phid)
elif phid.startswith("PHID-PROJ"):
# TODO: Support group reviewers somehow.
logger.info(f"Skipping group reviewer {phid}")
else:
raise Exception(f"Unsupported reviewer {phid}")
for patch in needed_stack:
revision = revisions[patch.phid]
message = "{}\n\n{}".format(revision["fields"]["title"],
revision["fields"]["summary"])
author_name = None
author_email = None
if patch.commits:
author_name = patch.commits[0]["author"]["name"]
author_email = patch.commits[0]["author"]["email"]
if author_name is None:
author = load_user(revision["fields"]["authorPHID"])
author_name = author["fields"]["realName"]
# XXX: Figure out a way to know the email address of the author.
author_email = author["fields"]["username"]
reviewers = list(
filter(
None,
(load_user(reviewer["reviewerPHID"]) for reviewer in
revision["attachments"]["reviewers"]["reviewers"]),
))
reviewers = set(reviewer["fields"]["username"]
for reviewer in reviewers)
if len(reviewers):
message = replace_reviewers(message, reviewers)
logger.info(
f"Applying {patch.phid} from revision {revision['id']}: {message}"
)
hg.import_(
patches=io.BytesIO(patch.patch.encode("utf-8")),
message=message.encode("utf-8"),
user=f"{author_name} <{author_email}>".encode("utf-8"),
)
with tempfile.TemporaryDirectory() as tmpdirname:
temp_file = os.path.join(tmpdirname, "temp.patch")
with open(temp_file, "w") as f:
f.write(patch.patch)
subprocess.run(
["git", "apply", "--3way", temp_file],
check=True,
cwd=self.git_repo_dir,
)
subprocess.run(
[
"git",
"-c",
f"user.name={author_name}",
"-c",
f"user.email={author_email}",
"commit",
"-am",
message,
],
check=True,
cwd=self.git_repo_dir,
)
def apply_phab(self, hg, diff_id):
def has_revision(revision):
if not revision:
return False
try:
hg.identify(revision)
return True
except hglib.error.CommandError:
return False
phabricator_api = PhabricatorAPI(
api_key=get_secret("PHABRICATOR_TOKEN"),
url=get_secret("PHABRICATOR_URL"))
# Get the stack of patches
stack = phabricator_api.load_patches_stack(diff_id)
assert len(stack) > 0, "No patches to apply"
# Find the first unknown base revision
needed_stack = []
revisions = {}
for patch in reversed(stack):
needed_stack.insert(0, patch)
# Stop as soon as a base revision is available
if has_revision(patch.base_revision):
logger.info(
f"Stopping at diff {patch.id} and revision {patch.base_revision}"
)
break
if not needed_stack:
logger.info("All the patches are already applied")
return
# Load all the diff revisions
diffs = phabricator_api.search_diffs(diff_phid=[p.phid for p in stack])
revisions = {
diff["phid"]:
phabricator_api.load_revision(rev_phid=diff["revisionPHID"])
for diff in diffs
}
# Update repo to base revision
hg_base = needed_stack[0].base_revision
if hg_base:
hg.update(rev=hg_base, clean=True)
logger.info(f"Updated repo to {hg_base}")
for patch in needed_stack:
revision = revisions[patch.phid]
if patch.commits:
message = patch.commits[0]["message"]
else:
message = revision["fields"]["title"]
logger.info(
f"Applying {patch.phid} from revision {revision['id']}: {message}"
)
hg.import_(
patches=io.BytesIO(patch.patch.encode("utf-8")),
message=message,
user="******",
)
def apply_phab(self, hg, diff_id):
def has_revision(revision):
if not revision:
return False
try:
hg.identify(revision)
return True
except hglib.error.CommandError:
return False
phabricator_api = PhabricatorAPI(
api_key=get_secret("PHABRICATOR_TOKEN"), url=get_secret("PHABRICATOR_URL")
)
# Get the stack of patches
stack = phabricator_api.load_patches_stack(diff_id)
assert len(stack) > 0, "No patches to apply"
# Find the first unknown base revision
needed_stack = []
revisions = {}
for patch in reversed(stack):
needed_stack.insert(0, patch)
# Stop as soon as a base revision is available
if has_revision(patch.base_revision):
logger.info(
f"Stopping at diff {patch.id} and revision {patch.base_revision}"
)
break
if not needed_stack:
logger.info("All the patches are already applied")
return
# Load all the diff revisions
diffs = phabricator_api.search_diffs(diff_phid=[p.phid for p in stack])
revisions = {
diff["phid"]: phabricator_api.load_revision(rev_phid=diff["revisionPHID"])
for diff in diffs
}
# Update repo to base revision
hg_base = needed_stack[0].base_revision
if not has_revision(hg_base):
logger.warning("Missing base revision {} from Phabricator".format(hg_base))
hg_base = "tip"
if hg_base:
hg.update(rev=hg_base, clean=True)
logger.info(f"Updated repo to {hg_base}")
try:
self.git_base = vcs_map.mercurial_to_git(hg_base)
subprocess.run(
["git", "checkout", "-b", "analysis_branch", self.git_base],
check=True,
cwd=self.git_repo_dir,
)
logger.info(f"Updated git repo to {self.git_base}")
except Exception as e:
logger.info(f"Updating git repo to Mercurial {hg_base} failed: {e}")
for patch in needed_stack:
revision = revisions[patch.phid]
if patch.commits:
message = patch.commits[0]["message"]
author_name = patch.commits[0]["author"]["name"]
author_email = patch.commits[0]["author"]["email"]
else:
message = revision["fields"]["title"]
author_name = "bugbug"
author_email = "*****@*****.**"
logger.info(
f"Applying {patch.phid} from revision {revision['id']}: {message}"
)
hg.import_(
patches=io.BytesIO(patch.patch.encode("utf-8")),
message=message.encode("utf-8"),
user=f"{author_name} <{author_email}>".encode("utf-8"),
)
with tempfile.TemporaryDirectory() as tmpdirname:
temp_file = os.path.join(tmpdirname, "temp.patch")
with open(temp_file, "w") as f:
f.write(patch.patch)
subprocess.run(
["git", "apply", "--3way", temp_file],
check=True,
cwd=self.git_repo_dir,
)
subprocess.run(
[
"git",
"-c",
f"user.name={author_name}",
"-c",
f"user.email={author_email}",
"commit",
"-am",
message,
],
check=True,
cwd=self.git_repo_dir,
)
