def apply_phab(self, hg, diff_id): phabricator_api = PhabricatorAPI( api_key=get_secret("PHABRICATOR_TOKEN"), url=get_secret("PHABRICATOR_URL")) diffs = phabricator_api.search_diffs(diff_id=diff_id) assert len(diffs) == 1, "No diff available for {}".format(diff_id) diff = diffs[0] # Get the stack of patches base, patches = phabricator_api.load_patches_stack(hg, diff) assert len(patches) > 0, "No patches to apply" # Load all the diffs details with commits messages diffs = phabricator_api.search_diffs(diff_phid=[p[0] for p in patches], attachments={"commits": True}) commits = { diff["phid"]: diff["attachments"]["commits"].get("commits", []) for diff in diffs } # First apply patches on local repo for diff_phid, patch in patches: commit = commits.get(diff_phid) message = "" if commit: message += "{}\n".format(commit[0]["message"]) logger.info(f"Applying {diff_phid}") hg.import_( patches=io.BytesIO(patch.encode("utf-8")), message=message, user="******", )
def apply_phab(self, hg, diff_id): phabricator_api = PhabricatorAPI( api_key=get_secret("PHABRICATOR_TOKEN"), url=get_secret("PHABRICATOR_URL")) diffs = phabricator_api.search_diffs(diff_id=diff_id) assert len(diffs) == 1, f"No diff available for {diff_id}" diff = diffs[0] # Get the stack of patches base, patches = phabricator_api.load_patches_stack(hg, diff) assert len(patches) > 0, "No patches to apply" # Load all the diffs details with commits messages diffs = phabricator_api.search_diffs(diff_phid=[p[0] for p in patches], attachments={"commits": True}) diffs_data = {} for diff in diffs: revision = phabricator_api.load_revision( rev_phid=diff["revisionPHID"]) logger.info("Diff {} linked to Revision {}".format( diff["id"], revision["id"])) diffs_data[diff["phid"]] = { "commits": diff["attachments"]["commits"].get("commits", []), "revision": revision, } # First apply patches on local repo for diff_phid, patch in patches: diff_data = diffs_data.get(diff_phid) commits = diff_data["commits"] revision = diff_data["revision"] if commits and commits[0]["message"]: message = commits[0]["message"] else: message = revision["fields"]["title"] logger.info(f"Applying {diff_phid}") hg.import_( patches=io.BytesIO(patch.encode("utf-8")), message=message, user="******", )
def from_autoland(autoland_task: dict, phabricator: PhabricatorAPI): """ Build a revision from a Mozilla autoland decision task """ assert ( autoland_task["payload"]["env"]["GECKO_HEAD_REPOSITORY"] == REPO_AUTOLAND ), "Not an autoland decision task" # Load mercurial revision mercurial_revision = autoland_task["payload"]["env"]["GECKO_HEAD_REV"] # Search phabricator revision from commit message commit_url = ( f"https://hg.mozilla.org/integration/autoland/json-rev/{mercurial_revision}" ) response = requests.get(commit_url) response.raise_for_status() description = response.json()["desc"] match = REGEX_PHABRICATOR_COMMIT.search(description) if match is not None: url, revision_id = match.groups() revision_id = int(revision_id) logger.info("Found phabricator revision", id=revision_id, url=url) else: raise Exception(f"No phabricator revision found in commit {commit_url}") # Lookup the Phabricator revision to get details (phid, title, bugzilla_id, ...) revision = phabricator.load_revision(rev_id=revision_id) # Search the Phabricator diff with same commit identifier diffs = phabricator.search_diffs( revision_phid=revision["phid"], attachments={"commits": True} ) diff = next( iter( d for d in diffs if d["attachments"]["commits"]["commits"][0]["identifier"] == mercurial_revision ), None, ) assert ( diff is not None ), f"No Phabricator diff found for D{revision_id} and mercurial revision {mercurial_revision}" logger.info("Found phabricator diff", id=diff["id"]) return Revision( id=revision_id, phid=revision["phid"], diff_id=diff["id"], diff_phid=diff["phid"], mercurial_revision=mercurial_revision, repository=REPO_AUTOLAND, target_repository=REPO_MOZILLA_CENTRAL, revision=revision, diff=diff, url=url, )
def from_try(try_task: dict, phabricator: PhabricatorAPI): """ Load identifiers from Phabricator, using the remote task description """ # Load build target phid from the task env code_review = try_task["extra"]["code-review"] build_target_phid = code_review.get("phabricator-diff") or code_review.get( "phabricator-build-target" ) assert ( build_target_phid is not None ), "Missing phabricator-build-target or phabricator-diff declaration" assert build_target_phid.startswith("PHID-HMBT-") # And get the diff from the phabricator api buildable = phabricator.find_target_buildable(build_target_phid) diff_phid = buildable["fields"]["objectPHID"] assert diff_phid.startswith("PHID-DIFF-") # Load diff details to get the diff revision # We also load the commits list in order to get the email of the author of the # patch for sending email if builds are failing. diffs = phabricator.search_diffs( diff_phid=diff_phid, attachments={"commits": True} ) assert len(diffs) == 1, "No diff available for {}".format(diff_phid) diff = diffs[0] diff_id = diff["id"] phid = diff["revisionPHID"] revision = phabricator.load_revision(phid) # Load repository detailed information repos = phabricator.request( "diffusion.repository.search", constraints={"phids": [revision["fields"]["repositoryPHID"]]}, ) assert len(repos["data"]) == 1, "Repository not found on Phabricator" # Load target patch from Phabricator for Try mode patch = phabricator.load_raw_diff(diff_id) # Build a revision without repositories as they are retrieved later # when analyzing the full task group return Revision( id=revision["id"], phid=phid, diff_id=diff_id, diff_phid=diff_phid, build_target_phid=build_target_phid, revision=revision, phabricator_repository=repos["data"][0], diff=diff, url="https://{}/D{}".format(phabricator.hostname, revision["id"]), patch=patch, )
def apply_phab(self, hg, phabricator_deployment, diff_id): if phabricator_deployment == PHAB_PROD: api_key = get_secret("PHABRICATOR_TOKEN") url = get_secret("PHABRICATOR_URL") else: api_key = get_secret("PHABRICATOR_DEV_TOKEN") url = get_secret("PHABRICATOR_DEV_URL") phabricator_api = PhabricatorAPI(api_key=api_key, url=url) # Get the stack of patches stack = phabricator_api.load_patches_stack(diff_id) assert len(stack) > 0, "No patches to apply" # Find the first unknown base revision needed_stack = [] revisions = {} for patch in reversed(stack): needed_stack.insert(0, patch) # Stop as soon as a base revision is available if self.has_revision(hg, patch.base_revision): logger.info( f"Stopping at diff {patch.id} and revision {patch.base_revision}" ) break if not needed_stack: logger.info("All the patches are already applied") return # Load all the diff revisions diffs = phabricator_api.search_diffs(diff_phid=[p.phid for p in stack]) revisions = { diff["phid"]: phabricator_api.load_revision(rev_phid=diff["revisionPHID"], attachments={"reviewers": True}) for diff in diffs } # Update repo to base revision hg_base = needed_stack[0].base_revision if not self.has_revision(hg, hg_base): logger.warning( "Missing base revision {} from Phabricator".format(hg_base)) hg_base = "tip" if hg_base: hg.update(rev=hg_base, clean=True) logger.info(f"Updated repo to {hg_base}") if self.git_repo_dir and hg_base != "tip": try: self.git_base = tuple( vcs_map.mercurial_to_git(self.git_repo_dir, [hg_base]))[0] subprocess.run( [ "git", "checkout", "-b", "analysis_branch", self.git_base ], check=True, cwd=self.git_repo_dir, ) logger.info(f"Updated git repo to {self.git_base}") except Exception as e: logger.info( f"Updating git repo to Mercurial {hg_base} failed: {e}" ) def load_user(phid): if phid.startswith("PHID-USER"): return phabricator_api.load_user(user_phid=phid) elif phid.startswith("PHID-PROJ"): # TODO: Support group reviewers somehow. logger.info(f"Skipping group reviewer {phid}") else: raise Exception(f"Unsupported reviewer {phid}") for patch in needed_stack: revision = revisions[patch.phid] message = "{}\n\n{}".format(revision["fields"]["title"], revision["fields"]["summary"]) author_name = None author_email = None if patch.commits: author_name = patch.commits[0]["author"]["name"] author_email = patch.commits[0]["author"]["email"] if author_name is None: author = load_user(revision["fields"]["authorPHID"]) author_name = author["fields"]["realName"] # XXX: Figure out a way to know the email address of the author. author_email = author["fields"]["username"] reviewers = list( filter( None, (load_user(reviewer["reviewerPHID"]) for reviewer in revision["attachments"]["reviewers"]["reviewers"]), )) reviewers = set(reviewer["fields"]["username"] for reviewer in reviewers) if len(reviewers): message = replace_reviewers(message, reviewers) logger.info( f"Applying {patch.phid} from revision {revision['id']}: {message}" ) hg.import_( patches=io.BytesIO(patch.patch.encode("utf-8")), message=message.encode("utf-8"), user=f"{author_name} <{author_email}>".encode("utf-8"), ) if self.git_repo_dir: patch_proc = subprocess.Popen( ["patch", "-p1", "--no-backup-if-mismatch", "--force"], stdin=subprocess.PIPE, cwd=self.git_repo_dir, ) patch_proc.communicate(patch.patch.encode("utf-8")) assert patch_proc.returncode == 0, "Failed to apply patch" subprocess.run( [ "git", "-c", f"user.name={author_name}", "-c", f"user.email={author_email}", "commit", "-am", message, ], check=True, cwd=self.git_repo_dir, )
def apply_phab(self, hg, diff_id): def has_revision(revision): if not revision: return False try: hg.identify(revision) return True except hglib.error.CommandError: return False phabricator_api = PhabricatorAPI( api_key=get_secret("PHABRICATOR_TOKEN"), url=get_secret("PHABRICATOR_URL")) # Get the stack of patches stack = phabricator_api.load_patches_stack(diff_id) assert len(stack) > 0, "No patches to apply" # Find the first unknown base revision needed_stack = [] revisions = {} for patch in reversed(stack): needed_stack.insert(0, patch) # Stop as soon as a base revision is available if has_revision(patch.base_revision): logger.info( f"Stopping at diff {patch.id} and revision {patch.base_revision}" ) break if not needed_stack: logger.info("All the patches are already applied") return # Load all the diff revisions diffs = phabricator_api.search_diffs(diff_phid=[p.phid for p in stack]) revisions = { diff["phid"]: phabricator_api.load_revision(rev_phid=diff["revisionPHID"], attachments={"reviewers": True}) for diff in diffs } # Update repo to base revision hg_base = needed_stack[0].base_revision if not has_revision(hg_base): logger.warning( "Missing base revision {} from Phabricator".format(hg_base)) hg_base = "tip" if hg_base: hg.update(rev=hg_base, clean=True) logger.info(f"Updated repo to {hg_base}") try: self.git_base = vcs_map.mercurial_to_git(hg_base) subprocess.run( [ "git", "checkout", "-b", "analysis_branch", self.git_base ], check=True, cwd=self.git_repo_dir, ) logger.info(f"Updated git repo to {self.git_base}") except Exception as e: logger.info( f"Updating git repo to Mercurial {hg_base} failed: {e}") def load_user(phid): if phid.startswith("PHID-USER"): return phabricator_api.load_user(user_phid=phid) elif phid.startswith("PHID-PROJ"): # TODO: Support group reviewers somehow. logger.info(f"Skipping group reviewer {phid}") else: raise Exception(f"Unsupported reviewer {phid}") for patch in needed_stack: revision = revisions[patch.phid] message = "{}\n\n{}".format(revision["fields"]["title"], revision["fields"]["summary"]) author_name = None author_email = None if patch.commits: author_name = patch.commits[0]["author"]["name"] author_email = patch.commits[0]["author"]["email"] if author_name is None: author = load_user(revision["fields"]["authorPHID"]) author_name = author["fields"]["realName"] # XXX: Figure out a way to know the email address of the author. author_email = author["fields"]["username"] reviewers = list( filter( None, (load_user(reviewer["reviewerPHID"]) for reviewer in revision["attachments"]["reviewers"]["reviewers"]), )) reviewers = set(reviewer["fields"]["username"] for reviewer in reviewers) if len(reviewers): message = replace_reviewers(message, reviewers) logger.info( f"Applying {patch.phid} from revision {revision['id']}: {message}" ) hg.import_( patches=io.BytesIO(patch.patch.encode("utf-8")), message=message.encode("utf-8"), user=f"{author_name} <{author_email}>".encode("utf-8"), ) with tempfile.TemporaryDirectory() as tmpdirname: temp_file = os.path.join(tmpdirname, "temp.patch") with open(temp_file, "w") as f: f.write(patch.patch) subprocess.run( ["git", "apply", "--3way", temp_file], check=True, cwd=self.git_repo_dir, ) subprocess.run( [ "git", "-c", f"user.name={author_name}", "-c", f"user.email={author_email}", "commit", "-am", message, ], check=True, cwd=self.git_repo_dir, )
def apply_phab(self, hg, diff_id): def has_revision(revision): if not revision: return False try: hg.identify(revision) return True except hglib.error.CommandError: return False phabricator_api = PhabricatorAPI( api_key=get_secret("PHABRICATOR_TOKEN"), url=get_secret("PHABRICATOR_URL")) # Get the stack of patches stack = phabricator_api.load_patches_stack(diff_id) assert len(stack) > 0, "No patches to apply" # Find the first unknown base revision needed_stack = [] revisions = {} for patch in reversed(stack): needed_stack.insert(0, patch) # Stop as soon as a base revision is available if has_revision(patch.base_revision): logger.info( f"Stopping at diff {patch.id} and revision {patch.base_revision}" ) break if not needed_stack: logger.info("All the patches are already applied") return # Load all the diff revisions diffs = phabricator_api.search_diffs(diff_phid=[p.phid for p in stack]) revisions = { diff["phid"]: phabricator_api.load_revision(rev_phid=diff["revisionPHID"]) for diff in diffs } # Update repo to base revision hg_base = needed_stack[0].base_revision if hg_base: hg.update(rev=hg_base, clean=True) logger.info(f"Updated repo to {hg_base}") for patch in needed_stack: revision = revisions[patch.phid] if patch.commits: message = patch.commits[0]["message"] else: message = revision["fields"]["title"] logger.info( f"Applying {patch.phid} from revision {revision['id']}: {message}" ) hg.import_( patches=io.BytesIO(patch.patch.encode("utf-8")), message=message, user="******", )
def apply_phab(self, hg, diff_id): def has_revision(revision): if not revision: return False try: hg.identify(revision) return True except hglib.error.CommandError: return False phabricator_api = PhabricatorAPI( api_key=get_secret("PHABRICATOR_TOKEN"), url=get_secret("PHABRICATOR_URL") ) # Get the stack of patches stack = phabricator_api.load_patches_stack(diff_id) assert len(stack) > 0, "No patches to apply" # Find the first unknown base revision needed_stack = [] revisions = {} for patch in reversed(stack): needed_stack.insert(0, patch) # Stop as soon as a base revision is available if has_revision(patch.base_revision): logger.info( f"Stopping at diff {patch.id} and revision {patch.base_revision}" ) break if not needed_stack: logger.info("All the patches are already applied") return # Load all the diff revisions diffs = phabricator_api.search_diffs(diff_phid=[p.phid for p in stack]) revisions = { diff["phid"]: phabricator_api.load_revision(rev_phid=diff["revisionPHID"]) for diff in diffs } # Update repo to base revision hg_base = needed_stack[0].base_revision if not has_revision(hg_base): logger.warning("Missing base revision {} from Phabricator".format(hg_base)) hg_base = "tip" if hg_base: hg.update(rev=hg_base, clean=True) logger.info(f"Updated repo to {hg_base}") try: self.git_base = vcs_map.mercurial_to_git(hg_base) subprocess.run( ["git", "checkout", "-b", "analysis_branch", self.git_base], check=True, cwd=self.git_repo_dir, ) logger.info(f"Updated git repo to {self.git_base}") except Exception as e: logger.info(f"Updating git repo to Mercurial {hg_base} failed: {e}") for patch in needed_stack: revision = revisions[patch.phid] if patch.commits: message = patch.commits[0]["message"] author_name = patch.commits[0]["author"]["name"] author_email = patch.commits[0]["author"]["email"] else: message = revision["fields"]["title"] author_name = "bugbug" author_email = "*****@*****.**" logger.info( f"Applying {patch.phid} from revision {revision['id']}: {message}" ) hg.import_( patches=io.BytesIO(patch.patch.encode("utf-8")), message=message.encode("utf-8"), user=f"{author_name} <{author_email}>".encode("utf-8"), ) with tempfile.TemporaryDirectory() as tmpdirname: temp_file = os.path.join(tmpdirname, "temp.patch") with open(temp_file, "w") as f: f.write(patch.patch) subprocess.run( ["git", "apply", "--3way", temp_file], check=True, cwd=self.git_repo_dir, ) subprocess.run( [ "git", "-c", f"user.name={author_name}", "-c", f"user.email={author_email}", "commit", "-am", message, ], check=True, cwd=self.git_repo_dir, )