def apply_phab(self, hg, diff_id): phabricator_api = PhabricatorAPI( api_key=get_secret("PHABRICATOR_TOKEN"), url=get_secret("PHABRICATOR_URL")) diffs = phabricator_api.search_diffs(diff_id=diff_id) assert len(diffs) == 1, "No diff available for {}".format(diff_id) diff = diffs[0] # Get the stack of patches base, patches = phabricator_api.load_patches_stack(hg, diff) assert len(patches) > 0, "No patches to apply" # Load all the diffs details with commits messages diffs = phabricator_api.search_diffs(diff_phid=[p[0] for p in patches], attachments={"commits": True}) commits = { diff["phid"]: diff["attachments"]["commits"].get("commits", []) for diff in diffs } # First apply patches on local repo for diff_phid, patch in patches: commit = commits.get(diff_phid) message = "" if commit: message += "{}\n".format(commit[0]["message"]) logger.info(f"Applying {diff_phid}") hg.import_( patches=io.BytesIO(patch.encode("utf-8")), message=message, user="******", )
def apply_phab(self, hg, diff_id): phabricator_api = PhabricatorAPI( api_key=get_secret("PHABRICATOR_TOKEN"), url=get_secret("PHABRICATOR_URL")) diffs = phabricator_api.search_diffs(diff_id=diff_id) assert len(diffs) == 1, f"No diff available for {diff_id}" diff = diffs[0] # Get the stack of patches base, patches = phabricator_api.load_patches_stack(hg, diff) assert len(patches) > 0, "No patches to apply" # Load all the diffs details with commits messages diffs = phabricator_api.search_diffs(diff_phid=[p[0] for p in patches], attachments={"commits": True}) diffs_data = {} for diff in diffs: revision = phabricator_api.load_revision( rev_phid=diff["revisionPHID"]) logger.info("Diff {} linked to Revision {}".format( diff["id"], revision["id"])) diffs_data[diff["phid"]] = { "commits": diff["attachments"]["commits"].get("commits", []), "revision": revision, } # First apply patches on local repo for diff_phid, patch in patches: diff_data = diffs_data.get(diff_phid) commits = diff_data["commits"] revision = diff_data["revision"] if commits and commits[0]["message"]: message = commits[0]["message"] else: message = revision["fields"]["title"] logger.info(f"Applying {diff_phid}") hg.import_( patches=io.BytesIO(patch.encode("utf-8")), message=message, user="******", )
def apply_phab(self, hg, phabricator_deployment, diff_id): if phabricator_deployment == PHAB_PROD: api_key = get_secret("PHABRICATOR_TOKEN") url = get_secret("PHABRICATOR_URL") else: api_key = get_secret("PHABRICATOR_DEV_TOKEN") url = get_secret("PHABRICATOR_DEV_URL") phabricator_api = PhabricatorAPI(api_key=api_key, url=url) # Get the stack of patches stack = phabricator_api.load_patches_stack(diff_id) assert len(stack) > 0, "No patches to apply" # Find the first unknown base revision needed_stack = [] revisions = {} for patch in reversed(stack): needed_stack.insert(0, patch) # Stop as soon as a base revision is available if self.has_revision(hg, patch.base_revision): logger.info( f"Stopping at diff {patch.id} and revision {patch.base_revision}" ) break if not needed_stack: logger.info("All the patches are already applied") return # Load all the diff revisions diffs = phabricator_api.search_diffs(diff_phid=[p.phid for p in stack]) revisions = { diff["phid"]: phabricator_api.load_revision(rev_phid=diff["revisionPHID"], attachments={"reviewers": True}) for diff in diffs } # Update repo to base revision hg_base = needed_stack[0].base_revision if not self.has_revision(hg, hg_base): logger.warning( "Missing base revision {} from Phabricator".format(hg_base)) hg_base = "tip" if hg_base: hg.update(rev=hg_base, clean=True) logger.info(f"Updated repo to {hg_base}") if self.git_repo_dir and hg_base != "tip": try: self.git_base = tuple( vcs_map.mercurial_to_git(self.git_repo_dir, [hg_base]))[0] subprocess.run( [ "git", "checkout", "-b", "analysis_branch", self.git_base ], check=True, cwd=self.git_repo_dir, ) logger.info(f"Updated git repo to {self.git_base}") except Exception as e: logger.info( f"Updating git repo to Mercurial {hg_base} failed: {e}" ) def load_user(phid): if phid.startswith("PHID-USER"): return phabricator_api.load_user(user_phid=phid) elif phid.startswith("PHID-PROJ"): # TODO: Support group reviewers somehow. logger.info(f"Skipping group reviewer {phid}") else: raise Exception(f"Unsupported reviewer {phid}") for patch in needed_stack: revision = revisions[patch.phid] message = "{}\n\n{}".format(revision["fields"]["title"], revision["fields"]["summary"]) author_name = None author_email = None if patch.commits: author_name = patch.commits[0]["author"]["name"] author_email = patch.commits[0]["author"]["email"] if author_name is None: author = load_user(revision["fields"]["authorPHID"]) author_name = author["fields"]["realName"] # XXX: Figure out a way to know the email address of the author. author_email = author["fields"]["username"] reviewers = list( filter( None, (load_user(reviewer["reviewerPHID"]) for reviewer in revision["attachments"]["reviewers"]["reviewers"]), )) reviewers = set(reviewer["fields"]["username"] for reviewer in reviewers) if len(reviewers): message = replace_reviewers(message, reviewers) logger.info( f"Applying {patch.phid} from revision {revision['id']}: {message}" ) hg.import_( patches=io.BytesIO(patch.patch.encode("utf-8")), message=message.encode("utf-8"), user=f"{author_name} <{author_email}>".encode("utf-8"), ) if self.git_repo_dir: patch_proc = subprocess.Popen( ["patch", "-p1", "--no-backup-if-mismatch", "--force"], stdin=subprocess.PIPE, cwd=self.git_repo_dir, ) patch_proc.communicate(patch.patch.encode("utf-8")) assert patch_proc.returncode == 0, "Failed to apply patch" subprocess.run( [ "git", "-c", f"user.name={author_name}", "-c", f"user.email={author_email}", "commit", "-am", message, ], check=True, cwd=self.git_repo_dir, )
def apply_phab(self, hg, diff_id): def has_revision(revision): if not revision: return False try: hg.identify(revision) return True except hglib.error.CommandError: return False phabricator_api = PhabricatorAPI( api_key=get_secret("PHABRICATOR_TOKEN"), url=get_secret("PHABRICATOR_URL")) # Get the stack of patches stack = phabricator_api.load_patches_stack(diff_id) assert len(stack) > 0, "No patches to apply" # Find the first unknown base revision needed_stack = [] revisions = {} for patch in reversed(stack): needed_stack.insert(0, patch) # Stop as soon as a base revision is available if has_revision(patch.base_revision): logger.info( f"Stopping at diff {patch.id} and revision {patch.base_revision}" ) break if not needed_stack: logger.info("All the patches are already applied") return # Load all the diff revisions diffs = phabricator_api.search_diffs(diff_phid=[p.phid for p in stack]) revisions = { diff["phid"]: phabricator_api.load_revision(rev_phid=diff["revisionPHID"], attachments={"reviewers": True}) for diff in diffs } # Update repo to base revision hg_base = needed_stack[0].base_revision if not has_revision(hg_base): logger.warning( "Missing base revision {} from Phabricator".format(hg_base)) hg_base = "tip" if hg_base: hg.update(rev=hg_base, clean=True) logger.info(f"Updated repo to {hg_base}") try: self.git_base = vcs_map.mercurial_to_git(hg_base) subprocess.run( [ "git", "checkout", "-b", "analysis_branch", self.git_base ], check=True, cwd=self.git_repo_dir, ) logger.info(f"Updated git repo to {self.git_base}") except Exception as e: logger.info( f"Updating git repo to Mercurial {hg_base} failed: {e}") def load_user(phid): if phid.startswith("PHID-USER"): return phabricator_api.load_user(user_phid=phid) elif phid.startswith("PHID-PROJ"): # TODO: Support group reviewers somehow. logger.info(f"Skipping group reviewer {phid}") else: raise Exception(f"Unsupported reviewer {phid}") for patch in needed_stack: revision = revisions[patch.phid] message = "{}\n\n{}".format(revision["fields"]["title"], revision["fields"]["summary"]) author_name = None author_email = None if patch.commits: author_name = patch.commits[0]["author"]["name"] author_email = patch.commits[0]["author"]["email"] if author_name is None: author = load_user(revision["fields"]["authorPHID"]) author_name = author["fields"]["realName"] # XXX: Figure out a way to know the email address of the author. author_email = author["fields"]["username"] reviewers = list( filter( None, (load_user(reviewer["reviewerPHID"]) for reviewer in revision["attachments"]["reviewers"]["reviewers"]), )) reviewers = set(reviewer["fields"]["username"] for reviewer in reviewers) if len(reviewers): message = replace_reviewers(message, reviewers) logger.info( f"Applying {patch.phid} from revision {revision['id']}: {message}" ) hg.import_( patches=io.BytesIO(patch.patch.encode("utf-8")), message=message.encode("utf-8"), user=f"{author_name} <{author_email}>".encode("utf-8"), ) with tempfile.TemporaryDirectory() as tmpdirname: temp_file = os.path.join(tmpdirname, "temp.patch") with open(temp_file, "w") as f: f.write(patch.patch) subprocess.run( ["git", "apply", "--3way", temp_file], check=True, cwd=self.git_repo_dir, ) subprocess.run( [ "git", "-c", f"user.name={author_name}", "-c", f"user.email={author_email}", "commit", "-am", message, ], check=True, cwd=self.git_repo_dir, )
class PhabricatorActions(object): ''' Common Phabricator actions shared across clients ''' def __init__(self, url, api_key, retries=5, sleep=10): self.api = PhabricatorAPI(url=url, api_key=api_key) # Phabricator secure revision retries configuration assert isinstance(retries, int) assert isinstance(sleep, int) self.retries = collections.defaultdict(lambda: (retries, None)) self.sleep = sleep logger.info('Will retry Phabricator secure revision queries', retries=retries, sleep=sleep) # Load secure projects projects = self.api.search_projects(slugs=['secure-revision']) self.secure_projects = { p['phid']: p['fields']['name'] for p in projects } logger.info('Loaded secure projects', projects=self.secure_projects.values()) def update_state(self, build): ''' Check the visibility of the revision, by retrying N times with a specified time This method is executed regularly by the client application to check on the status evolution as the BMO daemon can take several minutes to update the status ''' assert isinstance(build, PhabricatorBuild) # Only when queued if build.state != PhabricatorBuildState.Queued: return # Check this build has some retries left retries_left, last_try = self.retries[build.target_phid] if retries_left <= 0: return # Check this build has been awaited between tries now = time.time() if last_try is not None and now - last_try < self.sleep: return # Now we can check if this revision is public retries_left -= 1 self.retries[build.target_phid] = (retries_left, now) logger.info('Checking visibility status', build=str(build), retries_left=retries_left) if self.is_visible(build): build.state = PhabricatorBuildState.Public logger.info('Revision is public', build=str(build)) elif retries_left <= 0: # Mark as secured when no retries are left build.state = PhabricatorBuildState.Secured logger.info('Revision is marked as secure', build=str(build)) else: # Enqueue back to retry later build.state = PhabricatorBuildState.Queued def is_visible(self, build): ''' Check the visibility of the revision by loading its details ''' assert isinstance(build, PhabricatorBuild) assert build.state == PhabricatorBuildState.Queued try: # Load revision with projects build.revision = self.api.load_revision(rev_id=build.revision_id, attachments={ 'projects': True, 'reviewers': True }) if not build.revision: raise Exception('Not found') # Check against secure projects projects = set( build.revision['attachments']['projects']['projectPHIDs']) if projects.intersection(self.secure_projects): raise Exception('Secure revision') except Exception as e: logger.info('Revision not accessible', build=str(build), error=str(e)) return False return True def load_patches_stack(self, build): ''' Load a stack of patches for a public Phabricator build without hitting a local mercurial repository ''' build.stack = self.api.load_patches_stack(build.diff_id, build.diff) def load_reviewers(self, build): ''' Load details for reviewers found on a build ''' assert isinstance(build, PhabricatorBuild) assert build.state == PhabricatorBuildState.Public assert build.revision is not None reviewers = build.revision['attachments']['reviewers']['reviewers'] build.reviewers = [ self.api.load_user(user_phid=reviewer['reviewerPHID']) for reviewer in reviewers ]
def apply_phab(self, hg, diff_id): def has_revision(revision): if not revision: return False try: hg.identify(revision) return True except hglib.error.CommandError: return False phabricator_api = PhabricatorAPI( api_key=get_secret("PHABRICATOR_TOKEN"), url=get_secret("PHABRICATOR_URL")) # Get the stack of patches stack = phabricator_api.load_patches_stack(diff_id) assert len(stack) > 0, "No patches to apply" # Find the first unknown base revision needed_stack = [] revisions = {} for patch in reversed(stack): needed_stack.insert(0, patch) # Stop as soon as a base revision is available if has_revision(patch.base_revision): logger.info( f"Stopping at diff {patch.id} and revision {patch.base_revision}" ) break if not needed_stack: logger.info("All the patches are already applied") return # Load all the diff revisions diffs = phabricator_api.search_diffs(diff_phid=[p.phid for p in stack]) revisions = { diff["phid"]: phabricator_api.load_revision(rev_phid=diff["revisionPHID"]) for diff in diffs } # Update repo to base revision hg_base = needed_stack[0].base_revision if hg_base: hg.update(rev=hg_base, clean=True) logger.info(f"Updated repo to {hg_base}") for patch in needed_stack: revision = revisions[patch.phid] if patch.commits: message = patch.commits[0]["message"] else: message = revision["fields"]["title"] logger.info( f"Applying {patch.phid} from revision {revision['id']}: {message}" ) hg.import_( patches=io.BytesIO(patch.patch.encode("utf-8")), message=message, user="******", )
def apply_phab(self, hg, diff_id): def has_revision(revision): if not revision: return False try: hg.identify(revision) return True except hglib.error.CommandError: return False phabricator_api = PhabricatorAPI( api_key=get_secret("PHABRICATOR_TOKEN"), url=get_secret("PHABRICATOR_URL") ) # Get the stack of patches stack = phabricator_api.load_patches_stack(diff_id) assert len(stack) > 0, "No patches to apply" # Find the first unknown base revision needed_stack = [] revisions = {} for patch in reversed(stack): needed_stack.insert(0, patch) # Stop as soon as a base revision is available if has_revision(patch.base_revision): logger.info( f"Stopping at diff {patch.id} and revision {patch.base_revision}" ) break if not needed_stack: logger.info("All the patches are already applied") return # Load all the diff revisions diffs = phabricator_api.search_diffs(diff_phid=[p.phid for p in stack]) revisions = { diff["phid"]: phabricator_api.load_revision(rev_phid=diff["revisionPHID"]) for diff in diffs } # Update repo to base revision hg_base = needed_stack[0].base_revision if not has_revision(hg_base): logger.warning("Missing base revision {} from Phabricator".format(hg_base)) hg_base = "tip" if hg_base: hg.update(rev=hg_base, clean=True) logger.info(f"Updated repo to {hg_base}") try: self.git_base = vcs_map.mercurial_to_git(hg_base) subprocess.run( ["git", "checkout", "-b", "analysis_branch", self.git_base], check=True, cwd=self.git_repo_dir, ) logger.info(f"Updated git repo to {self.git_base}") except Exception as e: logger.info(f"Updating git repo to Mercurial {hg_base} failed: {e}") for patch in needed_stack: revision = revisions[patch.phid] if patch.commits: message = patch.commits[0]["message"] author_name = patch.commits[0]["author"]["name"] author_email = patch.commits[0]["author"]["email"] else: message = revision["fields"]["title"] author_name = "bugbug" author_email = "*****@*****.**" logger.info( f"Applying {patch.phid} from revision {revision['id']}: {message}" ) hg.import_( patches=io.BytesIO(patch.patch.encode("utf-8")), message=message.encode("utf-8"), user=f"{author_name} <{author_email}>".encode("utf-8"), ) with tempfile.TemporaryDirectory() as tmpdirname: temp_file = os.path.join(tmpdirname, "temp.patch") with open(temp_file, "w") as f: f.write(patch.patch) subprocess.run( ["git", "apply", "--3way", temp_file], check=True, cwd=self.git_repo_dir, ) subprocess.run( [ "git", "-c", f"user.name={author_name}", "-c", f"user.email={author_email}", "commit", "-am", message, ], check=True, cwd=self.git_repo_dir, )
class PhabricatorActions(object): """ Common Phabricator actions shared across clients """ def __init__(self, url, api_key, retries=5, sleep=10): self.api = PhabricatorAPI(url=url, api_key=api_key) # Phabricator secure revision retries configuration assert isinstance(retries, int) assert isinstance(sleep, int) self.max_retries = retries self.retries = collections.defaultdict(lambda: (retries, None)) self.sleep = sleep logger.info( "Will retry Phabricator secure revision queries", retries=retries, sleep=sleep, ) # Load secure projects projects = self.api.search_projects(slugs=["secure-revision"]) self.secure_projects = { p["phid"]: p["fields"]["name"] for p in projects } logger.info("Loaded secure projects", projects=self.secure_projects.values()) def update_state(self, build): """ Check the visibility of the revision, by retrying N times with an exponential backoff time This method is executed regularly by the client application to check on the status evolution as the BMO daemon can take several minutes to update the status """ assert isinstance(build, PhabricatorBuild) # Only when queued if build.state != PhabricatorBuildState.Queued: return # Check this build has some retries left retries_left, last_try = self.retries[build.target_phid] if retries_left <= 0: return # Check this build has been awaited between tries exp_backoff = (2**(self.max_retries - retries_left)) * self.sleep now = time.time() if last_try is not None and now - last_try < exp_backoff: return # Now we can check if this revision is public retries_left -= 1 self.retries[build.target_phid] = (retries_left, now) logger.info("Checking visibility status", build=str(build), retries_left=retries_left) if self.is_visible(build): build.state = PhabricatorBuildState.Public build.revision_url = self.build_revision_url(build) logger.info("Revision is public", build=str(build)) elif retries_left <= 0: # Mark as secured when no retries are left build.state = PhabricatorBuildState.Secured logger.info("Revision is marked as secure", build=str(build)) else: # Enqueue back to retry later build.state = PhabricatorBuildState.Queued def is_visible(self, build): """ Check the visibility of the revision by loading its details """ assert isinstance(build, PhabricatorBuild) assert build.state == PhabricatorBuildState.Queued try: # Load revision with projects build.revision = self.api.load_revision( rev_id=build.revision_id, attachments={ "projects": True, "reviewers": True }, ) if not build.revision: raise Exception("Not found") # Check against secure projects projects = set( build.revision["attachments"]["projects"]["projectPHIDs"]) if projects.intersection(self.secure_projects): raise Exception("Secure revision") except Exception as e: logger.info("Revision not accessible", build=str(build), error=str(e)) return False return True def load_patches_stack(self, build): """ Load a stack of patches for a public Phabricator build without hitting a local mercurial repository """ build.stack = self.api.load_patches_stack(build.diff_id, build.diff) def load_reviewers(self, build): """ Load details for reviewers found on a build """ assert isinstance(build, PhabricatorBuild) assert build.state == PhabricatorBuildState.Public assert build.revision is not None def load_user(phid): if phid.startswith("PHID-USER"): return self.api.load_user(user_phid=phid) elif phid.startswith("PHID-PROJ"): logger.info(f"Skipping group reviewer {phid}") else: raise Exception(f"Unsupported reviewer {phid}") reviewers = build.revision["attachments"]["reviewers"]["reviewers"] build.reviewers = list( filter(None, [ load_user(reviewer["reviewerPHID"]) for reviewer in reviewers ])) def build_revision_url(self, build): """ Build a Phabricator frontend url for a build's revision """ return "https://{}/D{}".format(self.api.hostname, build.revision_id)