def apply_phab(self, hg, diff_id):
phabricator_api = PhabricatorAPI(
api_key=get_secret("PHABRICATOR_TOKEN"),
url=get_secret("PHABRICATOR_URL"))
diffs = phabricator_api.search_diffs(diff_id=diff_id)
assert len(diffs) == 1, "No diff available for {}".format(diff_id)
diff = diffs[0]
# Get the stack of patches
base, patches = phabricator_api.load_patches_stack(hg, diff)
assert len(patches) > 0, "No patches to apply"
# Load all the diffs details with commits messages
diffs = phabricator_api.search_diffs(diff_phid=[p[0] for p in patches],
attachments={"commits": True})
commits = {
diff["phid"]: diff["attachments"]["commits"].get("commits", [])
for diff in diffs
}
# First apply patches on local repo
for diff_phid, patch in patches:
commit = commits.get(diff_phid)
message = ""
if commit:
message += "{}\n".format(commit[0]["message"])
logger.info(f"Applying {diff_phid}")
hg.import_(
patches=io.BytesIO(patch.encode("utf-8")),
message=message,
user="******",
)
def apply_phab(self, hg, diff_id):
phabricator_api = PhabricatorAPI(
api_key=get_secret("PHABRICATOR_TOKEN"),
url=get_secret("PHABRICATOR_URL"))
diffs = phabricator_api.search_diffs(diff_id=diff_id)
assert len(diffs) == 1, f"No diff available for {diff_id}"
diff = diffs[0]
# Get the stack of patches
base, patches = phabricator_api.load_patches_stack(hg, diff)
assert len(patches) > 0, "No patches to apply"
# Load all the diffs details with commits messages
diffs = phabricator_api.search_diffs(diff_phid=[p[0] for p in patches],
attachments={"commits": True})
diffs_data = {}
for diff in diffs:
revision = phabricator_api.load_revision(
rev_phid=diff["revisionPHID"])
logger.info("Diff {} linked to Revision {}".format(
diff["id"], revision["id"]))
diffs_data[diff["phid"]] = {
"commits": diff["attachments"]["commits"].get("commits", []),
"revision": revision,
}
# First apply patches on local repo
for diff_phid, patch in patches:
diff_data = diffs_data.get(diff_phid)
commits = diff_data["commits"]
revision = diff_data["revision"]
if commits and commits[0]["message"]:
message = commits[0]["message"]
else:
message = revision["fields"]["title"]
logger.info(f"Applying {diff_phid}")
hg.import_(
patches=io.BytesIO(patch.encode("utf-8")),
message=message,
user="******",
)
def apply_phab(self, hg, phabricator_deployment, diff_id):
if phabricator_deployment == PHAB_PROD:
api_key = get_secret("PHABRICATOR_TOKEN")
url = get_secret("PHABRICATOR_URL")
else:
api_key = get_secret("PHABRICATOR_DEV_TOKEN")
url = get_secret("PHABRICATOR_DEV_URL")
phabricator_api = PhabricatorAPI(api_key=api_key, url=url)
# Get the stack of patches
stack = phabricator_api.load_patches_stack(diff_id)
assert len(stack) > 0, "No patches to apply"
# Find the first unknown base revision
needed_stack = []
revisions = {}
for patch in reversed(stack):
needed_stack.insert(0, patch)
# Stop as soon as a base revision is available
if self.has_revision(hg, patch.base_revision):
logger.info(
f"Stopping at diff {patch.id} and revision {patch.base_revision}"
)
break
if not needed_stack:
logger.info("All the patches are already applied")
return
# Load all the diff revisions
diffs = phabricator_api.search_diffs(diff_phid=[p.phid for p in stack])
revisions = {
diff["phid"]:
phabricator_api.load_revision(rev_phid=diff["revisionPHID"],
attachments={"reviewers": True})
for diff in diffs
}
# Update repo to base revision
hg_base = needed_stack[0].base_revision
if not self.has_revision(hg, hg_base):
logger.warning(
"Missing base revision {} from Phabricator".format(hg_base))
hg_base = "tip"
if hg_base:
hg.update(rev=hg_base, clean=True)
logger.info(f"Updated repo to {hg_base}")
if self.git_repo_dir and hg_base != "tip":
try:
self.git_base = tuple(
vcs_map.mercurial_to_git(self.git_repo_dir,
[hg_base]))[0]
subprocess.run(
[
"git", "checkout", "-b", "analysis_branch",
self.git_base
],
check=True,
cwd=self.git_repo_dir,
)
logger.info(f"Updated git repo to {self.git_base}")
except Exception as e:
logger.info(
f"Updating git repo to Mercurial {hg_base} failed: {e}"
)
def load_user(phid):
if phid.startswith("PHID-USER"):
return phabricator_api.load_user(user_phid=phid)
elif phid.startswith("PHID-PROJ"):
# TODO: Support group reviewers somehow.
logger.info(f"Skipping group reviewer {phid}")
else:
raise Exception(f"Unsupported reviewer {phid}")
for patch in needed_stack:
revision = revisions[patch.phid]
message = "{}\n\n{}".format(revision["fields"]["title"],
revision["fields"]["summary"])
author_name = None
author_email = None
if patch.commits:
author_name = patch.commits[0]["author"]["name"]
author_email = patch.commits[0]["author"]["email"]
if author_name is None:
author = load_user(revision["fields"]["authorPHID"])
author_name = author["fields"]["realName"]
# XXX: Figure out a way to know the email address of the author.
author_email = author["fields"]["username"]
reviewers = list(
filter(
None,
(load_user(reviewer["reviewerPHID"]) for reviewer in
revision["attachments"]["reviewers"]["reviewers"]),
))
reviewers = set(reviewer["fields"]["username"]
for reviewer in reviewers)
if len(reviewers):
message = replace_reviewers(message, reviewers)
logger.info(
f"Applying {patch.phid} from revision {revision['id']}: {message}"
)
hg.import_(
patches=io.BytesIO(patch.patch.encode("utf-8")),
message=message.encode("utf-8"),
user=f"{author_name} <{author_email}>".encode("utf-8"),
)
if self.git_repo_dir:
patch_proc = subprocess.Popen(
["patch", "-p1", "--no-backup-if-mismatch", "--force"],
stdin=subprocess.PIPE,
cwd=self.git_repo_dir,
)
patch_proc.communicate(patch.patch.encode("utf-8"))
assert patch_proc.returncode == 0, "Failed to apply patch"
subprocess.run(
[
"git",
"-c",
f"user.name={author_name}",
"-c",
f"user.email={author_email}",
"commit",
"-am",
message,
],
check=True,
cwd=self.git_repo_dir,
)
def apply_phab(self, hg, diff_id):
def has_revision(revision):
if not revision:
return False
try:
hg.identify(revision)
return True
except hglib.error.CommandError:
return False
phabricator_api = PhabricatorAPI(
api_key=get_secret("PHABRICATOR_TOKEN"),
url=get_secret("PHABRICATOR_URL"))
# Get the stack of patches
stack = phabricator_api.load_patches_stack(diff_id)
assert len(stack) > 0, "No patches to apply"
# Find the first unknown base revision
needed_stack = []
revisions = {}
for patch in reversed(stack):
needed_stack.insert(0, patch)
# Stop as soon as a base revision is available
if has_revision(patch.base_revision):
logger.info(
f"Stopping at diff {patch.id} and revision {patch.base_revision}"
)
break
if not needed_stack:
logger.info("All the patches are already applied")
return
# Load all the diff revisions
diffs = phabricator_api.search_diffs(diff_phid=[p.phid for p in stack])
revisions = {
diff["phid"]:
phabricator_api.load_revision(rev_phid=diff["revisionPHID"],
attachments={"reviewers": True})
for diff in diffs
}
# Update repo to base revision
hg_base = needed_stack[0].base_revision
if not has_revision(hg_base):
logger.warning(
"Missing base revision {} from Phabricator".format(hg_base))
hg_base = "tip"
if hg_base:
hg.update(rev=hg_base, clean=True)
logger.info(f"Updated repo to {hg_base}")
try:
self.git_base = vcs_map.mercurial_to_git(hg_base)
subprocess.run(
[
"git", "checkout", "-b", "analysis_branch",
self.git_base
],
check=True,
cwd=self.git_repo_dir,
)
logger.info(f"Updated git repo to {self.git_base}")
except Exception as e:
logger.info(
f"Updating git repo to Mercurial {hg_base} failed: {e}")
def load_user(phid):
if phid.startswith("PHID-USER"):
return phabricator_api.load_user(user_phid=phid)
elif phid.startswith("PHID-PROJ"):
# TODO: Support group reviewers somehow.
logger.info(f"Skipping group reviewer {phid}")
else:
raise Exception(f"Unsupported reviewer {phid}")
for patch in needed_stack:
revision = revisions[patch.phid]
message = "{}\n\n{}".format(revision["fields"]["title"],
revision["fields"]["summary"])
author_name = None
author_email = None
if patch.commits:
author_name = patch.commits[0]["author"]["name"]
author_email = patch.commits[0]["author"]["email"]
if author_name is None:
author = load_user(revision["fields"]["authorPHID"])
author_name = author["fields"]["realName"]
# XXX: Figure out a way to know the email address of the author.
author_email = author["fields"]["username"]
reviewers = list(
filter(
None,
(load_user(reviewer["reviewerPHID"]) for reviewer in
revision["attachments"]["reviewers"]["reviewers"]),
))
reviewers = set(reviewer["fields"]["username"]
for reviewer in reviewers)
if len(reviewers):
message = replace_reviewers(message, reviewers)
logger.info(
f"Applying {patch.phid} from revision {revision['id']}: {message}"
)
hg.import_(
patches=io.BytesIO(patch.patch.encode("utf-8")),
message=message.encode("utf-8"),
user=f"{author_name} <{author_email}>".encode("utf-8"),
)
with tempfile.TemporaryDirectory() as tmpdirname:
temp_file = os.path.join(tmpdirname, "temp.patch")
with open(temp_file, "w") as f:
f.write(patch.patch)
subprocess.run(
["git", "apply", "--3way", temp_file],
check=True,
cwd=self.git_repo_dir,
)
subprocess.run(
[
"git",
"-c",
f"user.name={author_name}",
"-c",
f"user.email={author_email}",
"commit",
"-am",
message,
],
check=True,
cwd=self.git_repo_dir,
)
class PhabricatorActions(object):
'''
Common Phabricator actions shared across clients
'''
def __init__(self, url, api_key, retries=5, sleep=10):
self.api = PhabricatorAPI(url=url, api_key=api_key)
# Phabricator secure revision retries configuration
assert isinstance(retries, int)
assert isinstance(sleep, int)
self.retries = collections.defaultdict(lambda: (retries, None))
self.sleep = sleep
logger.info('Will retry Phabricator secure revision queries',
retries=retries,
sleep=sleep)
# Load secure projects
projects = self.api.search_projects(slugs=['secure-revision'])
self.secure_projects = {
p['phid']: p['fields']['name']
for p in projects
}
logger.info('Loaded secure projects',
projects=self.secure_projects.values())
def update_state(self, build):
'''
Check the visibility of the revision, by retrying N times with a specified time
This method is executed regularly by the client application to check on the status evolution
as the BMO daemon can take several minutes to update the status
'''
assert isinstance(build, PhabricatorBuild)
# Only when queued
if build.state != PhabricatorBuildState.Queued:
return
# Check this build has some retries left
retries_left, last_try = self.retries[build.target_phid]
if retries_left <= 0:
return
# Check this build has been awaited between tries
now = time.time()
if last_try is not None and now - last_try < self.sleep:
return
# Now we can check if this revision is public
retries_left -= 1
self.retries[build.target_phid] = (retries_left, now)
logger.info('Checking visibility status',
build=str(build),
retries_left=retries_left)
if self.is_visible(build):
build.state = PhabricatorBuildState.Public
logger.info('Revision is public', build=str(build))
elif retries_left <= 0:
# Mark as secured when no retries are left
build.state = PhabricatorBuildState.Secured
logger.info('Revision is marked as secure', build=str(build))
else:
# Enqueue back to retry later
build.state = PhabricatorBuildState.Queued
def is_visible(self, build):
'''
Check the visibility of the revision by loading its details
'''
assert isinstance(build, PhabricatorBuild)
assert build.state == PhabricatorBuildState.Queued
try:
# Load revision with projects
build.revision = self.api.load_revision(rev_id=build.revision_id,
attachments={
'projects': True,
'reviewers': True
})
if not build.revision:
raise Exception('Not found')
# Check against secure projects
projects = set(
build.revision['attachments']['projects']['projectPHIDs'])
if projects.intersection(self.secure_projects):
raise Exception('Secure revision')
except Exception as e:
logger.info('Revision not accessible',
build=str(build),
error=str(e))
return False
return True
def load_patches_stack(self, build):
'''
Load a stack of patches for a public Phabricator build
without hitting a local mercurial repository
'''
build.stack = self.api.load_patches_stack(build.diff_id, build.diff)
def load_reviewers(self, build):
'''
Load details for reviewers found on a build
'''
assert isinstance(build, PhabricatorBuild)
assert build.state == PhabricatorBuildState.Public
assert build.revision is not None
reviewers = build.revision['attachments']['reviewers']['reviewers']
build.reviewers = [
self.api.load_user(user_phid=reviewer['reviewerPHID'])
for reviewer in reviewers
]
def apply_phab(self, hg, diff_id):
def has_revision(revision):
if not revision:
return False
try:
hg.identify(revision)
return True
except hglib.error.CommandError:
return False
phabricator_api = PhabricatorAPI(
api_key=get_secret("PHABRICATOR_TOKEN"),
url=get_secret("PHABRICATOR_URL"))
# Get the stack of patches
stack = phabricator_api.load_patches_stack(diff_id)
assert len(stack) > 0, "No patches to apply"
# Find the first unknown base revision
needed_stack = []
revisions = {}
for patch in reversed(stack):
needed_stack.insert(0, patch)
# Stop as soon as a base revision is available
if has_revision(patch.base_revision):
logger.info(
f"Stopping at diff {patch.id} and revision {patch.base_revision}"
)
break
if not needed_stack:
logger.info("All the patches are already applied")
return
# Load all the diff revisions
diffs = phabricator_api.search_diffs(diff_phid=[p.phid for p in stack])
revisions = {
diff["phid"]:
phabricator_api.load_revision(rev_phid=diff["revisionPHID"])
for diff in diffs
}
# Update repo to base revision
hg_base = needed_stack[0].base_revision
if hg_base:
hg.update(rev=hg_base, clean=True)
logger.info(f"Updated repo to {hg_base}")
for patch in needed_stack:
revision = revisions[patch.phid]
if patch.commits:
message = patch.commits[0]["message"]
else:
message = revision["fields"]["title"]
logger.info(
f"Applying {patch.phid} from revision {revision['id']}: {message}"
)
hg.import_(
patches=io.BytesIO(patch.patch.encode("utf-8")),
message=message,
user="******",
)
def apply_phab(self, hg, diff_id):
def has_revision(revision):
if not revision:
return False
try:
hg.identify(revision)
return True
except hglib.error.CommandError:
return False
phabricator_api = PhabricatorAPI(
api_key=get_secret("PHABRICATOR_TOKEN"), url=get_secret("PHABRICATOR_URL")
)
# Get the stack of patches
stack = phabricator_api.load_patches_stack(diff_id)
assert len(stack) > 0, "No patches to apply"
# Find the first unknown base revision
needed_stack = []
revisions = {}
for patch in reversed(stack):
needed_stack.insert(0, patch)
# Stop as soon as a base revision is available
if has_revision(patch.base_revision):
logger.info(
f"Stopping at diff {patch.id} and revision {patch.base_revision}"
)
break
if not needed_stack:
logger.info("All the patches are already applied")
return
# Load all the diff revisions
diffs = phabricator_api.search_diffs(diff_phid=[p.phid for p in stack])
revisions = {
diff["phid"]: phabricator_api.load_revision(rev_phid=diff["revisionPHID"])
for diff in diffs
}
# Update repo to base revision
hg_base = needed_stack[0].base_revision
if not has_revision(hg_base):
logger.warning("Missing base revision {} from Phabricator".format(hg_base))
hg_base = "tip"
if hg_base:
hg.update(rev=hg_base, clean=True)
logger.info(f"Updated repo to {hg_base}")
try:
self.git_base = vcs_map.mercurial_to_git(hg_base)
subprocess.run(
["git", "checkout", "-b", "analysis_branch", self.git_base],
check=True,
cwd=self.git_repo_dir,
)
logger.info(f"Updated git repo to {self.git_base}")
except Exception as e:
logger.info(f"Updating git repo to Mercurial {hg_base} failed: {e}")
for patch in needed_stack:
revision = revisions[patch.phid]
if patch.commits:
message = patch.commits[0]["message"]
author_name = patch.commits[0]["author"]["name"]
author_email = patch.commits[0]["author"]["email"]
else:
message = revision["fields"]["title"]
author_name = "bugbug"
author_email = "*****@*****.**"
logger.info(
f"Applying {patch.phid} from revision {revision['id']}: {message}"
)
hg.import_(
patches=io.BytesIO(patch.patch.encode("utf-8")),
message=message.encode("utf-8"),
user=f"{author_name} <{author_email}>".encode("utf-8"),
)
with tempfile.TemporaryDirectory() as tmpdirname:
temp_file = os.path.join(tmpdirname, "temp.patch")
with open(temp_file, "w") as f:
f.write(patch.patch)
subprocess.run(
["git", "apply", "--3way", temp_file],
check=True,
cwd=self.git_repo_dir,
)
subprocess.run(
[
"git",
"-c",
f"user.name={author_name}",
"-c",
f"user.email={author_email}",
"commit",
"-am",
message,
],
check=True,
cwd=self.git_repo_dir,
)
class PhabricatorActions(object):
"""
Common Phabricator actions shared across clients
"""
def __init__(self, url, api_key, retries=5, sleep=10):
self.api = PhabricatorAPI(url=url, api_key=api_key)
# Phabricator secure revision retries configuration
assert isinstance(retries, int)
assert isinstance(sleep, int)
self.max_retries = retries
self.retries = collections.defaultdict(lambda: (retries, None))
self.sleep = sleep
logger.info(
"Will retry Phabricator secure revision queries",
retries=retries,
sleep=sleep,
)
# Load secure projects
projects = self.api.search_projects(slugs=["secure-revision"])
self.secure_projects = {
p["phid"]: p["fields"]["name"]
for p in projects
}
logger.info("Loaded secure projects",
projects=self.secure_projects.values())
def update_state(self, build):
"""
Check the visibility of the revision, by retrying N times with an exponential backoff time
This method is executed regularly by the client application to check on the status evolution
as the BMO daemon can take several minutes to update the status
"""
assert isinstance(build, PhabricatorBuild)
# Only when queued
if build.state != PhabricatorBuildState.Queued:
return
# Check this build has some retries left
retries_left, last_try = self.retries[build.target_phid]
if retries_left <= 0:
return
# Check this build has been awaited between tries
exp_backoff = (2**(self.max_retries - retries_left)) * self.sleep
now = time.time()
if last_try is not None and now - last_try < exp_backoff:
return
# Now we can check if this revision is public
retries_left -= 1
self.retries[build.target_phid] = (retries_left, now)
logger.info("Checking visibility status",
build=str(build),
retries_left=retries_left)
if self.is_visible(build):
build.state = PhabricatorBuildState.Public
build.revision_url = self.build_revision_url(build)
logger.info("Revision is public", build=str(build))
elif retries_left <= 0:
# Mark as secured when no retries are left
build.state = PhabricatorBuildState.Secured
logger.info("Revision is marked as secure", build=str(build))
else:
# Enqueue back to retry later
build.state = PhabricatorBuildState.Queued
def is_visible(self, build):
"""
Check the visibility of the revision by loading its details
"""
assert isinstance(build, PhabricatorBuild)
assert build.state == PhabricatorBuildState.Queued
try:
# Load revision with projects
build.revision = self.api.load_revision(
rev_id=build.revision_id,
attachments={
"projects": True,
"reviewers": True
},
)
if not build.revision:
raise Exception("Not found")
# Check against secure projects
projects = set(
build.revision["attachments"]["projects"]["projectPHIDs"])
if projects.intersection(self.secure_projects):
raise Exception("Secure revision")
except Exception as e:
logger.info("Revision not accessible",
build=str(build),
error=str(e))
return False
return True
def load_patches_stack(self, build):
"""
Load a stack of patches for a public Phabricator build
without hitting a local mercurial repository
"""
build.stack = self.api.load_patches_stack(build.diff_id, build.diff)
def load_reviewers(self, build):
"""
Load details for reviewers found on a build
"""
assert isinstance(build, PhabricatorBuild)
assert build.state == PhabricatorBuildState.Public
assert build.revision is not None
def load_user(phid):
if phid.startswith("PHID-USER"):
return self.api.load_user(user_phid=phid)
elif phid.startswith("PHID-PROJ"):
logger.info(f"Skipping group reviewer {phid}")
else:
raise Exception(f"Unsupported reviewer {phid}")
reviewers = build.revision["attachments"]["reviewers"]["reviewers"]
build.reviewers = list(
filter(None, [
load_user(reviewer["reviewerPHID"]) for reviewer in reviewers
]))
def build_revision_url(self, build):
"""
Build a Phabricator frontend url for a build's revision
"""
return "https://{}/D{}".format(self.api.hostname, build.revision_id)