def admin_remove_phab_project(revision_phid: str,
project_phid: str,
comment: Optional[str] = None):
"""Remove a project tag from the provided revision.
Note, this uses administrator privileges and should only be called
if permissions checking is handled elsewhere.
Args:
revision_phid: phid of the revision to remove the project tag from.
project_phid: phid of the project to remove.
comment: An optional comment to add when removing the project.
"""
transactions = [{"type": "projects.remove", "value": [project_phid]}]
if comment is not None:
transactions.append({"type": "comment", "value": comment})
privileged_phab = PhabricatorClient(
current_app.config["PHABRICATOR_URL"],
current_app.config["PHABRICATOR_ADMIN_API_KEY"],
)
# We only retry for PhabricatorCommunicationException, rather than the
# base PhabricatorAPIException to treat errors in this implementation as
# fatal.
privileged_phab.call_conduit(
"differential.revision.edit",
objectIdentifier=revision_phid,
transactions=transactions,
)
def collate_reviewer_attachments(reviewers, reviewers_extra):
"""Return collated reviewer data.
Args:
reviewers: Data from the 'reviewers' attachment of
differential.revision.search.
reviewers_extra: Data from the 'reviewers-extra'
attachment of differential.revision.search.
"""
phids = {}
for reviewer in reviewers:
data = {}
for k in ('reviewerPHID', 'isBlocking', 'actorPHID'):
data[k] = PhabricatorClient.expect(reviewer, k)
data['status'] = ReviewerStatus.from_status(
PhabricatorClient.expect(reviewer, 'status')
)
phids[data['reviewerPHID']] = data
for reviewer in reviewers_extra:
data = {}
for k in ('reviewerPHID', 'diffPHID', 'voidedPHID'):
data[k] = PhabricatorClient.expect(reviewer, k)
data.update(phids.get(data['reviewerPHID'], {}))
phids[data['reviewerPHID']] = data
if len(phids) > min(len(reviewers), len(reviewers_extra)):
raise PhabricatorCommunicationException(
'Phabricator responded with unexpected data'
)
return phids
def create_approval_request(phab: PhabricatorClient, revision: dict, form_content: str):
"""Update an existing revision with reviewers & form comment"""
release_managers = get_release_managers(phab)
rev = phab.call_conduit(
"differential.revision.edit",
objectIdentifier=revision["phid"],
transactions=[
# Set release managers as reviewers
{"type": "reviewers.add", "value": [release_managers["phid"]]},
# Post the form as a comment on the revision
{"type": "comment", "value": form_content},
],
)
rev_id = phab.expect(rev, "object", "id")
rev_phid = phab.expect(rev, "object", "phid")
assert rev_id == revision["id"], "Revision id mismatch"
logger.info("Updated Phabricator revision", extra={"id": rev_id, "phid": rev_phid})
return {
"mode": "approval",
"url": f"{phab.url_base}/D{rev_id}",
"revision_id": rev_id,
"revision_phid": rev_phid,
}
def get_landable_repos_for_revision_data(revision_data, supported_repos):
"""Return a dictionary mapping string PHID to landable Repo
Args:
revision_data: A RevisionData.
supported_repos: A dictionary mapping repository shortname to
a Repo for repositories lando supports.
Returns:
A dictionary where each key is a string PHID for a repository from
revision_data and the value is a Repo taken from supported_repos.
Repositories in revision_data which are unsupported will not be
present in the dictionary.
"""
repo_phids = {
PhabricatorClient.expect(revision, "fields", "repositoryPHID")
for revision in revision_data.revisions.values()
if PhabricatorClient.expect(revision, "fields", "repositoryPHID")
}
repos = {
phid: supported_repos.get(
PhabricatorClient.expect(
revision_data.repositories[phid], "fields", "shortName"
)
)
for phid in repo_phids
if phid in revision_data.repositories
}
return {phid: repo for phid, repo in repos.items() if repo}
def warning_previously_landed(*, revision, diff, **kwargs):
revision_id = PhabricatorClient.expect(revision, "id")
diff_id = PhabricatorClient.expect(diff, "id")
landed_transplant = (
Transplant.revisions_query([revision_id])
.filter_by(status=TransplantStatus.landed)
.order_by(Transplant.updated_at.desc())
.first()
)
if landed_transplant is None:
return None
landed_diff_id = landed_transplant.revision_to_diff_id[str(revision_id)]
same = diff_id == landed_diff_id
only_revision = len(landed_transplant.revision_order) == 1
return (
"Already landed with {is_same_string} diff ({landed_diff_id}), "
"pushed {push_string} {commit_sha}.".format(
is_same_string=("the same" if same else "an older"),
landed_diff_id=landed_diff_id,
push_string=("as" if only_revision else "with new tip"),
commit_sha=landed_transplant.result,
)
)
def _render_author_response(phid, user_search_data):
author = user_search_data.get(phid, {})
return {
'phid': PhabricatorClient.expect(author, 'phid'),
'username': PhabricatorClient.expect(author, 'fields', 'username'),
'real_name': PhabricatorClient.expect(author, 'fields', 'realName'),
}
def serialize_author(phid, user_search_data):
out = {"phid": phid, "username": None, "real_name": None}
author = user_search_data.get(phid)
if author is not None:
out["username"] = PhabricatorClient.expect(author, "fields", "username")
out["real_name"] = PhabricatorClient.expect(author, "fields", "realName")
return out
def check_landing_blockers(
auth0_user,
requested_path,
stack_data,
landable_paths,
landable_repos,
*,
user_blocks=[user_block_no_auth0_email, user_block_scm_level]):
revision_path = []
revision_to_diff_id = {}
for revision_phid, diff_id in requested_path:
revision_path.append(revision_phid)
revision_to_diff_id[revision_phid] = diff_id
# Check that the provided path is a prefix to, or equal to,
# a landable path.
for path in landable_paths:
if revision_path == path[:len(revision_path)]:
break
else:
return TransplantAssessment(
blocker="The requested set of revisions are not landable.")
# Check the requested diffs are the latest.
for revision_phid in revision_path:
latest_diff_phid = PhabricatorClient.expect(
stack_data.revisions[revision_phid], "fields", "diffPHID")
latest_diff_id = PhabricatorClient.expect(
stack_data.diffs[latest_diff_phid], "id")
if latest_diff_id != revision_to_diff_id[revision_phid]:
return TransplantAssessment(
blocker="A requested diff is not the latest.")
# Check if there is already a landing for something in the stack.
if (Transplant.revisions_query([
PhabricatorClient.expect(r, "id")
for r in stack_data.revisions.values()
]).filter_by(status=TransplantStatus.submitted).first() is not None):
return TransplantAssessment(blocker=(
"A landing for revisions in this stack is already in progress."))
# To be a landable path the entire path must have the same
# repository, so we can get away with checking only one.
repo = landable_repos[stack_data.revisions[revision_path[0]]["fields"]
["repositoryPHID"]]
# Check anything that would block the current user from
# landing this.
for block in user_blocks:
result = block(auth0_user=auth0_user, landing_repo=repo)
if result is not None:
return TransplantAssessment(blocker=result)
return TransplantAssessment()
def check(cls, *, get_revision, get_diff, **kwargs):
diff, _ = get_diff()
revision = get_revision()
if (PhabricatorClient.expect(revision, 'phid') !=
PhabricatorClient.expect(diff, 'fields', 'revisionPHID')):
raise ProblemException(
400,
'Diff not related to the revision',
'The requested diff is not related to the requested revision.',
type='https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/400' # noqa
) # yapf: disable
def lazy_get_reviewers(revision):
"""Return a dictionary mapping phid to collated reviewer attachment data.
Args:
revision: A dict of the revision data from differential.revision.search
with the 'reviewers' and 'reviewers-extra' attachments.
"""
attachments = PhabricatorClient.expect(revision, 'attachments')
return collate_reviewer_attachments(
PhabricatorClient.expect(attachments, 'reviewers', 'reviewers'),
PhabricatorClient.expect(attachments, 'reviewers-extra',
'reviewers-extra'))
def get_raw_comments(transaction):
"""Return a list of 'raw' comment bodies in a Phabricator transaction.
A single transaction can have multiple comment bodies: e.g. a top-level comment
and a couple of inline comments along with it.
See https://phabricator.services.mozilla.com/conduit/method/transaction.search/.
"""
return [
PhabricatorClient.expect(comment, "content", "raw")
for comment in PhabricatorClient.expect(transaction, "comments")
]
def serialize_author(phid, user_search_data):
out = {
'phid': phid,
'username': None,
'real_name': None,
}
author = user_search_data.get(phid)
if author is not None:
out['username'] = PhabricatorClient.expect(author, 'fields',
'username')
out['real_name'] = PhabricatorClient.expect(author, 'fields',
'realName')
return out
def _render_diff_response(querydiffs_data):
return {
'id': int(PhabricatorClient.expect(querydiffs_data, 'id')),
'date_created': _epoch_to_isoformat_time(
PhabricatorClient.expect(querydiffs_data, 'dateCreated')
),
'date_modified': _epoch_to_isoformat_time(
PhabricatorClient.expect(querydiffs_data, 'dateModified')
),
'author': {
'name': querydiffs_data.get('authorName', ''),
'email': querydiffs_data.get('authorEmail', ''),
},
} # yapf: disable
def _convert_path_id_to_phid(path, stack_data):
mapping = {
PhabricatorClient.expect(r, "id"): PhabricatorClient.expect(r, "phid")
for r in stack_data.revisions.values()
}
try:
return [(mapping[r], d) for r, d in path]
except IndexError:
ProblemException(
400,
"Landing Path Invalid",
"The provided landing_path is not valid.",
type="https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/400",
)
def _assess_transplant_request(phab, landing_path):
nodes, edges = _find_stack_from_landing_path(phab, landing_path)
stack_data = request_extended_revision_data(phab, [phid for phid in nodes])
landing_path = _convert_path_id_to_phid(landing_path, stack_data)
supported_repos = get_repos_for_env(current_app.config.get('ENVIRONMENT'))
landable_repos = get_landable_repos_for_revision_data(
stack_data, supported_repos)
landable, blocked = calculate_landable_subgraphs(
stack_data,
edges,
landable_repos,
other_checks=DEFAULT_OTHER_BLOCKER_CHECKS)
assessment = check_landing_blockers(
g.auth0_user,
landing_path,
stack_data,
landable,
landable_repos,
)
if assessment.blocker is not None:
return (assessment, None, None, None)
# We have now verified that landable_path is valid and is indeed
# landable (in the sense that it is a landable_subgraph, with no
# revisions being blocked). Make this clear by using a different
# value, and assume it going forward.
valid_path = landing_path
# Now that we know this is a valid path we can convert it into a list
# of (revision, diff) tuples.
to_land = [stack_data.revisions[r_phid] for r_phid, _ in valid_path]
to_land = [
(r, stack_data.diffs[PhabricatorClient.expect(r, 'fields',
'diffPHID')])
for r in to_land
]
# To be a landable path the entire path must have the same
# repository, so we can get away with checking only one.
repo = stack_data.repositories[to_land[0][0]['fields']['repositoryPHID']]
landing_repo = landable_repos[repo['phid']]
involved_phids = set()
for revision, _ in to_land:
involved_phids.update(gather_involved_phids(revision))
involved_phids = list(involved_phids)
users = lazy_user_search(phab, involved_phids)()
projects = lazy_project_search(phab, involved_phids)()
reviewers = {
revision['phid']: get_collated_reviewers(revision)
for revision, _ in to_land
}
assessment = check_landing_warnings(g.auth0_user, to_land, repo,
landing_repo, reviewers, users,
projects)
return (assessment, to_land, landing_repo, stack_data)
def test_find_txn_with_comment_in_phabricator(phabdouble):
phab = phabdouble.get_phabricator_client()
# A sec-approval request adds a comment to a revision.
mock_comment = phabdouble.comment("my sec-approval request")
revision = phabdouble.revision()
# Add the two sec-approval request transactions to Phabricator. This also links the
# comment transaction to the revision.
comment_txn = phabdouble.api_object_for(
phabdouble.transaction("comment", revision, comments=[mock_comment])
)
review_txn = phabdouble.api_object_for(
phabdouble.transaction("reviewers.add", revision)
)
# Fetch our comment transaction
comment = PhabricatorClient.single(comment_txn, "comments")
# Add the sec-approval request transactions to the database.
revision = phabdouble.api_object_for(revision)
sec_approval_request = SecApprovalRequest.build(revision, [comment_txn, review_txn])
# Search the list of sec-approval transactions for the comment.
matching_comment = search_sec_approval_request_for_comment(
phab, sec_approval_request
)
assert matching_comment == comment
def check_author_planned_changes(*, revision, **kwargs):
status = RevisionStatus.from_status(
PhabricatorClient.expect(revision, "fields", "status", "value"))
if status is not RevisionStatus.CHANGES_PLANNED:
return None
return "The author has indicated they are planning changes to this revision."
def get_client(api_key=None):
api_key = (
api_key or current_app.config['PHABRICATOR_UNPRIVILEGED_API_KEY']
)
return PhabricatorClient(
current_app.config['PHABRICATOR_URL'], api_key
)
def test_integrated_secure_stack_has_alternate_commit_message(
db,
client,
phabdouble,
mock_repo_config,
secure_project,
authed_headers,
monkeypatch,
):
sanitized_title = "my secure commit title"
revision_title = "my insecure revision title"
# Build a revision with an active sec-approval request.
diff, secure_revision = _make_sec_approval_request(
sanitized_title,
revision_title,
authed_headers,
client,
monkeypatch,
phabdouble,
secure_project,
)
# Request the revision from Lando. It should have our new title and summary.
response = client.get("/stacks/D{}".format(secure_revision["id"]))
assert response == 200
revision = PhabricatorClient.single(response.json, "revisions")
assert revision["is_secure"]
assert revision["is_using_secure_commit_message"]
assert revision["title"] == sanitized_title
assert revision["summary"] == ""
def warning_not_accepted(*, revision, **kwargs):
status = RevisionStatus.from_status(
PhabricatorClient.expect(revision, "fields", "status", "value"))
if status is RevisionStatus.ACCEPTED:
return None
return status.output_name
def wrapped(*args, **kwargs):
api_key = request.headers.get("X-Phabricator-API-Key")
if api_key is None and not self.optional:
return problem(
401,
"X-Phabricator-API-Key Required",
("Phabricator api key not provided in "
"X-Phabricator-API-Key header"),
type=
"https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/401",
)
g.phabricator = PhabricatorClient(
current_app.config["PHABRICATOR_URL"],
api_key
or current_app.config["PHABRICATOR_UNPRIVILEGED_API_KEY"],
)
if api_key is not None and not g.phabricator.verify_api_token():
return problem(
403,
"X-Phabricator-API-Key Invalid",
"Phabricator api key is not valid",
type=
"https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/403",
)
return f(*args, **kwargs)
def transaction_search(phabricator,
object_identifier,
transaction_phids=None,
limit=100):
"""Yield the Phabricator transactions related to an object.
See https://phabricator.services.mozilla.com/conduit/method/transaction.search/.
If the transaction list is larger that one page of API results then the generator
will call the Phabricator API successive times to fetch the full transaction list.
Args:
phabricator: A PhabricatorClient instance.
object_identifier: An object identifier (PHID or monogram) whose transactions
we want to fetch.
transaction_phids: An optional list of specific transactions PHIDs we want to
find for the given object_identifier.
limit: Integer keyword, limit the number of records retrieved per API call.
Default is 100 records.
Returns:
Yields individual transactions.
"""
next_page_start = None
if transaction_phids:
constraints = {"phids": transaction_phids}
else:
constraints = {}
while True:
transactions = phabricator.call_conduit(
"transaction.search",
objectIdentifier=object_identifier,
constraints=constraints,
limit=limit,
after=next_page_start,
)
yield from PhabricatorClient.expect(transactions, "data")
next_page_start = PhabricatorClient.expect(transactions, "cursor",
"after")
if next_page_start is None:
# This was the last page of results.
return
def reviewer_identity(phid, user_search_data, project_search_data):
if phid in user_search_data:
return ReviewerIdentity(
PhabricatorClient.expect(user_search_data, phid, "fields",
"username"),
PhabricatorClient.expect(user_search_data, phid, "fields",
"realName"),
)
if phid in project_search_data:
name = PhabricatorClient.expect(project_search_data, phid, "fields",
"name")
return ReviewerIdentity(name, name)
logger.info("reviewer was missing from user / project search data",
extra={"phid": phid})
return ReviewerIdentity("<unknown>", "Unknown User/Project")
def select_diff_author(diff):
commits = PhabricatorClient.expect(diff, "attachments", "commits", "commits")
if not commits:
return None, None
authors = [c.get("author", {}) for c in commits]
authors = Counter((a.get("name"), a.get("email")) for a in authors)
authors = authors.most_common(1)
return authors[0][0] if authors else (None, None)
def lazy_get_revision_status(revision):
"""Return a landoapi.phabricator.RevisionStatus.
Args:
revision: A dict of the revision data just as it is returned
by Phabricator.
"""
return RevisionStatus.from_status(
PhabricatorClient.expect(revision, 'fields', 'status', 'value'))
def _get_comment(phabdouble, msg):
"""Retrieve the Phabricator API representation of a raw comment string."""
revision = phabdouble.revision()
mock_comment = phabdouble.comment(msg)
phabdouble.transaction("dummy", revision, comments=[mock_comment])
transaction = phabdouble.api_object_for(
phabdouble.transaction("dummy", revision, comments=[mock_comment])
)
comment = PhabricatorClient.single(transaction, "comments")
return comment
def serialize_status(revision):
status_value = PhabricatorClient.expect(revision, "fields", "status", "value")
status = RevisionStatus.from_status(status_value)
if status is RevisionStatus.UNEXPECTED_STATUS:
logger.warning(
"Revision had unexpected status",
extra={
"id": PhabricatorClient.expection(revision, "id"),
"value": status_value,
},
)
return {"closed": False, "value": None, "display": "Unknown"}
return {
"closed": status.closed,
"value": status.value,
"display": status.output_name,
}
def reviewer_identity(phid, user_search_data, project_search_data):
if phid in user_search_data:
return ReviewerIdentity(
PhabricatorClient.expect(user_search_data, phid, 'fields',
'username'),
PhabricatorClient.expect(user_search_data, phid, 'fields',
'realName'))
if phid in project_search_data:
name = PhabricatorClient.expect(project_search_data, phid, 'fields',
'name')
return ReviewerIdentity(name, name)
logger.info(
{
'msg': 'A reviewer was missing from user / project search data',
'phid': phid,
}, 'reviewer.unknown')
return ReviewerIdentity('<unknown>', 'Unknown User/Project')
def check_phabricator():
try:
PhabricatorClient(
current_app.config['PHABRICATOR_URL'],
current_app.config['PHABRICATOR_UNPRIVILEGED_API_KEY']
).call_conduit('conduit.ping')
except PhabricatorAPIException as exc:
return ['PhabricatorAPIException: {!s}'.format(exc)]
return []
def heartbeat():
"""Perform an in-depth service health check.
This should check all the services that this service depends on
and return a 200 iff those services and the app itself are
performing normally. Return a 5XX if something goes wrong.
"""
phab = PhabricatorClient(
current_app.config['PHABRICATOR_URL'],
current_app.config['PHABRICATOR_UNPRIVILEGED_API_KEY'])
try:
phab.call_conduit('conduit.ping')
except PhabricatorAPIException:
logger.warning({
'msg': 'problem connecting to Phabricator',
}, 'heartbeat')
return 'heartbeat: problem', 502
logger.info({'msg': 'ok, all services are up'}, 'heartbeat')
return 'heartbeat: ok', 200
