def get_list(stack_revision_id): """Return a list of Transplant objects""" revision_id = revision_id_to_int(stack_revision_id) phab = g.phabricator revision = phab.call_conduit("differential.revision.search", constraints={"ids": [revision_id]}) revision = phab.single(revision, "data", none_when_empty=True) if revision is None: return problem( 404, "Revision not found", "The revision does not exist or you lack permission to see it.", type="https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404", ) # TODO: This assumes that all revisions and related objects in the stack # have uniform view permissions for the requesting user. Some revisions # being restricted could cause this to fail. nodes, edges = build_stack_graph(phab, phab.expect(revision, "phid")) revision_phids = list(nodes) revs = phab.call_conduit( "differential.revision.search", constraints={"phids": revision_phids}, limit=len(revision_phids), ) transplants = Transplant.revisions_query( [phab.expect(r, "id") for r in phab.expect(revs, "data")]).all() return [t.serialize() for t in transplants], 200
def warning_previously_landed(*, revision, diff, **kwargs): revision_id = PhabricatorClient.expect(revision, "id") diff_id = PhabricatorClient.expect(diff, "id") landed_transplant = ( Transplant.revisions_query([revision_id]) .filter_by(status=TransplantStatus.landed) .order_by(Transplant.updated_at.desc()) .first() ) if landed_transplant is None: return None landed_diff_id = landed_transplant.revision_to_diff_id[str(revision_id)] same = diff_id == landed_diff_id only_revision = len(landed_transplant.revision_order) == 1 return ( "Already landed with {is_same_string} diff ({landed_diff_id}), " "pushed {push_string} {commit_sha}.".format( is_same_string=("the same" if same else "an older"), landed_diff_id=landed_diff_id, push_string=("as" if only_revision else "with new tip"), commit_sha=landed_transplant.result, ) )
def check_landing_blockers( auth0_user, requested_path, stack_data, landable_paths, landable_repos, *, user_blocks=[user_block_no_auth0_email, user_block_scm_level]): revision_path = [] revision_to_diff_id = {} for revision_phid, diff_id in requested_path: revision_path.append(revision_phid) revision_to_diff_id[revision_phid] = diff_id # Check that the provided path is a prefix to, or equal to, # a landable path. for path in landable_paths: if revision_path == path[:len(revision_path)]: break else: return TransplantAssessment( blocker="The requested set of revisions are not landable.") # Check the requested diffs are the latest. for revision_phid in revision_path: latest_diff_phid = PhabricatorClient.expect( stack_data.revisions[revision_phid], "fields", "diffPHID") latest_diff_id = PhabricatorClient.expect( stack_data.diffs[latest_diff_phid], "id") if latest_diff_id != revision_to_diff_id[revision_phid]: return TransplantAssessment( blocker="A requested diff is not the latest.") # Check if there is already a landing for something in the stack. if (Transplant.revisions_query([ PhabricatorClient.expect(r, "id") for r in stack_data.revisions.values() ]).filter_by(status=TransplantStatus.submitted).first() is not None): return TransplantAssessment(blocker=( "A landing for revisions in this stack is already in progress.")) # To be a landable path the entire path must have the same # repository, so we can get away with checking only one. repo = landable_repos[stack_data.revisions[revision_path[0]]["fields"] ["repositoryPHID"]] # Check anything that would block the current user from # landing this. for block in user_blocks: result = block(auth0_user=auth0_user, landing_repo=repo) if result is not None: return TransplantAssessment(blocker=result) return TransplantAssessment()
def get_list(stack_revision_id): """Return a list of Transplant objects""" revision_id = revision_id_to_int(stack_revision_id) phab = g.phabricator revision = phab.call_conduit( "differential.revision.search", constraints={"ids": [revision_id]} ) revision = phab.single(revision, "data", none_when_empty=True) if revision is None: return problem( 404, "Revision not found", "The revision does not exist or you lack permission to see it.", type="https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404", ) # TODO: This assumes that all revisions and related objects in the stack # have uniform view permissions for the requesting user. Some revisions # being restricted could cause this to fail. nodes, edges = build_stack_graph(phab, phab.expect(revision, "phid")) revision_phids = list(nodes) revs = phab.call_conduit( "differential.revision.search", constraints={"phids": revision_phids}, limit=len(revision_phids), ) # Return both transplants and landing jobs, since for repos that were switched # both or either of these could be populated. rev_ids = [phab.expect(r, "id") for r in phab.expect(revs, "data")] transplants = Transplant.revisions_query(rev_ids).all() landing_jobs = LandingJob.revisions_query(rev_ids).all() if transplants and landing_jobs: logger.warning( "Both {} transplants and {} landing jobs found for this revision".format( str(len(transplants)), str(len(landing_jobs)) ) ) return ( [t.serialize() for t in transplants] + [j.serialize() for j in landing_jobs], 200, )
def get_list(revision_id): """API endpoint at GET /landings to return a list of Landing objects.""" # Verify that the client is permitted to see the associated revision. revision_id = revision_id_to_int(revision_id) revision = g.phabricator.call_conduit( 'differential.revision.search', constraints={'ids': [revision_id]}, ) revision = g.phabricator.expect(revision, 'data') revision = g.phabricator.single(revision, none_when_empty=True) if not revision: return problem( 404, 'Revision not found', 'The revision does not exist or you lack permission to see it.', type='https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404' ) transplants = Transplant.revisions_query([revision_id]).all() return [t.legacy_serialize() for t in transplants], 200
def warning_previously_landed(*, revision, diff, **kwargs): revision_id = PhabricatorClient.expect(revision, 'id') diff_id = PhabricatorClient.expect(diff, 'id') landed_transplant = Transplant.revisions_query([revision_id]).filter_by( status=TransplantStatus.landed ).order_by(Transplant.updated_at.desc()).first() if landed_transplant is None: return None landed_diff_id = landed_transplant.revision_to_diff_id[str(revision_id)] same = diff_id == landed_diff_id only_revision = len(landed_transplant.revision_order) == 1 return ( 'Already landed with {is_same_string} diff ({landed_diff_id}), ' 'pushed {push_string} {commit_sha}.'.format( is_same_string=('the same' if same else 'an older'), landed_diff_id=landed_diff_id, push_string=('as' if only_revision else 'with new tip'), commit_sha=landed_transplant.result ) )
def post(data): phab = g.phabricator landing_path, confirmation_token = _unmarshal_transplant_request(data) logger.info( "transplant requested by user", extra={ "has_confirmation_token": confirmation_token is not None, "landing_path": landing_path, }, ) assessment, to_land, landing_repo, stack_data = _assess_transplant_request( phab, landing_path ) assessment.raise_if_blocked_or_unacknowledged(confirmation_token) if not all((to_land, landing_repo, stack_data)): raise ValueError( "One or more values missing in access transplant request: " f"{to_land}, {landing_repo}, {stack_data}" ) if assessment.warnings: # Log any warnings that were acknowledged, for auditing. logger.info( "Transplant with acknowledged warnings is being requested", extra={ "landing_path": landing_path, "warnings": [ {"i": w.i, "revision_id": w.revision_id, "details": w.details} for w in assessment.warnings ], }, ) involved_phids = set() revisions = [r[0] for r in to_land] for revision in revisions: involved_phids.update(gather_involved_phids(revision)) involved_phids = list(involved_phids) users = user_search(phab, involved_phids) projects = project_search(phab, involved_phids) secure_project_phid = get_secure_project_phid(phab) # Take note of any revisions that the checkin project tag must be # removed from. checkin_phid = get_checkin_project_phid(phab) checkin_revision_phids = [ r["phid"] for r in revisions if checkin_phid in phab.expect(r, "attachments", "projects", "projectPHIDs") ] sec_approval_project_phid = get_sec_approval_project_phid(phab) # Build the patches to land. patch_urls = [] for revision, diff in to_land: reviewers = get_collated_reviewers(revision) accepted_reviewers = reviewers_for_commit_message( reviewers, users, projects, sec_approval_project_phid ) secure = revision_is_secure(revision, secure_project_phid) commit_description = find_title_and_summary_for_landing(phab, revision, secure) commit_message = format_commit_message( commit_description.title, get_bugzilla_bug(revision), accepted_reviewers, commit_description.summary, urllib.parse.urljoin( current_app.config["PHABRICATOR_URL"], "D{}".format(revision["id"]) ), )[1] author_name, author_email = select_diff_author(diff) date_modified = phab.expect(revision, "fields", "dateModified") # Construct the patch that will be sent to transplant. raw_diff = phab.call_conduit("differential.getrawdiff", diffID=diff["id"]) patch = build_patch_for_revision( raw_diff, author_name, author_email, commit_message, date_modified ) # Upload the patch to S3 patch_url = upload( revision["id"], diff["id"], patch, current_app.config["PATCH_BUCKET_NAME"], aws_access_key=current_app.config["AWS_ACCESS_KEY"], aws_secret_key=current_app.config["AWS_SECRET_KEY"], ) patch_urls.append(patch_url) ldap_username = g.auth0_user.email revision_to_diff_id = {str(r["id"]): d["id"] for r, d in to_land} revision_order = [str(r["id"]) for r in revisions] stack_ids = [r["id"] for r in stack_data.revisions.values()] submitted_assessment = TransplantAssessment( blocker=( "This stack was submitted for landing by another user at the same time." ) ) if landing_repo.transplant_locally: with db.session.begin_nested(): _lock_table_for(db.session, model=LandingJob) if ( LandingJob.revisions_query(stack_ids) .filter( LandingJob.status.in_( [LandingJobStatus.SUBMITTED, LandingJobStatus.IN_PROGRESS] ) ) .count() != 0 ): submitted_assessment.raise_if_blocked_or_unacknowledged(None) # Trigger a local transplant job = LandingJob( status=LandingJobStatus.SUBMITTED, requester_email=ldap_username, repository_name=landing_repo.tree, repository_url=landing_repo.url, revision_to_diff_id=revision_to_diff_id, revision_order=revision_order, ) db.session.add(job) db.session.commit() logger.info("New landing job {job.id} created for {landing_repo.tree} repo") # NOTE: the response body is not being used anywhere. return {"id": job.id}, 202 trans = TransplantClient( current_app.config["TRANSPLANT_URL"], current_app.config["TRANSPLANT_USERNAME"], current_app.config["TRANSPLANT_PASSWORD"], ) # We pass the revision id of the base of our landing path to # transplant in rev as it must be unique until the request # has been serviced. While this doesn't use Autoland Transplant # to enforce not requesting from the same stack again, Lando # ensures this itself. root_revision_id = to_land[0][0]["id"] try: # WARNING: Entering critical section, do not add additional # code unless absolutely necessary. Acquires a lock on the # transplants table which gives exclusive write access and # prevents readers who are entering this critical section. # See https://www.postgresql.org/docs/9.3/static/explicit-locking.html # for more details on the specifics of the lock mode. with db.session.begin_nested(): _lock_table_for(db.session, model=Transplant) if ( Transplant.revisions_query(stack_ids) .filter_by(status=TransplantStatus.submitted) .first() is not None ): submitted_assessment.raise_if_blocked_or_unacknowledged(None) transplant_request_id = trans.land( revision_id=root_revision_id, ldap_username=ldap_username, patch_urls=patch_urls, tree=landing_repo.tree, pingback=current_app.config["PINGBACK_URL"], push_bookmark=landing_repo.push_bookmark, ) transplant = Transplant( request_id=transplant_request_id, revision_to_diff_id=revision_to_diff_id, revision_order=revision_order, requester_email=ldap_username, tree=landing_repo.tree, repository_url=landing_repo.url, status=TransplantStatus.submitted, ) db.session.add(transplant) except TransplantError: logger.exception( "error creating transplant", extra={"landing_path": landing_path} ) return problem( 502, "Transplant not created", "The requested landing_path is valid, but transplant failed." "Please retry your request at a later time.", type="https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/502", ) # Transaction succeeded, commit the session. db.session.commit() logger.info( "transplant created", extra={"landing_path": landing_path, "transplant_id": transplant.id}, ) # Asynchronously remove the checkin project from any of the landing # revisions that had it. for r_phid in checkin_revision_phids: try: admin_remove_phab_project.apply_async( args=(r_phid, checkin_phid), kwargs=dict(comment=f"#{CHECKIN_PROJ_SLUG} handled, landing queued."), ) except kombu.exceptions.OperationalError: # Best effort is acceptable here, Transplant *is* going to land # these changes so it's better to return properly from the request. pass return {"id": transplant.id}, 202
def post(data): phab = g.phabricator landing_path, confirmation_token = _unmarshal_transplant_request(data) logger.info( "transplant requested by user", extra={ "has_confirmation_token": confirmation_token is not None, "landing_path": landing_path, }, ) assessment, to_land, landing_repo, stack_data = _assess_transplant_request( phab, landing_path) assessment.raise_if_blocked_or_unacknowledged(confirmation_token) assert to_land is not None assert landing_repo is not None assert stack_data is not None if assessment.warnings: # Log any warnings that were acknowledged, for auditing. logger.info( "Transplant with acknowledged warnings is being requested", extra={ "landing_path": landing_path, "warnings": [{ "i": w.i, "revision_id": w.revision_id, "details": w.details } for w in assessment.warnings], }, ) involved_phids = set() for revision, _ in to_land: involved_phids.update(gather_involved_phids(revision)) involved_phids = list(involved_phids) users = user_search(phab, involved_phids) projects = project_search(phab, involved_phids) # Build the patches to land. patch_urls = [] for revision, diff in to_land: reviewers = get_collated_reviewers(revision) accepted_reviewers = [ reviewer_identity(phid, users, projects).identifier for phid, r in reviewers.items() if r["status"] is ReviewerStatus.ACCEPTED ] _, commit_message = format_commit_message( phab.expect(revision, "fields", "title"), get_bugzilla_bug(revision), accepted_reviewers, phab.expect(revision, "fields", "summary"), urllib.parse.urljoin(current_app.config["PHABRICATOR_URL"], "D{}".format(revision["id"])), ) author_name, author_email = select_diff_author(diff) date_modified = phab.expect(revision, "fields", "dateModified") # Construct the patch that will be sent to transplant. raw_diff = phab.call_conduit("differential.getrawdiff", diffID=diff["id"]) patch = build_patch_for_revision(raw_diff, author_name, author_email, commit_message, date_modified) # Upload the patch to S3 patch_url = upload( revision["id"], diff["id"], patch, current_app.config["PATCH_BUCKET_NAME"], aws_access_key=current_app.config["AWS_ACCESS_KEY"], aws_secret_key=current_app.config["AWS_SECRET_KEY"], ) patch_urls.append(patch_url) trans = TransplantClient( current_app.config["TRANSPLANT_URL"], current_app.config["TRANSPLANT_USERNAME"], current_app.config["TRANSPLANT_PASSWORD"], ) submitted_assessment = TransplantAssessment(blocker=( "This stack was submitted for landing by another user at the same time." )) ldap_username = g.auth0_user.email revision_to_diff_id = {str(r["id"]): d["id"] for r, d in to_land} revision_order = [str(r["id"]) for r, _ in to_land] stack_ids = [r["id"] for r in stack_data.revisions.values()] # We pass the revision id of the base of our landing path to # transplant in rev as it must be unique until the request # has been serviced. While this doesn't use Autoland Transplant # to enforce not requesting from the same stack again, Lando # ensures this itself. root_revision_id = to_land[0][0]["id"] try: # WARNING: Entering critical section, do not add additional # code unless absolutely necessary. Acquires a lock on the # transplants table which gives exclusive write access and # prevents readers who are entering this critical section. # See https://www.postgresql.org/docs/9.3/static/explicit-locking.html # for more details on the specifics of the lock mode. with db.session.begin_nested(): db.session.execute( "LOCK TABLE transplants IN SHARE ROW EXCLUSIVE MODE;") if (Transplant.revisions_query(stack_ids).filter_by( status=TransplantStatus.submitted).first() is not None): submitted_assessment.raise_if_blocked_or_unacknowledged(None) transplant_request_id = trans.land( revision_id=root_revision_id, ldap_username=ldap_username, patch_urls=patch_urls, tree=landing_repo.tree, pingback=current_app.config["PINGBACK_URL"], push_bookmark=landing_repo.push_bookmark, ) transplant = Transplant( request_id=transplant_request_id, revision_to_diff_id=revision_to_diff_id, revision_order=revision_order, requester_email=ldap_username, tree=landing_repo.tree, repository_url=landing_repo.url, status=TransplantStatus.submitted, ) db.session.add(transplant) except TransplantError: logger.exception("error creating transplant", extra={"landing_path": landing_path}) return problem( 502, "Transplant not created", "The requested landing_path is valid, but transplant failed." "Please retry your request at a later time.", type="https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/502", ) # Transaction succeeded, commit the session. db.session.commit() logger.info( "transplant created", extra={ "landing_path": landing_path, "transplant_id": transplant.id }, ) return {"id": transplant.id}, 202