def upload(self, report, changesets): results = self.generate(report, changesets) if secrets[secrets.PHABRICATOR_ENABLED]: phabricator = PhabricatorAPI(secrets[secrets.PHABRICATOR_TOKEN], secrets[secrets.PHABRICATOR_URL]) else: phabricator = None for rev_id, coverage in results.items(): # Only upload raw coverage data to Phabricator, not stats coverage = { path: cov["coverage"] for path, cov in coverage.items() } logger.info("{} coverage: {}".format(rev_id, coverage)) if not phabricator or not coverage: continue try: rev_data = phabricator.load_revision(rev_id=rev_id) phabricator.upload_coverage_results( rev_data["fields"]["diffPHID"], coverage) # XXX: This is only necessary until https://bugzilla.mozilla.org/show_bug.cgi?id=1487843 is resolved. phabricator.upload_lint_results(rev_data["fields"]["diffPHID"], BuildState.Pass, []) except PhabricatorRevisionNotFoundException: logger.warn("Phabricator revision not found", rev_id=rev_id) return results
def get(rev_id): assert DEPLOYMENT_URL is not None assert API_KEY is not None phabricator_api = PhabricatorAPI(API_KEY, DEPLOYMENT_URL) return phabricator_api.load_revision(rev_id=rev_id, attachments={"projects": True})
def apply_phab(self, hg, diff_id): phabricator_api = PhabricatorAPI( api_key=get_secret("PHABRICATOR_TOKEN"), url=get_secret("PHABRICATOR_URL")) diffs = phabricator_api.search_diffs(diff_id=diff_id) assert len(diffs) == 1, "No diff available for {}".format(diff_id) diff = diffs[0] # Get the stack of patches base, patches = phabricator_api.load_patches_stack(hg, diff) assert len(patches) > 0, "No patches to apply" # Load all the diffs details with commits messages diffs = phabricator_api.search_diffs(diff_phid=[p[0] for p in patches], attachments={"commits": True}) commits = { diff["phid"]: diff["attachments"]["commits"].get("commits", []) for diff in diffs } # First apply patches on local repo for diff_phid, patch in patches: commit = commits.get(diff_phid) message = "" if commit: message += "{}\n".format(commit[0]["message"]) logger.info(f"Applying {diff_phid}") hg.import_( patches=io.BytesIO(patch.encode("utf-8")), message=message, user="******", )
def main(taskcluster_secret, taskcluster_client_id, taskcluster_access_token, cache_root, phab_build_target, ): secrets = get_secrets(taskcluster_secret, config.PROJECT_NAME, required=( 'PULSE_USER', 'PULSE_PASSWORD', 'HOOKS', 'ADMINS', 'PHABRICATOR', 'repositories', ), existing=dict( HOOKS=[], ADMINS=['*****@*****.**', '*****@*****.**'], repositories=[] ), taskcluster_client_id=taskcluster_client_id, taskcluster_access_token=taskcluster_access_token, ) init_logger(config.PROJECT_NAME, PAPERTRAIL_HOST=secrets.get('PAPERTRAIL_HOST'), PAPERTRAIL_PORT=secrets.get('PAPERTRAIL_PORT'), SENTRY_DSN=secrets.get('SENTRY_DSN'), MOZDEF=secrets.get('MOZDEF'), ) task_monitoring.emails = secrets['ADMINS'] phabricator = PhabricatorAPI( api_key=secrets['PHABRICATOR']['token'], url=secrets['PHABRICATOR']['url'], ) pl = PulseListener(secrets['PULSE_USER'], secrets['PULSE_PASSWORD'], secrets['HOOKS'], secrets['repositories'], phabricator, cache_root, secrets['PHABRICATOR'].get('publish', False), taskcluster_client_id, taskcluster_access_token, ) click.echo('Listening to pulse messages...') if phab_build_target: pl.add_build(phab_build_target) pl.run()
def apply_phab(self, hg, diff_id): phabricator_api = PhabricatorAPI( api_key=get_secret("PHABRICATOR_TOKEN"), url=get_secret("PHABRICATOR_URL")) diffs = phabricator_api.search_diffs(diff_id=diff_id) assert len(diffs) == 1, f"No diff available for {diff_id}" diff = diffs[0] # Get the stack of patches base, patches = phabricator_api.load_patches_stack(hg, diff) assert len(patches) > 0, "No patches to apply" # Load all the diffs details with commits messages diffs = phabricator_api.search_diffs(diff_phid=[p[0] for p in patches], attachments={"commits": True}) diffs_data = {} for diff in diffs: revision = phabricator_api.load_revision( rev_phid=diff["revisionPHID"]) logger.info("Diff {} linked to Revision {}".format( diff["id"], revision["id"])) diffs_data[diff["phid"]] = { "commits": diff["attachments"]["commits"].get("commits", []), "revision": revision, } # First apply patches on local repo for diff_phid, patch in patches: diff_data = diffs_data.get(diff_phid) commits = diff_data["commits"] revision = diff_data["revision"] if commits and commits[0]["message"]: message = commits[0]["message"] else: message = revision["fields"]["title"] logger.info(f"Applying {diff_phid}") hg.import_( patches=io.BytesIO(patch.encode("utf-8")), message=message, user="******", )
def __init__(self, url, api_key, retries=5, sleep=10): self.api = PhabricatorAPI(url=url, api_key=api_key) # Phabricator secure revision retries configuration assert isinstance(retries, int) assert isinstance(sleep, int) self.retries = collections.defaultdict(lambda: (retries, None)) self.sleep = sleep logger.info('Will retry Phabricator secure revision queries', retries=retries, sleep=sleep) # Load secure projects projects = self.api.search_projects(slugs=['secure-revision']) self.secure_projects = { p['phid']: p['fields']['name'] for p in projects } logger.info('Loaded secure projects', projects=self.secure_projects.values())
def __init__(self, url, api_key, retries=5, sleep=10): self.api = PhabricatorAPI(url=url, api_key=api_key) # Phabricator secure revision retries configuration assert isinstance(retries, int) assert isinstance(sleep, int) self.max_retries = retries self.retries = collections.defaultdict(lambda: (retries, None)) self.sleep = sleep logger.info( "Will retry Phabricator secure revision queries", retries=retries, sleep=sleep, ) # Load secure projects projects = self.api.search_projects(slugs=["secure-revision"]) self.secure_projects = { p["phid"]: p["fields"]["name"] for p in projects } logger.info("Loaded secure projects", projects=self.secure_projects.values())
def upload(self, report, changesets=None): results = self.generate(report, changesets) if secrets[secrets.PHABRICATOR_ENABLED]: phabricator = PhabricatorAPI(secrets[secrets.PHABRICATOR_TOKEN], secrets[secrets.PHABRICATOR_URL]) else: phabricator = None for rev_id, coverage in results.items(): logger.info('{} coverage: {}'.format(rev_id, coverage)) if not phabricator or not coverage: continue try: rev_data = phabricator.load_revision(rev_id=rev_id) phabricator.upload_coverage_results( rev_data['fields']['diffPHID'], coverage) # XXX: This is only necessary until https://bugzilla.mozilla.org/show_bug.cgi?id=1487843 is resolved. phabricator.upload_lint_results(rev_data['fields']['diffPHID'], BuildState.Pass, []) except PhabricatorRevisionNotFoundException: logger.warn('Phabricator revision not found', rev_id=rev_id)
def main(): args = parse_cli() taskcluster.auth(args.taskcluster_client_id, args.taskcluster_access_token) taskcluster.load_secrets( args.taskcluster_secret, prefixes=["common", "code-review-bot", "bot"], required=( "APP_CHANNEL", "REPORTERS", "PHABRICATOR", "ALLOWED_PATHS", "repositories", ), existing={ "APP_CHANNEL": "development", "REPORTERS": [], "ZERO_COVERAGE_ENABLED": True, "ALLOWED_PATHS": ["*"], "task_failures_ignored": [], }, local_secrets=yaml.safe_load(args.configuration) if args.configuration else None, ) init_logger( "bot", channel=taskcluster.secrets.get("APP_CHANNEL", "dev"), PAPERTRAIL_HOST=taskcluster.secrets.get("PAPERTRAIL_HOST"), PAPERTRAIL_PORT=taskcluster.secrets.get("PAPERTRAIL_PORT"), SENTRY_DSN=taskcluster.secrets.get("SENTRY_DSN"), ) # Setup settings before stats settings.setup( taskcluster.secrets["APP_CHANNEL"], taskcluster.secrets["ALLOWED_PATHS"], taskcluster.secrets["repositories"], ) # Setup statistics influx_conf = taskcluster.secrets.get("influxdb") if influx_conf: stats.auth(influx_conf) # Load reporters reporters = get_reporters(taskcluster.secrets["REPORTERS"]) # Load index service index_service = taskcluster.get_service("index") # Load queue service queue_service = taskcluster.get_service("queue") # Load Phabricator API phabricator = taskcluster.secrets["PHABRICATOR"] phabricator_reporting_enabled = "phabricator" in reporters phabricator_api = PhabricatorAPI(phabricator["api_key"], phabricator["url"]) if phabricator_reporting_enabled: reporters["phabricator"].setup_api(phabricator_api) # Load unique revision try: if settings.autoland_group_id: revision = Revision.from_autoland( queue_service.task(settings.autoland_group_id), phabricator_api ) else: revision = Revision.from_try( queue_service.task(settings.try_task_id), phabricator_api ) except Exception as e: # Report revision loading failure on production only # On testing or dev instances, we can use different Phabricator # configuration that do not match all the pulse messages sent if settings.on_production: raise else: logger.info( "Failed to load revision", task=settings.try_task_id, error=str(e), phabricator=phabricator["url"], ) return 1 # Run workflow according to source w = Workflow( reporters, index_service, queue_service, phabricator_api, taskcluster.secrets["ZERO_COVERAGE_ENABLED"], # Update build status only when phabricator reporting is enabled update_build=phabricator_reporting_enabled, task_failures_ignored=taskcluster.secrets["task_failures_ignored"], ) try: if revision.repository == REPO_AUTOLAND: w.ingest_autoland(revision) else: w.run(revision) except Exception as e: # Log errors to papertrail logger.error("Static analysis failure", revision=revision, error=e) # Index analysis state extras = {} if isinstance(e, AnalysisException): extras["error_code"] = e.code extras["error_message"] = str(e) w.index(revision, state="error", **extras) # Update Harbormaster status w.update_status(revision, state=BuildState.Fail) # Then raise to mark task as erroneous raise return 0
def __init__(self): super(NotLanded, self).__init__() self.nweeks = utils.get_config(self.name(), "number_of_weeks", 2) self.nyears = utils.get_config(self.name(), "number_of_years", 2) self.phab = PhabricatorAPI(utils.get_login_info()["phab_api_key"]) self.extra_ni = {}
def set_api_key(url: str, api_key: str) -> None: global PHABRICATOR_API PHABRICATOR_API = PhabricatorAPI(api_key, url)
def apply_phab(self, hg, diff_id): def has_revision(revision): if not revision: return False try: hg.identify(revision) return True except hglib.error.CommandError: return False phabricator_api = PhabricatorAPI( api_key=get_secret("PHABRICATOR_TOKEN"), url=get_secret("PHABRICATOR_URL") ) # Get the stack of patches stack = phabricator_api.load_patches_stack(diff_id) assert len(stack) > 0, "No patches to apply" # Find the first unknown base revision needed_stack = [] revisions = {} for patch in reversed(stack): needed_stack.insert(0, patch) # Stop as soon as a base revision is available if has_revision(patch.base_revision): logger.info( f"Stopping at diff {patch.id} and revision {patch.base_revision}" ) break if not needed_stack: logger.info("All the patches are already applied") return # Load all the diff revisions diffs = phabricator_api.search_diffs(diff_phid=[p.phid for p in stack]) revisions = { diff["phid"]: phabricator_api.load_revision(rev_phid=diff["revisionPHID"]) for diff in diffs } # Update repo to base revision hg_base = needed_stack[0].base_revision if not has_revision(hg_base): logger.warning("Missing base revision {} from Phabricator".format(hg_base)) hg_base = "tip" if hg_base: hg.update(rev=hg_base, clean=True) logger.info(f"Updated repo to {hg_base}") try: self.git_base = vcs_map.mercurial_to_git(hg_base) subprocess.run( ["git", "checkout", "-b", "analysis_branch", self.git_base], check=True, cwd=self.git_repo_dir, ) logger.info(f"Updated git repo to {self.git_base}") except Exception as e: logger.info(f"Updating git repo to Mercurial {hg_base} failed: {e}") for patch in needed_stack: revision = revisions[patch.phid] if patch.commits: message = patch.commits[0]["message"] author_name = patch.commits[0]["author"]["name"] author_email = patch.commits[0]["author"]["email"] else: message = revision["fields"]["title"] author_name = "bugbug" author_email = "*****@*****.**" logger.info( f"Applying {patch.phid} from revision {revision['id']}: {message}" ) hg.import_( patches=io.BytesIO(patch.patch.encode("utf-8")), message=message.encode("utf-8"), user=f"{author_name} <{author_email}>".encode("utf-8"), ) with tempfile.TemporaryDirectory() as tmpdirname: temp_file = os.path.join(tmpdirname, "temp.patch") with open(temp_file, "w") as f: f.write(patch.patch) subprocess.run( ["git", "apply", "--3way", temp_file], check=True, cwd=self.git_repo_dir, ) subprocess.run( [ "git", "-c", f"user.name={author_name}", "-c", f"user.email={author_email}", "commit", "-am", message, ], check=True, cwd=self.git_repo_dir, )
def mock_phabricator(mock_config): ''' Mock phabricator authentication process ''' def _response(name): path = os.path.join(MOCK_DIR, 'phabricator_{}.json'.format(name)) assert os.path.exists(path) return open(path).read() responses.add( responses.POST, 'http://phabricator.test/api/user.whoami', body=_response('auth'), content_type='application/json', ) responses.add( responses.POST, 'http://phabricator.test/api/differential.diff.search', body=_response('diff_search'), content_type='application/json', ) responses.add( responses.POST, 'http://phabricator.test/api/differential.revision.search', body=_response('revision_search'), content_type='application/json', ) responses.add( responses.POST, 'http://phabricator.test/api/differential.query', body=_response('diff_query'), content_type='application/json', ) responses.add( responses.POST, 'http://phabricator.test/api/differential.getrawdiff', body=_response('diff_raw'), content_type='application/json', ) responses.add( responses.POST, 'http://phabricator.test/api/differential.createinline', body=_response('createinline'), content_type='application/json', ) responses.add( responses.POST, 'http://phabricator.test/api/edge.search', body=_response('edge_search'), content_type='application/json', ) responses.add( responses.POST, 'http://phabricator.test/api/transaction.search', body=_response('transaction_search'), content_type='application/json', ) responses.add( responses.POST, 'http://phabricator.test/api/harbormaster.target.search', body=_response('target_search'), content_type='application/json', ) responses.add( responses.POST, 'http://phabricator.test/api/harbormaster.build.search', body=_response('build_search'), content_type='application/json', ) responses.add( responses.POST, 'http://phabricator.test/api/harbormaster.buildable.search', body=_response('buildable_search'), content_type='application/json', ) responses.add( responses.POST, 'http://phabricator.test/api/harbormaster.sendmessage', body=_response('send_message'), content_type='application/json', ) yield PhabricatorAPI( url='http://phabricator.test/api/', api_key='deadbeef', )
def main(): args = parse_cli() taskcluster.auth(args.taskcluster_client_id, args.taskcluster_access_token) taskcluster.load_secrets( name=args.taskcluster_secret, project_name=config.PROJECT_NAME, required=("APP_CHANNEL", "REPORTERS", "PHABRICATOR", "ALLOWED_PATHS"), existing={ "APP_CHANNEL": "development", "REPORTERS": [], "PUBLICATION": "IN_PATCH", "ZERO_COVERAGE_ENABLED": True, "ALLOWED_PATHS": ["*"], }, ) init_logger( config.PROJECT_NAME, PAPERTRAIL_HOST=taskcluster.secrets.get("PAPERTRAIL_HOST"), PAPERTRAIL_PORT=taskcluster.secrets.get("PAPERTRAIL_PORT"), SENTRY_DSN=taskcluster.secrets.get("SENTRY_DSN"), ) # Setup settings before stats settings.setup( taskcluster.secrets["APP_CHANNEL"], taskcluster.secrets["PUBLICATION"], taskcluster.secrets["ALLOWED_PATHS"], ) # Setup statistics datadog_api_key = taskcluster.secrets.get("DATADOG_API_KEY") if datadog_api_key: stats.auth(datadog_api_key) # Load reporters reporters = get_reporters(taskcluster.secrets["REPORTERS"]) # Load index service index_service = taskcluster.get_service("index") # Load queue service queue_service = taskcluster.get_service("queue") # Load Phabricator API phabricator = taskcluster.secrets["PHABRICATOR"] phabricator_reporting_enabled = "phabricator" in reporters phabricator_api = PhabricatorAPI(phabricator["api_key"], phabricator["url"]) if phabricator_reporting_enabled: reporters["phabricator"].setup_api(phabricator_api) # Load unique revision revision = Revision( phabricator_api, try_task=queue_service.task(settings.try_task_id), # Update build status only when phabricator reporting is enabled update_build=phabricator_reporting_enabled, ) # Run workflow according to source w = Workflow( reporters, index_service, queue_service, phabricator_api, taskcluster.secrets["ZERO_COVERAGE_ENABLED"], ) try: w.run(revision) except Exception as e: # Log errors to papertrail logger.error("Static analysis failure", revision=revision, error=e) # Index analysis state extras = {} if isinstance(e, AnalysisException): extras["error_code"] = e.code extras["error_message"] = str(e) w.index(revision, state="error", **extras) # Update Harbormaster status revision.update_status(state=BuildState.Fail) # Then raise to mark task as erroneous raise
def build_phabricator_api(name: str, url: str, token: str): assert url.endswith("/api/"), f"{name} Url {url} does not end with /api/" assert token is not None, f"Missing {name} Phabricator token" assert len( token) == 32, f"{name} Phabricator token must be 32 characters long" return PhabricatorAPI(token, url)
def apply_phab(self, hg, diff_id): def has_revision(revision): if not revision: return False try: hg.identify(revision) return True except hglib.error.CommandError: return False phabricator_api = PhabricatorAPI( api_key=get_secret("PHABRICATOR_TOKEN"), url=get_secret("PHABRICATOR_URL")) # Get the stack of patches stack = phabricator_api.load_patches_stack(diff_id) assert len(stack) > 0, "No patches to apply" # Find the first unknown base revision needed_stack = [] revisions = {} for patch in reversed(stack): needed_stack.insert(0, patch) # Stop as soon as a base revision is available if has_revision(patch.base_revision): logger.info( f"Stopping at diff {patch.id} and revision {patch.base_revision}" ) break if not needed_stack: logger.info("All the patches are already applied") return # Load all the diff revisions diffs = phabricator_api.search_diffs(diff_phid=[p.phid for p in stack]) revisions = { diff["phid"]: phabricator_api.load_revision(rev_phid=diff["revisionPHID"]) for diff in diffs } # Update repo to base revision hg_base = needed_stack[0].base_revision if hg_base: hg.update(rev=hg_base, clean=True) logger.info(f"Updated repo to {hg_base}") for patch in needed_stack: revision = revisions[patch.phid] if patch.commits: message = patch.commits[0]["message"] else: message = revision["fields"]["title"] logger.info( f"Applying {patch.phid} from revision {revision['id']}: {message}" ) hg.import_( patches=io.BytesIO(patch.patch.encode("utf-8")), message=message, user="******", )
def apply_phab(self, hg, diff_id): def has_revision(revision): if not revision: return False try: hg.identify(revision) return True except hglib.error.CommandError: return False phabricator_api = PhabricatorAPI( api_key=get_secret("PHABRICATOR_TOKEN"), url=get_secret("PHABRICATOR_URL")) # Get the stack of patches stack = phabricator_api.load_patches_stack(diff_id) assert len(stack) > 0, "No patches to apply" # Find the first unknown base revision needed_stack = [] revisions = {} for patch in reversed(stack): needed_stack.insert(0, patch) # Stop as soon as a base revision is available if has_revision(patch.base_revision): logger.info( f"Stopping at diff {patch.id} and revision {patch.base_revision}" ) break if not needed_stack: logger.info("All the patches are already applied") return # Load all the diff revisions diffs = phabricator_api.search_diffs(diff_phid=[p.phid for p in stack]) revisions = { diff["phid"]: phabricator_api.load_revision(rev_phid=diff["revisionPHID"], attachments={"reviewers": True}) for diff in diffs } # Update repo to base revision hg_base = needed_stack[0].base_revision if not has_revision(hg_base): logger.warning( "Missing base revision {} from Phabricator".format(hg_base)) hg_base = "tip" if hg_base: hg.update(rev=hg_base, clean=True) logger.info(f"Updated repo to {hg_base}") try: self.git_base = vcs_map.mercurial_to_git(hg_base) subprocess.run( [ "git", "checkout", "-b", "analysis_branch", self.git_base ], check=True, cwd=self.git_repo_dir, ) logger.info(f"Updated git repo to {self.git_base}") except Exception as e: logger.info( f"Updating git repo to Mercurial {hg_base} failed: {e}") def load_user(phid): if phid.startswith("PHID-USER"): return phabricator_api.load_user(user_phid=phid) elif phid.startswith("PHID-PROJ"): # TODO: Support group reviewers somehow. logger.info(f"Skipping group reviewer {phid}") else: raise Exception(f"Unsupported reviewer {phid}") for patch in needed_stack: revision = revisions[patch.phid] message = "{}\n\n{}".format(revision["fields"]["title"], revision["fields"]["summary"]) author_name = None author_email = None if patch.commits: author_name = patch.commits[0]["author"]["name"] author_email = patch.commits[0]["author"]["email"] if author_name is None: author = load_user(revision["fields"]["authorPHID"]) author_name = author["fields"]["realName"] # XXX: Figure out a way to know the email address of the author. author_email = author["fields"]["username"] reviewers = list( filter( None, (load_user(reviewer["reviewerPHID"]) for reviewer in revision["attachments"]["reviewers"]["reviewers"]), )) reviewers = set(reviewer["fields"]["username"] for reviewer in reviewers) if len(reviewers): message = replace_reviewers(message, reviewers) logger.info( f"Applying {patch.phid} from revision {revision['id']}: {message}" ) hg.import_( patches=io.BytesIO(patch.patch.encode("utf-8")), message=message.encode("utf-8"), user=f"{author_name} <{author_email}>".encode("utf-8"), ) with tempfile.TemporaryDirectory() as tmpdirname: temp_file = os.path.join(tmpdirname, "temp.patch") with open(temp_file, "w") as f: f.write(patch.patch) subprocess.run( ["git", "apply", "--3way", temp_file], check=True, cwd=self.git_repo_dir, ) subprocess.run( [ "git", "-c", f"user.name={author_name}", "-c", f"user.email={author_email}", "commit", "-am", message, ], check=True, cwd=self.git_repo_dir, )
def PhabricatorMock(): ''' Mock phabricator authentication process ''' json_headers = { 'Content-Type': 'application/json', } def _response(name): path = os.path.join(MOCK_DIR, 'phabricator', '{}.json'.format(name)) assert os.path.exists(path), 'Missing mock {}'.format(path) return open(path).read() def _phab_params(request): # What a weird way to send parameters return json.loads(urllib.parse.parse_qs(request.body)['params'][0]) def _diff_search(request): params = _phab_params(request) assert 'constraints' in params if 'revisionPHIDs' in params['constraints']: # Search from revision mock_name = 'search-{}'.format(params['constraints']['revisionPHIDs'][0]) elif 'phids' in params['constraints']: # Search from diffs diffs = '-'.join(params['constraints']['phids']) mock_name = 'search-{}'.format(diffs) else: raise Exception('Unsupported diff mock {}'.format(params)) return (200, json_headers, _response(mock_name)) def _diff_raw(request): params = _phab_params(request) assert 'diffID' in params return (200, json_headers, _response('raw-{}'.format(params['diffID']))) def _edges(request): params = _phab_params(request) assert 'sourcePHIDs' in params return (200, json_headers, _response('edges-{}'.format(params['sourcePHIDs'][0]))) def _create_artifact(request): params = _phab_params(request) assert 'buildTargetPHID' in params return (200, json_headers, _response('artifact-{}'.format(params['buildTargetPHID']))) def _send_message(request): params = _phab_params(request) assert 'buildTargetPHID' in params name = 'message-{}-{}'.format(params['buildTargetPHID'], params['type']) if params['unit']: name += '-unit' if params['lint']: name += '-lint' return (200, json_headers, _response(name)) with responses.RequestsMock(assert_all_requests_are_fired=False) as resp: resp.add( responses.POST, 'http://phabricator.test/api/user.whoami', body=_response('auth'), content_type='application/json', ) resp.add_callback( responses.POST, 'http://phabricator.test/api/edge.search', callback=_edges, ) resp.add_callback( responses.POST, 'http://phabricator.test/api/differential.diff.search', callback=_diff_search, ) resp.add_callback( responses.POST, 'http://phabricator.test/api/differential.getrawdiff', callback=_diff_raw, ) resp.add_callback( responses.POST, 'http://phabricator.test/api/harbormaster.createartifact', callback=_create_artifact, ) resp.add_callback( responses.POST, 'http://phabricator.test/api/harbormaster.sendmessage', callback=_send_message, ) resp.add( responses.POST, 'http://phabricator.test/api/diffusion.repository.search', body=_response('repositories'), content_type='application/json', ) api = PhabricatorAPI( url='http://phabricator.test/api/', api_key='deadbeef', ) api.mocks = resp # used to assert in tests on callbacks yield api
def apply_phab(self, hg, phabricator_deployment, diff_id): if phabricator_deployment == PHAB_PROD: api_key = get_secret("PHABRICATOR_TOKEN") url = get_secret("PHABRICATOR_URL") else: api_key = get_secret("PHABRICATOR_DEV_TOKEN") url = get_secret("PHABRICATOR_DEV_URL") phabricator_api = PhabricatorAPI(api_key=api_key, url=url) # Get the stack of patches stack = phabricator_api.load_patches_stack(diff_id) assert len(stack) > 0, "No patches to apply" # Find the first unknown base revision needed_stack = [] revisions = {} for patch in reversed(stack): needed_stack.insert(0, patch) # Stop as soon as a base revision is available if self.has_revision(hg, patch.base_revision): logger.info( f"Stopping at diff {patch.id} and revision {patch.base_revision}" ) break if not needed_stack: logger.info("All the patches are already applied") return # Load all the diff revisions diffs = phabricator_api.search_diffs(diff_phid=[p.phid for p in stack]) revisions = { diff["phid"]: phabricator_api.load_revision(rev_phid=diff["revisionPHID"], attachments={"reviewers": True}) for diff in diffs } # Update repo to base revision hg_base = needed_stack[0].base_revision if not self.has_revision(hg, hg_base): logger.warning( "Missing base revision {} from Phabricator".format(hg_base)) hg_base = "tip" if hg_base: hg.update(rev=hg_base, clean=True) logger.info(f"Updated repo to {hg_base}") if self.git_repo_dir and hg_base != "tip": try: self.git_base = tuple( vcs_map.mercurial_to_git(self.git_repo_dir, [hg_base]))[0] subprocess.run( [ "git", "checkout", "-b", "analysis_branch", self.git_base ], check=True, cwd=self.git_repo_dir, ) logger.info(f"Updated git repo to {self.git_base}") except Exception as e: logger.info( f"Updating git repo to Mercurial {hg_base} failed: {e}" ) def load_user(phid): if phid.startswith("PHID-USER"): return phabricator_api.load_user(user_phid=phid) elif phid.startswith("PHID-PROJ"): # TODO: Support group reviewers somehow. logger.info(f"Skipping group reviewer {phid}") else: raise Exception(f"Unsupported reviewer {phid}") for patch in needed_stack: revision = revisions[patch.phid] message = "{}\n\n{}".format(revision["fields"]["title"], revision["fields"]["summary"]) author_name = None author_email = None if patch.commits: author_name = patch.commits[0]["author"]["name"] author_email = patch.commits[0]["author"]["email"] if author_name is None: author = load_user(revision["fields"]["authorPHID"]) author_name = author["fields"]["realName"] # XXX: Figure out a way to know the email address of the author. author_email = author["fields"]["username"] reviewers = list( filter( None, (load_user(reviewer["reviewerPHID"]) for reviewer in revision["attachments"]["reviewers"]["reviewers"]), )) reviewers = set(reviewer["fields"]["username"] for reviewer in reviewers) if len(reviewers): message = replace_reviewers(message, reviewers) logger.info( f"Applying {patch.phid} from revision {revision['id']}: {message}" ) hg.import_( patches=io.BytesIO(patch.patch.encode("utf-8")), message=message.encode("utf-8"), user=f"{author_name} <{author_email}>".encode("utf-8"), ) if self.git_repo_dir: patch_proc = subprocess.Popen( ["patch", "-p1", "--no-backup-if-mismatch", "--force"], stdin=subprocess.PIPE, cwd=self.git_repo_dir, ) patch_proc.communicate(patch.patch.encode("utf-8")) assert patch_proc.returncode == 0, "Failed to apply patch" subprocess.run( [ "git", "-c", f"user.name={author_name}", "-c", f"user.email={author_email}", "commit", "-am", message, ], check=True, cwd=self.git_repo_dir, )
def mock_phabricator(mock_config): """ Mock phabricator authentication process """ def _response(name): path = os.path.join(MOCK_DIR, "phabricator_{}.json".format(name)) assert os.path.exists(path) return open(path).read() responses.add( responses.POST, "http://phabricator.test/api/user.whoami", body=_response("auth"), content_type="application/json", ) responses.add( responses.POST, "http://phabricator.test/api/differential.diff.search", body=_response("diff_search"), content_type="application/json", ) responses.add( responses.POST, "http://phabricator.test/api/differential.revision.search", body=_response("revision_search"), content_type="application/json", ) responses.add( responses.POST, "http://phabricator.test/api/differential.query", body=_response("diff_query"), content_type="application/json", ) responses.add( responses.POST, "http://phabricator.test/api/differential.getrawdiff", body=_response("diff_raw"), content_type="application/json", ) responses.add( responses.POST, "http://phabricator.test/api/differential.createinline", body=_response("createinline"), content_type="application/json", ) responses.add( responses.POST, "http://phabricator.test/api/edge.search", body=_response("edge_search"), content_type="application/json", ) responses.add( responses.POST, "http://phabricator.test/api/transaction.search", body=_response("transaction_search"), content_type="application/json", ) responses.add( responses.POST, "http://phabricator.test/api/harbormaster.target.search", body=_response("target_search"), content_type="application/json", ) responses.add( responses.POST, "http://phabricator.test/api/harbormaster.build.search", body=_response("build_search"), content_type="application/json", ) responses.add( responses.POST, "http://phabricator.test/api/harbormaster.buildable.search", body=_response("buildable_search"), content_type="application/json", ) responses.add( responses.POST, "http://phabricator.test/api/harbormaster.sendmessage", body=_response("send_message"), content_type="application/json", ) yield PhabricatorAPI(url="http://phabricator.test/api/", api_key="deadbeef")
def mock_phabricator(mock_config): """ Mock phabricator authentication process """ def _response(name): path = os.path.join(MOCK_DIR, "phabricator_{}.json".format(name)) assert os.path.exists(path) return open(path).read() def diff_search(request): payload = dict(urllib.parse.parse_qsl(request.body)) assert "params" in payload params = json.loads(payload["params"]) name = ["diff_search"] for values in params.get("constraints", {}).values(): name += values content = _response("_".join(name)) return (200, {"Content-Type": "application/json"}, content) responses.add( responses.POST, "http://phabricator.test/api/user.whoami", body=_response("auth"), content_type="application/json", ) responses.add_callback( responses.POST, "http://phabricator.test/api/differential.diff.search", callback=diff_search, ) responses.add( responses.POST, "http://phabricator.test/api/differential.revision.search", body=_response("revision_search"), content_type="application/json", ) responses.add( responses.POST, "http://phabricator.test/api/differential.query", body=_response("diff_query"), content_type="application/json", ) with open(os.path.join(os.path.join(MOCK_DIR, "phabricator_patch.diff"))) as f: test_patch = f.read() responses.add( responses.POST, "http://phabricator.test/api/differential.getrawdiff", body=json.dumps({ "error_code": None, "error_info": None, "result": test_patch }), content_type="application/json", ) responses.add( responses.POST, "http://phabricator.test/api/differential.createinline", body=_response("createinline"), content_type="application/json", ) responses.add( responses.POST, "http://phabricator.test/api/edge.search", body=_response("edge_search"), content_type="application/json", ) responses.add( responses.POST, "http://phabricator.test/api/transaction.search", body=_response("transaction_search"), content_type="application/json", ) responses.add( responses.POST, "http://phabricator.test/api/harbormaster.target.search", body=_response("target_search"), content_type="application/json", ) responses.add( responses.POST, "http://phabricator.test/api/harbormaster.build.search", body=_response("build_search"), content_type="application/json", ) responses.add( responses.POST, "http://phabricator.test/api/harbormaster.buildable.search", body=_response("buildable_search"), content_type="application/json", ) responses.add( responses.POST, "http://phabricator.test/api/harbormaster.sendmessage", body=_response("send_message"), content_type="application/json", ) responses.add( responses.POST, "http://phabricator.test/api/diffusion.repository.search", body=_response("repository_search"), content_type="application/json", ) yield PhabricatorAPI(url="http://phabricator.test/api/", api_key="deadbeef")
def main( taskcluster_secret, taskcluster_client_id, taskcluster_access_token, ): secrets = get_secrets( taskcluster_secret, config.PROJECT_NAME, required=( 'APP_CHANNEL', 'REPORTERS', 'PHABRICATOR', 'ALLOWED_PATHS', ), existing={ 'APP_CHANNEL': 'development', 'REPORTERS': [], 'PUBLICATION': 'IN_PATCH', 'ZERO_COVERAGE_ENABLED': True, 'ALLOWED_PATHS': [ '*', ], }, taskcluster_client_id=taskcluster_client_id, taskcluster_access_token=taskcluster_access_token, ) init_logger( config.PROJECT_NAME, PAPERTRAIL_HOST=secrets.get('PAPERTRAIL_HOST'), PAPERTRAIL_PORT=secrets.get('PAPERTRAIL_PORT'), SENTRY_DSN=secrets.get('SENTRY_DSN'), MOZDEF=secrets.get('MOZDEF'), timestamp=True, ) # Setup settings before stats phabricator = secrets['PHABRICATOR'] settings.setup( secrets['APP_CHANNEL'], secrets['PUBLICATION'], secrets['ALLOWED_PATHS'], ) # Setup statistics datadog_api_key = secrets.get('DATADOG_API_KEY') if datadog_api_key: stats.auth(datadog_api_key) # Load reporters reporters = get_reporters( secrets['REPORTERS'], taskcluster_client_id, taskcluster_access_token, ) # Load index service index_service = get_service( 'index', taskcluster_client_id, taskcluster_access_token, ) # Load queue service queue_service = get_service( 'queue', taskcluster_client_id, taskcluster_access_token, ) # Load Phabricator API phabricator_reporting_enabled = 'phabricator' in reporters phabricator_api = PhabricatorAPI(phabricator['api_key'], phabricator['url']) if phabricator_reporting_enabled: reporters['phabricator'].setup_api(phabricator_api) # Load unique revision revision = Revision( phabricator_api, try_task=queue_service.task(settings.try_task_id), # Update build status only when phabricator reporting is enabled update_build=phabricator_reporting_enabled, ) # Run workflow according to source w = Workflow(reporters, index_service, queue_service, phabricator_api, secrets['ZERO_COVERAGE_ENABLED']) try: w.run(revision) except Exception as e: # Log errors to papertrail logger.error( 'Static analysis failure', revision=revision, error=e, ) # Index analysis state extras = {} if isinstance(e, AnalysisException): extras['error_code'] = e.code extras['error_message'] = str(e) w.index(revision, state='error', **extras) # Update Harbormaster status revision.update_status(state=BuildState.Fail) # Then raise to mark task as erroneous raise