class Phab: def __init__(self, config): self.config = config self.phab = Phabricator(host=config.phabricator_url, token=config.phabricator_token) self.phab.connect() self.phab.update_interfaces() self.instance_string = config.phabricator_revision_url def notify_harbourmaster(self, key, success): key_number = re.findall(r"D(\d+)", key) differntial = self.phab.differential.query(ids=key_number) self.phab.harbormaster.sendmessage( buildTargetPHID=differntial[0]["phid"], type=success) def apply_patch(self, id): try: subprocess.check_output(["arc", "patch", "--nobranch", id], cwd=self.config.repository_path) except subprocess.CalledProcessError as e: raise Exception("git phab application failed: " + e.output) def get_id_from_commit_message(self, msg): revision_numbers = re.findall(r"" + self.instance_string + "/D(\d+)", msg) if len(revision_numbers) == 0: raise Exception( "Revision ID cannot be fetched from commit message") return "D" + revision_numbers[-1]
def init(): global phab, tag_map phab = Phabricator(host=os.environ['PHABRICATOR_API_URL'], token=os.environ.get('PHABRICATOR_API_TOKEN')) phab.update_interfaces() tag_map = task_tagger.resolve_tags(phab)
def setup_phab(projectphid): phab = Phabricator() phab.update_interfaces() print "Fetching tasks from Phab" tasks = phab.maniphest.query(projectPHIDs=[projectphid], status="status-open", limit=3000) return (phab, tasks)
def main(args): parser = argparse.ArgumentParser(prog=args[0], add_help=True, usage='%(prog)s [script] [reports...]', description=__doc__) parser.add_argument('--no-report', action='store_true', help='Do not report test results to phabricator') parser.add_argument('script', type=str, help='script to execute') parser.add_argument('reports', type=str, nargs='*', help='list of test result files') parsed = parser.parse_args(args[1:]) token = os.environ.get("TEAMCITY_CONDUIT_TOKEN", None) if not token and not parsed.no_report: print("Please provide a conduit token in the environment variable ""TEAMCITY_CONDUIT_TOKEN""") sys.exit(1) arcconfig = get_arcconfig() phabricatorUrl = urlparse.urljoin(arcconfig['conduit_uri'], "api/") buildUrl = os.environ.get('BUILD_URL', '') failures, exitcode = execute_and_parse(parsed.script) for file in parsed.reports: # All inputs may not exist if the build fails prematurely if not os.path.isfile(file): continue elif file.endswith(".xml"): failures.update(get_failures(file)) build_status = "success" if len(failures) != 0: build_status = "failure" if len(failures) != 0 and exitcode == 0: exitcode = 1 if parsed.no_report: print("Build terminated with {}".format(build_status)) sys.exit(exitcode) phab = Phabricator(host=phabricatorUrl, token=token) phab.update_interfaces() revisionID = get_revision(phab, get_commit_message()) authorPHID = get_author(phab, revisionID) if build_status != "success": task_body = create_task_body(buildUrl, revisionID, failures) create_task(phab, authorPHID, revisionID, task_body) create_comment(phab, revisionID, build_status, buildUrl) sys.exit(exitcode)
def main(args): if len(args) < 2: print( "Please provide a list of junit reports as command line arguments." ) sys.exit(1) token = os.getenv("TEAMCITY_CONDUIT_TOKEN", None) if not token: print("Please provide a conduit token in the environment variable " "TEAMCITY_CONDUIT_TOKEN" "") sys.exit(1) arcconfig = get_arcconfig() phabricatorUrl = urlparse.urljoin(arcconfig['conduit_uri'], "api/") buildUrl = os.getenv('BUILD_URL', '') build_status = "success" failures = {} for arg in args: # All inputs may not exist if the build fails prematurely if not os.path.isfile(arg): continue if arg.endswith(".status"): with open(arg, "r") as f: build_status = f.read().strip() elif arg.endswith(".xml"): failures.update(get_failures(arg)) if len(failures) != 0: build_status = "failure" phab = Phabricator(host=phabricatorUrl, token=token) phab.update_interfaces() revisionID = get_revision(phab, get_commit_message()) authorPHID = get_author(phab, revisionID) if build_status != "success": task_body = create_task_body(buildUrl, revisionID, failures) create_task(phab, authorPHID, revisionID, task_body) create_comment(phab, revisionID, build_status, buildUrl)
def main(): args = parse_args() fmt = '%(name)s - %(levelname)s - %(message)s' if not args.log_notime: fmt = '%(asctime)s - ' + fmt logging.basicConfig(format=fmt, level=logging.INFO) phab = Phabricator(host=args.phabricator, token=args.phabricator_token) logger.info('updating phabricator interfaces...') phab.update_interfaces() logger.info(f'working with {phab.host}') interactor = PhabGchat(phab, args.phabricator_hmac, args.gchat_webhook) ws = TornadoServer(int(args.port), [ tw.url('/post', PhabReciever, dict(pg=interactor, )), ]) ws.run()
def publish_log(): if not env.logger: return log_output = env.logger.getvalue() env.logger.close() phabricator = Phabricator() phabricator.update_interfaces() deployer = phabricator.user.whoami() info_dict = { "arguments": " ".join(sys.argv[1:]), "user": deployer.userName, "date": datetime.now().strftime('%Y-%m-%d %H:%M'), } paste_title = "fab %(arguments)s # %(user)s at %(date)s" % info_dict paste = phabricator.paste.create( content=log_output, title=paste_title, ) info_dict.update({ "paste_id": paste.id, "branch": env.branch, "branch_repo_url": "%(repo_browser_url)shistory/%(branch)s/" % { "repo_browser_url": env.repo_browser, "branch": env.branch, } }) msg = "P%(paste_id)d `fab %(arguments)s` # @%(user)s at %(date)s " \ "from branch [%(branch)s](%(branch_repo_url)s) " % info_dict phabricator.conpherence.updatethread( id=env.log_compherence, message=msg, )
def main(args): if len(args) < 2: print("Usage: {} <script> [<output> ...]".format(args[0])) print("") print(" <script>: Script to run and parse output from.") print(" <output>: Test result file for parsing.") sys.exit(1) token = os.environ.get("TEAMCITY_CONDUIT_TOKEN", None) if not token: print("Please provide a conduit token in the environment variable " "TEAMCITY_CONDUIT_TOKEN" "") sys.exit(1) arcconfig = get_arcconfig() phabricatorUrl = urlparse.urljoin(arcconfig['conduit_uri'], "api/") buildUrl = os.environ.get('BUILD_URL', '') failures = execute_and_parse(args[1]) for file in args[2:]: # All inputs may not exist if the build fails prematurely if not os.path.isfile(arg): continue elif arg.endswith(".xml"): failures.update(get_failures(arg)) build_status = "success" if len(failures) != 0: build_status = "failure" phab = Phabricator(host=phabricatorUrl, token=token) phab.update_interfaces() revisionID = get_revision(phab, get_commit_message()) authorPHID = get_author(phab, revisionID) if build_status != "success": task_body = create_task_body(buildUrl, revisionID, failures) create_task(phab, authorPHID, revisionID, task_body) create_comment(phab, revisionID, build_status, buildUrl)
def get_phabricator(payload): api_uri = get_conduit_uri(payload) token = get_token(payload) phab = Phabricator(host=api_uri, token=token) phab.update_interfaces() return phab
class TrelloImporter: # Map from Trello username to phabricator user. userMapPhab01 = { 'gtisza': 'Tgr', "legoktm": "legoktm", "matthewflaschen": "mattflaschen", "pauginer": None, "spage1": "spage", } userMapPhabWMF = { # from mingleterminator.py 'fflorin': 'Fabrice_Florin', 'gdubuc': 'Gilles', 'mholmquist': 'MarkTraceur', 'gtisza': 'Tgr', 'pginer': 'Pginer-WMF', # From ack username trabulous_dir/flow-current-iteration_lOh4XCy7_fm.json \ # | perl -pe 's/^\s+//' | sort | uniq # Collaboration-Team members: Eloquence, DannyH, Pginer-WMF, Spage, Quiddity, Mattflaschen, matthiasmullie, EBernhardson "alexandermonk": None, "antoinemusso": None, "benmq": None, "bsitu": None, "dannyhorn1": "DannyH", "erikbernhardson": "EBernhardson", "jaredmzimmerman": None, "jonrobson1": None, "kaityhammerstein": None, "legoktm": None, "matthewflaschen": "mattflaschen", "matthiasmullie": "matthiasmullie", "maygalloway": None, "moizsyed_": None, "oliverkeyes": None, "pauginer": "Pginer-WMF", "quiddity1": "Quiddity", "shahyarghobadpour": None, "spage1": "Spage", "wctaiwan": None, "werdnum": None, } def __init__(self, jsonName, args): self.jsonName = jsonName self.args = args # FIXME unpack all the args into member variables? self.verbose = args.verbose if config.phab_host.find('phab-01') != -1: self.host = 'phab-01' elif config.phab_host.find('phabricator.wikimedia.org') != -1: self.host = 'phabricator' else: self.json_error('Unrecognized host %s in config' % (config.phab_host)) sys.exit(3) self.board = TrelloDAO(self.jsonName) trelloBoardName = self.board.get_board_name(); vlog('Trello board = %s' % (trelloBoardName)) def connect_to_phab(self): self.phab = Phabricator(config.phab_user, config.phab_cert, config.phab_host) self.phab.update_interfaces() self.phabm = phabmacros('', '', '') self.phabm.con = self.phab # DEBUG to verify the API connection worked: print phab.user.whoami() vlog('Looks like we connected to the phab API \o/') def sanity_check(self): if not 'cards' in self.trelloJson: self.json_error('No cards in input file') sys.exit(1) def testify(self, str): if self.args.test_run: str = "TEST Trello_create RUN: " + str return str def json_error(self, str): elog('ERROR: %s in input file %s' % (str, self.jsonName)) # Determine projectPHID for the project name in which this will create tasks. def get_projectPHID(self, phabProjectName): # Similar conduit code in trello_makePHIDs.py get_trelloUserPHIDs response = self.phab.project.query(names = [phabProjectName]) for projInfo in response.data.values(): if projInfo["name"] == phabProjectName: vlog('Phabricator project %s has PHID %s' % (phabProjectName, projInfo["phid"] ) ) return projInfo["phid"] elog('Phabricator project %s not found' % (phabProjectName)) sys.exit(4) return # not reached # This is the workhorse def createTask(self, card): # Default some keys we always pass to createtask. taskinfo = { 'ownerPHID' : None, 'ccPHIDs' : [], 'projectPHIDs' : [self.projectPHID], } taskinfo["title"] = self.testify(card.name) # TODO: if Trello board is using scrum for Trello browser extension, # could extract story points /\s+\((\d+)\)' from card title to feed into Sprint extension. # TODO: process attachments # TODO: taskinfo["assignee"] desc = self.testify(card.desc) if card.checklist_strs: desc += '\n' + '\n\n'.join(card.checklist_strs) desc_tail = '\n--------------------------' desc_tail += '\n**Trello card**: [[ %s | %s ]]\n' % (card.url, card.shortLink) # Mention column the same way as the card.final_comment_fields below from export_trello.py. desc_tail += '\n * column: %s\n' % (unicode(card.column)) if len(card.final_comment_fields) > 0: s = '' s += '\n' for key in sorted(card.final_comment_fields): s += ' * %s: %s\n' % (str(key), unicode(card.final_comment_fields[key])) desc_tail += s # TODO: could add additional info (main attachment, etc.) to desc_tail. taskinfo["description"] = desc + '\n' + desc_tail # TODO: chasemp: what priority? taskinfo['priority'] = 50 # TODO: chasemp: can I put "Trello lOh4XCy7" in "Reference" field? # Take the set of members idMembers = card.idMembers # Get the Trello username for the idMember # memberNames = [ TrelloDAO.get_username(id) for id in idMembers if TrelloDAO.get_username(id)] # export_trello.py sets names it can't match to 'import-john-doe' if not 'FAILED' in card.owner and not card.owner == 'import-john-doe': taskinfo['ownerPHID'] = card.owner taskinfo['ccPHIDs'] = [u for u in card.subscribers if not 'FAILED' in u and not u == 'import-john-doe'] # TODO: Add any other members with a PHID to the ccPHIDs # TODO: Note remaining Trello members in desc_tail # TODO: bugzilla_create.py and wmfphablib/phabapi.py use axuiliary for # BZ ref, but it doesn't work for Trello ref? taskinfo["auxiliary"] = {"std:maniphest:external_reference":"Trello %s" % (card.shortLink)} if self.args.conduit: # This prints fields for maniphest.createtask print '"%s"\n"%s"\n\n' % (taskinfo["title"].encode('unicode-escape'), taskinfo["description"].encode('unicode-escape')) else: if self.args.dry_run: log("dry-run to create a task for Trello card %s ('%s')" % (card.shortLink, taskinfo["title"])) else: ticket = self.phab.maniphest.createtask( title = taskinfo['title'], description = taskinfo['description'], projectPHIDs = taskinfo['projectPHIDs'], ownerPHID = taskinfo['ownerPHID'], ccPHIDs = taskinfo['ccPHIDs'], auxiliary = taskinfo['auxiliary'] ) log("Created task: T%s (%s) from Trello card %s ('%s')" % (ticket['id'], ticket['phid'], card.shortLink, taskinfo["title"])) # Here bugzilla_create goes on to log actual creating user and view/edit policy, # then set_task_ctime to creation_time. # Should I add comments to the card here, # or a separate step that goes through action in self.board.blob["actions"] # handling type="commentCard"? # Here are the types of objects in the "actions" array. # 20 "type": "addAttachmentToCard", # 9 "type": "addChecklistToCard", # 2 "type": "addMemberToBoard", # 38 "type": "addMemberToCard", # 69 "type": "commentCard", # 1 "type": "copyCard", # 25 "type": "createCard", # 3 "type": "createList", # 6 "type": "deleteAttachmentFromCard", # 29 "type": "moveCardFromBoard", # 18 "type": "moveCardToBoard", # 4 "type": "moveListFromBoard", # 2 "type": "moveListToBoard", # 3 "type": "removeChecklistFromCard", # 14 "type": "removeMemberFromCard", # 3 "type": "updateBoard", # 698 "type": "updateCard", # 48 "type": "updateCheckItemStateOnCard", # 8 "type": "updateList", # def getCardCreationMeta(self, cardId): # Look around in JSON for ["actions"] array for member with type:"createCard" # with ["card"]["id"] = cardId # and use the siblings ["date"] and ["memberCreator"]["id"] # def getCardComments(self, cardId): # Look around in JSON ["actions"] for member with type:""commentCard" # with ["card"]["id"] = cardId # and use the siblings ["date"] and ["memberCreator"]["id"] def process_cards(self): self.connect_to_phab() self.projectPHID = self.get_projectPHID(self.args.phab_project); # This file has Trello_username: user_PHID mapping created by trello_makePHIDs.py. scrubber = TrelloScrubber('conf/trello-scrub.yaml') for j_card in self.board.blob["cards"]: card = TrelloCard(j_card, scrubber) card.figure_stuff_out(self.board) if self.args.column and not card.column == self.args.column: continue # Skip archived cards ("closed" seems to correspond?) # But I think archive all cards in column doesn't set this. if card.closed: continue # TODO: skip cards that are bugs # TODO: skip cards that already exist in Phab. self.createTask(card)
class PhabEventListener(object): ignore_list = [ "added inline comments to D", "added a comment to D", "added a reviewer for D", "added reviewers for D", "removed a reviewer for D", "removed reviewers for D", "requested review of D", "requested changes to D", "added a subscriber to D", "added a project to D", "edited reviewers for D", "updated the summary of D", # Maybe useful to upstream info? "accepted D", # Maybe useful to upstream info? "retitled D", # Maybe useful to upstream info? "blocking reviewer(s) for D", "planned changes to D", "updated subscribers of D", "resigned from D", "changed the edit policy for D", "removed a project from D", "updated D", "changed the visibility for D", "updated the test plan for D" ] event_mapping = { "updated the diff for D": "commit", "created D": "commit", "closed D": "closed", "abandoned D": "abandoned", "added a reverting change for D": None, # Not sure what this is yet "reopened D": "commit", # This may need its own event type } def __init__(self, config): self.running = True self.timer_in_seconds = config['phabricator']['listener']['interval'] self.latest = None self.phab = Phabricator( host='https://phabricator.services.mozilla.com/api/', token=config['phabricator']['token']) self.phab.update_interfaces() def run(self): # Run until told to stop. while self.running: feed = self.get_feed() self.parse(feed) time.sleep(self.timer_in_seconds) @newrelic.agent.background_task(name='feed-fetching', group='Phabricator') def get_feed(self, before=None): """ """ if self.latest and before is None: before = int(self.latest['chronologicalKey']) feed = [] def chrono_key(feed_story_tuple): return int(feed_story_tuple[1]["chronologicalKey"]) # keep fetching stories from Phabricator until there are no more stories to fetch while True: result = self.phab.feed.query(before=before, view='text') if result.response: results = sorted(result.response.items(), key=chrono_key) results = map(self.map_feed_tuple, results) feed.extend(results) if len(results) == 100 and before is not None: # There may be more events we wish to fetch before = int(results[-1]["chronologicalKey"]) continue break return feed @newrelic.agent.background_task(name='feed-parsing', group='Phabricator') def parse(self, feed): # Go through rows in reverse order, and ignore first row as it has the table headers for event in feed: if RE_COMMIT.search(event['text']): # This is a commit event, ignore it continue # Split the text to get the part that describes the event type event_text = RE_EVENT.split(event['text'])[0] # Check if this is an event we wish to ignore if any(event_type in event_text for event_type in PhabEventListener.ignore_list): continue # Map the event text to an event type so we know how to handle it event['type'] = self.map_event_type(event_text, event) if event['type'] is None: continue # Add the event to the queue, and set this as the latest parsed handle.apply_async(("phabricator", event)) self.latest = event @staticmethod def map_event_type(event_text, event): # Could use compiled regex expression instead for event_type, mapping in PhabEventListener.event_mapping.items(): if event_type in event_text: return mapping logger.warning("Unknown phabricator event type: %s" % event_text) newrelic.agent.record_custom_event( "unknown_phabricator_event", params={ "event_text": event_text, "event": event, }, application=newrelic.agent.application()) @staticmethod def map_feed_tuple(feed_tuple): story_phid, feed_story = feed_tuple feed_story.update({"storyPHID": story_phid}) return feed_story
def init_phab_connection(): phab = Phabricator() phab.update_interfaces() return phab
def versions_from_targets(phab, targets): builds = get_builds_from_targets(phab, targets) buildables = get_buildables_from_builds(phab, builds) diffs = get_diffs_from_buildables(phab, buildables) return [ Version(target.id, target.phid, diff.id, diff.branch, diff.base) for target, diff in zip(targets, diffs) ] def get_new_versions(phab, payload, step=None): last_version = get_version_from_payload(payload) new_targets = get_targets_since(phab, last_version.target) if step: filtered_targets = [ target for target in new_targets if target.buildStepPHID == step ] else: filtered_targets = new_targets return versions_from_targets(phab, filtered_targets) if __name__ == "__main__": payload = json.loads(input()) source = Source(payload) phab = Phabricator(host=source.conduit_uri, token=source.conduit_token) phab.update_interfaces() new_versions = get_new_versions(phab, payload, source.buildStepPHID) print(json.dumps(versions_to_json(new_versions)))
def request_reviews(self, host, token, user_names=None, age=None, show_last_comment=None, **kwargs): """ Returns revision requests for specified username and repo name. If user_names are not provided then requests pulls all open revisions Args: host (str): The host Phabricator server url token (str): Phabricator token for authentication (Looks like 'api-1234567890x') user_names (lst(str)): Phabricator user names we want to query age (Age): Contains the filter state for pull requests, e.g, older or newer and date show_last_comment (int): Show text of last comment and filter out pull requests in which last comments are newer than specified number of days Returns: response (list): Returns list of list of pull requests for specified username and reponame or all reponame for given username Note: We will use the list 'raw_response' to keep track of the raw queries we make to phabricator. We do this to keep API calls minimal, and first search through raw_response before making another API call. """ # Create Phabricator object with token phab = Phabricator(host=urljoin(host, "/api/"), token=token) phab.update_interfaces() # Create response list response = [] # Create raw response list to keep track of users we've come across raw_response = [] if user_names: # Find open reviews for all users (aka the list user_names) # Generate a list of user phids based on their username # Also begin keeping track of queried users in raw_response user_phids, raw_response = self.generate_phids(user_names, phab) # Query phabricator based on all users passed reviews = self.differential_query(status='status-open', responsibleUsers=user_phids, phab=phab) else: # Find all open reviews reviews = self.differential_query(status='status-open', responsibleUsers=[], phab=phab) # Format and go through all reviews for a user res = self.get_reviews(phab=phab, reviews=reviews, raw_response=raw_response, host=host, age=age, show_last_comment=show_last_comment) # extend in case of non-empty results # If we've come across a revision that's dated < duration # (i.e. too far in the past) we would have skipped it and # returned nothing if res: response.extend(res) return response
def init_phab_connection(): phab = Phabricator() phab.update_interfaces() return phab
def create_phab(self): phab = Phabricator(token=self.conduit_token, host=self.host) phab.update_interfaces() return phab
class PhabTalk: """Talk to Phabricator to upload build results. See https://secure.phabricator.com/conduit/method/harbormaster.sendmessage/ """ def __init__(self, token: Optional[str], host: Optional[str] = 'https://reviews.llvm.org/api/', dryrun: bool = False): self._phab = None # type: Optional[Phabricator] if not dryrun: self._phab = Phabricator(token=token, host=host) self.update_interfaces() @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) def update_interfaces(self): self._phab.update_interfaces() @property def dryrun(self): return self._phab is None @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) def get_revision_id(self, diff: str) -> Optional[str]: """Get the revision ID for a diff from Phabricator.""" if self.dryrun: return None result = self._phab.differential.querydiffs(ids=[diff]) return 'D' + result[diff]['revisionID'] def comment_on_diff(self, diff_id: str, text: str): """Add a comment to a differential based on the diff_id""" print('Sending comment to diff {}:'.format(diff_id)) print(text) revision_id = self.get_revision_id(diff_id) if revision_id is not None: self._comment_on_revision(revision_id, text) @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) def _comment_on_revision(self, revision: str, text: str): """Add comment on a differential based on the revision id.""" transactions = [{ 'type': 'comment', 'value': text }] if self.dryrun: print('differential.revision.edit =================') print('Transactions: {}'.format(transactions)) return # API details at # https://secure.phabricator.com/conduit/method/differential.revision.edit/ self._phab.differential.revision.edit(objectIdentifier=revision, transactions=transactions) print('Uploaded comment to Revision D{}:{}'.format(revision, text)) @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) def update_build_status(self, phid: str, working: bool, success: bool, lint: {}, unit: []): """Submit collected report to Phabricator. """ result_type = 'pass' if working: result_type = 'working' elif not success: result_type = 'fail' # Group lint messages by file and line. lint_messages = [] for v in lint.values(): path = '' line = 0 descriptions = [] for e in v: path = e['path'] line = e['line'] descriptions.append('{}: {}'.format(e['name'], e['description'])) lint_message = { 'name': 'Pre-merge checks', 'severity': 'warning', 'code': 'llvm-premerge-checks', 'path': path, 'line': line, 'description': '\n'.join(descriptions), } lint_messages.append(lint_message) if self.dryrun: print('harbormaster.sendmessage =================') print('type: {}'.format(result_type)) print('unit: {}'.format(unit)) print('lint: {}'.format(lint_messages)) return self._phab.harbormaster.sendmessage( buildTargetPHID=phid, type=result_type, unit=unit, lint=lint_messages) print('Uploaded build status {}, {} test results and {} lint results'.format( result_type, len(unit), len(lint_messages))) @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) def create_artifact(self, phid, artifact_key, artifact_type, artifact_data): if self.dryrun: print('harbormaster.createartifact =================') print('artifactKey: {}'.format(artifact_key)) print('artifactType: {}'.format(artifact_type)) print('artifactData: {}'.format(artifact_data)) return self._phab.harbormaster.createartifact( buildTargetPHID=phid, artifactKey=artifact_key, artifactType=artifact_type, artifactData=artifact_data) def maybe_add_url_artifact(self, phid: str, url: str, name: str): if phid is None: logging.warning('PHID is not provided, cannot create URL artifact') return self.create_artifact(phid, str(uuid.uuid4()), 'uri', {'uri': url, 'ui.external': True, 'name': name})
def setup_phab(projectphid): phab = Phabricator() phab.update_interfaces() print "Fetching tasks from Phab" tasks = phab.maniphest.query(projectPHIDs=[projectphid], status="status-open", limit=3000) return(phab, tasks)
class ApplyPatch: def __init__(self, comment_file_path: str, git_hash: str): # TODO: turn os.environ parameter into command line arguments # this would be much clearer and easier for testing self.comment_file_path = comment_file_path self.conduit_token = os.environ.get('CONDUIT_TOKEN') # type: Optional[str] self.host = os.environ.get('PHABRICATOR_HOST') # type: Optional[str] self._load_arcrc() self.diff_id = os.environ['DIFF_ID'] # type: str self.diff_json_path = os.environ['DIFF_JSON'] # type: str if not self.host.endswith('/api/'): self.host += '/api/' self.phab = Phabricator(token=self.conduit_token, host=self.host) self.git_hash = git_hash # type: Optional[str] self.msg = [] # type: List[str] def _load_arcrc(self): """Load arc configuration from file if not set.""" if self.conduit_token is not None or self.host is not None: return print('Loading configuration from ~/.arcrc file') with open(os.path.expanduser('~/.arcrc'), 'r') as arcrc_file: arcrc = json.load(arcrc_file) # use the first host configured in the file self.host = next(iter(arcrc['hosts'])) self.conduit_token = arcrc['hosts'][self.host]['token'] @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) def update_interfaces(self): self.phab.update_interfaces() def run(self): """try to apply the patch from phabricator """ self.update_interfaces() try: if self.git_hash is None: self._get_parent_hash() else: print('Use provided commit {}'.format(self.git_hash)) self._git_checkout() self._apply_patch() finally: self._write_error_message() def _get_parent_hash(self): diff = self.phab.differential.getdiff(diff_id=self.diff_id) # Keep a copy of the Phabricator answer for later usage in a json file try: with open(self.diff_json_path, 'w') as json_file: json.dump(diff.response, json_file, sort_keys=True, indent=4) print('Wrote diff details to "{}".'.format(self.diff_json_path)) except Exception: print('WARNING: could not write build/diff.json log file') self.git_hash = diff['sourceControlBaseRevision'] @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) def _git_checkout(self): try: print('Checking out git hash {}'.format(self.git_hash)) subprocess.check_call('git reset --hard {}'.format(self.git_hash), stdout=sys.stdout, stderr=sys.stderr, shell=True) except subprocess.CalledProcessError: print('WARNING: checkout of hash failed, using master branch instead.') self.msg += [ 'Could not check out parent git hash "{}". It was not found in ' 'the repository. Did you configure the "Parent Revision" in ' 'Phabricator properly? Trying to apply the patch to the ' 'master branch instead...'.format(self.git_hash)] subprocess.check_call('git checkout master', stdout=sys.stdout, stderr=sys.stderr, shell=True) subprocess.check_call('git show -s', stdout=sys.stdout, stderr=sys.stderr, shell=True) print('git checkout completed.') @backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3) def _apply_patch(self): print('running arc patch...') cmd = 'arc patch --force --nobranch --no-ansi --diff "{}" --nocommit ' \ '--conduit-token "{}" --conduit-uri "{}"'.format( self.diff_id, self.conduit_token, self.host) result = subprocess.run(cmd, capture_output=True, shell=True, text=True) print(result.stdout + result.stderr) if result.returncode != 0: msg = ( 'ERROR: arc patch failed with error code {}. ' 'Check build log for details.'.format(result.returncode)) self.msg += [msg] raise Exception(msg) print('Patching completed.') def _write_error_message(self): """Write the log message to a file.""" if self.comment_file_path is None: return if len(self.msg) == 0: return print('writing error message to {}'.format(self.comment_file_path)) with open(self.comment_file_path, 'a') as comment_file: text = '\n\n'.join(self.msg) comment_file.write(text)
#!/usr/bin/env python import sys from phabricator import Phabricator if len(sys.argv) == 1: print "Syntax: %s ProjectName" % sys.argv[0] sys.exit(1) project_name = sys.argv[1] phab = Phabricator() phab.update_interfaces() proj = phab.project def phid_from_name(name): all_projects = proj.query() for phid in all_projects: if all_projects[phid]['name'] == name: return phid projectPHID = phid_from_name(project_name) tasks = phab.maniphest.find( projectPHIDs=[projectPHID], ) print "Project %s" % project_name
class PhabTalk: """Talk to Phabricator to upload build results. See https://secure.phabricator.com/conduit/method/harbormaster.sendmessage/ """ def __init__(self, token: Optional[str], host: Optional[str], dryrun: bool): self._phab = None # type: Optional[Phabricator] if not dryrun: self._phab = Phabricator(token=token, host=host) self._phab.update_interfaces() @property def dryrun(self): return self._phab is None def get_revision_id(self, diff: str) -> Optional[str]: """Get the revision ID for a diff from Phabricator.""" if self.dryrun: return None result = self._phab.differential.querydiffs(ids=[diff]) return 'D' + result[diff]['revisionID'] def _comment_on_diff(self, diff_id: str, text: str): """Add a comment to a differential based on the diff_id""" print('Sending comment to diff {}:'.format(diff_id)) print(text) revision_id = self.get_revision_id(diff_id) if revision_id is not None: self._comment_on_revision(revision_id, text) def _comment_on_revision(self, revision: str, text: str): """Add comment on a differential based on the revision id.""" transactions = [{'type': 'comment', 'value': text}] if self.dryrun: print('differential.revision.edit =================') print('Transactions: {}'.format(transactions)) return # API details at # https://secure.phabricator.com/conduit/method/differential.revision.edit/ self._phab.differential.revision.edit(objectIdentifier=revision, transactions=transactions) def submit_report(self, diff_id: str, phid: str, report: BuildReport, build_result: str): """Submit collected report to Phabricator. """ result_type = 'pass' if report.working: result_type = 'working' elif not report.success: result_type = 'fail' # Group lint messages by file and line. lint_messages = [] for v in report.lint.values(): path = '' line = 0 descriptions = [] for e in v: path = e['path'] line = e['line'] descriptions.append('{}: {}'.format(e['name'], e['description'])) lint_message = { 'name': 'Pre-merge checks', 'severity': 'warning', 'code': 'llvm-premerge-checks', 'path': path, 'line': line, 'description': '\n'.join(descriptions), } lint_messages.append(lint_message) if self.dryrun: print('harbormaster.sendmessage =================') print('type: {}'.format(result_type)) print('unit: {}'.format(report.unit)) print('lint: {}'.format(lint_messages)) else: _try_call(lambda: self._phab.harbormaster.sendmessage( buildTargetPHID=phid, type=result_type, unit=report.unit, lint=lint_messages)) if len(report.comments) > 0: _try_call(lambda: self._comment_on_diff( diff_id, '\n\n'.join(report.comments)))
def _create_phab(self) -> Phabricator: """Create Phabricator API instance and update it.""" phab = Phabricator(token=self.conduit_token, host=self.host) # TODO: retry on communication error phab.update_interfaces() return phab
def _create_phab(self): phab = Phabricator(token=self.conduit_token, host=self.host) try_call(lambda: phab.update_interfaces()) return phab
class Phidify: def __init__(self, args): self.args = args # FIXME unpack all the args into member variables? if config.phab_host.find('phab-01') != -1: self.host = 'phab-01' elif config.phab_host.find('phabricator.wikimedia.org') != -1: self.host = 'phabricator' else: self.json_error('Unrecognized host %s in config' % (config.phab_host)) sys.exit(1) self.userMap = 'data/trello_names_' + self.host + '.yaml' with open(self.userMap) as f: yam = yaml.load(f) self.trelloUserMap = yam['trelloUserMap'] vlog('Looks like we loaded self.trelloUserMap OK \o/') def connect_to_phab(self): self.phab = Phabricator(config.phab_user, config.phab_cert, config.phab_host) self.phab.update_interfaces() # DEBUG to verify the API connection worked: print phab.user.whoami() vlog('Looks like we connected to the phab API \o/') # Returns dict mapping Trello usernames to phab PHIDs def get_trelloUserPHIDs(self, trelloUserMap): self.connect_to_phab() # trelloUserMap maps trello usernames to phabricator usernames, e.g. # {'spage1': 'spage', 'Tisza': 'Tgr'} # We can query to get the PHID of the phab username # {'spage': 'PHID-USER-rwvw2dwvvjiyrlzy4iho', 'Tisza': 'PHID-USER-66kvyuekkkwkqbpze2uk'} # we want to return {'spage1' : 'PHID-USER-rwvw2dwvvjiyrlzy4iho'} # Get list of unique Phabricator usernames to query, excluding 'None' and empty lookupNames = [u for u in set(trelloUserMap.values()) if u] vlog('get_trelloUserPHIDs querying Phab for %d usernames: %s ' % (len(lookupNames), lookupNames)) response = self.phab.user.query(usernames = lookupNames) # That conduit query returns an array of users' data, each containing phabUsername and phid. # Turn it into a dict. phabUserPHIDMap = {} for userInfo in response: if userInfo["userName"] and userInfo["phid"]: phabUserPHIDMap[userInfo["userName"]] = userInfo["phid"] # Now create what we want, a dict of trelloUsername -> phid trelloUserPHIDMap = {} for trelloUsername, phabUsername in trelloUserMap.iteritems(): if (phabUsername and phabUsername in phabUserPHIDMap and phabUserPHIDMap[phabUsername] and phabUserPHIDMap[phabUsername] != 'None'): # XXX fires? trelloUserPHIDMap[trelloUsername] = phabUserPHIDMap[phabUsername] return trelloUserPHIDMap def writeUserPHIDs(self): trelloUserPHIDMap = self.get_trelloUserPHIDs(self.trelloUserMap) fname = 'conf/trello-scrub_' + self.host + '.yaml' if os.path.exists(fname): elog('ERROR: ' + fname + ' already exists') sys.exit(2) stream = file(fname, 'w') trelloScrub = { 'uid-map': trelloUserPHIDMap, 'uid-cheatsheet': {} } # safe_dump() avoids gtisza: !!python/unicode 'PHID-USER-66kvyuekkkwkqbpze2uk' yaml.safe_dump( trelloScrub, stream, width=30) log('SUCCESS: wrote trello usernames->PHIDs to ' + fname)