class Phab:
def __init__(self, config):
self.config = config
self.phab = Phabricator(host=config.phabricator_url,
token=config.phabricator_token)
self.phab.connect()
self.phab.update_interfaces()
self.instance_string = config.phabricator_revision_url
def notify_harbourmaster(self, key, success):
key_number = re.findall(r"D(\d+)", key)
differntial = self.phab.differential.query(ids=key_number)
self.phab.harbormaster.sendmessage(
buildTargetPHID=differntial[0]["phid"], type=success)
def apply_patch(self, id):
try:
subprocess.check_output(["arc", "patch", "--nobranch", id],
cwd=self.config.repository_path)
except subprocess.CalledProcessError as e:
raise Exception("git phab application failed: " + e.output)
def get_id_from_commit_message(self, msg):
revision_numbers = re.findall(r"" + self.instance_string + "/D(\d+)",
msg)
if len(revision_numbers) == 0:
raise Exception(
"Revision ID cannot be fetched from commit message")
return "D" + revision_numbers[-1]
def init():
global phab, tag_map
phab = Phabricator(host=os.environ['PHABRICATOR_API_URL'],
token=os.environ.get('PHABRICATOR_API_TOKEN'))
phab.update_interfaces()
tag_map = task_tagger.resolve_tags(phab)
def setup_phab(projectphid):
phab = Phabricator()
phab.update_interfaces()
print "Fetching tasks from Phab"
tasks = phab.maniphest.query(projectPHIDs=[projectphid],
status="status-open",
limit=3000)
return (phab, tasks)
def main(args):
parser = argparse.ArgumentParser(prog=args[0], add_help=True,
usage='%(prog)s [script] [reports...]',
description=__doc__)
parser.add_argument('--no-report', action='store_true',
help='Do not report test results to phabricator')
parser.add_argument('script', type=str, help='script to execute')
parser.add_argument('reports', type=str, nargs='*',
help='list of test result files')
parsed = parser.parse_args(args[1:])
token = os.environ.get("TEAMCITY_CONDUIT_TOKEN", None)
if not token and not parsed.no_report:
print("Please provide a conduit token in the environment variable ""TEAMCITY_CONDUIT_TOKEN""")
sys.exit(1)
arcconfig = get_arcconfig()
phabricatorUrl = urlparse.urljoin(arcconfig['conduit_uri'], "api/")
buildUrl = os.environ.get('BUILD_URL', '')
failures, exitcode = execute_and_parse(parsed.script)
for file in parsed.reports:
# All inputs may not exist if the build fails prematurely
if not os.path.isfile(file):
continue
elif file.endswith(".xml"):
failures.update(get_failures(file))
build_status = "success"
if len(failures) != 0:
build_status = "failure"
if len(failures) != 0 and exitcode == 0:
exitcode = 1
if parsed.no_report:
print("Build terminated with {}".format(build_status))
sys.exit(exitcode)
phab = Phabricator(host=phabricatorUrl, token=token)
phab.update_interfaces()
revisionID = get_revision(phab, get_commit_message())
authorPHID = get_author(phab, revisionID)
if build_status != "success":
task_body = create_task_body(buildUrl, revisionID, failures)
create_task(phab, authorPHID, revisionID, task_body)
create_comment(phab, revisionID, build_status, buildUrl)
sys.exit(exitcode)
def main(args):
if len(args) < 2:
print(
"Please provide a list of junit reports as command line arguments."
)
sys.exit(1)
token = os.getenv("TEAMCITY_CONDUIT_TOKEN", None)
if not token:
print("Please provide a conduit token in the environment variable "
"TEAMCITY_CONDUIT_TOKEN"
"")
sys.exit(1)
arcconfig = get_arcconfig()
phabricatorUrl = urlparse.urljoin(arcconfig['conduit_uri'], "api/")
buildUrl = os.getenv('BUILD_URL', '')
build_status = "success"
failures = {}
for arg in args:
# All inputs may not exist if the build fails prematurely
if not os.path.isfile(arg):
continue
if arg.endswith(".status"):
with open(arg, "r") as f:
build_status = f.read().strip()
elif arg.endswith(".xml"):
failures.update(get_failures(arg))
if len(failures) != 0:
build_status = "failure"
phab = Phabricator(host=phabricatorUrl, token=token)
phab.update_interfaces()
revisionID = get_revision(phab, get_commit_message())
authorPHID = get_author(phab, revisionID)
if build_status != "success":
task_body = create_task_body(buildUrl, revisionID, failures)
create_task(phab, authorPHID, revisionID, task_body)
create_comment(phab, revisionID, build_status, buildUrl)
def main():
args = parse_args()
fmt = '%(name)s - %(levelname)s - %(message)s'
if not args.log_notime:
fmt = '%(asctime)s - ' + fmt
logging.basicConfig(format=fmt, level=logging.INFO)
phab = Phabricator(host=args.phabricator, token=args.phabricator_token)
logger.info('updating phabricator interfaces...')
phab.update_interfaces()
logger.info(f'working with {phab.host}')
interactor = PhabGchat(phab, args.phabricator_hmac, args.gchat_webhook)
ws = TornadoServer(int(args.port), [
tw.url('/post', PhabReciever, dict(pg=interactor, )),
])
ws.run()
def publish_log():
if not env.logger:
return
log_output = env.logger.getvalue()
env.logger.close()
phabricator = Phabricator()
phabricator.update_interfaces()
deployer = phabricator.user.whoami()
info_dict = {
"arguments": " ".join(sys.argv[1:]),
"user": deployer.userName,
"date": datetime.now().strftime('%Y-%m-%d %H:%M'),
}
paste_title = "fab %(arguments)s # %(user)s at %(date)s" % info_dict
paste = phabricator.paste.create(
content=log_output,
title=paste_title,
)
info_dict.update({
"paste_id": paste.id,
"branch": env.branch,
"branch_repo_url": "%(repo_browser_url)shistory/%(branch)s/" % {
"repo_browser_url": env.repo_browser,
"branch": env.branch,
}
})
msg = "P%(paste_id)d `fab %(arguments)s` # @%(user)s at %(date)s " \
"from branch [%(branch)s](%(branch_repo_url)s) " % info_dict
phabricator.conpherence.updatethread(
id=env.log_compherence,
message=msg,
)
def main(args):
if len(args) < 2:
print("Usage: {} <script> [<output> ...]".format(args[0]))
print("")
print(" <script>: Script to run and parse output from.")
print(" <output>: Test result file for parsing.")
sys.exit(1)
token = os.environ.get("TEAMCITY_CONDUIT_TOKEN", None)
if not token:
print("Please provide a conduit token in the environment variable "
"TEAMCITY_CONDUIT_TOKEN"
"")
sys.exit(1)
arcconfig = get_arcconfig()
phabricatorUrl = urlparse.urljoin(arcconfig['conduit_uri'], "api/")
buildUrl = os.environ.get('BUILD_URL', '')
failures = execute_and_parse(args[1])
for file in args[2:]:
# All inputs may not exist if the build fails prematurely
if not os.path.isfile(arg):
continue
elif arg.endswith(".xml"):
failures.update(get_failures(arg))
build_status = "success"
if len(failures) != 0:
build_status = "failure"
phab = Phabricator(host=phabricatorUrl, token=token)
phab.update_interfaces()
revisionID = get_revision(phab, get_commit_message())
authorPHID = get_author(phab, revisionID)
if build_status != "success":
task_body = create_task_body(buildUrl, revisionID, failures)
create_task(phab, authorPHID, revisionID, task_body)
create_comment(phab, revisionID, build_status, buildUrl)
def get_phabricator(payload):
api_uri = get_conduit_uri(payload)
token = get_token(payload)
phab = Phabricator(host=api_uri, token=token)
phab.update_interfaces()
return phab
class TrelloImporter:
# Map from Trello username to phabricator user.
userMapPhab01 = {
'gtisza': 'Tgr',
"legoktm": "legoktm",
"matthewflaschen": "mattflaschen",
"pauginer": None,
"spage1": "spage",
}
userMapPhabWMF = {
# from mingleterminator.py
'fflorin': 'Fabrice_Florin',
'gdubuc': 'Gilles',
'mholmquist': 'MarkTraceur',
'gtisza': 'Tgr',
'pginer': 'Pginer-WMF',
# From ack username trabulous_dir/flow-current-iteration_lOh4XCy7_fm.json \
# | perl -pe 's/^\s+//' | sort | uniq
# Collaboration-Team members: Eloquence, DannyH, Pginer-WMF, Spage, Quiddity, Mattflaschen, matthiasmullie, EBernhardson
"alexandermonk": None,
"antoinemusso": None,
"benmq": None,
"bsitu": None,
"dannyhorn1": "DannyH",
"erikbernhardson": "EBernhardson",
"jaredmzimmerman": None,
"jonrobson1": None,
"kaityhammerstein": None,
"legoktm": None,
"matthewflaschen": "mattflaschen",
"matthiasmullie": "matthiasmullie",
"maygalloway": None,
"moizsyed_": None,
"oliverkeyes": None,
"pauginer": "Pginer-WMF",
"quiddity1": "Quiddity",
"shahyarghobadpour": None,
"spage1": "Spage",
"wctaiwan": None,
"werdnum": None,
}
def __init__(self, jsonName, args):
self.jsonName = jsonName
self.args = args # FIXME unpack all the args into member variables?
self.verbose = args.verbose
if config.phab_host.find('phab-01') != -1:
self.host = 'phab-01'
elif config.phab_host.find('phabricator.wikimedia.org') != -1:
self.host = 'phabricator'
else:
self.json_error('Unrecognized host %s in config' % (config.phab_host))
sys.exit(3)
self.board = TrelloDAO(self.jsonName)
trelloBoardName = self.board.get_board_name();
vlog('Trello board = %s' % (trelloBoardName))
def connect_to_phab(self):
self.phab = Phabricator(config.phab_user,
config.phab_cert,
config.phab_host)
self.phab.update_interfaces()
self.phabm = phabmacros('', '', '')
self.phabm.con = self.phab
# DEBUG to verify the API connection worked: print phab.user.whoami()
vlog('Looks like we connected to the phab API \o/')
def sanity_check(self):
if not 'cards' in self.trelloJson:
self.json_error('No cards in input file')
sys.exit(1)
def testify(self, str):
if self.args.test_run:
str = "TEST Trello_create RUN: " + str
return str
def json_error(self, str):
elog('ERROR: %s in input file %s' % (str, self.jsonName))
# Determine projectPHID for the project name in which this will create tasks.
def get_projectPHID(self, phabProjectName):
# Similar conduit code in trello_makePHIDs.py get_trelloUserPHIDs
response = self.phab.project.query(names = [phabProjectName])
for projInfo in response.data.values():
if projInfo["name"] == phabProjectName:
vlog('Phabricator project %s has PHID %s' % (phabProjectName, projInfo["phid"] ) )
return projInfo["phid"]
elog('Phabricator project %s not found' % (phabProjectName))
sys.exit(4)
return # not reached
# This is the workhorse
def createTask(self, card):
# Default some keys we always pass to createtask.
taskinfo = {
'ownerPHID' : None,
'ccPHIDs' : [],
'projectPHIDs' : [self.projectPHID],
}
taskinfo["title"] = self.testify(card.name)
# TODO: if Trello board is using scrum for Trello browser extension,
# could extract story points /\s+\((\d+)\)' from card title to feed into Sprint extension.
# TODO: process attachments
# TODO: taskinfo["assignee"]
desc = self.testify(card.desc)
if card.checklist_strs:
desc += '\n' + '\n\n'.join(card.checklist_strs)
desc_tail = '\n--------------------------'
desc_tail += '\n**Trello card**: [[ %s | %s ]]\n' % (card.url, card.shortLink)
# Mention column the same way as the card.final_comment_fields below from export_trello.py.
desc_tail += '\n * column: %s\n' % (unicode(card.column))
if len(card.final_comment_fields) > 0:
s = ''
s += '\n'
for key in sorted(card.final_comment_fields):
s += ' * %s: %s\n' % (str(key), unicode(card.final_comment_fields[key]))
desc_tail += s
# TODO: could add additional info (main attachment, etc.) to desc_tail.
taskinfo["description"] = desc + '\n' + desc_tail
# TODO: chasemp: what priority?
taskinfo['priority'] = 50
# TODO: chasemp: can I put "Trello lOh4XCy7" in "Reference" field?
# Take the set of members
idMembers = card.idMembers
# Get the Trello username for the idMember
# memberNames = [ TrelloDAO.get_username(id) for id in idMembers if TrelloDAO.get_username(id)]
# export_trello.py sets names it can't match to 'import-john-doe'
if not 'FAILED' in card.owner and not card.owner == 'import-john-doe':
taskinfo['ownerPHID'] = card.owner
taskinfo['ccPHIDs'] = [u for u in card.subscribers if not 'FAILED' in u and not u == 'import-john-doe']
# TODO: Add any other members with a PHID to the ccPHIDs
# TODO: Note remaining Trello members in desc_tail
# TODO: bugzilla_create.py and wmfphablib/phabapi.py use axuiliary for
# BZ ref, but it doesn't work for Trello ref?
taskinfo["auxiliary"] = {"std:maniphest:external_reference":"Trello %s" % (card.shortLink)}
if self.args.conduit:
# This prints fields for maniphest.createtask
print '"%s"\n"%s"\n\n' % (taskinfo["title"].encode('unicode-escape'),
taskinfo["description"].encode('unicode-escape'))
else:
if self.args.dry_run:
log("dry-run to create a task for Trello card %s ('%s')" %
(card.shortLink, taskinfo["title"]))
else:
ticket = self.phab.maniphest.createtask(
title = taskinfo['title'],
description = taskinfo['description'],
projectPHIDs = taskinfo['projectPHIDs'],
ownerPHID = taskinfo['ownerPHID'],
ccPHIDs = taskinfo['ccPHIDs'],
auxiliary = taskinfo['auxiliary']
)
log("Created task: T%s (%s) from Trello card %s ('%s')" %
(ticket['id'], ticket['phid'], card.shortLink, taskinfo["title"]))
# Here bugzilla_create goes on to log actual creating user and view/edit policy,
# then set_task_ctime to creation_time.
# Should I add comments to the card here,
# or a separate step that goes through action in self.board.blob["actions"]
# handling type="commentCard"?
# Here are the types of objects in the "actions" array.
# 20 "type": "addAttachmentToCard",
# 9 "type": "addChecklistToCard",
# 2 "type": "addMemberToBoard",
# 38 "type": "addMemberToCard",
# 69 "type": "commentCard",
# 1 "type": "copyCard",
# 25 "type": "createCard",
# 3 "type": "createList",
# 6 "type": "deleteAttachmentFromCard",
# 29 "type": "moveCardFromBoard",
# 18 "type": "moveCardToBoard",
# 4 "type": "moveListFromBoard",
# 2 "type": "moveListToBoard",
# 3 "type": "removeChecklistFromCard",
# 14 "type": "removeMemberFromCard",
# 3 "type": "updateBoard",
# 698 "type": "updateCard",
# 48 "type": "updateCheckItemStateOnCard",
# 8 "type": "updateList",
# def getCardCreationMeta(self, cardId):
# Look around in JSON for ["actions"] array for member with type:"createCard"
# with ["card"]["id"] = cardId
# and use the siblings ["date"] and ["memberCreator"]["id"]
# def getCardComments(self, cardId):
# Look around in JSON ["actions"] for member with type:""commentCard"
# with ["card"]["id"] = cardId
# and use the siblings ["date"] and ["memberCreator"]["id"]
def process_cards(self):
self.connect_to_phab()
self.projectPHID = self.get_projectPHID(self.args.phab_project);
# This file has Trello_username: user_PHID mapping created by trello_makePHIDs.py.
scrubber = TrelloScrubber('conf/trello-scrub.yaml')
for j_card in self.board.blob["cards"]:
card = TrelloCard(j_card, scrubber)
card.figure_stuff_out(self.board)
if self.args.column and not card.column == self.args.column:
continue
# Skip archived cards ("closed" seems to correspond?)
# But I think archive all cards in column doesn't set this.
if card.closed:
continue
# TODO: skip cards that are bugs
# TODO: skip cards that already exist in Phab.
self.createTask(card)
class PhabEventListener(object):
ignore_list = [
"added inline comments to D",
"added a comment to D",
"added a reviewer for D",
"added reviewers for D",
"removed a reviewer for D",
"removed reviewers for D",
"requested review of D",
"requested changes to D",
"added a subscriber to D",
"added a project to D",
"edited reviewers for D",
"updated the summary of D", # Maybe useful to upstream info?
"accepted D", # Maybe useful to upstream info?
"retitled D", # Maybe useful to upstream info?
"blocking reviewer(s) for D",
"planned changes to D",
"updated subscribers of D",
"resigned from D",
"changed the edit policy for D",
"removed a project from D",
"updated D",
"changed the visibility for D",
"updated the test plan for D"
]
event_mapping = {
"updated the diff for D": "commit",
"created D": "commit",
"closed D": "closed",
"abandoned D": "abandoned",
"added a reverting change for D": None, # Not sure what this is yet
"reopened D": "commit", # This may need its own event type
}
def __init__(self, config):
self.running = True
self.timer_in_seconds = config['phabricator']['listener']['interval']
self.latest = None
self.phab = Phabricator(
host='https://phabricator.services.mozilla.com/api/',
token=config['phabricator']['token'])
self.phab.update_interfaces()
def run(self):
# Run until told to stop.
while self.running:
feed = self.get_feed()
self.parse(feed)
time.sleep(self.timer_in_seconds)
@newrelic.agent.background_task(name='feed-fetching', group='Phabricator')
def get_feed(self, before=None):
""" """
if self.latest and before is None:
before = int(self.latest['chronologicalKey'])
feed = []
def chrono_key(feed_story_tuple):
return int(feed_story_tuple[1]["chronologicalKey"])
# keep fetching stories from Phabricator until there are no more stories to fetch
while True:
result = self.phab.feed.query(before=before, view='text')
if result.response:
results = sorted(result.response.items(), key=chrono_key)
results = map(self.map_feed_tuple, results)
feed.extend(results)
if len(results) == 100 and before is not None:
# There may be more events we wish to fetch
before = int(results[-1]["chronologicalKey"])
continue
break
return feed
@newrelic.agent.background_task(name='feed-parsing', group='Phabricator')
def parse(self, feed):
# Go through rows in reverse order, and ignore first row as it has the table headers
for event in feed:
if RE_COMMIT.search(event['text']):
# This is a commit event, ignore it
continue
# Split the text to get the part that describes the event type
event_text = RE_EVENT.split(event['text'])[0]
# Check if this is an event we wish to ignore
if any(event_type in event_text
for event_type in PhabEventListener.ignore_list):
continue
# Map the event text to an event type so we know how to handle it
event['type'] = self.map_event_type(event_text, event)
if event['type'] is None:
continue
# Add the event to the queue, and set this as the latest parsed
handle.apply_async(("phabricator", event))
self.latest = event
@staticmethod
def map_event_type(event_text, event):
# Could use compiled regex expression instead
for event_type, mapping in PhabEventListener.event_mapping.items():
if event_type in event_text:
return mapping
logger.warning("Unknown phabricator event type: %s" % event_text)
newrelic.agent.record_custom_event(
"unknown_phabricator_event",
params={
"event_text": event_text,
"event": event,
},
application=newrelic.agent.application())
@staticmethod
def map_feed_tuple(feed_tuple):
story_phid, feed_story = feed_tuple
feed_story.update({"storyPHID": story_phid})
return feed_story
def init_phab_connection():
phab = Phabricator()
phab.update_interfaces()
return phab
def versions_from_targets(phab, targets):
builds = get_builds_from_targets(phab, targets)
buildables = get_buildables_from_builds(phab, builds)
diffs = get_diffs_from_buildables(phab, buildables)
return [
Version(target.id, target.phid, diff.id, diff.branch, diff.base)
for target, diff in zip(targets, diffs)
]
def get_new_versions(phab, payload, step=None):
last_version = get_version_from_payload(payload)
new_targets = get_targets_since(phab, last_version.target)
if step:
filtered_targets = [
target for target in new_targets if target.buildStepPHID == step
]
else:
filtered_targets = new_targets
return versions_from_targets(phab, filtered_targets)
if __name__ == "__main__":
payload = json.loads(input())
source = Source(payload)
phab = Phabricator(host=source.conduit_uri, token=source.conduit_token)
phab.update_interfaces()
new_versions = get_new_versions(phab, payload, source.buildStepPHID)
print(json.dumps(versions_to_json(new_versions)))
def request_reviews(self, host, token, user_names=None, age=None, show_last_comment=None,
**kwargs):
"""
Returns revision requests for specified username and repo name.
If user_names are not provided then requests pulls all open
revisions
Args:
host (str): The host Phabricator server url
token (str): Phabricator token for authentication
(Looks like 'api-1234567890x')
user_names (lst(str)): Phabricator user names we want to query
age (Age): Contains the filter state for pull requests,
e.g, older or newer and date
show_last_comment (int): Show text of last comment and
filter out pull requests in which
last comments are newer than
specified number of days
Returns:
response (list): Returns list of list of pull requests for
specified username and reponame or all reponame
for given username
Note:
We will use the list 'raw_response' to keep track of the
raw queries we make to phabricator. We do this to keep
API calls minimal, and first search through raw_response
before making another API call.
"""
# Create Phabricator object with token
phab = Phabricator(host=urljoin(host, "/api/"), token=token)
phab.update_interfaces()
# Create response list
response = []
# Create raw response list to keep track of users we've come across
raw_response = []
if user_names:
# Find open reviews for all users (aka the list user_names)
# Generate a list of user phids based on their username
# Also begin keeping track of queried users in raw_response
user_phids, raw_response = self.generate_phids(user_names, phab)
# Query phabricator based on all users passed
reviews = self.differential_query(status='status-open',
responsibleUsers=user_phids,
phab=phab)
else:
# Find all open reviews
reviews = self.differential_query(status='status-open',
responsibleUsers=[], phab=phab)
# Format and go through all reviews for a user
res = self.get_reviews(phab=phab, reviews=reviews,
raw_response=raw_response,
host=host,
age=age,
show_last_comment=show_last_comment)
# extend in case of non-empty results
# If we've come across a revision that's dated < duration
# (i.e. too far in the past) we would have skipped it and
# returned nothing
if res:
response.extend(res)
return response
def init_phab_connection():
phab = Phabricator()
phab.update_interfaces()
return phab
def create_phab(self):
phab = Phabricator(token=self.conduit_token, host=self.host)
phab.update_interfaces()
return phab
class PhabTalk:
"""Talk to Phabricator to upload build results.
See https://secure.phabricator.com/conduit/method/harbormaster.sendmessage/
"""
def __init__(self, token: Optional[str], host: Optional[str] = 'https://reviews.llvm.org/api/',
dryrun: bool = False):
self._phab = None # type: Optional[Phabricator]
if not dryrun:
self._phab = Phabricator(token=token, host=host)
self.update_interfaces()
@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
def update_interfaces(self):
self._phab.update_interfaces()
@property
def dryrun(self):
return self._phab is None
@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
def get_revision_id(self, diff: str) -> Optional[str]:
"""Get the revision ID for a diff from Phabricator."""
if self.dryrun:
return None
result = self._phab.differential.querydiffs(ids=[diff])
return 'D' + result[diff]['revisionID']
def comment_on_diff(self, diff_id: str, text: str):
"""Add a comment to a differential based on the diff_id"""
print('Sending comment to diff {}:'.format(diff_id))
print(text)
revision_id = self.get_revision_id(diff_id)
if revision_id is not None:
self._comment_on_revision(revision_id, text)
@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
def _comment_on_revision(self, revision: str, text: str):
"""Add comment on a differential based on the revision id."""
transactions = [{
'type': 'comment',
'value': text
}]
if self.dryrun:
print('differential.revision.edit =================')
print('Transactions: {}'.format(transactions))
return
# API details at
# https://secure.phabricator.com/conduit/method/differential.revision.edit/
self._phab.differential.revision.edit(objectIdentifier=revision,
transactions=transactions)
print('Uploaded comment to Revision D{}:{}'.format(revision, text))
@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
def update_build_status(self, phid: str, working: bool, success: bool, lint: {}, unit: []):
"""Submit collected report to Phabricator.
"""
result_type = 'pass'
if working:
result_type = 'working'
elif not success:
result_type = 'fail'
# Group lint messages by file and line.
lint_messages = []
for v in lint.values():
path = ''
line = 0
descriptions = []
for e in v:
path = e['path']
line = e['line']
descriptions.append('{}: {}'.format(e['name'], e['description']))
lint_message = {
'name': 'Pre-merge checks',
'severity': 'warning',
'code': 'llvm-premerge-checks',
'path': path,
'line': line,
'description': '\n'.join(descriptions),
}
lint_messages.append(lint_message)
if self.dryrun:
print('harbormaster.sendmessage =================')
print('type: {}'.format(result_type))
print('unit: {}'.format(unit))
print('lint: {}'.format(lint_messages))
return
self._phab.harbormaster.sendmessage(
buildTargetPHID=phid,
type=result_type,
unit=unit,
lint=lint_messages)
print('Uploaded build status {}, {} test results and {} lint results'.format(
result_type, len(unit), len(lint_messages)))
@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
def create_artifact(self, phid, artifact_key, artifact_type, artifact_data):
if self.dryrun:
print('harbormaster.createartifact =================')
print('artifactKey: {}'.format(artifact_key))
print('artifactType: {}'.format(artifact_type))
print('artifactData: {}'.format(artifact_data))
return
self._phab.harbormaster.createartifact(
buildTargetPHID=phid,
artifactKey=artifact_key,
artifactType=artifact_type,
artifactData=artifact_data)
def maybe_add_url_artifact(self, phid: str, url: str, name: str):
if phid is None:
logging.warning('PHID is not provided, cannot create URL artifact')
return
self.create_artifact(phid, str(uuid.uuid4()), 'uri', {'uri': url, 'ui.external': True, 'name': name})
def setup_phab(projectphid): phab = Phabricator() phab.update_interfaces() print "Fetching tasks from Phab" tasks = phab.maniphest.query(projectPHIDs=[projectphid], status="status-open", limit=3000) return(phab, tasks)
class ApplyPatch:
def __init__(self, comment_file_path: str, git_hash: str):
# TODO: turn os.environ parameter into command line arguments
# this would be much clearer and easier for testing
self.comment_file_path = comment_file_path
self.conduit_token = os.environ.get('CONDUIT_TOKEN') # type: Optional[str]
self.host = os.environ.get('PHABRICATOR_HOST') # type: Optional[str]
self._load_arcrc()
self.diff_id = os.environ['DIFF_ID'] # type: str
self.diff_json_path = os.environ['DIFF_JSON'] # type: str
if not self.host.endswith('/api/'):
self.host += '/api/'
self.phab = Phabricator(token=self.conduit_token, host=self.host)
self.git_hash = git_hash # type: Optional[str]
self.msg = [] # type: List[str]
def _load_arcrc(self):
"""Load arc configuration from file if not set."""
if self.conduit_token is not None or self.host is not None:
return
print('Loading configuration from ~/.arcrc file')
with open(os.path.expanduser('~/.arcrc'), 'r') as arcrc_file:
arcrc = json.load(arcrc_file)
# use the first host configured in the file
self.host = next(iter(arcrc['hosts']))
self.conduit_token = arcrc['hosts'][self.host]['token']
@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
def update_interfaces(self):
self.phab.update_interfaces()
def run(self):
"""try to apply the patch from phabricator
"""
self.update_interfaces()
try:
if self.git_hash is None:
self._get_parent_hash()
else:
print('Use provided commit {}'.format(self.git_hash))
self._git_checkout()
self._apply_patch()
finally:
self._write_error_message()
def _get_parent_hash(self):
diff = self.phab.differential.getdiff(diff_id=self.diff_id)
# Keep a copy of the Phabricator answer for later usage in a json file
try:
with open(self.diff_json_path, 'w') as json_file:
json.dump(diff.response, json_file, sort_keys=True, indent=4)
print('Wrote diff details to "{}".'.format(self.diff_json_path))
except Exception:
print('WARNING: could not write build/diff.json log file')
self.git_hash = diff['sourceControlBaseRevision']
@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
def _git_checkout(self):
try:
print('Checking out git hash {}'.format(self.git_hash))
subprocess.check_call('git reset --hard {}'.format(self.git_hash),
stdout=sys.stdout, stderr=sys.stderr, shell=True)
except subprocess.CalledProcessError:
print('WARNING: checkout of hash failed, using master branch instead.')
self.msg += [
'Could not check out parent git hash "{}". It was not found in '
'the repository. Did you configure the "Parent Revision" in '
'Phabricator properly? Trying to apply the patch to the '
'master branch instead...'.format(self.git_hash)]
subprocess.check_call('git checkout master', stdout=sys.stdout,
stderr=sys.stderr, shell=True)
subprocess.check_call('git show -s', stdout=sys.stdout,
stderr=sys.stderr, shell=True)
print('git checkout completed.')
@backoff.on_exception(backoff.expo, Exception, max_tries=5, logger='', factor=3)
def _apply_patch(self):
print('running arc patch...')
cmd = 'arc patch --force --nobranch --no-ansi --diff "{}" --nocommit ' \
'--conduit-token "{}" --conduit-uri "{}"'.format(
self.diff_id, self.conduit_token, self.host)
result = subprocess.run(cmd, capture_output=True, shell=True, text=True)
print(result.stdout + result.stderr)
if result.returncode != 0:
msg = (
'ERROR: arc patch failed with error code {}. '
'Check build log for details.'.format(result.returncode))
self.msg += [msg]
raise Exception(msg)
print('Patching completed.')
def _write_error_message(self):
"""Write the log message to a file."""
if self.comment_file_path is None:
return
if len(self.msg) == 0:
return
print('writing error message to {}'.format(self.comment_file_path))
with open(self.comment_file_path, 'a') as comment_file:
text = '\n\n'.join(self.msg)
comment_file.write(text)
#!/usr/bin/env python
import sys
from phabricator import Phabricator
if len(sys.argv) == 1:
print "Syntax: %s ProjectName" % sys.argv[0]
sys.exit(1)
project_name = sys.argv[1]
phab = Phabricator()
phab.update_interfaces()
proj = phab.project
def phid_from_name(name):
all_projects = proj.query()
for phid in all_projects:
if all_projects[phid]['name'] == name:
return phid
projectPHID = phid_from_name(project_name)
tasks = phab.maniphest.find(
projectPHIDs=[projectPHID],
)
print "Project %s" % project_name
class PhabTalk:
"""Talk to Phabricator to upload build results.
See https://secure.phabricator.com/conduit/method/harbormaster.sendmessage/
"""
def __init__(self, token: Optional[str], host: Optional[str],
dryrun: bool):
self._phab = None # type: Optional[Phabricator]
if not dryrun:
self._phab = Phabricator(token=token, host=host)
self._phab.update_interfaces()
@property
def dryrun(self):
return self._phab is None
def get_revision_id(self, diff: str) -> Optional[str]:
"""Get the revision ID for a diff from Phabricator."""
if self.dryrun:
return None
result = self._phab.differential.querydiffs(ids=[diff])
return 'D' + result[diff]['revisionID']
def _comment_on_diff(self, diff_id: str, text: str):
"""Add a comment to a differential based on the diff_id"""
print('Sending comment to diff {}:'.format(diff_id))
print(text)
revision_id = self.get_revision_id(diff_id)
if revision_id is not None:
self._comment_on_revision(revision_id, text)
def _comment_on_revision(self, revision: str, text: str):
"""Add comment on a differential based on the revision id."""
transactions = [{'type': 'comment', 'value': text}]
if self.dryrun:
print('differential.revision.edit =================')
print('Transactions: {}'.format(transactions))
return
# API details at
# https://secure.phabricator.com/conduit/method/differential.revision.edit/
self._phab.differential.revision.edit(objectIdentifier=revision,
transactions=transactions)
def submit_report(self, diff_id: str, phid: str, report: BuildReport,
build_result: str):
"""Submit collected report to Phabricator.
"""
result_type = 'pass'
if report.working:
result_type = 'working'
elif not report.success:
result_type = 'fail'
# Group lint messages by file and line.
lint_messages = []
for v in report.lint.values():
path = ''
line = 0
descriptions = []
for e in v:
path = e['path']
line = e['line']
descriptions.append('{}: {}'.format(e['name'],
e['description']))
lint_message = {
'name': 'Pre-merge checks',
'severity': 'warning',
'code': 'llvm-premerge-checks',
'path': path,
'line': line,
'description': '\n'.join(descriptions),
}
lint_messages.append(lint_message)
if self.dryrun:
print('harbormaster.sendmessage =================')
print('type: {}'.format(result_type))
print('unit: {}'.format(report.unit))
print('lint: {}'.format(lint_messages))
else:
_try_call(lambda: self._phab.harbormaster.sendmessage(
buildTargetPHID=phid,
type=result_type,
unit=report.unit,
lint=lint_messages))
if len(report.comments) > 0:
_try_call(lambda: self._comment_on_diff(
diff_id, '\n\n'.join(report.comments)))
def _create_phab(self) -> Phabricator:
"""Create Phabricator API instance and update it."""
phab = Phabricator(token=self.conduit_token, host=self.host)
# TODO: retry on communication error
phab.update_interfaces()
return phab
def _create_phab(self):
phab = Phabricator(token=self.conduit_token, host=self.host)
try_call(lambda: phab.update_interfaces())
return phab
class Phidify:
def __init__(self, args):
self.args = args # FIXME unpack all the args into member variables?
if config.phab_host.find('phab-01') != -1:
self.host = 'phab-01'
elif config.phab_host.find('phabricator.wikimedia.org') != -1:
self.host = 'phabricator'
else:
self.json_error('Unrecognized host %s in config' % (config.phab_host))
sys.exit(1)
self.userMap = 'data/trello_names_' + self.host + '.yaml'
with open(self.userMap) as f:
yam = yaml.load(f)
self.trelloUserMap = yam['trelloUserMap']
vlog('Looks like we loaded self.trelloUserMap OK \o/')
def connect_to_phab(self):
self.phab = Phabricator(config.phab_user,
config.phab_cert,
config.phab_host)
self.phab.update_interfaces()
# DEBUG to verify the API connection worked: print phab.user.whoami()
vlog('Looks like we connected to the phab API \o/')
# Returns dict mapping Trello usernames to phab PHIDs
def get_trelloUserPHIDs(self, trelloUserMap):
self.connect_to_phab()
# trelloUserMap maps trello usernames to phabricator usernames, e.g.
# {'spage1': 'spage', 'Tisza': 'Tgr'}
# We can query to get the PHID of the phab username
# {'spage': 'PHID-USER-rwvw2dwvvjiyrlzy4iho', 'Tisza': 'PHID-USER-66kvyuekkkwkqbpze2uk'}
# we want to return {'spage1' : 'PHID-USER-rwvw2dwvvjiyrlzy4iho'}
# Get list of unique Phabricator usernames to query, excluding 'None' and empty
lookupNames = [u for u in set(trelloUserMap.values()) if u]
vlog('get_trelloUserPHIDs querying Phab for %d usernames: %s ' % (len(lookupNames), lookupNames))
response = self.phab.user.query(usernames = lookupNames)
# That conduit query returns an array of users' data, each containing phabUsername and phid.
# Turn it into a dict.
phabUserPHIDMap = {}
for userInfo in response:
if userInfo["userName"] and userInfo["phid"]:
phabUserPHIDMap[userInfo["userName"]] = userInfo["phid"]
# Now create what we want, a dict of trelloUsername -> phid
trelloUserPHIDMap = {}
for trelloUsername, phabUsername in trelloUserMap.iteritems():
if (phabUsername
and phabUsername in phabUserPHIDMap
and phabUserPHIDMap[phabUsername]
and phabUserPHIDMap[phabUsername] != 'None'): # XXX fires?
trelloUserPHIDMap[trelloUsername] = phabUserPHIDMap[phabUsername]
return trelloUserPHIDMap
def writeUserPHIDs(self):
trelloUserPHIDMap = self.get_trelloUserPHIDs(self.trelloUserMap)
fname = 'conf/trello-scrub_' + self.host + '.yaml'
if os.path.exists(fname):
elog('ERROR: ' + fname + ' already exists')
sys.exit(2)
stream = file(fname, 'w')
trelloScrub = {
'uid-map': trelloUserPHIDMap,
'uid-cheatsheet': {}
}
# safe_dump() avoids gtisza: !!python/unicode 'PHID-USER-66kvyuekkkwkqbpze2uk'
yaml.safe_dump( trelloScrub, stream, width=30)
log('SUCCESS: wrote trello usernames->PHIDs to ' + fname)
