def __init__(self, config):
self.config = config
self.phab = Phabricator(host=config.phabricator_url,
token=config.phabricator_token)
self.phab.connect()
self.phab.update_interfaces()
self.instance_string = config.phabricator_revision_url
class Phab:
def __init__(self, config):
self.config = config
self.phab = Phabricator(host=config.phabricator_url,
token=config.phabricator_token)
self.phab.connect()
self.phab.update_interfaces()
self.instance_string = config.phabricator_revision_url
def notify_harbourmaster(self, key, success):
key_number = re.findall(r"D(\d+)", key)
differntial = self.phab.differential.query(ids=key_number)
self.phab.harbormaster.sendmessage(
buildTargetPHID=differntial[0]["phid"], type=success)
def apply_patch(self, id):
try:
subprocess.check_output(["arc", "patch", "--nobranch", id],
cwd=self.config.repository_path)
except subprocess.CalledProcessError as e:
raise Exception("git phab application failed: " + e.output)
def get_id_from_commit_message(self, msg):
revision_numbers = re.findall(r"" + self.instance_string + "/D(\d+)",
msg)
if len(revision_numbers) == 0:
raise Exception(
"Revision ID cannot be fetched from commit message")
return "D" + revision_numbers[-1]
def __init__(self, config):
self.running = True
self.timer_in_seconds = config['phabricator']['listener']['interval']
self.latest = None
self.phab = Phabricator(host='https://phabricator.services.mozilla.com/api/',
token=config['phabricator']['token'])
self.phab.update_interfaces()
def init():
global phab, tag_map
phab = Phabricator(host=os.environ['PHABRICATOR_API_URL'],
token=os.environ.get('PHABRICATOR_API_TOKEN'))
phab.update_interfaces()
tag_map = task_tagger.resolve_tags(phab)
def setup_phab(projectphid):
phab = Phabricator()
phab.update_interfaces()
print "Fetching tasks from Phab"
tasks = phab.maniphest.query(projectPHIDs=[projectphid],
status="status-open",
limit=3000)
return (phab, tasks)
def __init__(self,
token: Optional[str],
host: Optional[str] = 'https://reviews.llvm.org/api/',
dry_run_updates: bool = False):
self._phab = None # type: Optional[Phabricator]
self.dry_run_updates = dry_run_updates
self._phab = Phabricator(token=token, host=host)
self.update_interfaces()
def main(args):
parser = argparse.ArgumentParser(prog=args[0], add_help=True,
usage='%(prog)s [script] [reports...]',
description=__doc__)
parser.add_argument('--no-report', action='store_true',
help='Do not report test results to phabricator')
parser.add_argument('script', type=str, help='script to execute')
parser.add_argument('reports', type=str, nargs='*',
help='list of test result files')
parsed = parser.parse_args(args[1:])
token = os.environ.get("TEAMCITY_CONDUIT_TOKEN", None)
if not token and not parsed.no_report:
print("Please provide a conduit token in the environment variable ""TEAMCITY_CONDUIT_TOKEN""")
sys.exit(1)
arcconfig = get_arcconfig()
phabricatorUrl = urlparse.urljoin(arcconfig['conduit_uri'], "api/")
buildUrl = os.environ.get('BUILD_URL', '')
failures, exitcode = execute_and_parse(parsed.script)
for file in parsed.reports:
# All inputs may not exist if the build fails prematurely
if not os.path.isfile(file):
continue
elif file.endswith(".xml"):
failures.update(get_failures(file))
build_status = "success"
if len(failures) != 0:
build_status = "failure"
if len(failures) != 0 and exitcode == 0:
exitcode = 1
if parsed.no_report:
print("Build terminated with {}".format(build_status))
sys.exit(exitcode)
phab = Phabricator(host=phabricatorUrl, token=token)
phab.update_interfaces()
revisionID = get_revision(phab, get_commit_message())
authorPHID = get_author(phab, revisionID)
if build_status != "success":
task_body = create_task_body(buildUrl, revisionID, failures)
create_task(phab, authorPHID, revisionID, task_body)
create_comment(phab, revisionID, build_status, buildUrl)
sys.exit(exitcode)
def __init__(self, comment_file_path: str, git_hash: str):
# TODO: turn os.environ parameter into command line arguments
# this would be much clearer and easier for testing
self.comment_file_path = comment_file_path
self.conduit_token = os.environ.get('CONDUIT_TOKEN') # type: Optional[str]
self.host = os.environ.get('PHABRICATOR_HOST') # type: Optional[str]
self._load_arcrc()
self.diff_id = os.environ['DIFF_ID'] # type: str
self.diff_json_path = os.environ['DIFF_JSON'] # type: str
if not self.host.endswith('/api/'):
self.host += '/api/'
self.phab = Phabricator(token=self.conduit_token, host=self.host)
self.git_hash = git_hash # type: Optional[str]
self.msg = [] # type: List[str]
def get_comments(phid, author_phids, exclude_author_phid, days=None):
phab = Phabricator()
constraints = {
'authorPHIDs': author_phids,
}
transactions = phab.transaction.search(
objectIdentifier='D%s' % phid,
constraints=constraints,
)
comments = []
for transaction in transactions.get('data'):
if not transaction.get('comments'):
continue
created = transaction.get('comments')[0].get('dateCreated')
if days and created < get_epoc_days_ago(days):
continue
author_phid = transaction.get('authorPHID')
if author_phid == exclude_author_phid:
continue
comments.append({
'authorPHID':
author_phid,
'comments':
transaction.get('comments')[0].get('content').get('raw'),
})
list.reverse(comments)
return comments
def __init__(self):
# Get super-early debugging by `export PHABFIVE_DEBUG=1`
if "PHABFIVE_DEBUG" in os.environ:
log.setLevel(logging.DEBUG)
log.info("Loglevel is: {}".format(
logging.getLevelName(log.getEffectiveLevel())))
self.conf = self.load_config()
maxlen = 8 + len(max(dict(self.conf).keys(), key=len))
for k, v in dict(self.conf).items():
log.debug("{} {} {}".format(k, "." * (maxlen - len(k)), v))
# check for required configurables
for k, v in dict(self.conf).items():
if k in REQUIRED and not v:
error = "{} is not configured".format(k)
example = CONFIG_EXAMPLES.get(k)
if example:
error += ", " + example
raise PhabfiveConfigException(error)
# check validity of configurables
for k in VALIDATORS.keys():
if not re.match(VALIDATORS[k], self.conf[k]):
error = "{} is malformed".format(k)
example = VALID_EXAMPLES.get(k)
if example:
error += ", " + example
raise PhabfiveConfigException(error)
self.phab = Phabricator(host=self.conf.get("PHAB_URL"),
token=self.conf.get("PHAB_TOKEN"))
self.verify_connection()
def get_task(version):
p = Phabricator()
constraints = {
'query': 'title:"{} deployment blockers"'.format(version)
}
phid = p.maniphest.search(constraints=constraints)['data'][0]['id']
return 'https://phabricator.wikimedia.org/T{}'.format(phid)
def loadConfig():
global PuppetRoot
global PhabricatorAPIToken
global PhabricatorHost
global WireguardManifestFilePath
global RepositoryName
global RepositoryCommit
global PHIDsOfProjectsToNotify
click.echo('Loading config...')
config = configparser.ConfigParser()
config.read('config.ini')
params = config['Params']
PuppetRoot = params['puppet_root']
PhabricatorAPIToken = params['phabricator_api_token']
PhabricatorHost = params['phabricator_host']
WireguardManifestFilePath = params['wireguard_manifest_file_path']
RepositoryName = params['puppet_repository_name']
RepositoryCommit = params['puppet_repository_commit']
projects = config['Projects_To_Tag']
PHIDsOfProjectsToNotify = [projects[slug] for slug in projects.keys()]
repo = git.Repo(PuppetRoot)
phab = Phabricator(host=PhabricatorHost, token=PhabricatorAPIToken)
return (repo, phab)
def get_phabricator_client():
"""Return a Phabricator client instance"""
client = Phabricator(username='******',
token='<%= @icingabot_password %>',
host='https://phabricator.miraheze.org/api/')
return client
def connect_to_phab(self):
self.phab = Phabricator(config.phab_user,
config.phab_cert,
config.phab_host)
self.phab.update_interfaces()
# DEBUG to verify the API connection worked: print phab.user.whoami()
vlog('Looks like we connected to the phab API \o/')
def test_author_data_full(self):
phab = Phabricator(host='dummmy.com', token='dummy.token')
response, raw_response = PhabricatorService().author_data(
author_phid="PHID-USER-test",
raw_response=test_mock.mock_phabricator_raw_response(),
phab=phab)
print(response)
self.assertTrue(True, False)
def main(args):
if len(args) < 2:
print(
"Please provide a list of junit reports as command line arguments."
)
sys.exit(1)
token = os.getenv("TEAMCITY_CONDUIT_TOKEN", None)
if not token:
print("Please provide a conduit token in the environment variable "
"TEAMCITY_CONDUIT_TOKEN"
"")
sys.exit(1)
arcconfig = get_arcconfig()
phabricatorUrl = urlparse.urljoin(arcconfig['conduit_uri'], "api/")
buildUrl = os.getenv('BUILD_URL', '')
build_status = "success"
failures = {}
for arg in args:
# All inputs may not exist if the build fails prematurely
if not os.path.isfile(arg):
continue
if arg.endswith(".status"):
with open(arg, "r") as f:
build_status = f.read().strip()
elif arg.endswith(".xml"):
failures.update(get_failures(arg))
if len(failures) != 0:
build_status = "failure"
phab = Phabricator(host=phabricatorUrl, token=token)
phab.update_interfaces()
revisionID = get_revision(phab, get_commit_message())
authorPHID = get_author(phab, revisionID)
if build_status != "success":
task_body = create_task_body(buildUrl, revisionID, failures)
create_task(phab, authorPHID, revisionID, task_body)
create_comment(phab, revisionID, build_status, buildUrl)
def test_resolve_tags__should_find_standard_tags():
phab = Phabricator(host=os.environ['PHABRICATOR_API_URL'],
token=os.environ.get('PHABRICATOR_API_TOKEN'))
tag_map = resolve_tags(phab)
eq_(tag_map['has_revision_required_diff_tag'],
"PHID-PROJ-kx4rkibaeu6rt72oupll")
eq_(tag_map['has_diff_tag'], "PHID-PROJ-segawqxql2j3qedl7cqc")
eq_(tag_map['has_accepted_diff_tag'], "PHID-PROJ-72kxzxxqoak5bp2llx3m")
def test_get_diff():
phab = Phabricator(host=api_url_base, token=credentials.api_key)
response = phab.differential.getdiff(diff_id="178")
parsed = response.__dict__
filename = 'diff.json'
if filename:
with open(filename, 'r') as f:
data = json.load(f)
assert parsed == data
def main():
args = parse_args()
fmt = '%(name)s - %(levelname)s - %(message)s'
if not args.log_notime:
fmt = '%(asctime)s - ' + fmt
logging.basicConfig(format=fmt, level=logging.INFO)
phab = Phabricator(host=args.phabricator, token=args.phabricator_token)
logger.info('updating phabricator interfaces...')
phab.update_interfaces()
logger.info(f'working with {phab.host}')
interactor = PhabGchat(phab, args.phabricator_hmac, args.gchat_webhook)
ws = TornadoServer(int(args.port), [
tw.url('/post', PhabReciever, dict(pg=interactor, )),
])
ws.run()
def main(args):
if len(args) < 2:
print("Usage: {} <script> [<output> ...]".format(args[0]))
print("")
print(" <script>: Script to run and parse output from.")
print(" <output>: Test result file for parsing.")
sys.exit(1)
token = os.environ.get("TEAMCITY_CONDUIT_TOKEN", None)
if not token:
print("Please provide a conduit token in the environment variable "
"TEAMCITY_CONDUIT_TOKEN"
"")
sys.exit(1)
arcconfig = get_arcconfig()
phabricatorUrl = urlparse.urljoin(arcconfig['conduit_uri'], "api/")
buildUrl = os.environ.get('BUILD_URL', '')
failures = execute_and_parse(args[1])
for file in args[2:]:
# All inputs may not exist if the build fails prematurely
if not os.path.isfile(arg):
continue
elif arg.endswith(".xml"):
failures.update(get_failures(arg))
build_status = "success"
if len(failures) != 0:
build_status = "failure"
phab = Phabricator(host=phabricatorUrl, token=token)
phab.update_interfaces()
revisionID = get_revision(phab, get_commit_message())
authorPHID = get_author(phab, revisionID)
if build_status != "success":
task_body = create_task_body(buildUrl, revisionID, failures)
create_task(phab, authorPHID, revisionID, task_body)
create_comment(phab, revisionID, build_status, buildUrl)
def publish_log():
if not env.logger:
return
log_output = env.logger.getvalue()
env.logger.close()
phabricator = Phabricator()
phabricator.update_interfaces()
deployer = phabricator.user.whoami()
info_dict = {
"arguments": " ".join(sys.argv[1:]),
"user": deployer.userName,
"date": datetime.now().strftime('%Y-%m-%d %H:%M'),
}
paste_title = "fab %(arguments)s # %(user)s at %(date)s" % info_dict
paste = phabricator.paste.create(
content=log_output,
title=paste_title,
)
info_dict.update({
"paste_id": paste.id,
"branch": env.branch,
"branch_repo_url": "%(repo_browser_url)shistory/%(branch)s/" % {
"repo_browser_url": env.repo_browser,
"branch": env.branch,
}
})
msg = "P%(paste_id)d `fab %(arguments)s` # @%(user)s at %(date)s " \
"from branch [%(branch)s](%(branch_repo_url)s) " % info_dict
phabricator.conpherence.updatethread(
id=env.log_compherence,
message=msg,
)
def test_get_last_comment(self, mock_phabricator_author_data):
phab = Phabricator(host='dummy.com', token='dummy.token')
comments = test_mock.mock_phabricator_get_comments(1201, phab)
response = PhabricatorService().get_last_comment(comments,
phab,
raw_response=[])
createdAt = datetime.datetime.fromtimestamp(float(1551763640))
res = LastComment(author='dummy_user',
body='This is some content',
created_at=createdAt)
self.assertEqual(response, res)
def test_get_raw_diff():
phab = Phabricator(host=api_url_base, token=credentials.api_key)
response = phab.differential.getrawdiff(diffID="178")
parsed = response.__dict__
with open('rawdiff.json', 'w') as outfile:
json.dump(parsed, outfile)
filename = 'rawdiff.json'
if filename:
with open(filename, 'r') as f:
data = json.load(f)
assert parsed == data
def get_phabricator_client():
"""Return a Phabricator client instance"""
parser = ConfigParser.SafeConfigParser()
parser_mode = 'phabricator_bot'
parser.read(PHABRICATOR_CONFIG_FILE)
client = Phabricator(username=parser.get(parser_mode, 'username'),
token=parser.get(parser_mode, 'token'),
host=parser.get(parser_mode, 'host'))
return client
def phid_to_name(phid):
if phid is None:
return None
if cache.has(phid):
return cache.get(phid)
phab = Phabricator()
data = phab.phid.lookup(names=[
phid,
])
name = data[phid]['name']
cache.set(phid, name)
return name
def __init__(self, version, crs):
self.version = version
self.crs = crs
self.last_five_version_ids = None
self.version = version
phab = Phabricator()
self.train_blocker = trainblockers.TrainBlockers(self.version, phab)
try:
self.conductor = trainblockers.get_phab_user(
self.train_blocker.blocker_task['fields']['ownerPHID'], phab)
except APIError:
self.conductor = 'Release Engineer Team'
backups = self.train_blocker.blocker_task['fields'][
'custom.train.backup']
if backups:
self.backup = trainblockers.get_phab_user(backups[0], phab)
else:
self.backup = 'Nobody .·´¯`(>▂<)´¯`·.'
self.stats = TrainsStats(
self.crs.execute(
'''
SELECT
id,
version,
patches,
rollbacks,
(group0_delay_days + group1_delay_days + group2_delay_days) as days_delay,
(SELECT count(*) from blocker b WHERE b.train_id = train.id) as blocker_count
FROM train
WHERE start_time <= (
SELECT start_time
FROM train
WHERE version = ?
)
ORDER BY start_time desc
LIMIT 5
''', (self.version, )).fetchall())
self.thanks = set()
for row in self.crs.execute(
'''
SELECT
blocker,
unblocker
FROM train t
JOIN blocker b ON b.train_id = t.id
WHERE version = ?
''', (self.version, )).fetchall():
if row[0] not in [self.conductor, self.backup, 'null']:
self.thanks.add(row[0])
if row[1] not in [self.conductor, self.backup, 'null']:
self.thanks.add(row[1])
def test_get_reviews_no_raw(self, mock_get_comments,
mock_phabricator_get_last_comment,
mock_phabricator_author_data,
mock_phabricator_time_from_epoch):
reviews = test_mock.mock_phabricator_differential_query(
None, None, None)
phab = Phabricator(host='dummmy.com', token='dummy.token')
response = PhabricatorService().get_reviews(phab,
reviews,
host="www.google.com",
raw_response=[])
self.assertEqual(response[0]._format_oneline(1, 1),
self.config['msg2'])
def get_diffs(author_phids, days=None):
phab = Phabricator()
constraints = {
'authorPHIDs': author_phids,
}
if settings.DIFF_STATUS:
constraints['statuses'] = settings.DIFF_STATUS
if days:
constraints['createdStart'] = get_epoc_days_ago(days)
results = phab.differential.revision.search(constraints=constraints,
#order='newest',
)
return results.get('data')
def test_on_task_update__for_task_with_needs_changes_diff__should_remove_undesired_tags(
cassette):
phab = Phabricator(host=os.environ['PHABRICATOR_API_URL'],
token=os.environ.get('PHABRICATOR_API_TOKEN'))
tag_map = resolve_tags(phab)
on_task_update(phab, 2895, tag_map=tag_map)
eq_(cassette.requests[-1].url,
"https://truecode.trueship.com/api/maniphest.edit")
eq_(
cassette.requests[-1].body,
b"params=%7B%22objectIdentifier%22%3A+%22PHID-TASK-bdaz2w2ue4nlotls6wqn%22%2C+%22transactions%22%3A+%5B%7B%22value%22%3A+%5B%22PHID-PROJ-kx4rkibaeu6rt72oupll%22%5D%2C+%22type%22%3A+%22projects.add%22%7D%2C+%7B%22value%22%3A+%5B%22PHID-PROJ-segawqxql2j3qedl7cqc%22%2C+%22PHID-PROJ-72kxzxxqoak5bp2llx3m%22%5D%2C+%22type%22%3A+%22projects.remove%22%7D%5D%2C+%22__conduit__%22%3A+%7B%22token%22%3A+%22123%22%7D%7D&output=json"
)
def test_on_task_update__for_task_with_unreviewed_diff__should_tag_with_diff_tag(
cassette):
phab = Phabricator(host=os.environ['PHABRICATOR_API_URL'],
token=os.environ.get('PHABRICATOR_API_TOKEN'))
tag_map = resolve_tags(phab)
on_task_update(phab, 3109, tag_map=tag_map)
eq_(cassette.requests[-1].url,
"https://truecode.trueship.com/api/maniphest.edit")
eq_(
cassette.requests[-1].body,
b"params=%7B%22transactions%22%3A+%5B%7B%22value%22%3A+%5B%22PHID-PROJ-segawqxql2j3qedl7cqc%22%5D%2C+%22type%22%3A+%22projects.add%22%7D%5D%2C+%22objectIdentifier%22%3A+%22PHID-TASK-s3scdbecgvk23b4uwsfp%22%2C+%22__conduit__%22%3A+%7B%22token%22%3A+%22123%22%7D%7D&output=json"
)
def test_on_task_update__for_landed_task_with_accepted_diff__should_tag_with_accepted_tag(
cassette):
phab = Phabricator(host=os.environ['PHABRICATOR_API_URL'],
token=os.environ.get('PHABRICATOR_API_TOKEN'))
tag_map = resolve_tags(phab)
on_task_update(phab, 3135, tag_map=tag_map)
eq_(cassette.requests[-1].url,
"https://truecode.trueship.com/api/maniphest.edit")
eq_(
cassette.requests[-1].body,
b"output=json¶ms=%7B%22transactions%22%3A+%5B%7B%22type%22%3A+%22projects.add%22%2C+%22value%22%3A+%5B%22PHID-PROJ-t2mvgyqoolxpakgeyngn%22%5D%7D%2C+%7B%22type%22%3A+%22projects.remove%22%2C+%22value%22%3A+%5B%22PHID-PROJ-segawqxql2j3qedl7cqc%22%5D%7D%5D%2C+%22__conduit__%22%3A+%7B%22token%22%3A+%22123%22%7D%2C+%22objectIdentifier%22%3A+%22PHID-TASK-zjxys23b7zuuojohnfrl%22%7D"
)
def test_on_task_update__for_task_with_accepted_diff__should_tag_with_accepted_tag(
cassette):
phab = Phabricator(host=os.environ['PHABRICATOR_API_URL'],
token=os.environ.get('PHABRICATOR_API_TOKEN'))
tag_map = resolve_tags(phab)
on_task_update(phab, 3070, tag_map=tag_map)
eq_(cassette.requests[-1].url,
"https://truecode.trueship.com/api/maniphest.edit")
eq_(
cassette.requests[-1].body,
b"params=%7B%22objectIdentifier%22%3A+%22PHID-TASK-djleagj2aiegvb7kzivn%22%2C+%22__conduit__%22%3A+%7B%22token%22%3A+%22123%22%7D%2C+%22transactions%22%3A+%5B%7B%22value%22%3A+%5B%22PHID-PROJ-72kxzxxqoak5bp2llx3m%22%5D%2C+%22type%22%3A+%22projects.add%22%7D%5D%7D&output=json"
)
#!/usr/bin/env python
import sys
from phabricator import Phabricator
if len(sys.argv) == 1:
print "Syntax: %s ProjectName" % sys.argv[0]
sys.exit(1)
project_name = sys.argv[1]
phab = Phabricator()
phab.update_interfaces()
proj = phab.project
def phid_from_name(name):
all_projects = proj.query()
for phid in all_projects:
if all_projects[phid]['name'] == name:
return phid
projectPHID = phid_from_name(project_name)
tasks = phab.maniphest.find(
projectPHIDs=[projectPHID],
)
print "Project %s" % project_name
def setup_phab(projectphid): phab = Phabricator() phab.update_interfaces() print "Fetching tasks from Phab" tasks = phab.maniphest.query(projectPHIDs=[projectphid], status="status-open", limit=3000) return(phab, tasks)
class Phidify:
def __init__(self, args):
self.args = args # FIXME unpack all the args into member variables?
if config.phab_host.find('phab-01') != -1:
self.host = 'phab-01'
elif config.phab_host.find('phabricator.wikimedia.org') != -1:
self.host = 'phabricator'
else:
self.json_error('Unrecognized host %s in config' % (config.phab_host))
sys.exit(1)
self.userMap = 'data/trello_names_' + self.host + '.yaml'
with open(self.userMap) as f:
yam = yaml.load(f)
self.trelloUserMap = yam['trelloUserMap']
vlog('Looks like we loaded self.trelloUserMap OK \o/')
def connect_to_phab(self):
self.phab = Phabricator(config.phab_user,
config.phab_cert,
config.phab_host)
self.phab.update_interfaces()
# DEBUG to verify the API connection worked: print phab.user.whoami()
vlog('Looks like we connected to the phab API \o/')
# Returns dict mapping Trello usernames to phab PHIDs
def get_trelloUserPHIDs(self, trelloUserMap):
self.connect_to_phab()
# trelloUserMap maps trello usernames to phabricator usernames, e.g.
# {'spage1': 'spage', 'Tisza': 'Tgr'}
# We can query to get the PHID of the phab username
# {'spage': 'PHID-USER-rwvw2dwvvjiyrlzy4iho', 'Tisza': 'PHID-USER-66kvyuekkkwkqbpze2uk'}
# we want to return {'spage1' : 'PHID-USER-rwvw2dwvvjiyrlzy4iho'}
# Get list of unique Phabricator usernames to query, excluding 'None' and empty
lookupNames = [u for u in set(trelloUserMap.values()) if u]
vlog('get_trelloUserPHIDs querying Phab for %d usernames: %s ' % (len(lookupNames), lookupNames))
response = self.phab.user.query(usernames = lookupNames)
# That conduit query returns an array of users' data, each containing phabUsername and phid.
# Turn it into a dict.
phabUserPHIDMap = {}
for userInfo in response:
if userInfo["userName"] and userInfo["phid"]:
phabUserPHIDMap[userInfo["userName"]] = userInfo["phid"]
# Now create what we want, a dict of trelloUsername -> phid
trelloUserPHIDMap = {}
for trelloUsername, phabUsername in trelloUserMap.iteritems():
if (phabUsername
and phabUsername in phabUserPHIDMap
and phabUserPHIDMap[phabUsername]
and phabUserPHIDMap[phabUsername] != 'None'): # XXX fires?
trelloUserPHIDMap[trelloUsername] = phabUserPHIDMap[phabUsername]
return trelloUserPHIDMap
def writeUserPHIDs(self):
trelloUserPHIDMap = self.get_trelloUserPHIDs(self.trelloUserMap)
fname = 'conf/trello-scrub_' + self.host + '.yaml'
if os.path.exists(fname):
elog('ERROR: ' + fname + ' already exists')
sys.exit(2)
stream = file(fname, 'w')
trelloScrub = {
'uid-map': trelloUserPHIDMap,
'uid-cheatsheet': {}
}
# safe_dump() avoids gtisza: !!python/unicode 'PHID-USER-66kvyuekkkwkqbpze2uk'
yaml.safe_dump( trelloScrub, stream, width=30)
log('SUCCESS: wrote trello usernames->PHIDs to ' + fname)
class TrelloImporter:
# Map from Trello username to phabricator user.
userMapPhab01 = {
'gtisza': 'Tgr',
"legoktm": "legoktm",
"matthewflaschen": "mattflaschen",
"pauginer": None,
"spage1": "spage",
}
userMapPhabWMF = {
# from mingleterminator.py
'fflorin': 'Fabrice_Florin',
'gdubuc': 'Gilles',
'mholmquist': 'MarkTraceur',
'gtisza': 'Tgr',
'pginer': 'Pginer-WMF',
# From ack username trabulous_dir/flow-current-iteration_lOh4XCy7_fm.json \
# | perl -pe 's/^\s+//' | sort | uniq
# Collaboration-Team members: Eloquence, DannyH, Pginer-WMF, Spage, Quiddity, Mattflaschen, matthiasmullie, EBernhardson
"alexandermonk": None,
"antoinemusso": None,
"benmq": None,
"bsitu": None,
"dannyhorn1": "DannyH",
"erikbernhardson": "EBernhardson",
"jaredmzimmerman": None,
"jonrobson1": None,
"kaityhammerstein": None,
"legoktm": None,
"matthewflaschen": "mattflaschen",
"matthiasmullie": "matthiasmullie",
"maygalloway": None,
"moizsyed_": None,
"oliverkeyes": None,
"pauginer": "Pginer-WMF",
"quiddity1": "Quiddity",
"shahyarghobadpour": None,
"spage1": "Spage",
"wctaiwan": None,
"werdnum": None,
}
def __init__(self, jsonName, args):
self.jsonName = jsonName
self.args = args # FIXME unpack all the args into member variables?
self.verbose = args.verbose
if config.phab_host.find('phab-01') != -1:
self.host = 'phab-01'
elif config.phab_host.find('phabricator.wikimedia.org') != -1:
self.host = 'phabricator'
else:
self.json_error('Unrecognized host %s in config' % (config.phab_host))
sys.exit(3)
self.board = TrelloDAO(self.jsonName)
trelloBoardName = self.board.get_board_name();
vlog('Trello board = %s' % (trelloBoardName))
def connect_to_phab(self):
self.phab = Phabricator(config.phab_user,
config.phab_cert,
config.phab_host)
self.phab.update_interfaces()
self.phabm = phabmacros('', '', '')
self.phabm.con = self.phab
# DEBUG to verify the API connection worked: print phab.user.whoami()
vlog('Looks like we connected to the phab API \o/')
def sanity_check(self):
if not 'cards' in self.trelloJson:
self.json_error('No cards in input file')
sys.exit(1)
def testify(self, str):
if self.args.test_run:
str = "TEST Trello_create RUN: " + str
return str
def json_error(self, str):
elog('ERROR: %s in input file %s' % (str, self.jsonName))
# Determine projectPHID for the project name in which this will create tasks.
def get_projectPHID(self, phabProjectName):
# Similar conduit code in trello_makePHIDs.py get_trelloUserPHIDs
response = self.phab.project.query(names = [phabProjectName])
for projInfo in response.data.values():
if projInfo["name"] == phabProjectName:
vlog('Phabricator project %s has PHID %s' % (phabProjectName, projInfo["phid"] ) )
return projInfo["phid"]
elog('Phabricator project %s not found' % (phabProjectName))
sys.exit(4)
return # not reached
# This is the workhorse
def createTask(self, card):
# Default some keys we always pass to createtask.
taskinfo = {
'ownerPHID' : None,
'ccPHIDs' : [],
'projectPHIDs' : [self.projectPHID],
}
taskinfo["title"] = self.testify(card.name)
# TODO: if Trello board is using scrum for Trello browser extension,
# could extract story points /\s+\((\d+)\)' from card title to feed into Sprint extension.
# TODO: process attachments
# TODO: taskinfo["assignee"]
desc = self.testify(card.desc)
if card.checklist_strs:
desc += '\n' + '\n\n'.join(card.checklist_strs)
desc_tail = '\n--------------------------'
desc_tail += '\n**Trello card**: [[ %s | %s ]]\n' % (card.url, card.shortLink)
# Mention column the same way as the card.final_comment_fields below from export_trello.py.
desc_tail += '\n * column: %s\n' % (unicode(card.column))
if len(card.final_comment_fields) > 0:
s = ''
s += '\n'
for key in sorted(card.final_comment_fields):
s += ' * %s: %s\n' % (str(key), unicode(card.final_comment_fields[key]))
desc_tail += s
# TODO: could add additional info (main attachment, etc.) to desc_tail.
taskinfo["description"] = desc + '\n' + desc_tail
# TODO: chasemp: what priority?
taskinfo['priority'] = 50
# TODO: chasemp: can I put "Trello lOh4XCy7" in "Reference" field?
# Take the set of members
idMembers = card.idMembers
# Get the Trello username for the idMember
# memberNames = [ TrelloDAO.get_username(id) for id in idMembers if TrelloDAO.get_username(id)]
# export_trello.py sets names it can't match to 'import-john-doe'
if not 'FAILED' in card.owner and not card.owner == 'import-john-doe':
taskinfo['ownerPHID'] = card.owner
taskinfo['ccPHIDs'] = [u for u in card.subscribers if not 'FAILED' in u and not u == 'import-john-doe']
# TODO: Add any other members with a PHID to the ccPHIDs
# TODO: Note remaining Trello members in desc_tail
# TODO: bugzilla_create.py and wmfphablib/phabapi.py use axuiliary for
# BZ ref, but it doesn't work for Trello ref?
taskinfo["auxiliary"] = {"std:maniphest:external_reference":"Trello %s" % (card.shortLink)}
if self.args.conduit:
# This prints fields for maniphest.createtask
print '"%s"\n"%s"\n\n' % (taskinfo["title"].encode('unicode-escape'),
taskinfo["description"].encode('unicode-escape'))
else:
if self.args.dry_run:
log("dry-run to create a task for Trello card %s ('%s')" %
(card.shortLink, taskinfo["title"]))
else:
ticket = self.phab.maniphest.createtask(
title = taskinfo['title'],
description = taskinfo['description'],
projectPHIDs = taskinfo['projectPHIDs'],
ownerPHID = taskinfo['ownerPHID'],
ccPHIDs = taskinfo['ccPHIDs'],
auxiliary = taskinfo['auxiliary']
)
log("Created task: T%s (%s) from Trello card %s ('%s')" %
(ticket['id'], ticket['phid'], card.shortLink, taskinfo["title"]))
# Here bugzilla_create goes on to log actual creating user and view/edit policy,
# then set_task_ctime to creation_time.
# Should I add comments to the card here,
# or a separate step that goes through action in self.board.blob["actions"]
# handling type="commentCard"?
# Here are the types of objects in the "actions" array.
# 20 "type": "addAttachmentToCard",
# 9 "type": "addChecklistToCard",
# 2 "type": "addMemberToBoard",
# 38 "type": "addMemberToCard",
# 69 "type": "commentCard",
# 1 "type": "copyCard",
# 25 "type": "createCard",
# 3 "type": "createList",
# 6 "type": "deleteAttachmentFromCard",
# 29 "type": "moveCardFromBoard",
# 18 "type": "moveCardToBoard",
# 4 "type": "moveListFromBoard",
# 2 "type": "moveListToBoard",
# 3 "type": "removeChecklistFromCard",
# 14 "type": "removeMemberFromCard",
# 3 "type": "updateBoard",
# 698 "type": "updateCard",
# 48 "type": "updateCheckItemStateOnCard",
# 8 "type": "updateList",
# def getCardCreationMeta(self, cardId):
# Look around in JSON for ["actions"] array for member with type:"createCard"
# with ["card"]["id"] = cardId
# and use the siblings ["date"] and ["memberCreator"]["id"]
# def getCardComments(self, cardId):
# Look around in JSON ["actions"] for member with type:""commentCard"
# with ["card"]["id"] = cardId
# and use the siblings ["date"] and ["memberCreator"]["id"]
def process_cards(self):
self.connect_to_phab()
self.projectPHID = self.get_projectPHID(self.args.phab_project);
# This file has Trello_username: user_PHID mapping created by trello_makePHIDs.py.
scrubber = TrelloScrubber('conf/trello-scrub.yaml')
for j_card in self.board.blob["cards"]:
card = TrelloCard(j_card, scrubber)
card.figure_stuff_out(self.board)
if self.args.column and not card.column == self.args.column:
continue
# Skip archived cards ("closed" seems to correspond?)
# But I think archive all cards in column doesn't set this.
if card.closed:
continue
# TODO: skip cards that are bugs
# TODO: skip cards that already exist in Phab.
self.createTask(card)
def init_phab_connection():
phab = Phabricator()
phab.update_interfaces()
return phab
