def fetch(bugid):
pmig = phabdb.phdb(db=config.bzmigrate_db,
user=config.bzmigrate_user,
passwd=config.bzmigrate_passwd)
server = xmlrpclib.ServerProxy(config.Bugzilla_url, use_datetime=True)
token_data = server.User.login({'login': config.Bugzilla_login,
'password': config.Bugzilla_password})
token = token_data['token']
kwargs = { 'ids': [bugid], 'Bugzilla_token': token }
#grabbing one bug at a time for now
buginfo = server.Bug.get(kwargs)['bugs']
buginfo = buginfo[0]
# some bugs have header data but no actual content
# https://bugzilla.wikimedia.org/show_bug.cgi?id=32738
com = server.Bug.comments(kwargs)['bugs'][str(bugid)]['comments']
#have to do for json
buginfo['last_change_time'] = datetime_to_epoch(buginfo['last_change_time'])
buginfo['creation_time'] = datetime_to_epoch(buginfo['creation_time'])
if 'flags' in buginfo:
for flag in buginfo['flags']:
for k, v in flag.iteritems():
if isinstance(v, datetime.datetime):
flag[k] = datetime_to_epoch(v)
for c in com:
c['creation_time'] = datetime_to_epoch(c['creation_time'])
c['time'] = datetime_to_epoch(c['time'])
# set ticket status for priority import
status = bzlib.status_convert(buginfo['status'], buginfo['resolution'])
if status == 'open':
creation_priority = ipriority['unresolved']
else:
creation_priority = ipriority['na']
current = pmig.sql_x("SELECT * from bugzilla_meta where id = %s", bugid)
if current:
update_values = (creation_priority,
json.dumps(buginfo),
json.dumps(com),
now(),
bugid)
vlog('update: ' + str(update_values))
pmig.sql_x("UPDATE bugzilla_meta SET priority=%s, header=%s, comments=%s, modified=%s WHERE id = %s",
update_values)
else:
insert_values = (bugid, creation_priority, json.dumps(buginfo), json.dumps(com), now(), now())
vlog('insert: ' + str(insert_values))
sql = "INSERT INTO bugzilla_meta (id, priority, header, comments, created, modified) VALUES (%s, %s, %s, %s, %s, %s)"
pmig.sql_x(sql,
insert_values)
pmig.close()
return True
def create(bugid):
phab = Phabricator(config.phab_user,
config.phab_cert,
config.phab_host)
phabm = phabmacros('', '', '')
phabm.con = phab
pmig = phabdb.phdb(db=config.bzmigrate_db,
user=config.bzmigrate_user,
passwd=config.bzmigrate_passwd)
current = pmig.sql_x("SELECT priority, \
header, \
comments, \
created, \
modified \
FROM bugzilla_meta WHERE id = %s",
(bugid,))
if current:
import_priority, buginfo, com, created, modified = current[0]
else:
pmig.close()
elog('%s not present for migration' % (bugid,))
return False
def get_ref(id):
refexists = phabdb.reference_ticket('%s%s' % (bzlib.prepend,
id))
if refexists:
pmig.close()
return refexists
if get_ref(bugid):
log('reference ticket %s already exists' % (bugid,))
return True
buginfo = json.loads(buginfo)
com = json.loads(com)
bugid = int(bugid)
vlog(bugid)
vlog(buginfo)
buginfo["secstate"] = 'none'
# And the relevant herald rule must be in place.
if bzlib.is_sensitive(buginfo["product"]):
buginfo["secstate"] = 'security-bug'
log("found security-bug issue %s" % (bugid,))
#this allows testing while respecting security queue
if config.bz_security.lower() != 'true':
log("ignoring security issue %s" % (bugid,))
return True
server = xmlrpclib.ServerProxy(config.Bugzilla_url, use_datetime=True)
token_data = server.User.login({'login': config.Bugzilla_login,
'password': config.Bugzilla_password})
token = token_data['token']
#http://www.bugzilla.org/docs/tip/en/html/api/Bugzilla/WebService/Bug.html#attachments
kwargs = { 'ids': [bugid], 'Bugzilla_token': token }
bzdata= open("data/bugzilla.yaml", 'r')
bzdata_yaml = yaml.load(bzdata)
tag_keys = bzdata_yaml['keywords_to_tags'].split(' ')
mlists = bzdata_yaml['assigned_to_lists'].split(' ')
vlog("Mailinglists: " + str(mlists))
attached = server.Bug.attachments(kwargs)['bugs'][str(bugid)]
vlog("bz attached items %s" % (str(attached)))
#process ticket uploads to map attach id to phab file id
uploads = {}
for a in attached:
vlog("processing bz attachment %s" % (str(a)))
if a['is_private']:
vlog('ignoring private attachment: %s' % (str(a)))
a['ignore'] = 'private'
elif a['is_obsolete'] == 1:
vlog('ignoring obsolete attachment: %s' % (str(a)))
a['ignore'] = 'obsolete'
else:
if buginfo["secstate"] == 'none':
viewpolicy = 'public'
else:
viewpolicy = phabdb.get_project_phid('security')
try:
#upload = phabm.upload_file(a['file_name'], a['data'].data)
upload = phabm.upload_file(a['file_name'], a['data'].data, viewpolicy)
a['phid'] = upload['phid']
a['name'] = upload['name']
a['objectName'] = upload['objectName']
except Exception as e:
print "\n%s ATTACHMENT CORRUPTED -- %s\n" % (str(bugid), str(a))
print "%s --" % (str(e))
a['ignore'] = 'corrupt on retrieval'
uploads[a['id']] = a
log('%s attachment count: %s' % (bugid, str(len(uploads.keys()))))
vlog("phab upload details: %s" % (str(uploads)))
#list of projects to add to ticket
ptags = []
#mask emails for public consumption
buginfo['cc'] = [c.split('@')[0] for c in buginfo['cc']]
# Convert bugzilla source to phabricator
buginfo['status'] = bzlib.status_convert(buginfo['status'],
buginfo['resolution'])
buginfo['priority'] = bzlib.priority_convert(buginfo['priority'])
if '-d' in sys.argv:
with open('dump', 'w') as d:
d.write(str(json.dumps(buginfo)))
if buginfo['status'].lower() == 'patch_to_review':
ptags.append(('patch_to_review', 'tags', 'green'))
if buginfo['status'] == 'verified':
ptags.append(('verified', 'tags'))
if buginfo['cf_browser'] not in ['---', "Other"]:
btag = "Browser_Support_%s" % (buginfo['cf_browser'].replace(' ', '-'),)
log('Adding browser tag: %s' % (btag,))
ptags.append((btag, 'tags'))
if buginfo['target_milestone'] != '---':
log('Creating milestone: %s' % (buginfo['target_milestone'],))
ptags.append((buginfo['target_milestone'], 'truck'))
#set defaults to be overridden by sec if needed
buginfo['viewPolicy'] = 'public'
buginfo['editPolicy'] = 'users'
buginfo['project'] = bzlib.sanitize_project_name(buginfo["product"],
buginfo["component"])
vlog(buginfo['project'])
ptags.append((buginfo['project'], None))
title = buginfo['summary']
clean_com = []
for c in com:
if not isinstance(c, dict):
c = ast.literal_eval(c)
clean_c = bzlib.build_comment(c, buginfo['secstate'])
clean_com.append(clean_c)
log('project: ' + buginfo['project'])
try:
# strip out comment 0 as description
description = clean_com[0]
del clean_com[0]
except IndexError:
log("%s has no comment 0" % (str(bugid)))
# some random tasks were created at a point in bugzilla
# history with metadata but no comment 0
# https://bugzilla.wikimedia.org/show_bug.cgi?id=32056
description = {'author': buginfo['creator'].split('@')[0],
'text': '//this issue has no description//',
'creation_time': buginfo['creation_time']}
created = epoch_to_datetime(description['creation_time'])
desc_block = "**Author:** `%s`\n\n**Description:**\n%s\n" % (description['author'],
description['text'])
# https://phabricator.wikimedia.org/T694
desc_tail = '--------------------------'
desc_tail += "\n**Version**: %s" % (buginfo['version'])
desc_tail += "\n**Severity**: %s" % (buginfo['severity'] or 'none')
if buginfo['op_sys'] != 'All':
desc_tail += "\n**OS**: %s" % (buginfo['op_sys'])
if "platform" in buginfo and buginfo['platform'] != 'All':
desc_tail += "\n**Platform**: %s" % (buginfo['platform'])
if buginfo['whiteboard']:
desc_tail += "\n**Whiteboard**: %s" % (buginfo['whiteboard'])
if buginfo['url']:
desc_tail += "\n**URL**: %s" % (buginfo['url'])
if buginfo['see_also']:
desc_tail += "\n**See Also**:\n%s" % ('\n'.join(buginfo['see_also']))
attachments = ''
if 'attachment' in description:
attached = int(description['attachment'])
if attached in uploads:
cattached = uploads[int(description['attachment'])]
if 'objectName' in cattached:
attachments = "\n\n**Attached**: {%s}" % (cattached['objectName'])
if 'ignore' in cattached:
attachments = "\n\n//attachment %s ignored as %s//" % (cattached['file_name'],
cattached['ignore'])
else:
attachments = "\n\n//attachment missing in source//"
desc_tail += attachments
full_description = desc_block + '\n' + desc_tail
keys = buginfo['keywords']
for k in keys:
if k in tag_keys:
if k == 'ops':
k = 'operations'
ptags.append((k, 'tags'))
def project_security_settings(pname):
if bzlib.is_sensitive(pname):
ephid = phabdb.get_project_phid('security')
edit = ephid
else:
edit = 'public'
view = 'public'
return edit, view
phids = []
for p in ptags:
edit, view = project_security_settings(p[0])
phid = phabm.ensure_project(p[0], edit=edit, view=view)
phids.append(phid)
if p[1] is not None:
vlog("setting project %s icon to %s" % (p[0], p[1]))
set_project_icon(p[0], icon=p[1])
log("ptags: " + str(ptags))
vlog("phids: " + str(phids))
#buginfo'assigned_to': u'*****@*****.**'
assignee = buginfo['assigned_to']
ccphids = []
if assignee in mlists:
ccphids.append(mailinglist_phid(assignee))
# viewPolicy = buginfo['viewPolicy'],
# editPolicy = buginfo['editPolicy'],
vlog("Ticket Info: %s" % (desc_block,))
ticket = phab.maniphest.createtask(title=buginfo['summary'],
description=full_description,
projectPHIDs=phids,
ccPHIDs=ccphids,
priority=buginfo['priority'],
auxiliary={"std:maniphest:external_reference":"bz%s" % (bugid,),
"std:maniphest:security_topic":"%s" % (buginfo["secstate"],)})
log("Created task: T%s (%s)" % (ticket['id'], ticket['phid']))
botphid = phabdb.get_phid_by_username(config.phab_user)
phabdb.set_task_title_transaction(ticket['phid'],
botphid,
buginfo['viewPolicy'],
buginfo['editPolicy'])
phabdb.set_task_ctime(ticket['phid'],
int(buginfo['creation_time'].split('.')[0]))
fmt_comments = {}
for c in clean_com:
fmt_comment = {}
created = epoch_to_datetime(c['creation_time'])
comment_header = "**%s** wrote:\n\n" % (c['author'],)
comment_body = c['text']
attachments = ''
if 'attachment' in c:
attached = int(c['attachment'])
if attached in uploads:
cattached = uploads[int(c['attachment'])]
if 'objectName' in cattached:
attachments += "\n\n**Attached**: {%s}" % (cattached['objectName'])
if 'ignore' in cattached:
attachments += "\n\n//attachment %s ignored as %s//" % (cattached['file_name'],
cattached['ignore'])
else:
attachments += "\n\n//attachment missing in source//"
fmt_comment['xpreamble'] = comment_header
fmt_comment['xattached'] = attachments
phabm.task_comment(ticket['id'],
comment_header + comment_body + attachments)
ctransaction = phabdb.last_comment(ticket['phid'])
phabdb.set_comment_time(ctransaction, c['creation_time'])
fmt_comment['xctransaction'] = ctransaction
fmt_comments[c['count']] = fmt_comment
if buginfo['status'] != 'open':
log("setting status for T%s to %s" % (ticket['id'], buginfo['status']))
phabdb.set_issue_status(ticket['phid'], buginfo['status'])
phabdb.set_task_mtime(ticket['phid'],
int(buginfo['last_change_time'].split('.')[0]))
xcomments = json.dumps(fmt_comments)
pmig.sql_x("UPDATE bugzilla_meta \
SET xcomments=%s WHERE id = %s", (xcomments, bugid))
pmig.sql_x("UPDATE bugzilla_meta \
SET priority=%s, modified=%s WHERE id = %s",
(ipriority['creation_success'], now(), bugid))
pmig.close()
return True
