def main(): phab = Phabricator(config.phab_user, config.phab_cert, config.phab_host) phabm = phabmacros('', '', '') phabm.con = phab pmig = phabdb.phdb(db=config.bzmigrate_db, user=config.bzmigrate_user, passwd=config.bzmigrate_passwd) server = xmlrpclib.ServerProxy(config.Bugzilla_url, use_datetime=True) bzdata = open("data/bugzilla.yaml", 'r') bzdata_yaml = yaml.load(bzdata) #product = 'Wikimedia' kwargs = { 'Bugzilla_login': config.Bugzilla_login, 'Bugzilla_password': config.Bugzilla_password} products = server.Product.get_selectable_products(kwargs)['ids'] print products for p in products: kwargs = { 'ids': p, 'Bugzilla_login': config.Bugzilla_login, 'Bugzilla_password': config.Bugzilla_password} pi = server.Product.get(kwargs)['products'][0] print pi['name'], pi['description'] for c in pi['components']: pname = "\n%s-%s\n\n%s" % (pi['name'], c['name']) print pname
def update(user): phab = Phabricator(config.phab_user, config.phab_cert, config.phab_host) pmig = phabdb.phdb(db=config.rtmigrate_db, user=config.rtmigrate_user, passwd=config.rtmigrate_passwd) phabm = phabmacros('', '', '') phabm.con = phab if phabdb.is_bot(user['userphid']): log("%s is a bot no action" % (user['user'])) return True epriority = phabdb.get_user_relations_comments_priority(user['user'], pmig) if epriority and len(epriority[0]) > 0: if epriority[0][0] == ipriority['update_success_comments']: log('Skipping %s as already updated' % (user['user'])) return True if not user['issues']: log("%s has no issues to update" % (user['user'],)) return True for i in user['issues']: comdetails = pmig.sql_x("SELECT comments, \ xcomments \ FROM rt_meta \ WHERE id = %s", (int(i),)) jcom, jxcom = comdetails[0] coms = json.loads(jcom) xcoms = json.loads(jxcom) for key, xi in xcoms.iteritems(): content = xi['content'] if xi["creator"] == user['user']: log("Updating comment %s for %s" % (xi['xctransaction'], user['user'])) phabdb.set_comment_author(xi['xctransaction'], user['userphid']) phabdb.set_comment_content(xi['xctransaction'], xi['content']) current = phabdb.get_user_migration_comment_history(user['user'], pmig) if current: success = ipriority['update_success_comments'] log(phabdb.set_user_relations_comments_priority(success, user['user'], pmig)) else: log('%s user does not exist to update' % (user['user'])) return False pmig.close() log(util.purge_cache()) return True
def connect_to_phab(self): self.phab = Phabricator(config.phab_user, config.phab_cert, config.phab_host) self.phab.update_interfaces() self.phabm = phabmacros('', '', '') self.phabm.con = self.phab # DEBUG to verify the API connection worked: print phab.user.whoami() vlog('Looks like we connected to the phab API \o/')
def update(user): phab = Phabricator(config.phab_user, config.phab_cert, config.phab_host) pmig = phabdb.phdb(host=config.dbhost, db=config.bzmigrate_db, user=config.bzmigrate_user, passwd=config.bzmigrate_passwd) phabm = phabmacros('', '', '') phabm.con = phab if phabdb.is_bot(user['userphid']): log("%s is a bot no action" % (user['user'])) return True epriority = phabdb.get_user_relations_priority(user['user'], pmig) if epriority and len(epriority[0]) > 0: if epriority[0][0] == ipriority['update_success']: log('Skipping %s as already updated' % (user['user'])) return True # 'author': [409, 410, 411, 404, 412], # 'cc': [221, 69, 203, 268, 261, 8], # 'created': 1410276037L, # 'modified': 1410276060L, # 'assigned': [97, 64, 150, 59, 6], # 'userphid': 'PHID-USER-4hsexplytovmqmcb7tq2', # 'user': u'*****@*****.**'} if user['assigned']: for ag in user['assigned']: vlog(phabm.sync_assigned(user['userphid'], ag, bzlib.prepend)) if user['author']: for a in user['author']: vlog(phabm.synced_authored(user['userphid'], a, bzlib.prepend)) if user['cc']: for ccd in user['cc']: vlog(phabdb.add_task_cc_by_ref(user['userphid'], ccd, bzlib.prepend)) current = phabdb.get_user_migration_history(user['user'], pmig) if current: log(phabdb.set_user_relations_priority(ipriority['update_success'], user['user'], pmig)) else: elog('%s user does not exist to update' % (user['user'])) return False pmig.close() return True
def fetch(PHABTICKETID): PHABTICKETID = int(PHABTICKETID) parser = ConfigParser.SafeConfigParser() parser_mode = 'oldfab' parser.read(configfile) oldfab = Phabricator(parser.get(parser_mode, 'user'), parser.get(parser_mode, 'cert'), parser.get(parser_mode, 'host')) #dummy instance of phabapi phabm = phabmacros('', '', '') #assign newphab instance as self.con for dummyphab phabm.con = oldfab """ <Result: {u'authorPHID': u'PHID-USER-qbtllnzb6pwl3ttzqa3m', u'status': u'open', u'phid': u'PHID-TASK-qr3fpbtk6kdx4slhgnsd', u'description': u'', u'objectName': u'T10', u'title': u'Get icinga alerts into logstash', u'priorityColor': u'red', u'dependsOnTaskPHIDs': [], u'auxiliary': [], u'uri': u'http://fab.wmflabs.org/T10', u'ccPHIDs': [u'PHID-USER-qbtllnzb6pwl3ttzqa3m'], u'isClosed': False, u'dateModified': u'1399311492', u'ownerPHID': None, u'statusName': u'Open', u'dateCreated': u'1391716779', u'projectPHIDs': [u'PHID-PROJ-5ncvaivs3upngr7ijqy2'], u'id': u'10', u'priority': u'High'}> """ tinfo = oldfab.maniphest.info(task_id=PHABTICKETID).response vlog(tinfo) if 'objectName' in tinfo: log("Fetching %s" % (tinfo['objectName'])) comments = comments_by_task(tinfo['phid']) for i, c in comments.iteritems(): comments[i]['xcommenter'] = dict(oldfab.user.info(phid=c['xuserphid'])) ordered_comments = collections.OrderedDict(sorted(comments.items())) vlog(str(ordered_comments)) """ <Result: {u'userName': u'bd808', u'phid': u'PHID-USER-qbtllnzb6pwl3ttzqa3m', u'realName': u'Bryan Davis', u'roles': [u'admin',u'verified', u'approved', u'activated'], u'image': u'http://fab.wmflabs.org/file/data/fijwoqt62w6atpond4vb/PHID-FILE-37htsfegn7bnlfvzwsts/profile-profile-gravatar', u'uri': u'http://fab.wmflabs.org/p/bd808/'}> """ authorInfo = oldfab.user.info(phid=tinfo['authorPHID']) tinfo['xauthor'] = phabdb.email_by_userphid(authorInfo['phid']) lauthor = tinfo['xauthor'] or 'no author' vlog('author: ' + lauthor) ccs = [] if tinfo['ccPHIDs']: for c in tinfo['ccPHIDs']: ccInfo = oldfab.user.info(phid=c) ccs.append(phabdb.email_by_userphid(ccInfo['phid'])) tinfo['xccs'] = ccs vlog('ccs: ' + str(ccs)) if tinfo['ownerPHID']: tinfo['xowner'] = phabdb.email_by_userphid(tinfo['ownerPHID']) else: tinfo['xowner'] = None """ u'data': {u'PHID-PROJ-5ncvaivs3upngr7ijqy2': {u'phid': u'PHID-PROJ-5ncvaivs3upngr7ijqy2', u'name': u'logstash', u'dateCreated': u'1391641549', u'members': [u'PHID-USER-65zhggegfvhojb4nynay'], u'id': u'3', u'dateModified': u'1398282408', u'slugs': [u'logstash']}}, u'slugMap': []}> """ project_names = [] associated_projects = tinfo['projectPHIDs'] vlog('associated projects: %s' % (str(tinfo['projectPHIDs']))) #if we try to query for an empty list we get back ALLLLLL if associated_projects: pinfo = oldfab.project.query(phids=associated_projects) if pinfo['data']: for p in pinfo['data'].values(): project_names.append(p['name']) def norm(pname): return pname.lower().replace(' ', '_').replace('-', '_') norm_projects = [norm(p) for p in fablib.saved] saved_projects = [norm(p) in norm_projects for p in project_names] if not any(saved_projects): print "Skipping %s as it's not in a saved project" % (PHABTICKETID) return True vlog('project names: ' + str(project_names)) tinfo['xprojects'] = project_names status = tinfo['status'] if status != 'open': creation_priority = ipriority['na'] else: creation_priority = ipriority['unresolved'] blocked_tasks = phabdb.get_tasks_blocked(tinfo['phid']) if blocked_tasks: blocked = [] block_details = oldfab.maniphest.find(phids=blocked_tasks) for k, v in block_details.iteritems(): blocked.append(v['id']) else: blocked = [] vlog('blocking: %s' % (str(blocked))) tinfo['xblocking'] = blocked pmig = phabdb.phdb(db='fab_migration') current = pmig.sql_x("SELECT * from fab_meta where id = %s", PHABTICKETID) if current: log('updating current record %s' % (PHABTICKETID,)) update_values = (creation_priority, json.dumps(tinfo), json.dumps(comments), now(), PHABTICKETID) pmig.sql_x("UPDATE fab_meta SET priority=%s, header=%s, comments=%s, modified=%s WHERE id = %s", update_values) else: log('inserting new record %s' % (PHABTICKETID,)) insert_values = (PHABTICKETID, creation_priority, json.dumps(tinfo), json.dumps(comments), now(), now()) pmig.sql_x("INSERT INTO fab_meta (id, priority, header, comments, created, modified) VALUES (%s, %s, %s, %s, %s, %s)", insert_values) pmig.close() return True
def create(bugid): phab = Phabricator(config.phab_user, config.phab_cert, config.phab_host) phabm = phabmacros('', '', '') phabm.con = phab pmig = phabdb.phdb(db=config.bzmigrate_db, user=config.bzmigrate_user, passwd=config.bzmigrate_passwd) current = pmig.sql_x("SELECT priority, \ header, \ comments, \ created, \ modified \ FROM bugzilla_meta WHERE id = %s", (bugid,)) if current: import_priority, buginfo, com, created, modified = current[0] else: elog('%s not present for migration' % (bugid,)) return False def get_ref(id): refexists = phabdb.reference_ticket('%s%s' % (bzlib.prepend, id)) if refexists: return refexists[0] buginfo = json.loads(buginfo) com = json.loads(com) bugid = int(bugid) vlog(bugid) vlog(buginfo) ticket = get_ref(bugid) print 'TICKET ', ticket def is_sensitive(name): return name.strip().lower().startswith('security') def project_security_settings(pname): if is_sensitive(pname): ephid = phabdb.get_project_phid('security') edit = ephid else: edit = 'users' view = 'public' return edit, view server = xmlrpclib.ServerProxy(config.Bugzilla_url, use_datetime=True) token_data = server.User.login({'login': config.Bugzilla_login, 'password': config.Bugzilla_password}) token = token_data['token'] #http://www.bugzilla.org/docs/tip/en/html/api/Bugzilla/WebService/Bug.html#attachments kwargs = { 'ids': [bugid], 'Bugzilla_token': token } #list of projects to add to ticket ptags = [] if buginfo['status'] == 'VERIFIED': vlog("Adding 'verified' to %s" % (ticket,)) ptags.append(('verified', 'tags')) if buginfo['status'].lower() == 'patch_to_review': vlog("Adding 'Patch-For-Review' to %s" % (ticket,)) ptags.append(('Patch-For-Review', 'tags', 'green')) log("status recognized as %s" % (buginfo['status'],)) phids = [] for p in ptags: edit, view = project_security_settings(p[0]) phid = phabm.ensure_project(p[0], edit=edit, view=view) phids.append(phid) if p[1] is not None: vlog("setting project %s icon to %s" % (p[0], p[1])) set_project_icon(p[0], icon=p[1]) for phid in phids: phabdb.set_related_project(ticket, phid) pmig.close() return True
def create(rtid): phab = Phabricator(config.phab_user, config.phab_cert, config.phab_host) phabm = phabmacros('', '', '') phabm.con = phab pmig = phdb(db=config.rtmigrate_db) response = resource.RTResource(config.rt_url, config.rt_login, config.rt_passwd, authenticators.CookieAuthenticator) current = pmig.sql_x("SELECT priority, header, \ comments, created, modified \ FROM rt_meta WHERE id = %s", (rtid,)) if current: import_priority, rtinfo, com, created, modified = current[0] else: log('%s not present for migration' % (rtid,)) return 'missing' if not rtinfo: log("ignoring invalid data for issue %s" % (rtid,)) return False def get_ref(id): refexists = phabdb.reference_ticket('%s%s' % (rtlib.prepend, id)) if refexists: return refexists if get_ref(rtid): log('reference ticket %s already exists' % (rtid,)) return True def remove_sig(content): return re.split('--\s?\n', content)[0] def uob(obj, encoding='utf-8'): """ unicode or bust""" if isinstance(obj, basestring): if not isinstance(obj, unicode): obj = unicode(obj, encoding) return obj def sanitize_text(line): if line.strip() and not line.lstrip().startswith('>'): # in remarkup having '--' on a new line seems to bold last # line so signatures really cause issues if all(map(lambda c: c in '-', line.strip())): return '%%%{0}%%%'.format(line.strip()) elif line.strip() == '-------- Original Message --------': return '%%%{0}%%%'.format(line.strip()) elif line.strip() == '---------- Forwarded message ----------': return '%%%{0}%%%'.format(unicode(line.strip())) elif line.strip().startswith('#'): return uob('%%%') + uob(line.strip()) + uob('%%%') else: return uob('%%%') + uob(line).strip() + uob('%%%') elif line.strip().startswith('>'): quoted_content = line.lstrip('>').strip() if not quoted_content.lstrip('>').strip(): return line.strip() if all(map(lambda c: c in '-', quoted_content.lstrip('>').strip())): return "> ~~" else: return uob(line.strip()) else: vlog("ignoring content line %s" % (line,)) return None botphid = phabdb.get_phid_by_username(config.phab_user) viewpolicy = phabdb.get_project_phid('WMF-NDA') if not viewpolicy: elog("View policy group not present: %s" % (viewpolicy,)) return False # Example: # id: ticket/8175/attachments\n # Attachments: 141490: (Unnamed) (multipart/mixed / 0b), # 141491: (Unnamed) (text/html / 23b), # 141492: 0jp9B09.jpg (image/jpeg / 117.4k), attachments = response.get(path="ticket/%s/attachments/" % (rtid,)) if not attachments: raise Exception("no attachment response: %s" % (rtid)) history = response.get(path="ticket/%s/history?format=l" % (rtid,)) rtinfo = json.loads(rtinfo) comments = json.loads(com) vlog(rtid) vlog(rtinfo) comment_dict = {} for i, c in enumerate(comments): cwork = {} comment_dict[i] = cwork if not 'Attachments:' in c: pass attachsplit = c.split('Attachments:') if len(attachsplit) > 1: body, attached = attachsplit[0], attachsplit[1] else: body, attached = c, '0' comment_dict[i]['text_body'] = unicode(body) comment_dict[i]['attached'] = attached # Example: # Ticket: 8175\nTimeTaken: 0\n # Type: # Create\nField: # Data: \nDescription: Ticket created by cpettet\n\n # Content: test ticket description\n\n\n # Creator: cpettet\nCreated: 2014-08-21 21:21:38\n\n'} params = {'id': 'id:(.*)', 'ticket': 'Ticket:(.*)', 'timetaken': 'TimeTaken:(.*)', 'content': 'Content:(.*)', 'creator': 'Creator:(.*)', 'description': 'Description:(.*)', 'created': 'Created:(.*)', 'ovalue': 'OldValue:(.*)', 'nvalue': 'NewValue:(.*)'} for k, v in comment_dict.iteritems(): text_body = v['text_body'] comment_dict[k]['body'] = {} for paramkey, regex in params.iteritems(): value = re.search(regex, text_body) if value: comment_dict[k]['body'][paramkey] = value.group(1).strip() else: comment_dict[k]['body'][paramkey] = None if 'Content' in text_body: content = text_body.split('Content:')[1] content = content.split('Creator:') comment_dict[k]['body']['content'] = content creator = comment_dict[k]['body']['creator'] if creator and '@' in creator: comment_dict[k]['body']['creator'] = rtlib.sanitize_email(creator) #15475: untitled (18.7k) comment_attachments= re.findall('(\d+):\s', v['attached']) comment_dict[k]['body']['attached'] = comment_attachments # due to the nature of the RT api sometimes whitespacing becomes # a noise comment if not any(comment_dict[comment_dict.keys()[0]]['body'].values()): vlog('dropping %s comment' % (str(comment_dict[comment_dict.keys()[0]],))) del comment_dict[0] #attachments into a dict def attach_to_kv(attachments_output): attached = re.split('Attachments:', attachments_output, 1)[1] ainfo = {} for at in attached.strip().splitlines(): if not at: continue k, v = re.split(':', at, 1) ainfo[k.strip()] = v.strip() return ainfo ainfo = attach_to_kv(attachments) #lots of junk attachments from emailing comments and ticket creation ainfo_f = {} for k, v in ainfo.iteritems(): if '(Unnamed)' not in v: ainfo_f[k] = v #taking attachment text and convert to tuple (name, content type, size) ainfo_ext = {} comments = re.split("\d+\/\d+\s+\(id\/.\d+\/total\)", history) for k, v in ainfo_f.iteritems(): # Handle general attachment case: # NO: 686318802.html (application/octet-stream / 19.5k), # YES: Summary_686318802.pdf (application/unknown / 215.3k), extract = re.search('(.*)\.(\S{3,4})\s\((.*)\s\/\s(.*)\)', v) # due to goofy email handling of signature/x-header/meta info # it seems they sometimes # become malformed attachments. Such as when a response into # rt was directed to a mailinglist # Example: # ->Attached Message Part (text/plain / 158b) # # Private-l mailing list # [email protected] # https://lists.wikimedia.org/mailman/listinfo/private-l if extract: fdetails = extract.groups() if not extract and v.startswith('Attached Message Part'): continue if not extract: extract = re.match('(\S+)\s\((.*)\/(.*)\),.*', v) if not extract: elog("attachment CORRUPT or FAILED extraction: %s %s (%s)" % (k, v, rtid)) continue fdetails = extract.group(1), '', extract.group(2), extract.group(3) if not fdetails: elog("attachment CORRUPT or FAILED extraction: %s %s (%s)" % (k, v, rtid)) continue ainfo_ext[k] = fdetails vlog(ainfo_ext[k]) # deb # cgi attachment_types = ['pdf', 'jpeg', 'asc', 'tgz', 'csr', 'jpg', 'png', 'xls', 'xls', 'csv', 'docx', 'gif', 'html', 'htm', 'txt', 'diff', 'log', 'zip', 'rtf', 'tmpl', 'vcf', 'pub', 'sql', 'odt', 'p7s', 'iso', 'ods', 'conf', 'doc', 'xff', 'eml'] #Uploading attachment dl = [] #('Quote Summary_686318802', 'pdf', 'application/unknown', '215.3k') uploaded = {} for k, v in ainfo_ext.iteritems(): file_extension = v[1].lower() # vendors have this weird habit of capitalizing extension names # make sure we can handle the extension type otherwise #if file_extension not in attachment_types: # elog("Unknown Exception (%s) %s %s" % (rtid, v, file_extension)) # #raise Exception('unknown extension: %s (%s)' % (v, rtid)) full = "ticket/%s/attachments/%s/content" % (rtid, k) vcontent = response.get(path=full, headers={'Content-Type': v[2], 'Content-Length': v[3] }) #PDF's don't react well to stripping header -- fine without it if file_extension.strip() == 'pdf': sanscontent = ''.join(vcontent.readlines()) else: vcontent = vcontent.readlines() sanscontent = ''.join(vcontent[2:]) if file_extension: fname = "%s.%s" % (v[0], file_extension) else: fname = v[0] upload = phabm.upload_file(fname, sanscontent, viewpolicy) uploaded[k] = upload if rtinfo['Queue'] not in rtlib.enabled: log("%s not in an enabled queue" % (rtid,)) return True ptags = [] # In a practical sense ops-requets seemed to get tagged # with straight Operations group in Phab so we backfill # this for consistency. if rtinfo['Queue'] == 'ops-requests': ptags.append('operations') pname = rtlib.project_translate(rtinfo['Queue']) ptags.append(pname) phids = [] for p in ptags: phids.append(phabm.ensure_project(p)) rtinfo['xpriority'] = rtlib.priority_convert(rtinfo['Priority']) rtinfo['xstatus'] = rtlib.status_convert(rtinfo['Status']) import collections # {'ovalue': u'open', # 'description': u"Status changed from 'open' to 'resolved' by robh", # 'nvalue': None, 'creator': u'robh', 'attached': [], # 'timetaken': u'0', 'created': u'2011-07-01 02:47:24', # 'content': [u' This transaction appears to have no content\n', u' # robh\nCreated: 2011-07-01 02:47:24\n'], # 'ticket': u'1000', 'id': u'23192'} ordered_comments = collections.OrderedDict(sorted(comment_dict.items())) upfiles = uploaded.keys() # much like bugzilla comment 0 is the task description header = comment_dict[comment_dict.keys()[0]] del comment_dict[comment_dict.keys()[0]] dtext_san = [] dtext_list = header['body']['content'][0].splitlines() for t in dtext_list: dtext_san.append(sanitize_text(rtlib.shadow_emails(t))) dtext = '\n'.join(filter(None, dtext_san)) #dtext = '\n'.join(filter(None, sanitize_text(rtlib.shadow_emails(dtext_list)))) full_description = "**Author:** `%s`\n\n**Description:**\n%s\n" % (rtinfo['Creator'].strip(), dtext) hafound = header['body']['attached'] header_attachments = [] for at in hafound: if at in upfiles: header_attachments.append('{F%s}' % uploaded[at]['id']) if 'CF.{Bugzilla ticket}' in rtinfo and rtinfo['CF.{Bugzilla ticket}'] or header_attachments: full_description += '\n__________________________\n\n' if 'CF.{Bugzilla ticket}' in rtinfo and rtinfo['CF.{Bugzilla ticket}']: obzurl = 'https://old-bugzilla.wikimedia.org/show_bug.cgi?id=' obz = "[[ %s%s | %s ]]" % (obzurl, rtinfo['CF.{Bugzilla ticket}'], rtinfo['CF.{Bugzilla ticket}'],) bzref = int(rtinfo['CF.{Bugzilla ticket}'].strip()) newbzref = bzref + 2000 full_description += "**Bugzilla Ticket**: %s => %s\n" % (obz, '{T%s}' % (newbzref,)) if header_attachments: full_description += '\n'.join(header_attachments) vlog("Ticket Info: %s" % (full_description,)) ticket = phab.maniphest.createtask(title=rtinfo['Subject'], description=full_description, projectPHIDs=phids, ccPHIDs=[], priority=rtinfo['xpriority'], auxiliary={"std:maniphest:external_reference":"rt%s" % (rtid,)}) # XXX: perms phabdb.set_task_title_transaction(ticket['phid'], botphid, 'public', 'public') phabdb.set_task_ctime(ticket['phid'], rtlib.str_to_epoch(rtinfo['Created'])) phabdb.set_task_policy(ticket['phid'], viewpolicy) #vlog(str(ordered_comments)) fmt_comments = {} for comment, contents in comment_dict.iteritems(): fmt_comment = {} dbody = contents['body'] if dbody['content'] is None and dbody['creator'] is None: continue elif dbody['content'] is None: content = 'no content found' else: mailsan = rtlib.shadow_emails(dbody['content'][0]) content_literal = [] for c in mailsan.splitlines(): content_literal.append(sanitize_text(c)) content = '\n'.join(filter(None, content_literal)) # In case of attachment but not much else if not content and dbody['attached']: content = True void_content = 'This transaction appears to have no content' if not content == True and void_content in content: content = None auto_actions = ['Outgoing email about a comment recorded by RT_System', 'Outgoing email recorded by RT_System'] if dbody['description'] in auto_actions: vlog("ignoring comment: %s/%s" % (dbody['description'], content)) continue preamble = '' cbody = '' if content: if dbody['creator'] is None: dbody['creator'] = '//creator field not set in source//' preamble += "`%s wrote:`\n\n" % (dbody['creator'].strip(),) if content == True: content = '' cbody += "%s" % (content.strip() or '//no content//',) if dbody['nvalue'] or dbody['ovalue']: value_update = '' value_update_text = rtlib.shadow_emails(dbody['description']) value_update_text = value_update_text.replace('fsck.com-rt', 'https') relations = ['Reference by ticket', 'Dependency by', 'Reference to ticket', 'Dependency on', 'Merged into ticket', 'Membership in'] states = ['open', 'resolved', 'new', 'stalled'] if any(map(lambda x: x in dbody['description'], relations)): value_update = value_update_text elif re.search('tags\s\S+\sadded', dbody['description']): value_update = "%s added tag %s" % (dbody['creator'], dbody['nvalue']) elif re.search('Taken\sby\s\S+', dbody['description']): value_update = "Issue taken by **%s**" % (dbody['creator'],) else: value_update = "//%s//" % (value_update_text,) cbody += value_update afound = contents['body']['attached'] cbody_attachments = [] for a in afound: if a in upfiles: cbody_attachments.append('{F%s}' % uploaded[a]['id']) if cbody_attachments: cbody += '\n__________________________\n\n' cbody += '\n'.join(cbody_attachments) fmt_comment['xattached'] = cbody_attachments phabm.task_comment(ticket['id'], preamble + cbody) ctransaction = phabdb.last_comment(ticket['phid']) try: created = rtlib.str_to_epoch_comments(dbody['created']) except (ValueError, TypeError): # A handful of issues seems to show NULL creation times # for now reason: see 1953 for example of NULL # 3001 for example of None elog("Could not determine comment time for %s" % (rtid,)) dbody['created'] = rtlib.str_to_epoch(rtinfo['Created']) phabdb.set_comment_time(ctransaction, created) fmt_comment['xctransaction'] = ctransaction fmt_comment['preamble'] = preamble fmt_comment['content'] = cbody fmt_comment['created'] = created fmt_comment['author'] = dbody['creator'] fmt_comment['creator'] = rtlib.user_lookup(dbody['creator']) # XXX TRX both ways? cid = len(fmt_comments.keys()) + 1 fmt_comment['count'] = cid fmt_comments[cid] = fmt_comment if rtinfo['Status'].lower() != 'open': log('setting %s to status %s' % (rtid, rtinfo['xstatus'].lower())) phabdb.set_issue_status(ticket['phid'], rtinfo['xstatus'].lower()) log("Created task: T%s (%s)" % (ticket['id'], ticket['phid'])) phabdb.set_task_mtime(ticket['phid'], rtlib.str_to_epoch(rtinfo['LastUpdated'])) xcomments = json.dumps(fmt_comments) pmig.sql_x("UPDATE rt_meta SET xcomments=%s WHERE id = %s", (xcomments, rtid)) pmig.sql_x("UPDATE rt_meta SET priority=%s, modified=%s WHERE id = %s", (ipriority['creation_success'], now(), rtid)) pmig.close() return True
def main(): parser = argparse.ArgumentParser(description='Updates user header metadata from bugzilla') parser.add_argument('-a', action="store_true", default=False) parser.add_argument('-e', action="store", dest='email') parser.add_argument('-m', action="store", dest="starting_epoch", default=None) parser.add_argument('-v', action="store_true", default=False) args = parser.parse_args() pmig = phabdb.phdb(db=config.rtmigrate_db, user=config.rtmigrate_user, passwd=config.rtmigrate_passwd) if args.a: starting_epoch = phabdb.get_user_relations_last_finish(pmig) users, finish_epoch = phabdb.get_verified_users(starting_epoch, config.bz_updatelimit) elif args.email: users = phabdb.get_verified_user(args.email) starting_epoch = 0 finish_epoch = 0 elif args.starting_epoch: users, finish_epoch = phabdb.get_verified_users(args.starting_epoch) starting_epoch = args.starting_epoch else: parser.print_help() sys.exit(1) if not any(users): log("Existing as there are no new verified users") sys.exit() histories = get_user_histories(filter(bool, users)) user_count = len(histories) icounts = [] for u in histories: c = 0 if u['cc']: c += len(u['cc']) if u['author']: c += len(u['author']) if u['assigned']: c += len(u['assigned']) icounts.append(c) issue_count = sum(icounts) log("User Count %s" % (str(user_count))) log("Issue Count %s" % (str(issue_count))) pid = os.getpid() source = util.source_name(sys.argv[0]) phabdb.user_relations_start(pid, source, int(time.time()), ipriority['na'], starting_epoch, user_count, issue_count, pmig) from multiprocessing import Pool pool = Pool(processes=int(config.bz_updatemulti)) _ = pool.map(run_update, histories) complete = len(filter(bool, _)) failed = len(_) - complete phabdb.user_relations_finish(pid, int(time.time()), ipriority['update_success'], finish_epoch, complete, failed, pmig) pmig.close() pm = phabmacros(config.phab_user, config.phab_cert, config.phab_host) vlog(util.update_blog(source, complete, failed, user_count, issue_count, pm)) print '%s completed %s, failed %s' % (sys.argv[0], complete, failed)
def fetch(PHABTICKETID): PHABTICKETID = int(PHABTICKETID) parser = ConfigParser.SafeConfigParser() parser_mode = 'phab' parser.read(configfile) phab = Phabricator(parser.get(parser_mode, 'username'), parser.get(parser_mode, 'certificate'), parser.get(parser_mode, 'host')) #dummy instance of phabapi phabm = phabmacros('', '', '') phabm.con = phab pmig = phabdb.phdb(db='fab_migration') issue = pmig.sql_x("SELECT id FROM fab_meta WHERE id = %s", PHABTICKETID) if not issue: log('%s not present for migration' % (PHABTICKETID,)) return True exists = phabdb.reference_ticket('%s%s' % (fablib.prepend, PHABTICKETID)) if exists: log('reference ticket %s already exists' % (PHABTICKETID,)) return True tid, import_priority, header, com, created, modified = pmig.sql_x("SELECT * FROM fab_meta WHERE id = %s", PHABTICKETID) vlog('priority: %d' % (import_priority,)) tinfo = json.loads(header) comments = json.loads(com) proj_phids = [] for pn in tinfo['xprojects']: proj_phids.append(phabm.ensure_project(pn)) vlog(proj_phids) priorities = {"Unbreak Now!": 100, "Needs Triage": 90, "High": 80, "Normal": 50, "Low": 25, "Needs Volunteer": 10, 0: 10, '0': 10} newticket = phab.maniphest.createtask(title=tinfo['title'], description=tinfo['description'], projectPHIDs=proj_phids, priority=priorities[tinfo['priority']], auxiliary={"std:maniphest:external_reference":"fl%s" % (PHABTICKETID,)}) phabdb.set_task_ctime(newticket['phid'], tinfo['dateCreated']) log('setting ctime of %s for %s' % (tinfo['dateCreated'], newticket['id'])) log('Created phab ticket %s for %s' % (newticket['id'], PHABTICKETID)) vlog(newticket) # 0 {'xcommenter': {u'userName': u'uvhooligan', # u'phid': u'PHID-USER-lb2dbts4cdunqxzjqf2d', # u'realName': u'Un Ver Hooligan', # u'roles': [u'unverified', u'approved', u'activated'], # u'image': u'http://fabapi.wmflabs.org/res/phabricator/3eb28cd9/rsrc/image/avatar.png', # u'uri': u'http://fabapi.wmflabs.org/p/uvhooligan/'}, # 'created': 1409875492L, 'xuseremail': None, # 'text': 'hi guys I hate email', 'last_edit': 1409875492L, # 'xuserphid': 'PHID-USER-lb2dbts4cdunqxzjqf2d'} csorted = sorted(comments.values(), key=lambda k: k['created']) for k, v in enumerate(csorted): created = epoch_to_datetime(v['created']) user = v['xcommenter']['userName'] comment_body = "**%s** wrote on `%s`\n\n%s" % (user, created, v['text']) vlog(phabm.task_comment(newticket['id'], comment_body)) if tinfo["status"] == "wontfix": tinfo["status"] = 'resolved' if tinfo['status'] != 'open': log('set status %s' % (tinfo['status'])) vlog(phabm.task_comment(newticket['id'], '//importing issue status//')) vlog(phabm.set_status(newticket['id'], tinfo['status'])) phabdb.set_task_mtime(newticket['phid'], tinfo['dateModified']) log('setting modtime of %s for %s' % (tinfo['dateModified'], newticket['id'])) pmig.close() time.sleep(1) return True
def create(bugid): phab = Phabricator(config.phab_user, config.phab_cert, config.phab_host) phabm = phabmacros('', '', '') phabm.con = phab pmig = phabdb.phdb(db=config.bzmigrate_db, user=config.bzmigrate_user, passwd=config.bzmigrate_passwd) current = pmig.sql_x("SELECT priority, \ header, \ comments, \ created, \ modified \ FROM bugzilla_meta WHERE id = %s", (bugid,)) if current: import_priority, buginfo, com, created, modified = current[0] else: pmig.close() elog('%s not present for migration' % (bugid,)) return False def get_ref(id): refexists = phabdb.reference_ticket('%s%s' % (bzlib.prepend, id)) if refexists: pmig.close() return refexists if get_ref(bugid): log('reference ticket %s already exists' % (bugid,)) return True buginfo = json.loads(buginfo) com = json.loads(com) bugid = int(bugid) vlog(bugid) vlog(buginfo) buginfo["secstate"] = 'none' # And the relevant herald rule must be in place. if bzlib.is_sensitive(buginfo["product"]): buginfo["secstate"] = 'security-bug' log("found security-bug issue %s" % (bugid,)) #this allows testing while respecting security queue if config.bz_security.lower() != 'true': log("ignoring security issue %s" % (bugid,)) return True server = xmlrpclib.ServerProxy(config.Bugzilla_url, use_datetime=True) token_data = server.User.login({'login': config.Bugzilla_login, 'password': config.Bugzilla_password}) token = token_data['token'] #http://www.bugzilla.org/docs/tip/en/html/api/Bugzilla/WebService/Bug.html#attachments kwargs = { 'ids': [bugid], 'Bugzilla_token': token } bzdata= open("data/bugzilla.yaml", 'r') bzdata_yaml = yaml.load(bzdata) tag_keys = bzdata_yaml['keywords_to_tags'].split(' ') mlists = bzdata_yaml['assigned_to_lists'].split(' ') vlog("Mailinglists: " + str(mlists)) attached = server.Bug.attachments(kwargs)['bugs'][str(bugid)] vlog("bz attached items %s" % (str(attached))) #process ticket uploads to map attach id to phab file id uploads = {} for a in attached: vlog("processing bz attachment %s" % (str(a))) if a['is_private']: vlog('ignoring private attachment: %s' % (str(a))) a['ignore'] = 'private' elif a['is_obsolete'] == 1: vlog('ignoring obsolete attachment: %s' % (str(a))) a['ignore'] = 'obsolete' else: if buginfo["secstate"] == 'none': viewpolicy = 'public' else: viewpolicy = phabdb.get_project_phid('security') try: #upload = phabm.upload_file(a['file_name'], a['data'].data) upload = phabm.upload_file(a['file_name'], a['data'].data, viewpolicy) a['phid'] = upload['phid'] a['name'] = upload['name'] a['objectName'] = upload['objectName'] except Exception as e: print "\n%s ATTACHMENT CORRUPTED -- %s\n" % (str(bugid), str(a)) print "%s --" % (str(e)) a['ignore'] = 'corrupt on retrieval' uploads[a['id']] = a log('%s attachment count: %s' % (bugid, str(len(uploads.keys())))) vlog("phab upload details: %s" % (str(uploads))) #list of projects to add to ticket ptags = [] #mask emails for public consumption buginfo['cc'] = [c.split('@')[0] for c in buginfo['cc']] # Convert bugzilla source to phabricator buginfo['status'] = bzlib.status_convert(buginfo['status'], buginfo['resolution']) buginfo['priority'] = bzlib.priority_convert(buginfo['priority']) if '-d' in sys.argv: with open('dump', 'w') as d: d.write(str(json.dumps(buginfo))) if buginfo['status'].lower() == 'patch_to_review': ptags.append(('patch_to_review', 'tags', 'green')) if buginfo['status'] == 'verified': ptags.append(('verified', 'tags')) if buginfo['cf_browser'] not in ['---', "Other"]: btag = "Browser_Support_%s" % (buginfo['cf_browser'].replace(' ', '-'),) log('Adding browser tag: %s' % (btag,)) ptags.append((btag, 'tags')) if buginfo['target_milestone'] != '---': log('Creating milestone: %s' % (buginfo['target_milestone'],)) ptags.append((buginfo['target_milestone'], 'truck')) #set defaults to be overridden by sec if needed buginfo['viewPolicy'] = 'public' buginfo['editPolicy'] = 'users' buginfo['project'] = bzlib.sanitize_project_name(buginfo["product"], buginfo["component"]) vlog(buginfo['project']) ptags.append((buginfo['project'], None)) title = buginfo['summary'] clean_com = [] for c in com: if not isinstance(c, dict): c = ast.literal_eval(c) clean_c = bzlib.build_comment(c, buginfo['secstate']) clean_com.append(clean_c) log('project: ' + buginfo['project']) try: # strip out comment 0 as description description = clean_com[0] del clean_com[0] except IndexError: log("%s has no comment 0" % (str(bugid))) # some random tasks were created at a point in bugzilla # history with metadata but no comment 0 # https://bugzilla.wikimedia.org/show_bug.cgi?id=32056 description = {'author': buginfo['creator'].split('@')[0], 'text': '//this issue has no description//', 'creation_time': buginfo['creation_time']} created = epoch_to_datetime(description['creation_time']) desc_block = "**Author:** `%s`\n\n**Description:**\n%s\n" % (description['author'], description['text']) # https://phabricator.wikimedia.org/T694 desc_tail = '--------------------------' desc_tail += "\n**Version**: %s" % (buginfo['version']) desc_tail += "\n**Severity**: %s" % (buginfo['severity'] or 'none') if buginfo['op_sys'] != 'All': desc_tail += "\n**OS**: %s" % (buginfo['op_sys']) if "platform" in buginfo and buginfo['platform'] != 'All': desc_tail += "\n**Platform**: %s" % (buginfo['platform']) if buginfo['whiteboard']: desc_tail += "\n**Whiteboard**: %s" % (buginfo['whiteboard']) if buginfo['url']: desc_tail += "\n**URL**: %s" % (buginfo['url']) if buginfo['see_also']: desc_tail += "\n**See Also**:\n%s" % ('\n'.join(buginfo['see_also'])) attachments = '' if 'attachment' in description: attached = int(description['attachment']) if attached in uploads: cattached = uploads[int(description['attachment'])] if 'objectName' in cattached: attachments = "\n\n**Attached**: {%s}" % (cattached['objectName']) if 'ignore' in cattached: attachments = "\n\n//attachment %s ignored as %s//" % (cattached['file_name'], cattached['ignore']) else: attachments = "\n\n//attachment missing in source//" desc_tail += attachments full_description = desc_block + '\n' + desc_tail keys = buginfo['keywords'] for k in keys: if k in tag_keys: if k == 'ops': k = 'operations' ptags.append((k, 'tags')) def project_security_settings(pname): if bzlib.is_sensitive(pname): ephid = phabdb.get_project_phid('security') edit = ephid else: edit = 'public' view = 'public' return edit, view phids = [] for p in ptags: edit, view = project_security_settings(p[0]) phid = phabm.ensure_project(p[0], edit=edit, view=view) phids.append(phid) if p[1] is not None: vlog("setting project %s icon to %s" % (p[0], p[1])) set_project_icon(p[0], icon=p[1]) log("ptags: " + str(ptags)) vlog("phids: " + str(phids)) #buginfo'assigned_to': u'*****@*****.**' assignee = buginfo['assigned_to'] ccphids = [] if assignee in mlists: ccphids.append(mailinglist_phid(assignee)) # viewPolicy = buginfo['viewPolicy'], # editPolicy = buginfo['editPolicy'], vlog("Ticket Info: %s" % (desc_block,)) ticket = phab.maniphest.createtask(title=buginfo['summary'], description=full_description, projectPHIDs=phids, ccPHIDs=ccphids, priority=buginfo['priority'], auxiliary={"std:maniphest:external_reference":"bz%s" % (bugid,), "std:maniphest:security_topic":"%s" % (buginfo["secstate"],)}) log("Created task: T%s (%s)" % (ticket['id'], ticket['phid'])) botphid = phabdb.get_phid_by_username(config.phab_user) phabdb.set_task_title_transaction(ticket['phid'], botphid, buginfo['viewPolicy'], buginfo['editPolicy']) phabdb.set_task_ctime(ticket['phid'], int(buginfo['creation_time'].split('.')[0])) fmt_comments = {} for c in clean_com: fmt_comment = {} created = epoch_to_datetime(c['creation_time']) comment_header = "**%s** wrote:\n\n" % (c['author'],) comment_body = c['text'] attachments = '' if 'attachment' in c: attached = int(c['attachment']) if attached in uploads: cattached = uploads[int(c['attachment'])] if 'objectName' in cattached: attachments += "\n\n**Attached**: {%s}" % (cattached['objectName']) if 'ignore' in cattached: attachments += "\n\n//attachment %s ignored as %s//" % (cattached['file_name'], cattached['ignore']) else: attachments += "\n\n//attachment missing in source//" fmt_comment['xpreamble'] = comment_header fmt_comment['xattached'] = attachments phabm.task_comment(ticket['id'], comment_header + comment_body + attachments) ctransaction = phabdb.last_comment(ticket['phid']) phabdb.set_comment_time(ctransaction, c['creation_time']) fmt_comment['xctransaction'] = ctransaction fmt_comments[c['count']] = fmt_comment if buginfo['status'] != 'open': log("setting status for T%s to %s" % (ticket['id'], buginfo['status'])) phabdb.set_issue_status(ticket['phid'], buginfo['status']) phabdb.set_task_mtime(ticket['phid'], int(buginfo['last_change_time'].split('.')[0])) xcomments = json.dumps(fmt_comments) pmig.sql_x("UPDATE bugzilla_meta \ SET xcomments=%s WHERE id = %s", (xcomments, bugid)) pmig.sql_x("UPDATE bugzilla_meta \ SET priority=%s, modified=%s WHERE id = %s", (ipriority['creation_success'], now(), bugid)) pmig.close() return True
def main(): parser = argparse.ArgumentParser(description='Updates user header metadata from bugzilla') parser.add_argument('-a', action="store_true", default=False) parser.add_argument('-e', action="store", dest='email') parser.add_argument('-m', action="store", dest="backlog", default=None) parser.add_argument('-v', action="store_true", default=False) args = parser.parse_args() pmig = phabdb.phdb(db=config.bzmigrate_db, user=config.bzmigrate_user, passwd=config.bzmigrate_passwd) if args.a: backlog = time.time() - config.bz_update_lookback log("Backlog since %d" % (backlog),) users, finish_epoch = phabdb.get_verified_users(backlog, config.bz_updatelimit) elif args.email: users = phabdb.get_verified_user(args.email) backlog = 0 finish_epoch = 0 elif args.backlog: users, finish_epoch = phabdb.get_verified_users(args.backlog) backlog = args.backlog else: parser.print_help() sys.exit(1) if not any(users): log("Existing as there are no new verified users") sys.exit() histories = get_user_histories(filter(bool, users)) user_count = len(histories) icounts = [] for u in histories: c = 0 if u['issues']: c += len(u['issues']) icounts.append(c) issue_count = sum(icounts) log("User Count %s" % (str(user_count))) log("Issue Count %s" % (str(issue_count))) pid = os.getpid() source = util.source_name(sys.argv[0]) phabdb.user_relations_start(pid, source, int(time.time()), ipriority['na'], backlog, user_count, issue_count, pmig) results = [] for h in histories: results.append(run_update(h)) complete = len(filter(bool, results)) failed = len(results) - complete phabdb.user_relations_finish(pid, int(time.time()), ipriority['update_success'], finish_epoch, complete, failed, pmig) pm = phabmacros(config.phab_user, config.phab_cert, config.phab_host) #vlog(util.update_blog(source, complete, failed, user_count, issue_count, pm)) pmig.close() print '%s completed %s, failed %s' % (sys.argv[0], complete, failed)