phabdb.set_task_id(issue.id, phid)
        id = issue.id
    else:
        id = new_task['id']

    if config.have_db_access:
        if author_phid:
            phabdb.set_task_author(author_phid, id)
        phabdb.set_task_ctime(phid, issue.created_at.strftime("%s"))
        phabdb.set_task_mtime(phid, issue.updated_at.strftime("%s"))

    if issue.state != "open":
        api.set_status(id, "resolved")
        if config.have_db_access:
            if issue.closed_at:
                tphid = phabdb.last_state_change(phid)
                phabdb.set_transaction_time(tphid, issue.closed_at.strftime("%s"))
    for (author, date, comment) in issue.comments:
        print "Adding comment from %s" % author
        author_phid = api.get_phid_by_username(author)
        if author_phid is None or config.have_db_access is False:
            comment = "> Comment originally made by **%s** on //%s//\n\n%s" % (author, date, comment)
        api.task_comment(id, comment)
        if config.have_db_access:
            tphid = phabdb.last_comment(phid)
            phabdb.set_comment_time(tphid, date.strftime("%s"))
            if author_phid:
                phabdb.set_comment_author(tphid, author_phid)

util.purge_cache()
Example #2
0
    else:
        id = new_task['id']

    if config.have_db_access:
        if author_phid:
            phabdb.set_task_author(author_phid, id)
        phabdb.set_task_ctime(phid, issue.created_at.strftime("%s"))
        phabdb.set_task_mtime(phid, issue.updated_at.strftime("%s"))

    if issue.state != "open":
        api.set_status(id, "resolved")
        if config.have_db_access:
            if issue.closed_at:
                tphid = phabdb.last_state_change(phid)
                phabdb.set_transaction_time(tphid,
                                            issue.closed_at.strftime("%s"))
    for (author, date, comment) in issue.comments:
        print "Adding comment from %s" % author
        author_phid = api.get_phid_by_username(author)
        if author_phid is None or config.have_db_access is False:
            comment = "> Comment originaly made by **%s** on //%s//\n\n%s" % (
                author, date, comment)
        api.task_comment(id, comment)
        if config.have_db_access:
            tphid = phabdb.last_comment(phid)
            phabdb.set_comment_time(tphid, date.strftime("%s"))
            if author_phid:
                phabdb.set_comment_author(tphid, author_phid)

util.purge_cache()
def create(rtid):

    phab = Phabricator(config.phab_user,
                       config.phab_cert,
                       config.phab_host)

    phabm = phabmacros('', '', '')
    phabm.con = phab

    pmig = phdb(db=config.rtmigrate_db)

    response = resource.RTResource(config.rt_url,
                                   config.rt_login,
                                   config.rt_passwd,
                                   authenticators.CookieAuthenticator)

    current = pmig.sql_x("SELECT priority, header, \
                          comments, created, modified \
                          FROM rt_meta WHERE id = %s",
                          (rtid,))
    if current:
        import_priority, rtinfo, com, created, modified = current[0]
    else:
        log('%s not present for migration' % (rtid,))
        return 'missing'

    if not rtinfo:
        log("ignoring invalid data for issue %s" % (rtid,))
        return False
        
    def get_ref(id):
        refexists = phabdb.reference_ticket('%s%s' % (rtlib.prepend, id))
        if refexists:
            return refexists

    if get_ref(rtid):
        log('reference ticket %s already exists' % (rtid,))
        return True

    def remove_sig(content):
        return re.split('--\s?\n', content)[0]

    def uob(obj, encoding='utf-8'):
        """ unicode or bust"""
        if isinstance(obj, basestring):
            if not isinstance(obj, unicode):
                obj = unicode(obj, encoding)
        return obj

    def sanitize_text(line):
        if line.strip() and not line.lstrip().startswith('>'):
            # in remarkup having '--' on a new line seems to bold last
            # line so signatures really cause issues
            if all(map(lambda c: c in '-', line.strip())):
                return '%%%{0}%%%'.format(line.strip())
            elif line.strip() == '-------- Original Message --------':
                return '%%%{0}%%%'.format(line.strip())
            elif line.strip() == '---------- Forwarded message ----------':
                return '%%%{0}%%%'.format(unicode(line.strip()))
            elif line.strip().startswith('#'):
                return uob('%%%') + uob(line.strip()) + uob('%%%')
            else:
                return uob('%%%') + uob(line).strip() + uob('%%%')
        elif line.strip().startswith('>'):
            quoted_content = line.lstrip('>').strip()
            if not quoted_content.lstrip('>').strip():
                return line.strip()
            if all(map(lambda c: c in '-', quoted_content.lstrip('>').strip())):
                return "> ~~"
            else:
                return uob(line.strip())
        else:
            vlog("ignoring content line %s" % (line,))
            return None

    botphid = phabdb.get_phid_by_username(config.phab_user)
    viewpolicy = phabdb.get_project_phid('WMF-NDA')
    if not viewpolicy:
        elog("View policy group not present: %s" % (viewpolicy,))
        return False

    # Example:
    # id: ticket/8175/attachments\n
    # Attachments: 141490: (Unnamed) (multipart/mixed / 0b),
    #              141491: (Unnamed) (text/html / 23b),
    #              141492: 0jp9B09.jpg (image/jpeg / 117.4k),
    attachments = response.get(path="ticket/%s/attachments/" % (rtid,))
    if not attachments:
        raise Exception("no attachment response: %s" % (rtid))

    history = response.get(path="ticket/%s/history?format=l" % (rtid,))

    rtinfo = json.loads(rtinfo)
    comments = json.loads(com)
    vlog(rtid)
    vlog(rtinfo)

    comment_dict = {}
    for i, c in enumerate(comments):
        cwork = {}
        comment_dict[i] = cwork
        if not 'Attachments:' in c:
            pass
        attachsplit = c.split('Attachments:')
        if len(attachsplit) > 1:
            body, attached = attachsplit[0], attachsplit[1]
        else:
            body, attached = c, '0'
        comment_dict[i]['text_body'] = unicode(body)
        comment_dict[i]['attached'] = attached

    # Example:
    # Ticket: 8175\nTimeTaken: 0\n
    # Type: 
    # Create\nField:
    # Data: \nDescription: Ticket created by cpettet\n\n
    # Content: test ticket description\n\n\n
    # Creator: cpettet\nCreated: 2014-08-21 21:21:38\n\n'}
    params = {'id': 'id:(.*)',
              'ticket': 'Ticket:(.*)',
              'timetaken': 'TimeTaken:(.*)',
              'content': 'Content:(.*)',
              'creator': 'Creator:(.*)',
              'description': 'Description:(.*)',
              'created': 'Created:(.*)',
              'ovalue': 'OldValue:(.*)',
              'nvalue': 'NewValue:(.*)'}

    for k, v in comment_dict.iteritems():
        text_body = v['text_body']
        comment_dict[k]['body'] = {}
        for paramkey, regex in params.iteritems():
            value = re.search(regex, text_body)
            if value:
                comment_dict[k]['body'][paramkey] = value.group(1).strip()
            else:
                comment_dict[k]['body'][paramkey] = None

        if 'Content' in text_body:
            content = text_body.split('Content:')[1]
            content = content.split('Creator:')
            comment_dict[k]['body']['content'] = content

        creator = comment_dict[k]['body']['creator']
        if creator and '@' in creator:
            comment_dict[k]['body']['creator'] = rtlib.sanitize_email(creator)

        #15475: untitled (18.7k)
        comment_attachments= re.findall('(\d+):\s', v['attached'])
        comment_dict[k]['body']['attached'] = comment_attachments

    # due to the nature of the RT api sometimes whitespacing becomes
    # a noise comment
    if not any(comment_dict[comment_dict.keys()[0]]['body'].values()):
        vlog('dropping %s comment' % (str(comment_dict[comment_dict.keys()[0]],)))
        del comment_dict[0]

    #attachments into a dict
    def attach_to_kv(attachments_output):
        attached = re.split('Attachments:', attachments_output, 1)[1]
        ainfo = {}
        for at in attached.strip().splitlines():
            if not at:
                continue
            k, v = re.split(':', at, 1)
            ainfo[k.strip()] = v.strip()
        return ainfo

    ainfo = attach_to_kv(attachments)
    #lots of junk attachments from emailing comments and ticket creation
    ainfo_f = {}
    for k, v in ainfo.iteritems():
        if '(Unnamed)' not in v:
            ainfo_f[k] = v

    #taking attachment text and convert to tuple (name, content type, size)
    ainfo_ext = {}
    comments = re.split("\d+\/\d+\s+\(id\/.\d+\/total\)", history)
    for k, v in ainfo_f.iteritems():
        # Handle general attachment case:
        # NO: 686318802.html (application/octet-stream / 19.5k),
        # YES: Summary_686318802.pdf (application/unknown / 215.3k),
        extract = re.search('(.*)\.(\S{3,4})\s\((.*)\s\/\s(.*)\)', v)
        # due to goofy email handling of signature/x-header/meta info
        # it seems they sometimes
        # become malformed attachments.  Such as when a response into
        # rt was directed to a mailinglist
        # Example:
        #     ->Attached Message Part (text/plain / 158b)
        #
        #    Private-l mailing list
        #    [email protected]
        #    https://lists.wikimedia.org/mailman/listinfo/private-l
        if extract:
            fdetails = extract.groups()
        if not extract and v.startswith('Attached Message Part'):
            continue
        if not extract:
            extract = re.match('(\S+)\s\((.*)\/(.*)\),.*', v)
            if not extract:
                elog("attachment CORRUPT or FAILED extraction: %s %s (%s)" % (k, v, rtid))
                continue

            fdetails = extract.group(1), '', extract.group(2), extract.group(3)

        if not fdetails:
            elog("attachment CORRUPT or FAILED extraction: %s %s (%s)" % (k, v, rtid))
            continue
        ainfo_ext[k] = fdetails
        vlog(ainfo_ext[k])

    # deb
    # cgi
    attachment_types = ['pdf',
                        'jpeg',
                        'asc',
                        'tgz',
                        'csr',
                        'jpg',
                        'png',
                        'xls',
                        'xls',
                        'csv',
                        'docx',
                        'gif',
                        'html',
                        'htm',
                        'txt',
                        'diff',
                        'log',
                        'zip',
                        'rtf',
                        'tmpl',
                        'vcf',
                        'pub',
                        'sql',
                        'odt',
                        'p7s',
                        'iso',
                        'ods',
                        'conf',
                        'doc',
                        'xff',
                        'eml']

    #Uploading attachment
    dl = []
    #('Quote Summary_686318802', 'pdf', 'application/unknown', '215.3k')
    uploaded = {}
    for k, v in ainfo_ext.iteritems():
        file_extension = v[1].lower()

        # vendors have this weird habit of capitalizing extension names
        # make sure we can handle the extension type otherwise
        #if file_extension not in attachment_types:
        #    elog("Unknown Exception (%s) %s %s" % (rtid, v, file_extension))
        #    #raise Exception('unknown extension: %s (%s)' % (v, rtid))

        full = "ticket/%s/attachments/%s/content" % (rtid, k)
        vcontent = response.get(path=full, headers={'Content-Type': v[2], 'Content-Length': v[3] })
        #PDF's don't react well to stripping header -- fine without it
        if file_extension.strip() == 'pdf':
            sanscontent = ''.join(vcontent.readlines())
        else:
            vcontent = vcontent.readlines()
            sanscontent = ''.join(vcontent[2:])

        if file_extension:
            fname = "%s.%s" % (v[0], file_extension)
        else:
            fname = v[0]

        upload = phabm.upload_file(fname,
                                   sanscontent,
                                   viewpolicy)
        uploaded[k] = upload

    if rtinfo['Queue'] not in rtlib.enabled:
        log("%s not in an enabled queue" % (rtid,))
        return True

    ptags = []

    # In a practical sense ops-requets seemed to get tagged
    # with straight Operations group in Phab so we backfill
    # this for consistency.
    if rtinfo['Queue'] == 'ops-requests':
        ptags.append('operations')

    pname = rtlib.project_translate(rtinfo['Queue'])
    ptags.append(pname)

    phids = []
    for p in ptags:
        phids.append(phabm.ensure_project(p))

    rtinfo['xpriority'] = rtlib.priority_convert(rtinfo['Priority'])
    rtinfo['xstatus'] = rtlib.status_convert(rtinfo['Status'])

    import collections
    # {'ovalue': u'open',
    # 'description': u"Status changed from 'open' to 'resolved' by robh",
    # 'nvalue': None, 'creator': u'robh', 'attached': [],
    # 'timetaken': u'0', 'created': u'2011-07-01 02:47:24', 
    # 'content': [u' This transaction appears to have no content\n', u'
    #              robh\nCreated: 2011-07-01 02:47:24\n'],
    # 'ticket': u'1000', 'id': u'23192'}
    ordered_comments = collections.OrderedDict(sorted(comment_dict.items()))
    upfiles = uploaded.keys()

    # much like bugzilla comment 0 is the task description
    header = comment_dict[comment_dict.keys()[0]]
    del comment_dict[comment_dict.keys()[0]]

    dtext_san = []
    dtext_list = header['body']['content'][0].splitlines()
    for t in dtext_list:
        dtext_san.append(sanitize_text(rtlib.shadow_emails(t)))
    dtext = '\n'.join(filter(None, dtext_san))
    #dtext = '\n'.join(filter(None, sanitize_text(rtlib.shadow_emails(dtext_list))))
    full_description = "**Author:** `%s`\n\n**Description:**\n%s\n" % (rtinfo['Creator'].strip(),
                                                                       dtext)


    hafound = header['body']['attached']
    header_attachments = []
    for at in hafound:
        if at in upfiles:
            header_attachments.append('{F%s}' % uploaded[at]['id'])
    if 'CF.{Bugzilla ticket}' in rtinfo and rtinfo['CF.{Bugzilla ticket}'] or header_attachments: 
        full_description += '\n__________________________\n\n'
        if 'CF.{Bugzilla ticket}' in rtinfo and rtinfo['CF.{Bugzilla ticket}']:
            obzurl = 'https://old-bugzilla.wikimedia.org/show_bug.cgi?id='
            obz = "[[ %s%s | %s ]]" % (obzurl,
                                       rtinfo['CF.{Bugzilla ticket}'],
                                       rtinfo['CF.{Bugzilla ticket}'],)
            bzref = int(rtinfo['CF.{Bugzilla ticket}'].strip())
            newbzref = bzref + 2000
            full_description += "**Bugzilla Ticket**: %s => %s\n" % (obz, '{T%s}' % (newbzref,))
        if header_attachments:
            full_description += '\n'.join(header_attachments)

    vlog("Ticket Info: %s" % (full_description,))
    ticket =  phab.maniphest.createtask(title=rtinfo['Subject'],
                                        description=full_description,
                                        projectPHIDs=phids,
                                        ccPHIDs=[],
                                        priority=rtinfo['xpriority'],
                                        auxiliary={"std:maniphest:external_reference":"rt%s" % (rtid,)})

    # XXX: perms
    phabdb.set_task_title_transaction(ticket['phid'],
                                      botphid,
                                      'public',
                                      'public')

    phabdb.set_task_ctime(ticket['phid'], rtlib.str_to_epoch(rtinfo['Created']))
    phabdb.set_task_policy(ticket['phid'], viewpolicy)

    #vlog(str(ordered_comments))
    fmt_comments = {}
    for comment, contents in comment_dict.iteritems():
        fmt_comment = {}
        dbody = contents['body']
        if dbody['content'] is None and dbody['creator'] is None:
            continue
        elif dbody['content'] is None:
            content = 'no content found'
        else:
            mailsan = rtlib.shadow_emails(dbody['content'][0])
            content_literal = []
            for c in mailsan.splitlines():
                content_literal.append(sanitize_text(c))
            content = '\n'.join(filter(None, content_literal))

            # In case of attachment but not much else
            if not content and dbody['attached']:
                content = True

        void_content = 'This transaction appears to have no content'
        if not content == True and void_content in content:
            content = None

        auto_actions = ['Outgoing email about a comment recorded by RT_System',
                        'Outgoing email recorded by RT_System']

        if dbody['description'] in auto_actions:
            vlog("ignoring comment: %s/%s" % (dbody['description'], content))
            continue

        preamble = ''
        cbody = ''
        if content:
            if dbody['creator'] is None:
                dbody['creator'] = '//creator field not set in source//'
            preamble += "`%s  wrote:`\n\n" % (dbody['creator'].strip(),)

            if content == True:
                content = ''
            cbody += "%s" % (content.strip() or '//no content//',)

        
        if dbody['nvalue'] or dbody['ovalue']:
            value_update = ''
            value_update_text = rtlib.shadow_emails(dbody['description'])
            value_update_text = value_update_text.replace('fsck.com-rt', 'https')

            relations = ['Reference by ticket',
                         'Dependency by',
                         'Reference to ticket',
                         'Dependency on',
                         'Merged into ticket',
                         'Membership in']

            states = ['open', 'resolved', 'new', 'stalled']
            if any(map(lambda x: x in dbody['description'], relations)):
                value_update = value_update_text
            elif re.search('tags\s\S+\sadded', dbody['description']):
                value_update = "%s added tag %s" % (dbody['creator'], dbody['nvalue'])
            elif re.search('Taken\sby\s\S+', dbody['description']):
                value_update = "Issue taken by **%s**" % (dbody['creator'],)
            else:
                value_update = "//%s//" % (value_update_text,)
            cbody += value_update

        afound = contents['body']['attached']
        cbody_attachments = []
        for a in afound:
            if a in upfiles:
                cbody_attachments.append('{F%s}' % uploaded[a]['id'])
        if cbody_attachments:
            cbody += '\n__________________________\n\n'
            cbody += '\n'.join(cbody_attachments)
            fmt_comment['xattached'] = cbody_attachments

        phabm.task_comment(ticket['id'], preamble + cbody)
        ctransaction = phabdb.last_comment(ticket['phid'])

        try:    
            created = rtlib.str_to_epoch_comments(dbody['created'])
        except (ValueError, TypeError):
            # A handful of issues seems to show NULL creation times
            # for now reason: see 1953 for example of NULL
            # 3001 for example of None
            elog("Could not determine comment time for %s" % (rtid,))
            dbody['created'] = rtlib.str_to_epoch(rtinfo['Created'])

        phabdb.set_comment_time(ctransaction,
                                created)

        fmt_comment['xctransaction'] = ctransaction
        fmt_comment['preamble'] = preamble
        fmt_comment['content'] = cbody
        fmt_comment['created'] = created
        fmt_comment['author'] = dbody['creator']
        fmt_comment['creator'] = rtlib.user_lookup(dbody['creator'])

        # XXX TRX both ways?
        cid =  len(fmt_comments.keys()) + 1
        fmt_comment['count'] = cid
        fmt_comments[cid] = fmt_comment

    if rtinfo['Status'].lower() != 'open':
        log('setting %s to status %s' % (rtid, rtinfo['xstatus'].lower()))
        phabdb.set_issue_status(ticket['phid'], rtinfo['xstatus'].lower())


    log("Created task: T%s (%s)" % (ticket['id'], ticket['phid']))
    phabdb.set_task_mtime(ticket['phid'], rtlib.str_to_epoch(rtinfo['LastUpdated']))
    xcomments = json.dumps(fmt_comments)
    pmig.sql_x("UPDATE rt_meta SET xcomments=%s WHERE id = %s", (xcomments, rtid))
    pmig.sql_x("UPDATE rt_meta SET priority=%s, modified=%s WHERE id = %s",
               (ipriority['creation_success'], now(), rtid))
    pmig.close()
    return True
def create(bugid):
    phab = Phabricator(config.phab_user,
                       config.phab_cert,
                       config.phab_host)

    phabm = phabmacros('', '', '')
    phabm.con = phab

    pmig = phabdb.phdb(db=config.bzmigrate_db,
                       user=config.bzmigrate_user,
                       passwd=config.bzmigrate_passwd)

    current = pmig.sql_x("SELECT priority, \
                          header, \
                          comments, \
                          created, \
                          modified \
                          FROM bugzilla_meta WHERE id = %s",
                          (bugid,))
    if current:
        import_priority, buginfo, com, created, modified = current[0]
    else:    
        pmig.close()
        elog('%s not present for migration' % (bugid,))
        return False

    def get_ref(id):
        refexists = phabdb.reference_ticket('%s%s' % (bzlib.prepend,
                                                      id))
        if refexists:
            pmig.close()
            return refexists

    if get_ref(bugid):
        log('reference ticket %s already exists' % (bugid,))
        return True

    buginfo = json.loads(buginfo)
    com = json.loads(com)
    bugid = int(bugid)
    vlog(bugid)
    vlog(buginfo)

    buginfo["secstate"] = 'none'
    # And the relevant herald rule must be in place.
    if bzlib.is_sensitive(buginfo["product"]):
        buginfo["secstate"] = 'security-bug'
        log("found security-bug issue %s" % (bugid,))

        #this allows testing while respecting security queue
        if config.bz_security.lower() != 'true':
            log("ignoring security issue %s" % (bugid,))
            return True

    server = xmlrpclib.ServerProxy(config.Bugzilla_url, use_datetime=True)
    token_data = server.User.login({'login': config.Bugzilla_login,
                                    'password': config.Bugzilla_password})

    token = token_data['token']
    #http://www.bugzilla.org/docs/tip/en/html/api/Bugzilla/WebService/Bug.html#attachments
    kwargs = { 'ids': [bugid], 'Bugzilla_token': token }

    bzdata= open("data/bugzilla.yaml", 'r')
    bzdata_yaml = yaml.load(bzdata)
    tag_keys = bzdata_yaml['keywords_to_tags'].split(' ')
    mlists = bzdata_yaml['assigned_to_lists'].split(' ')
    vlog("Mailinglists: " + str(mlists))

    attached = server.Bug.attachments(kwargs)['bugs'][str(bugid)]
    vlog("bz attached items %s" % (str(attached)))

    #process ticket uploads to map attach id to phab file id
    uploads = {}
    for a in attached:
        vlog("processing bz attachment %s" % (str(a)))
        if a['is_private']:
            vlog('ignoring private attachment: %s' %  (str(a)))
            a['ignore'] = 'private'
        elif a['is_obsolete'] == 1:
            vlog('ignoring obsolete attachment: %s' %  (str(a)))
            a['ignore'] = 'obsolete'
        else:
            if buginfo["secstate"] == 'none':
                viewpolicy = 'public'
            else:
                viewpolicy = phabdb.get_project_phid('security')
            try:
                #upload = phabm.upload_file(a['file_name'], a['data'].data)
                upload = phabm.upload_file(a['file_name'], a['data'].data, viewpolicy)
                a['phid'] = upload['phid']
                a['name'] = upload['name']
                a['objectName'] = upload['objectName']
            except Exception as e:
                 print "\n%s ATTACHMENT CORRUPTED -- %s\n" % (str(bugid), str(a))
                 print "%s --" % (str(e))
                 a['ignore'] = 'corrupt on retrieval'

        uploads[a['id']] = a

    log('%s attachment count: %s' % (bugid, str(len(uploads.keys()))))
    vlog("phab upload details: %s" % (str(uploads)))

    #list of projects to add to ticket
    ptags = []

    #mask emails for public consumption
    buginfo['cc'] = [c.split('@')[0] for c in buginfo['cc']]

    # Convert bugzilla source to phabricator
    buginfo['status'] = bzlib.status_convert(buginfo['status'],
                                             buginfo['resolution'])

    buginfo['priority'] = bzlib.priority_convert(buginfo['priority'])

    if '-d' in sys.argv:
        with open('dump', 'w') as d:
            d.write(str(json.dumps(buginfo)))

    if buginfo['status'].lower() == 'patch_to_review':
        ptags.append(('patch_to_review', 'tags', 'green'))

    if buginfo['status'] == 'verified':
        ptags.append(('verified', 'tags'))

    if buginfo['cf_browser'] not in ['---', "Other"]:
        btag = "Browser_Support_%s" % (buginfo['cf_browser'].replace(' ', '-'),)
        log('Adding browser tag: %s' % (btag,))
        ptags.append((btag, 'tags'))

    if buginfo['target_milestone'] != '---':
        log('Creating milestone: %s' % (buginfo['target_milestone'],))
        ptags.append((buginfo['target_milestone'], 'truck'))

    #set defaults to be overridden by sec if needed
    buginfo['viewPolicy'] = 'public'
    buginfo['editPolicy'] = 'users'
    buginfo['project'] = bzlib.sanitize_project_name(buginfo["product"],
                                                     buginfo["component"])
    vlog(buginfo['project'])
    ptags.append((buginfo['project'], None))

    title = buginfo['summary']

    clean_com = []
    for c in com:
        if not isinstance(c, dict):
            c = ast.literal_eval(c)
        clean_c = bzlib.build_comment(c, buginfo['secstate'])
        clean_com.append(clean_c)

    log('project: ' + buginfo['project'])

    try:
        # strip out comment 0 as description
        description = clean_com[0]
        del clean_com[0]
    except IndexError:
        log("%s has no comment 0" % (str(bugid)))
        # some random tasks were created at a point in bugzilla
        # history with metadata but no comment 0
        # https://bugzilla.wikimedia.org/show_bug.cgi?id=32056
        description = {'author': buginfo['creator'].split('@')[0],
                       'text': '//this issue has no description//',
                       'creation_time': buginfo['creation_time']}

    created = epoch_to_datetime(description['creation_time'])
    desc_block = "**Author:** `%s`\n\n**Description:**\n%s\n" % (description['author'],
                                                                 description['text'])

    # https://phabricator.wikimedia.org/T694
    desc_tail = '--------------------------'
    desc_tail += "\n**Version**: %s" % (buginfo['version'])
    desc_tail += "\n**Severity**: %s" % (buginfo['severity'] or 'none')

    if buginfo['op_sys'] != 'All':
        desc_tail += "\n**OS**: %s" % (buginfo['op_sys'])

    if "platform" in buginfo and buginfo['platform'] != 'All':
        desc_tail += "\n**Platform**: %s" % (buginfo['platform'])

    if buginfo['whiteboard']:
        desc_tail += "\n**Whiteboard**: %s" % (buginfo['whiteboard'])

    if buginfo['url']:
        desc_tail += "\n**URL**: %s" % (buginfo['url'])

    if buginfo['see_also']:
        desc_tail += "\n**See Also**:\n%s" % ('\n'.join(buginfo['see_also']))

    attachments = ''
    if 'attachment' in description:
        attached = int(description['attachment'])
        if attached in uploads:
            cattached = uploads[int(description['attachment'])]
            if 'objectName' in cattached:
               attachments = "\n\n**Attached**: {%s}" % (cattached['objectName'])

            if 'ignore' in cattached:
                attachments = "\n\n//attachment %s ignored as %s//" % (cattached['file_name'],
                                                                       cattached['ignore'])
        else:
            attachments = "\n\n//attachment missing in source//"

    desc_tail += attachments
    full_description = desc_block + '\n' + desc_tail

    keys = buginfo['keywords']
    for k in keys:
        if k in tag_keys:
            if k == 'ops':
                k = 'operations'
            ptags.append((k, 'tags'))

    def project_security_settings(pname):
        if bzlib.is_sensitive(pname):
            ephid = phabdb.get_project_phid('security')
            edit = ephid
        else:
            edit = 'public'
        view = 'public'
        return edit, view

    phids = []
    for p in ptags:
        edit, view = project_security_settings(p[0])
        phid = phabm.ensure_project(p[0], edit=edit, view=view)
        phids.append(phid)
        if p[1] is not None:
            vlog("setting project %s icon to %s" % (p[0], p[1]))
            set_project_icon(p[0], icon=p[1])

    log("ptags: " + str(ptags))
    vlog("phids: " + str(phids))

    #buginfo'assigned_to': u'*****@*****.**'
    assignee = buginfo['assigned_to']

    ccphids = []
    if assignee in mlists:
        ccphids.append(mailinglist_phid(assignee))

    # viewPolicy = buginfo['viewPolicy'],
    # editPolicy = buginfo['editPolicy'],
    vlog("Ticket Info: %s" % (desc_block,))
    ticket = phab.maniphest.createtask(title=buginfo['summary'],
                                 description=full_description,
                                 projectPHIDs=phids,
                                 ccPHIDs=ccphids,
                                 priority=buginfo['priority'],
                                 auxiliary={"std:maniphest:external_reference":"bz%s" % (bugid,),
                                            "std:maniphest:security_topic":"%s" % (buginfo["secstate"],)})

    log("Created task: T%s (%s)" % (ticket['id'], ticket['phid']))
    botphid = phabdb.get_phid_by_username(config.phab_user)
    phabdb.set_task_title_transaction(ticket['phid'],
                                      botphid,
                                      buginfo['viewPolicy'],
                                      buginfo['editPolicy'])

    phabdb.set_task_ctime(ticket['phid'],
                          int(buginfo['creation_time'].split('.')[0]))

    fmt_comments = {}
    for c in clean_com:
        fmt_comment = {}
        created = epoch_to_datetime(c['creation_time'])
        comment_header = "**%s** wrote:\n\n" % (c['author'],)
        comment_body = c['text']
        attachments = ''
        if 'attachment' in c:
            attached = int(c['attachment'])
            if attached in uploads:
                cattached = uploads[int(c['attachment'])]
                if 'objectName' in cattached:
                    attachments += "\n\n**Attached**: {%s}" % (cattached['objectName'])

                if 'ignore' in cattached:
                    attachments += "\n\n//attachment %s ignored as %s//" % (cattached['file_name'],
                                                                          cattached['ignore'])
            else:
                attachments += "\n\n//attachment missing in source//"
        fmt_comment['xpreamble'] = comment_header
        fmt_comment['xattached'] = attachments
        phabm.task_comment(ticket['id'],
                           comment_header + comment_body + attachments)
        ctransaction = phabdb.last_comment(ticket['phid'])
        phabdb.set_comment_time(ctransaction, c['creation_time'])
        fmt_comment['xctransaction'] = ctransaction
        fmt_comments[c['count']] = fmt_comment

    if buginfo['status'] != 'open':
        log("setting status for T%s to %s" % (ticket['id'], buginfo['status']))
        phabdb.set_issue_status(ticket['phid'], buginfo['status'])

    phabdb.set_task_mtime(ticket['phid'],
                          int(buginfo['last_change_time'].split('.')[0]))
    xcomments = json.dumps(fmt_comments)
    pmig.sql_x("UPDATE bugzilla_meta \
                SET xcomments=%s WHERE id = %s", (xcomments, bugid))
    pmig.sql_x("UPDATE bugzilla_meta \
                SET priority=%s, modified=%s WHERE id = %s",
               (ipriority['creation_success'], now(), bugid))
    pmig.close()
    return True