def run_fetch(fabid, tries=1):
    if tries == 0:
        pmig = phabdb.phdb(db='fab_migration')
        import_priority = pmig.sql_x("SELECT priority FROM fab_meta WHERE id = %s", (fabid,))
        if import_priority:
            log('updating existing record')
            pmig.sql_x("UPDATE fab_meta SET priority=%s, modified=%s WHERE id = %s", (ipriority['creation_failed'],
                                                                                     now(),
                                                                                     fabid))
        else:
            print "%s does not seem to exist" % (fabid)
        pmig.close()
        print 'failed to grab %s' % (fabid,)
        return False
    try:
        if fetch(fabid):
            try:
                pmig = phabdb.phdb(db='fab_migration')
                pandmupdate = "UPDATE fab_meta SET priority=%s, modified=%s WHERE id = %s"
                pmig.sql_x(pandmupdate, (ipriority['creation_success'],
                                         now(),
                                         fabid))
                print time.time()
                print 'done with %s' % (fabid,)
            except:
                return False
            return True
    except Exception as e:
        import traceback
        tries -= 1
        time.sleep(5)
        traceback.print_exc(file=sys.stdout)
        print 'failed to grab %s (%s)' % (fabid, e)
        return run_fetch(fabid, tries=tries)
def run_create(rtid, tries=1):
    if tries == 0:
        pmig = phabdb.phdb(db=config.rtmigrate_db)
        import_priority = pmig.sql_x("SELECT priority \
                                      FROM rt_meta \
                                      WHERE id = %s", \
                                      (rtid,))
        if import_priority:
            pmig.sql_x("UPDATE rt_meta \
                       SET priority=%s, modified=%s \
                       WHERE id = %s",
                       (ipriority['creation_failed'],
                       now(),
                       rtid))
        else:
            elog("%s does not seem to exist" % (rtid))
        elog('failed to create %s' % (rtid,))
        pmig.close()
        return False
    try:
        return create(rtid)
    except Exception as e:
        import traceback
        tries -= 1
        time.sleep(5)
        traceback.print_exc(file=sys.stdout)
        elog('failed to grab %s (%s)' % (rtid, e))
        return run_create(rtid, tries=tries)
def fetch(bugid):

    pmig = phabdb.phdb(db=config.bzmigrate_db,
                       user=config.bzmigrate_user,
                       passwd=config.bzmigrate_passwd)

    server = xmlrpclib.ServerProxy(config.Bugzilla_url, use_datetime=True)
    token_data = server.User.login({'login': config.Bugzilla_login,
                                    'password': config.Bugzilla_password})

    token = token_data['token']
    kwargs = { 'ids': [bugid], 'Bugzilla_token': token }

    #grabbing one bug at a time for now
    buginfo = server.Bug.get(kwargs)['bugs']	
    buginfo =  buginfo[0]
    # some bugs have header data but no actual content
    # https://bugzilla.wikimedia.org/show_bug.cgi?id=32738
    com = server.Bug.comments(kwargs)['bugs'][str(bugid)]['comments']

    #have to do for json
    buginfo['last_change_time'] = datetime_to_epoch(buginfo['last_change_time'])
    buginfo['creation_time'] = datetime_to_epoch(buginfo['creation_time'])

    if 'flags' in buginfo:
        for flag in buginfo['flags']:
            for k, v in flag.iteritems():
                if isinstance(v, datetime.datetime):
                    flag[k] = datetime_to_epoch(v)

    for c in com:
        c['creation_time'] = datetime_to_epoch(c['creation_time'])
        c['time'] = datetime_to_epoch(c['time'])

    # set ticket status for priority import
    status = bzlib.status_convert(buginfo['status'], buginfo['resolution'])

    if status == 'open':
        creation_priority = ipriority['unresolved']
    else:
        creation_priority = ipriority['na']

    current = pmig.sql_x("SELECT * from bugzilla_meta where id = %s", bugid)
    if current:
        update_values = (creation_priority,
                         json.dumps(buginfo),
                         json.dumps(com),
                         now(),
                         bugid)
        vlog('update: ' + str(update_values))
        pmig.sql_x("UPDATE bugzilla_meta SET priority=%s, header=%s, comments=%s, modified=%s WHERE id = %s",
                   update_values)
    else:
        insert_values =  (bugid, creation_priority, json.dumps(buginfo), json.dumps(com), now(), now())
        vlog('insert: ' + str(insert_values))
        sql = "INSERT INTO bugzilla_meta (id, priority, header, comments, created, modified) VALUES (%s, %s, %s, %s, %s, %s)"
        pmig.sql_x(sql,
                   insert_values)
    pmig.close()
    return True
def run_create(bugid, tries=1):
    if tries == 0:
        pmig = phabdb.phdb(db=config.bzmigrate_db,
                       user=config.bzmigrate_user,
                       passwd=config.bzmigrate_passwd)

        import_priority = pmig.sql_x("SELECT priority \
                                      FROM bugzilla_meta \
                                      WHERE id = %s", (bugid,))
        if import_priority:
            pmig.sql_x("UPDATE bugzilla_meta \
                        SET priority=%s \
                        WHERE id = %s", (ipriority['update_failed'],
                                         bugid))
        else:
            elog("%s does not seem to exist" % (bugid))
        pmig.close()
        elog('failed to create %s' % (bugid,))
        return False
    try:
        return create(bugid)
    except Exception as e:
        import traceback
        tries -= 1
        time.sleep(5)
        traceback.print_exc(file=sys.stdout)
        elog('failed to create %s (%s)' % (bugid, e))
        return run_create(bugid, tries=tries)
def run_fetch(bugid, tries=1):
    if tries == 0:
        pmig = phabdb.phdb(db=config.bzmigrate_db,
                       user=config.bzmigrate_user,
                       passwd=config.bzmigrate_passwd)

        current = pmig.sql_x("SELECT * from bugzilla_meta \
                              where id = %s", bugid)
        if current:
            update_values =  (ipriority['fetch_failed'], '', '', now(), bugid)
            pmig.sql_x("UPDATE bugzilla_meta SET priority=%s, \
                                                 header=%s, \
                                                 comments=%s, \
                                                 modified=%s \
                                                 WHERE id = %s",
                       update_values)
        else:
            insert_values =  (bugid, ipriority['fetch_failed'], '', '', now(), now())
            pmig.sql_x("INSERT INTO bugzilla_meta \
                        (id, priority, header, comments, modified, created) \
                        VALUES (%s, %s, %s, %s, %s, %s)",
                        insert_values)
            pmig.close()
        elog('failed to grab %s' % (bugid,))
        return False
    try:
        return fetch(bugid)
    except Exception as e:
        import traceback
        tries -= 1
        time.sleep(5)
        traceback.print_exc(file=sys.stdout)
        elog('failed to fetch %s (%s)' % (bugid, e))
        return run_fetch(bugid, tries=tries)
def main():

    phab = Phabricator(config.phab_user,
                       config.phab_cert,
                       config.phab_host)

    phabm = phabmacros('', '', '')
    phabm.con = phab
    pmig = phabdb.phdb(db=config.bzmigrate_db,
                       user=config.bzmigrate_user,
                       passwd=config.bzmigrate_passwd)

    server = xmlrpclib.ServerProxy(config.Bugzilla_url, use_datetime=True)

    bzdata = open("data/bugzilla.yaml", 'r')
    bzdata_yaml = yaml.load(bzdata)
    #product = 'Wikimedia'

    kwargs = {
              'Bugzilla_login': config.Bugzilla_login,
              'Bugzilla_password': config.Bugzilla_password}
    products = server.Product.get_selectable_products(kwargs)['ids']
    print products
    for p in products:
        kwargs = { 'ids': p,
                   'Bugzilla_login': config.Bugzilla_login,
                   'Bugzilla_password': config.Bugzilla_password}
        pi = server.Product.get(kwargs)['products'][0]
        print pi['name'], pi['description']
        for c in pi['components']:
            pname = "\n%s-%s\n\n%s" % (pi['name'], c['name'])
            print pname
def run_update(user, tries=1):
    if tries == 0:
        pmig = phabdb.phdb(db=config.rtmigrate_db,
                       user=config.rtmigrate_user,
                       passwd=config.rtmigrate_passwd)
        current = phabdb.get_user_migration_history(user['user'],
                                                    pmig)
        if current:
           failed = ipriority['update_failed_comments']
           log(phabdb.set_user_relations_priority(failed,
                                                  user['user'],
                                                  pmig))
        else:
            log('%s user does not exist to update' % (user['user']))
        pmig.close()
        elog('final fail to update %s' % (user['user'],))
        return False
    try:
        return update(user)
    except Exception as e:
        import traceback
        tries -= 1
        time.sleep(5)
        traceback.print_exc(file=sys.stdout)
        elog('failed to update %s' % (user,))
        return run_update(user, tries=tries)
def main():

    def remove(bugid):
        notice("Removing bugid %s" % (bugid,))
        log(util.remove_issue_by_bugid(bugid, bzlib.prepend))

    if not util.can_edit_ref:
        elog('%s reference field not editable on this install' % (bugid,))
        sys.exit(1)

    if 'failed' in sys.argv:
        priority = ipriority['creation_failed']
    else:
        priority = 0

    pmig = phabdb.phdb(db=config.bzmigrate_db,
                       user=config.bzmigrate_user,
                       passwd=config.bzmigrate_passwd)
    bugs = return_bug_list(dbcon=pmig, priority=priority)
    pmig.close()

    #Serious business
    if 'failed' in sys.argv or '-r' in sys.argv:
        for b in bugs:
            remove(b)

    from multiprocessing import Pool
    pool = Pool(processes=int(config.bz_createmulti))
    _ =  pool.map(run_create, bugs)
    complete = len(filter(bool, _))
    failed = len(_) - complete
    print '%s completed %s, failed %s' % (sys.argv[0], complete, failed)
def run_fetch(fabid, tries=1):
    if tries == 0:
        pmig = phabdb.phdb(db='fab_migration')
        import_priority = pmig.sql_x("SELECT priority FROM fab_meta WHERE id = %s", (fabid,))
        if import_priority:
            log('updating existing record')
            pmig.sql_x("UPDATE fab_meta SET priority=%s, modified=%s WHERE id = %s", (ipriority['fetch_failed'],
                                                                                     now(),
                                                                                     fabid))
        else:
            insert_values =  (fabid, ipriority['fetch_failed'], 'nan', 'nan', now(), now())
            pmig.sql_x("INSERT INTO fab_meta (id, priority, header, comments, created, modified) VALUES (%s, %s, %s, %s, %s, %s)",
                   insert_values)
        pmig.close()
        log('failed to grab %s' % (fabid,))
        return False
    try:
        if fetch(fabid):
            vlog(str(time.time()))
            log('done fetching %s' % (fabid,))
            return True
    except Exception as e:
        tries -= 1
        time.sleep(5)
        import traceback
        traceback.print_exc(file=sys.stdout)
        log('failed to grab %s (%s)' % (fabid, e))
        return run_fetch(fabid, tries=tries)
def get_user_histories(verified):
    histories = []
    pmig = phabdb.phdb(db=config.fabmigrate_db,
                       user=config.fabmigrate_user,
                       passwd=config.fabmigrate_passwd)
    #print 'verified', verified
    for v in verified:
        vlog(str(v))
        # Get user history from old fab system
        saved_history = phabdb.get_user_migration_history(v[1], pmig)
        if not saved_history:
            log('%s verified email has no saved history' % (v[1],))
            continue
        log('%s is being processed' % (v[1],))
        history = {}
        history['user'] = v[1]
        history['userphid'] = v[0]
        history['assigned'] = saved_history[0]
        history['cc'] = saved_history[1]
        history['author'] = saved_history[2]
        history['created'] = saved_history[3]
        history['modified'] = saved_history[4]
        histories.append(history)
    pmig.close()
    return [util.translate_json_dict_items(d) for d in histories]
def run_update(bugid, tries=1):
    if tries == 0:
        pmig = phabdb.phdb(db=config.rtmigrate_db)
        current = pmig.sql_x("SELECT * from \
                              task_relations \
                              where id = %s", bugid)
        if current:
            pmig.sql_x("UPDATE task_relations \
                        SET priority=%s, \
                        blocks=%s, \
                        modified=%s \
                        WHERE id = %s",
                        (ipriority['creation_failed'],
                        json.dumps([]), now(), bugid))
        else:
            sql = "INSERT INTO task_relations \
                   (id, priority, blocks, modified) \
                   VALUES (%s, %s, %s, %s)"
            pmig.sql_x(sql, (bugid,
                             ipriority['creation_failed'],
                             json.dumps([]),
                             now()))
        pmig.close()
        elog('final fail to update %s' % (bugid,))
        return False
    try:
        return update(bugid)
    except Exception as e:
        import traceback
        tries -= 1
        time.sleep(5)
        traceback.print_exc(file=sys.stdout)
        elog('failed to update %s' % (bugid,))
        return run_update(bugid, tries=tries)
def populate(bugid):

    def add_comment_ref(owner):    
        """ adds an issue reference to a user or later updating their comments
        """
        ouser = pmig.sql_x("SELECT user FROM user_relations_comments WHERE user = %s", (owner,))
        if ouser:
            jcommed = pmig.sql_x("SELECT issues FROM user_relations_comments WHERE user = %s", (owner,))
            if jcommed and any(tflatten(jcommed)):
                issues = json.loads(jcommed[0][0])
            else:
                issues = []

            if bugid not in issues:
                log("Comment reference %s to %s" % (str(bugid), owner))
                issues.append(bugid)
            pmig.sql_x("UPDATE user_relations_comments SET issues=%s, modified=%s WHERE user = %s", (json.dumps(issues),
                                                                                                     now(),
                                                                                                     owner))
        else:
            issues = json.dumps([bugid])
            insert_values =  (owner,
                              issues,
                              now(),
                              now())

            pmig.sql_x("INSERT INTO user_relations_comments (user, issues, created, modified) VALUES (%s, %s, %s, %s)",
                       insert_values)

    pmig = phabdb.phdb(db=config.bzmigrate_db,
                       user=config.bzmigrate_user,
                       passwd=config.bzmigrate_passwd)

    issue = pmig.sql_x("SELECT id FROM bugzilla_meta WHERE id = %s", bugid)
    if not issue:
        log('issue %s does not exist for user population' % (bugid,))
        return True

    fpriority= pmig.sql_x("SELECT priority FROM bugzilla_meta WHERE id = %s", bugid)
    if fpriority[0] == ipriority['fetch_failed']:
        log('issue %s does not fetched successfully for user population (failed fetch)' % (bugid,))
        return True

    current = pmig.sql_x("SELECT comments, xcomments, modified FROM bugzilla_meta WHERE id = %s", bugid)
    if current:
        comments, xcomments, modified = current[0]
    else:
        log('%s not present for migration' % (bugid,))
        return True

    com = json.loads(comments)
    xcom = json.loads(xcomments)
    commenters = [c['author'] for c in com if c['count'] > 0]
    commenters = set(commenters)
    log("commenters for issue %s: %s" % (bugid, str(commenters)))
    for c in commenters:
        add_comment_ref(c)
    pmig.close()
    return True
def update(user):

    phab = Phabricator(config.phab_user,
                       config.phab_cert,
                       config.phab_host)

    pmig = phabdb.phdb(db=config.rtmigrate_db,
                       user=config.rtmigrate_user,
                       passwd=config.rtmigrate_passwd)

    phabm = phabmacros('', '', '')
    phabm.con = phab

    if phabdb.is_bot(user['userphid']):
        log("%s is a bot no action" % (user['user']))
        return True

    epriority = phabdb.get_user_relations_comments_priority(user['user'],
                                                            pmig)
    if epriority and len(epriority[0]) > 0:
        if epriority[0][0] == ipriority['update_success_comments']:
            log('Skipping %s as already updated' % (user['user']))
            return True

    if not user['issues']:
        log("%s has no issues to update" % (user['user'],))
        return True

    for i in user['issues']:
        comdetails = pmig.sql_x("SELECT comments, \
                                        xcomments \
                                        FROM rt_meta \
                                        WHERE id = %s", (int(i),))
        jcom, jxcom = comdetails[0]
        coms = json.loads(jcom)
        xcoms = json.loads(jxcom)

        for key, xi in xcoms.iteritems():
            content = xi['content']
            if xi["creator"] == user['user']:
                log("Updating comment %s for %s" % (xi['xctransaction'], user['user']))
                phabdb.set_comment_author(xi['xctransaction'], user['userphid'])
                phabdb.set_comment_content(xi['xctransaction'], xi['content'])

    current = phabdb.get_user_migration_comment_history(user['user'],
                                                        pmig)
    if current:
        success = ipriority['update_success_comments']
        log(phabdb.set_user_relations_comments_priority(success,
                                                        user['user'],
                                                        pmig))
    else:
        log('%s user does not exist to update' % (user['user']))
        return False
    pmig.close()
    log(util.purge_cache())
    return True
def update(bugid):

    pmig = phabdb.phdb(db=config.rtmigrate_db)

    epriority = pmig.sql_x(
        "SELECT priority \
                            from task_relations \
                            where id = %s",
        bugid,
    )

    if epriority and epriority[0] == ipriority["update_success"]:
        log("skipping %s as blockers already updated" % (bugid,))
        return True

    query = "SELECT header FROM rt_meta WHERE id = %s"
    header = pmig.sql_x(query, (bugid,))
    if not header:
        log("no header found for %s" % (bugid,))
        return "missing"

    def extref(ticket):
        refid = phabdb.reference_ticket("%s%s" % (rtlib.prepend, ticket))
        if not refid:
            return ""
        return refid[0]

    blocker_ref = extref(bugid)
    tinfo = json.loads(header[0][0])

    if "refers_to" in tinfo["links"] and tinfo["links"]["refers_to"]:
        refers_to = []
        for b in tinfo["links"]["refers_to"]:
            refersto_ref = extref(b)
            if not refersto_ref:
                continue
            refers_to.append(phabdb.get_task_id_by_phid(refersto_ref))
        if refers_to:
            refers_block = "\n\n**Refers To:**\n"
            refers_block += "\n".join(["{T%s}" % r for r in refers_to])
            log(phabdb.append_to_task_description(blocker_ref, refers_block))

    if "refers_toby" in tinfo["links"] and tinfo["links"]["refers_toby"]:
        refers_toby = []
        for b in tinfo["links"]["refers_toby"]:
            referstoby_ref = extref(b)
            if not referstoby_ref:
                continue
            refers_toby.append(phabdb.get_task_id_by_phid(referstoby_ref))
        if refers_toby:
            refer_block = "\n\n**Referred To By:**\n"
            refer_block += "\n".join(["{T%s}" % r for r in refers_toby])
            log(phabdb.append_to_task_description(blocker_ref, refer_block))
    return True
def update(user):

    phab = Phabricator(config.phab_user,
                       config.phab_cert,
                       config.phab_host)

    pmig = phabdb.phdb(host=config.dbhost,
                       db=config.bzmigrate_db,
                       user=config.bzmigrate_user,
                       passwd=config.bzmigrate_passwd)

    phabm = phabmacros('', '', '')
    phabm.con = phab

    if phabdb.is_bot(user['userphid']):
        log("%s is a bot no action" % (user['user']))
        return True

    epriority = phabdb.get_user_relations_priority(user['user'], pmig)
    if epriority and len(epriority[0]) > 0:
        if epriority[0][0] == ipriority['update_success']:
            log('Skipping %s as already updated' % (user['user']))
            return True

    # 'author': [409, 410, 411, 404, 412],
    # 'cc': [221, 69, 203, 268, 261, 8],
    # 'created': 1410276037L,
    # 'modified': 1410276060L,
    # 'assigned': [97, 64, 150, 59, 6],
    # 'userphid': 'PHID-USER-4hsexplytovmqmcb7tq2',
    # 'user': u'*****@*****.**'}

    if user['assigned']:
        for ag in user['assigned']:
             vlog(phabm.sync_assigned(user['userphid'], ag, bzlib.prepend))

    if user['author']:
        for a in user['author']:
            vlog(phabm.synced_authored(user['userphid'], a, bzlib.prepend))

    if user['cc']:
        for ccd in user['cc']:
            vlog(phabdb.add_task_cc_by_ref(user['userphid'], ccd, bzlib.prepend))

    current = phabdb.get_user_migration_history(user['user'], pmig)
    if current:
        log(phabdb.set_user_relations_priority(ipriority['update_success'], user['user'], pmig))
    else:
        elog('%s user does not exist to update' % (user['user']))
        return False
    pmig.close()
    return True
def update(bugid):

    pmig = phabdb.phdb(db=config.bzmigrate_db,
                       user=config.bzmigrate_user,
                       passwd=config.bzmigrate_passwd)

    epriority = pmig.sql_x("SELECT priority from task_relations where id = %s", bugid)
    if epriority and epriority[0] == ipriority['update_success']:
        log('skipping %s as blockers already updated' % (bugid,))
        return True

    hq = "SELECT header FROM bugzilla_meta WHERE id = %s"
    header = pmig.sql_x(hq, (bugid,))
    if not header:
       elog('no header found for %s' % (bugid,))
       return False

    def extref(ticket):
        refid = phabdb.reference_ticket("%s%s" % (bzlib.prepend, ticket))
        if not refid:
            return ''
        return refid[0]

    blocker_ref = extref(bugid)

    tinfo = json.loads(header[0][0])
    if not tinfo['blocks']:
        log("%s doesn't block anything" % (str(bugid),))
        return True

    for b in tinfo["blocks"]:
        blocked_ref = extref(b)
        log("%s is blocking %s" % (blocker_ref, blocked_ref))
        if blocked_ref:
            log(phabdb.set_blocked_task(blocker_ref, blocked_ref))
        else:
            log('%s is missing blocker %s' % (blocked_ref, blocker_ref))

    blocks = phabdb.get_tasks_blocked(blocker_ref)
    vlog('%s is blocking %s' % (blocker_ref, str(blocks)))
    current = pmig.sql_x("SELECT * from task_relations where id = %s", bugid)
    if current:
        pmig.sql_x("UPDATE task_relations SET priority=%s, blocks=%s, modified=%s WHERE id = %s",
                   (ipriority['update_success'], json.dumps(blocks), now(), bugid))
    else:
        sql = "INSERT INTO task_relations (id, priority, blocks, modified) VALUES (%s, %s, %s, %s)"
        pmig.sql_x(sql, (bugid, ipriority['update_success'], json.dumps(blocks), now()))
    pmig.close()
    return True
def update(id):
    fabdb = phabdb.phdb(db='fab_migration')

    epriority = fabdb.sql_x("SELECT priority from task_relations where id = %s", id)
    if epriority and epriority[0] == ipriority['creation_success']:
        log('Skipping %s as blockers already updated' % (id,))
        return True

    hq = "SELECT header FROM fab_meta WHERE id = %s"
    header = fabdb.sql_x(hq, (id,))
    if not header:
       vlog('no header found for %s' % (id,))
       return True

    def extref(ticket):
        refid = phabdb.reference_ticket("%s%s" % (fablib.prepend, ticket))
        if not refid:
            return ''
        return refid[0]

    blocker_ref = extref(id)
    tinfo = json.loads(header[0])
    vlog(tinfo)
    for b in tinfo['xblocking']:
        blocked_ref = extref(b)
        log("%s is blocking %s" % (blocker_ref, blocked_ref))
        if blocked_ref:
            log(phabdb.set_blocked_task(blocker_ref, blocked_ref))
        else:
            log('%s is missing blocker %s' % (blocked_ref, blocker_ref))

    blocks = phabdb.get_tasks_blocked(blocker_ref)
    log('%s is blocking %s' % (blocker_ref, str(blocks)))
    current = fabdb.sql_x("SELECT * from task_relations where id = %s", id)
    if current:
        fabdb.sql_x("UPDATE task_relations SET priority=%s, blocks=%s, modified=%s WHERE id = %s",
                   (ipriority['creation_success'], json.dumps(blocks), now(), id))
    else:
        sql = "INSERT INTO task_relations (id, priority, blocks, modified) VALUES (%s, %s, %s, %s)"
        fabdb.sql_x(sql, (id, ipriority['creation_success'], json.dumps(blocks), now()))
    fabdb.close()
    return True
def get_user_histories(verified):
    histories = []
    pmig = phabdb.phdb(db=config.bzmigrate_db,
                       user=config.bzmigrate_user,
                       passwd=config.bzmigrate_passwd)

    for v in verified:
        vlog(str(v))
        saved_history = phabdb.get_user_migration_comment_history(v[1], pmig)
        if not saved_history:
            log('%s verified email has no saved history' % (v[1],))
            continue
        log('%s is being processed' % (v[1],))
        history = {}
        history['user'] = v[1]
        history['userphid'] = v[0]
        history['issues'] = saved_history[0]
        history['created'] = saved_history[1]
        history['modified'] = saved_history[2]
        histories.append(history)

    pmig.close()
    return [util.translate_json_dict_items(d) for d in histories]
def run_fetch(tid, tries=1):
    if tries == 0:
        pmig = phabdb.phdb(db=config.rtmigrate_db,
                       user=config.rtmigrate_user,
                       passwd=config.rtmigrate_passwd)
        insert_values =  (tid, ipriority['fetch_failed'], '', '', now(), now())

        pmig.sql_x("INSERT INTO rt_meta \
                (id, priority, header, comments, created, modified) \
                VALUES (%s, %s, %s, %s, %s, %s)",
                insert_values)
        pmig.close()
        elog('failed to grab %s' % (tid,))
        return False
    try:
        return fetch(tid)
    except Exception as e:
        import traceback
        tries -= 1
        time.sleep(5)
        traceback.print_exc(file=sys.stdout)
        elog('failed to grab %s (%s)' % (tid, e))
        return run_fetch(tid, tries=tries)
def run_update(fabid, tries=1):
    if tries == 0:
        log('final fail to grab %s' % (fabid,))
        pmig = phabdb.phdb(db='fab_migration')
        current = pmig.sql_x("SELECT * from task_relations where id = %s", fabid)
        if current:
            pmig.sql_x("UPDATE task_relations SET priority=%s, blocks=%s, modified=%s WHERE id = %s",
                      (ipriority['creation_failed'], json.dumps([]), now(), fabid))
        else:
            sql = "INSERT INTO task_relations (id, priority, blocks, modified) VALUES (%s, %s, %s, %s)"
            pmig.sql_x(sql, (fabid, ipriority['creation_failed'], json.dumps([]), now()))
        pmig.close()
        return False
    try:
        if update(fabid):
            log('%s done with %s' % (str(int(time.time())), fabid,))
            return True
    except Exception as e:
        import traceback
        tries -= 1
        time.sleep(5)
        traceback.print_exc(file=sys.stdout)
        log('failed to grab %s (%s)' % (fabid, e))
        return run_update(fabid, tries=tries)
def run_update(user, tries=1):
    if tries == 0:
        pmig = phabdb.phdb(db=config.fabmigrate_db,
                       user=config.fabmigrate_user,
                       passwd=config.fabmigrate_passwd)
        current = phabdb.get_user_migration_history(user['user'], pmig)
        if current:
           log(phabdb.set_user_relations_priority(ipriority['update_failed'], user['user'], pmig))
        else:
            log('%s user does not exist to update' % (user['user']))
        pmig.close()
        log('final fail to update %s' % (user['user'],))
        return False
    try:
        if update(user):
            log('%s done with %s' % (str(int(time.time())), user,))
            return True
    except Exception as e:
        import traceback
        tries -= 1
        time.sleep(5)
        traceback.print_exc(file=sys.stdout)
        log('failed to update %s (%s)' % (user, e))
        return run_update(user, tries=tries)
def update(bugid):

    pmig = phabdb.phdb(db=config.rtmigrate_db)

    epriority = pmig.sql_x("SELECT priority \
                            from task_relations \
                            where id = %s", bugid)

    if epriority and epriority[0] == ipriority['update_success']:
        log('skipping %s as blockers already updated' % (bugid,))
        return True

    query = "SELECT header FROM rt_meta WHERE id = %s"
    header = pmig.sql_x(query, (bugid,))
    if not header:
       elog('no header found for %s' % (bugid,))
       return 'missing'

    def extref(ticket):
        refid = phabdb.reference_ticket("%s%s" % (rtlib.prepend, ticket))
        if not refid:
            return ''
        return refid[0]

    blocker_ref = extref(bugid)
    tinfo = json.loads(header[0][0])
    
    upstream = []

    if 'parent' in tinfo['links']:
        upstream += tinfo['links']['parent']

    if 'blocks' in tinfo['links']:
        upstream += tinfo['links']['blocks']

    if upstream:
        for b in upstream:
            blocked_ref = extref(b)
            log("%s is blocking %s" % (blocker_ref,
                                       blocked_ref))
            if blocked_ref:
                log(phabdb.set_blocked_task(blocker_ref,
                                        blocked_ref))
            else:
                log('%s is missing blocker %s' % (blocked_ref,
                                              blocker_ref))
    blocks = phabdb.get_tasks_blocked(blocker_ref)
    vlog('%s is blocking %s' % (blocker_ref, str(blocks)))

    current = pmig.sql_x("SELECT * \
                          from task_relations \
                          WHERE id = %s", bugid)
    if current:
        pmig.sql_x("UPDATE task_relations \
                    SET priority=%s, blocks=%s, modified=%s \
                    WHERE id = %s",
                    (ipriority['update_success'],
                    json.dumps(blocks),
                    now(), bugid))
    else:
        sql = "INSERT INTO task_relations \
               (id, priority, blocks, modified) \
               VALUES (%s, %s, %s, %s)"
        pmig.sql_x(sql, (bugid, ipriority['update_success'],
                   json.dumps(blocks), now()))
    pmig.close()
    return True
def populate(bugid):

    pmig = phabdb.phdb(db=config.bzmigrate_db,
                       user=config.bzmigrate_user,
                       passwd=config.bzmigrate_passwd)

    issue = pmig.sql_x("SELECT id FROM bugzilla_meta WHERE id = %s", bugid)
    if not issue:
        log('issue %s does not exist for user population' % (bugid,))
        return True

    fpriority= pmig.sql_x("SELECT priority FROM bugzilla_meta WHERE id = %s", bugid)
    if fpriority[0] == ipriority['fetch_failed']:
        log('issue %s does not fetched successfully for user population (failed fetch)' % (bugid,))
        return True

    current = pmig.sql_x("SELECT priority, header, comments, created, modified FROM bugzilla_meta WHERE id = %s", bugid)
    if current:
        import_priority, buginfo, com, created, modified = current[0]
    else:
        log('%s not present for migration' % (bugid,))
        return True

    bzdata = open("data/bugzilla.yaml", 'r')
    bzdata_yaml = yaml.load(bzdata)
    mlists = bzdata_yaml['assigned_to_lists'].split(' ')
    vlog(mlists)
    header = json.loads(buginfo)
    vlog(str(header))
    relations = {}
    relations['author'] = header["creator"]
    relations['cc'] = header['cc']

    if header['assigned_to'] not in mlists:
        vlog("adding assignee %s to %s" % (header['assigned_to'], bugid))
        relations['owner'] = header['assigned_to']
    else:
        vlog("skipping %s assigned to %s" % (bugid, header['assigned_to']))
        relations['owner'] = ''


    for k, v in relations.iteritems():
        if relations[k]:
            relations[k] = filter(bool, v)

    def add_owner(owner):    
        ouser = pmig.sql_x("SELECT user FROM user_relations WHERE user = %s", (owner,))
        if ouser:
            jassigned = pmig.sql_x("SELECT assigned FROM user_relations WHERE user = %s", (owner,))
            jflat = tflatten(jassigned)
            if any(jflat):
                assigned = json.loads(jassigned[0][0])
            else:
                assigned = []
            if bugid not in assigned:
                log("Assigning %s to %s" % (str(bugid), owner))
                assigned.append(bugid)
            vlog("owner %s" % (str(assigned),))
            pmig.sql_x("UPDATE user_relations SET assigned=%s, modified=%s WHERE user = %s", (json.dumps(assigned),
                                                                                              now(),
                                                                                              owner))
        else:
            vlog('inserting new record')
            assigned = json.dumps([bugid])
            insert_values =  (owner,
                              assigned,
                              now(),
                              now())

            pmig.sql_x("INSERT INTO user_relations (user, assigned, created, modified) VALUES (%s, %s, %s, %s)",
                       insert_values)


    def add_author(author):
        euser = pmig.sql_x("SELECT user FROM user_relations WHERE user = %s", (relations['author'],))
        if euser:
            jauthored = pmig.sql_x("SELECT author FROM user_relations WHERE user = %s", (relations['author'],))
            jflat = tflatten(jauthored)
            if any(jflat):
                authored = json.loads(jauthored[0][0])
            else:
               authored = []
            if bugid not in authored:
                authored.append(bugid)
            vlog("author %s" % (str(authored),))
            pmig.sql_x("UPDATE user_relations SET author=%s, modified=%s WHERE user = %s", (json.dumps(authored),
                                                                                        now(),
                                                                                        relations['author']))
        else:
            vlog('inserting new record')
            authored = json.dumps([bugid])
            insert_values =  (relations['author'],
                              authored,
                              now(),
                              now())
            pmig.sql_x("INSERT INTO user_relations (user, author, created, modified) VALUES (%s, %s, %s, %s)",
                       insert_values)


    def add_cc(ccuser):
        eccuser = pmig.sql_x("SELECT user FROM user_relations WHERE user = %s", (ccuser,))
        if eccuser:
            jcc = pmig.sql_x("SELECT cc FROM user_relations WHERE user = %s", (ccuser,))
            jflat = tflatten(jcc)
            if any(jflat):
               cc = json.loads(jcc[0][0])
            else:
               cc = []
            if bugid not in cc:
                cc.append(bugid)
            vlog("cc %s" % (str(cc),))
            pmig.sql_x("UPDATE user_relations SET cc=%s, modified=%s WHERE user = %s", (json.dumps(cc),
                                                                                        now(),
                                                                                        ccuser))
        else:
            vlog('inserting new record')
            cc = json.dumps([bugid])
            insert_values =  (ccuser,
                              cc,
                              now(),
                              now())
            pmig.sql_x("INSERT INTO user_relations (user, cc, created, modified) VALUES (%s, %s, %s, %s)",
                   insert_values)

    if relations['author']:
        add_author(relations['author'])

    if relations['owner']:
        add_owner(relations['owner'])

    if relations['cc']:
        for u in filter(bool, relations['cc']):
            add_cc(u)

    pmig.close()
    return True
def dbcon(db):
    return phabdb.phdb(db=db,
                       host=c.dbslave,
                       user=c.phuser_user,
                       passwd=c.phuser_passwd)
def fetch(PHABTICKETID):

    PHABTICKETID = int(PHABTICKETID)

    parser = ConfigParser.SafeConfigParser()
    parser_mode = 'oldfab'
    parser.read(configfile)
    oldfab = Phabricator(parser.get(parser_mode, 'user'),
                         parser.get(parser_mode, 'cert'),
                         parser.get(parser_mode, 'host'))

    #dummy instance of phabapi
    phabm = phabmacros('', '', '')
    #assign newphab instance as self.con for dummyphab
    phabm.con = oldfab

    """
    <Result: {u'authorPHID': u'PHID-USER-qbtllnzb6pwl3ttzqa3m',
                   u'status': u'open',
                     u'phid': u'PHID-TASK-qr3fpbtk6kdx4slhgnsd',
              u'description': u'',
               u'objectName': u'T10',
                    u'title': u'Get icinga alerts into logstash',
            u'priorityColor': u'red',
       u'dependsOnTaskPHIDs': [],
                u'auxiliary': [],
                      u'uri': u'http://fab.wmflabs.org/T10',
                  u'ccPHIDs': [u'PHID-USER-qbtllnzb6pwl3ttzqa3m'],
                 u'isClosed': False,
             u'dateModified': u'1399311492',
                u'ownerPHID': None,
               u'statusName': u'Open',
              u'dateCreated': u'1391716779',
             u'projectPHIDs': [u'PHID-PROJ-5ncvaivs3upngr7ijqy2'],
                       u'id': u'10',
                 u'priority': u'High'}>
    """

    tinfo = oldfab.maniphest.info(task_id=PHABTICKETID).response
    vlog(tinfo)
    if 'objectName' in tinfo:
        log("Fetching %s" % (tinfo['objectName']))

    comments = comments_by_task(tinfo['phid'])
    for i, c in comments.iteritems():
        comments[i]['xcommenter'] = dict(oldfab.user.info(phid=c['xuserphid']))
    ordered_comments =  collections.OrderedDict(sorted(comments.items()))
    vlog(str(ordered_comments))

    """
    <Result: {u'userName': u'bd808',
                  u'phid': u'PHID-USER-qbtllnzb6pwl3ttzqa3m',
              u'realName': u'Bryan Davis',
                 u'roles': [u'admin',u'verified', u'approved', u'activated'],
                 u'image': u'http://fab.wmflabs.org/file/data/fijwoqt62w6atpond4vb/PHID-FILE-37htsfegn7bnlfvzwsts/profile-profile-gravatar',
                   u'uri': u'http://fab.wmflabs.org/p/bd808/'}>
    """

    authorInfo = oldfab.user.info(phid=tinfo['authorPHID'])
    tinfo['xauthor'] = phabdb.email_by_userphid(authorInfo['phid'])
    lauthor = tinfo['xauthor'] or 'no author'
    vlog('author: ' + lauthor)

    ccs = []
    if tinfo['ccPHIDs']:
        for c in tinfo['ccPHIDs']:
            ccInfo = oldfab.user.info(phid=c)
            ccs.append(phabdb.email_by_userphid(ccInfo['phid']))
    tinfo['xccs'] = ccs
    vlog('ccs: ' + str(ccs))

    if tinfo['ownerPHID']:
        tinfo['xowner'] = phabdb.email_by_userphid(tinfo['ownerPHID'])
    else:
        tinfo['xowner'] = None

    """
    u'data':
      {u'PHID-PROJ-5ncvaivs3upngr7ijqy2':
        {u'phid': u'PHID-PROJ-5ncvaivs3upngr7ijqy2',
         u'name': u'logstash',
    u'dateCreated': u'1391641549',
      u'members': [u'PHID-USER-65zhggegfvhojb4nynay'],
           u'id': u'3',
    u'dateModified': u'1398282408',
        u'slugs': [u'logstash']}}, u'slugMap': []}>
    """

    project_names = []
    associated_projects = tinfo['projectPHIDs']

    vlog('associated projects: %s' % (str(tinfo['projectPHIDs'])))
    #if we try to query for an empty list we get back ALLLLLL
    if associated_projects:
        pinfo = oldfab.project.query(phids=associated_projects)
        if pinfo['data']:
            for p in pinfo['data'].values():
                project_names.append(p['name'])

    def norm(pname):
        return pname.lower().replace(' ', '_').replace('-', '_')

    norm_projects = [norm(p) for p in fablib.saved]
    saved_projects = [norm(p) in norm_projects for p in project_names]
    if not any(saved_projects):
        print "Skipping %s as it's not in a saved project" % (PHABTICKETID)
        return True

    vlog('project names: ' + str(project_names))
    tinfo['xprojects'] = project_names

    status = tinfo['status']
    if status != 'open':
        creation_priority = ipriority['na']
    else:
        creation_priority = ipriority['unresolved']


    blocked_tasks = phabdb.get_tasks_blocked(tinfo['phid'])
    if blocked_tasks:
        blocked = []
        block_details = oldfab.maniphest.find(phids=blocked_tasks)
        for k, v in block_details.iteritems():
            blocked.append(v['id'])
    else:
        blocked = []

    vlog('blocking: %s' % (str(blocked)))
    tinfo['xblocking'] = blocked

    pmig = phabdb.phdb(db='fab_migration')
    current = pmig.sql_x("SELECT * from fab_meta where id = %s", PHABTICKETID)
    if current:
        log('updating current record %s' % (PHABTICKETID,))
        update_values = (creation_priority,
                         json.dumps(tinfo),
                         json.dumps(comments),
                         now(),
                         PHABTICKETID)
        pmig.sql_x("UPDATE fab_meta SET priority=%s, header=%s, comments=%s, modified=%s WHERE id = %s",
                    update_values)
    else:
        log('inserting new record %s' % (PHABTICKETID,))
        insert_values =  (PHABTICKETID,
                          creation_priority,
                          json.dumps(tinfo),
                          json.dumps(comments),
                          now(),
                          now())

        pmig.sql_x("INSERT INTO fab_meta (id, priority, header, comments, created, modified) VALUES (%s, %s, %s, %s, %s, %s)",
                   insert_values)
    pmig.close()
    return True
def create(bugid):
    phab = Phabricator(config.phab_user,
                       config.phab_cert,
                       config.phab_host)

    phabm = phabmacros('', '', '')
    phabm.con = phab

    pmig = phabdb.phdb(db=config.bzmigrate_db,
                       user=config.bzmigrate_user,
                       passwd=config.bzmigrate_passwd)

    current = pmig.sql_x("SELECT priority, \
                          header, \
                          comments, \
                          created, \
                          modified \
                          FROM bugzilla_meta WHERE id = %s",
                          (bugid,))
    if current:
        import_priority, buginfo, com, created, modified = current[0]
    else:    
        elog('%s not present for migration' % (bugid,))
        return False

    def get_ref(id):
        refexists = phabdb.reference_ticket('%s%s' % (bzlib.prepend,
                                                      id))
        if refexists:
            return refexists[0]

    buginfo = json.loads(buginfo)
    com = json.loads(com)
    bugid = int(bugid)
    vlog(bugid)
    vlog(buginfo)

    ticket = get_ref(bugid)
    print 'TICKET ', ticket

    def is_sensitive(name):
        return name.strip().lower().startswith('security')

    def project_security_settings(pname):
        if is_sensitive(pname):
            ephid = phabdb.get_project_phid('security')
            edit = ephid
        else:
            edit = 'users'
        view = 'public'
        return edit, view

    server = xmlrpclib.ServerProxy(config.Bugzilla_url, use_datetime=True)
    token_data = server.User.login({'login': config.Bugzilla_login,
                                    'password': config.Bugzilla_password})

    token = token_data['token']
    #http://www.bugzilla.org/docs/tip/en/html/api/Bugzilla/WebService/Bug.html#attachments
    kwargs = { 'ids': [bugid], 'Bugzilla_token': token }

    #list of projects to add to ticket
    ptags = []

    if buginfo['status'] == 'VERIFIED':
        vlog("Adding 'verified' to %s" % (ticket,))
        ptags.append(('verified', 'tags'))

    if buginfo['status'].lower() == 'patch_to_review':
        vlog("Adding 'Patch-For-Review' to %s" % (ticket,))
        ptags.append(('Patch-For-Review', 'tags', 'green'))

    log("status recognized as %s" % (buginfo['status'],))

    phids = []
    for p in ptags:
        edit, view = project_security_settings(p[0])
        phid = phabm.ensure_project(p[0], edit=edit, view=view)
        phids.append(phid)
        if p[1] is not None:
            vlog("setting project %s icon to %s" % (p[0], p[1]))
            set_project_icon(p[0], icon=p[1])

    for phid in phids:
        phabdb.set_related_project(ticket, phid)

    pmig.close()
    return True
def populate(rtid):

    pmig = phabdb.phdb(db=config.rtmigrate_db, user=config.rtmigrate_user, passwd=config.rtmigrate_passwd)

    issue = pmig.sql_x("SELECT id FROM rt_meta WHERE id = %s", rtid)
    if not issue:
        log("issue %s does not exist for user population" % (rtid,))
        return "missing"

    fpriority = pmig.sql_x("SELECT priority FROM rt_meta WHERE id = %s", rtid)
    if fpriority[0] == ipriority["fetch_failed"]:
        log("issue %s does not fetched successfully for user population (failed fetch)" % (rtid,))
        return True

    current = pmig.sql_x("SELECT priority, header, comments, created, modified FROM rt_meta WHERE id = %s", rtid)
    if current:
        import_priority, buginfo, com, created, modified = current[0]
    else:
        log("%s not present for migration" % (rtid,))
        return True

    header = json.loads(buginfo)
    vlog(str(header))
    relations = {}
    relations["author"] = rtlib.user_lookup(header["Creator"])
    ccusers = header["AdminCc"].split(",") + header["Cc"].split(",")
    relations["cc"] = ccusers
    relations["cc"] = [cc.strip() for cc in relations["cc"] if cc]
    # RT uses a literal nobody for no assigned
    if header["Owner"] == "Nobody":
        relations["owner"] = ""
    else:
        relations["owner"] = rtlib.user_lookup(header["Owner"])

    for k, v in relations.iteritems():
        if relations[k]:
            relations[k] = filter(bool, v)

    def add_owner(owner):
        ouser = pmig.sql_x("SELECT user FROM user_relations WHERE user = %s", (owner,))
        if ouser:
            jassigned = pmig.sql_x("SELECT assigned FROM user_relations WHERE user = %s", (owner,))
            jflat = tflatten(jassigned)
            if any(jflat):
                assigned = json.loads(jassigned[0][0])
            else:
                assigned = []
            if rtid not in assigned:
                log("Assigning %s to %s" % (str(rtid), owner))
                assigned.append(rtid)
            vlog("owner %s" % (str(assigned),))
            pmig.sql_x(
                "UPDATE user_relations SET assigned=%s, modified=%s WHERE user = %s",
                (json.dumps(assigned), now(), owner),
            )
        else:
            vlog("inserting new record")
            assigned = json.dumps([rtid])
            insert_values = (owner, assigned, now(), now())

            pmig.sql_x(
                "INSERT INTO user_relations (user, assigned, created, modified) VALUES (%s, %s, %s, %s)", insert_values
            )

    def add_author(author):
        euser = pmig.sql_x("SELECT user FROM user_relations WHERE user = %s", (relations["author"],))
        if euser:
            jauthored = pmig.sql_x("SELECT author FROM user_relations WHERE user = %s", (relations["author"],))
            jflat = tflatten(jauthored)
            if any(jflat):
                authored = json.loads(jauthored[0][0])
            else:
                authored = []
            if rtid not in authored:
                authored.append(rtid)
            vlog("author %s" % (str(authored),))
            pmig.sql_x(
                "UPDATE user_relations SET author=%s, modified=%s WHERE user = %s",
                (json.dumps(authored), now(), relations["author"]),
            )
        else:
            vlog("inserting new record")
            authored = json.dumps([rtid])
            insert_values = (relations["author"], authored, now(), now())
            pmig.sql_x(
                "INSERT INTO user_relations (user, author, created, modified) VALUES (%s, %s, %s, %s)", insert_values
            )

    def add_cc(ccuser):
        eccuser = pmig.sql_x("SELECT user FROM user_relations WHERE user = %s", (ccuser,))
        if eccuser:
            jcc = pmig.sql_x("SELECT cc FROM user_relations WHERE user = %s", (ccuser,))
            jflat = tflatten(jcc)
            if any(jflat):
                cc = json.loads(jcc[0][0])
            else:
                cc = []
            if rtid not in cc:
                cc.append(rtid)
            vlog("cc %s" % (str(cc),))
            pmig.sql_x("UPDATE user_relations SET cc=%s, modified=%s WHERE user = %s", (json.dumps(cc), now(), ccuser))
        else:
            vlog("inserting new record")
            cc = json.dumps([rtid])
            insert_values = (ccuser, cc, now(), now())
            pmig.sql_x(
                "INSERT INTO user_relations (user, cc, created, modified) VALUES (%s, %s, %s, %s)", insert_values
            )

    if relations["author"]:
        add_author(relations["author"])

    if relations["owner"]:
        add_owner(relations["owner"])

    if relations["cc"]:
        for u in filter(bool, relations["cc"]):
            add_cc(u)

    pmig.close()
    return True
def fetch(PHABTICKETID):

    PHABTICKETID = int(PHABTICKETID)

    parser = ConfigParser.SafeConfigParser()
    parser_mode = 'phab'
    parser.read(configfile)
    phab = Phabricator(parser.get(parser_mode, 'username'),
                       parser.get(parser_mode, 'certificate'),
                       parser.get(parser_mode, 'host'))

    #dummy instance of phabapi
    phabm = phabmacros('', '', '')
    phabm.con = phab

    pmig = phabdb.phdb(db='fab_migration')

    issue = pmig.sql_x("SELECT id FROM fab_meta WHERE id = %s", PHABTICKETID)
    if not issue:
        log('%s not present for migration' % (PHABTICKETID,))
        return True

    exists = phabdb.reference_ticket('%s%s' % (fablib.prepend, PHABTICKETID))
    if exists:
        log('reference ticket %s already exists' % (PHABTICKETID,))
        return True

    tid, import_priority, header, com, created, modified = pmig.sql_x("SELECT * FROM fab_meta WHERE id = %s", PHABTICKETID)

    vlog('priority: %d' % (import_priority,))

    tinfo = json.loads(header)
    comments = json.loads(com)

    proj_phids = []
    for pn in tinfo['xprojects']:
        proj_phids.append(phabm.ensure_project(pn))
    vlog(proj_phids)
    priorities = {"Unbreak Now!": 100,
                  "Needs Triage": 90,
                  "High": 80,
                  "Normal": 50,
                  "Low": 25,
                  "Needs Volunteer": 10,
                  0: 10,
                  '0': 10}

    newticket =  phab.maniphest.createtask(title=tinfo['title'],
                                 description=tinfo['description'],
                                 projectPHIDs=proj_phids,
                                 priority=priorities[tinfo['priority']],
                                 auxiliary={"std:maniphest:external_reference":"fl%s" % (PHABTICKETID,)})

    phabdb.set_task_ctime(newticket['phid'], tinfo['dateCreated'])
    log('setting ctime of %s for %s' % (tinfo['dateCreated'], newticket['id']))
    log('Created phab ticket %s for %s' % (newticket['id'], PHABTICKETID))
    vlog(newticket)

    #  0 {'xcommenter': {u'userName': u'uvhooligan', 
    #  u'phid': u'PHID-USER-lb2dbts4cdunqxzjqf2d', 
    #  u'realName': u'Un Ver Hooligan', 
    #  u'roles': [u'unverified', u'approved', u'activated'],
    #  u'image': u'http://fabapi.wmflabs.org/res/phabricator/3eb28cd9/rsrc/image/avatar.png',
    #  u'uri': u'http://fabapi.wmflabs.org/p/uvhooligan/'}, 
    #  'created': 1409875492L, 'xuseremail': None, 
    #  'text': 'hi guys I hate email', 'last_edit': 1409875492L,
    #  'xuserphid': 'PHID-USER-lb2dbts4cdunqxzjqf2d'}
    csorted = sorted(comments.values(), key=lambda k: k['created']) 
    for k, v in enumerate(csorted):
        created = epoch_to_datetime(v['created'])
        user = v['xcommenter']['userName']
        comment_body = "**%s** wrote on `%s`\n\n%s" % (user, created, v['text'])
        vlog(phabm.task_comment(newticket['id'], comment_body))

    if tinfo["status"] == "wontfix":
        tinfo["status"] = 'resolved'

    if tinfo['status'] != 'open':
        log('set status %s' % (tinfo['status']))
        vlog(phabm.task_comment(newticket['id'], '//importing issue status//'))
        vlog(phabm.set_status(newticket['id'], tinfo['status']))

    phabdb.set_task_mtime(newticket['phid'], tinfo['dateModified'])
    log('setting modtime of %s for %s' % (tinfo['dateModified'], newticket['id']))
    pmig.close()
    time.sleep(1)
    return True
def fetch(fabid):
    ausers = {}
    pmig = phabdb.phdb(db='fab_migration')
    issue = pmig.sql_x("SELECT id FROM fab_meta WHERE id = %s", fabid)

    if not issue:
        log('issue %s does not exist for user population' % (fabid,))
        return True

    fpriority= pmig.sql_x("SELECT priority FROM fab_meta WHERE id = %s", fabid)
    if fpriority[0] == ipriority['fetch_failed']:
        log('issue %s does not fetched successfully for user population (failed fetch)' % (fabid,))
        return True


    tid, import_priority, jheader, com, created, modified = pmig.sql_x("SELECT * FROM fab_meta WHERE id = %s", fabid)
    header = json.loads(jheader)
    vlog(str(header))
    relations = {}
    relations['author'] = header['xauthor']
    relations['cc'] = header['xccs']
    relations['owner'] = header['xowner']

    for k, v in relations.iteritems():
        if relations[k]:
            relations[k] = filter(bool, v)

    def add_owner(owner):    
        ouser = pmig.sql_x("SELECT user FROM user_relations WHERE user = %s", (owner,))
        if ouser:
            jassigned = pmig.sql_x("SELECT assigned FROM user_relations WHERE user = %s", (owner,))

            if jassigned[0]:
                assigned = json.loads(jassigned[0])
            else:
                assigned = []
            if fabid not in assigned:
                log("Assigning %s to %s" % (str(fabid), owner))
                assigned.append(fabid)
            vlog("owner %s" % (str(assigned),))
            pmig.sql_x("UPDATE user_relations SET assigned=%s, modified=%s WHERE user = %s", (json.dumps(assigned),
                                                                                              now(),
                                                                                              owner))
        else:
            vlog('inserting new record')
            assigned = json.dumps([fabid])
            insert_values =  (owner,
                              assigned,
                              now(),
                              now())

            pmig.sql_x("INSERT INTO user_relations (user, assigned, created, modified) VALUES (%s, %s, %s, %s)",
                       insert_values)


    def add_author(author):
        euser = pmig.sql_x("SELECT user FROM user_relations WHERE user = %s", (relations['author'],))
        if euser:
            jauthored = pmig.sql_x("SELECT author FROM user_relations WHERE user = %s", (relations['author'],))
            if jauthored[0]:
                authored = json.loads(jauthored[0])
            else:
               authored = []
            if fabid not in authored:
                authored.append(fabid)
            vlog("author %s" % (str(authored),))
            pmig.sql_x("UPDATE user_relations SET author=%s, modified=%s WHERE user = %s", (json.dumps(authored),
                                                                                        now(),
                                                                                        relations['author']))
        else:
            vlog('inserting new record')
            authored = json.dumps([fabid])
            insert_values =  (relations['author'],
                              authored,
                              now(),
                              now())
            pmig.sql_x("INSERT INTO user_relations (user, author, created, modified) VALUES (%s, %s, %s, %s)",
                       insert_values)


    def add_cc(ccuser):
        eccuser = pmig.sql_x("SELECT user FROM user_relations WHERE user = %s", (ccuser,))
        if eccuser:
            jcc = pmig.sql_x("SELECT cc FROM user_relations WHERE user = %s", (ccuser,))
            if jcc[0]:
               cc = json.loads(jcc[0])
            else:
               cc = []
            if fabid not in cc:
                cc.append(fabid)
            vlog("cc %s" % (str(cc),))
            pmig.sql_x("UPDATE user_relations SET cc=%s, modified=%s WHERE user = %s", (json.dumps(cc),
                                                                                        now(),
                                                                                        ccuser))
        else:
            vlog('inserting new record')
            cc = json.dumps([fabid])
            insert_values =  (ccuser,
                              cc,
                              now(),
                              now())
            pmig.sql_x("INSERT INTO user_relations (user, cc, created, modified) VALUES (%s, %s, %s, %s)",
                   insert_values)

    if relations['author']:
        add_author(relations['author'])

    if relations['owner']:
        add_owner(relations['owner'])

    if relations['cc']:
        for u in filter(bool, relations['cc']):
            add_cc(u)

    pmig.close()
    return True
def main():
    parser = argparse.ArgumentParser(description='Updates user header metadata from bugzilla')
    parser.add_argument('-a', action="store_true", default=False)
    parser.add_argument('-e', action="store", dest='email')
    parser.add_argument('-m', action="store", dest="starting_epoch", default=None)
    parser.add_argument('-v', action="store_true", default=False)
    args =  parser.parse_args()

    pmig = phabdb.phdb(db=config.rtmigrate_db,
                       user=config.rtmigrate_user,
                       passwd=config.rtmigrate_passwd)

    if args.a:
        starting_epoch = phabdb.get_user_relations_last_finish(pmig)
        users, finish_epoch = phabdb.get_verified_users(starting_epoch, config.bz_updatelimit)
    elif args.email:
        users = phabdb.get_verified_user(args.email)
        starting_epoch = 0
        finish_epoch = 0
    elif args.starting_epoch:
        users, finish_epoch = phabdb.get_verified_users(args.starting_epoch)
        starting_epoch = args.starting_epoch
    else:
        parser.print_help()
        sys.exit(1)

    if not any(users):
        log("Existing as there are no new verified users")
        sys.exit()

    histories = get_user_histories(filter(bool, users))
    user_count = len(histories)

    icounts = []
    for u in histories:
        c = 0
        if u['cc']:
            c += len(u['cc'])
        if u['author']:
            c += len(u['author'])
        if u['assigned']:
            c += len(u['assigned'])
        icounts.append(c)
    issue_count = sum(icounts)

    log("User Count %s" % (str(user_count)))
    log("Issue Count %s" % (str(issue_count)))

    pid = os.getpid()
    source = util.source_name(sys.argv[0])
    phabdb.user_relations_start(pid,
                                source,
                                int(time.time()),
                                ipriority['na'],
                                starting_epoch,
                                user_count, issue_count, pmig)


    from multiprocessing import Pool
    pool = Pool(processes=int(config.bz_updatemulti))
    _ =  pool.map(run_update, histories)
    complete = len(filter(bool, _))
    failed = len(_) - complete
    phabdb.user_relations_finish(pid,
                                 int(time.time()),
                                 ipriority['update_success'],
                                 finish_epoch,
                                 complete,
                                 failed,
                                 pmig)
    pmig.close()
    pm = phabmacros(config.phab_user, config.phab_cert, config.phab_host)
    vlog(util.update_blog(source, complete, failed, user_count, issue_count, pm))
    print '%s completed %s, failed %s' % (sys.argv[0], complete, failed)