def uname2uid(uname):
    try:
        uid = uname2uid_redis.hget(uname2uid_hash, uname)
    except:
        uid = None
    return uid
Esempio n. 2
0
def uname2uid(uname):
    uid = uname2uid_redis.hget(uname2uid_hash, uname)
    if not uid:
        uid = ''
    return uid
def cal_propage_work(item):
    cluster_redis = R_CLUSTER_FLOW1
    user = str(item['uid'])
    followers_count = item['user_fansnum']
    friends_count = item.get("user_friendsnum", 0)
    cluster_redis.hset(user, 'user_fansnum', followers_count)
    cluster_redis.hset(user, 'user_friendsnum', friends_count)

    retweeted_uid = str(item['root_uid'])
    retweeted_mid = str(item['root_mid'])

    message_type = int(item['message_type'])
    mid = str(item['mid'])
    timestamp = item['timestamp']
    text = item['text']

    if message_type == 1:
        cluster_redis.sadd('user_set', user)
        cluster_redis.sadd(user + '_origin_weibo', mid)
        #cluster_redis.hset(user, mid + '_origin_weibo_retweeted', 0)
        #cluster_redis.hset(user, mid + '_origin_weibo_comment', 0)
        #cluster_redis.hset(user, mid + '_origin_weibo_timestamp', timestamp) # origin weibo mid and timestamp

    elif message_type == 2: # comment weibo
        cluster_redis.sadd('user_set', user)
        if cluster_redis.sismember(user + '_comment_weibo', retweeted_mid):
            return 
        cluster_redis.sadd(user + '_comment_weibo', retweeted_mid)
        RE = re.compile(u'//@([a-zA-Z-_⺀-⺙⺛-⻳⼀-⿕々〇〡-〩〸-〺〻㐀-䶵一-鿃豈-鶴侮-頻並-龎]+):', re.UNICODE)
        nicknames = RE.findall(text)
        queue_index = get_queue_index(timestamp)
        #cluster_redis.hincrby(user, 'comment_weibo', 1)

        #if 1:
        if len(nicknames) == 0:
            cluster_redis.hincrby(retweeted_uid, retweeted_mid + '_origin_weibo_comment', 1) 
            cluster_redis.hincrby(retweeted_uid, 'origin_weibo_comment_timestamp_%s' % queue_index, 1)
            #cluster_redis.hset(retweeted_uid, retweeted_mid + '_origin_weibo_comment_timestamp', timestamp)

        else:
            nick_id = nicknames[0]
            _id = r_name.hget("weibo_user", nick_id)
            if _id:
                cluster_redis.hincrby(str(_id), retweeted_mid + '_retweeted_weibo_comment', 1) 
                cluster_redis.hincrby(str(_id), 'retweeted_weibo_comment_timestamp_%s' % queue_index, 1)
                #cluster_redis.hset(str(_id), retweeted_mid + '_retweeted_weibo_comment_timestamp', timestamp)

    elif message_type == 3:
        cluster_redis.sadd('user_set', user)
        if cluster_redis.sismember(user + '_retweeted_weibo', retweeted_mid):
            return

        cluster_redis.sadd(user + '_retweeted_weibo', retweeted_mid)
        #cluster_redis.hset(user, retweeted_mid + '_retweeted_weibo_timestamp', timestamp) 
        queue_index = get_queue_index(timestamp)
        cluster_redis.hincrby(retweeted_uid, 'origin_weibo_retweeted_timestamp_%s' % queue_index, 1)
        cluster_redis.hincrby(retweeted_uid, retweeted_mid + '_origin_weibo_retweeted', 1) 
        #cluster_redis.hset(user, retweeted_mid + '_retweeted_weibo_retweeted', 0)
        #cluster_redis.hset(user, retweeted_mid + '_retweeted_weibo_comment', 0)
        RE = re.compile(u'//@([a-zA-Z-_⺀-⺙⺛-⻳⼀-⿕々〇〡-〩〸-〺〻㐀-䶵一-鿃豈-鶴侮-頻並-龎]+):', re.UNICODE)
        nicknames = RE.findall(text)
        if len(nicknames) != 0:
            for nick_id in nicknames:
                _id = r_name.hget("weibo_user", nick_id)
                if _id:
                    cluster_redis.hincrby(str(_id), retweeted_mid+'_retweeted_weibo_retweeted', 1) 
                    #cluster_redis.hset(str(_id), retweeted_mid+'_retweeted_weibo_retweeted_timestamp', timestamp)
                    cluster_redis.hincrby(str(_id), 'retweeted_weibo_retweeted_timestamp_%s' % queue_index, 1)
def uname2uid(uname):
    uid = uname2uid_redis.hget(uname2uid_hash, uname)
    if not uid:
        uid = ''
    return uid
def uname2uid(uname):
    try:
        uid = uname2uid_redis.hget(uname2uid_hash, uname)
    except:
        uid = None
    return uid
Esempio n. 6
0
def cal_propage_work(item):
    cluster_redis = R_CLUSTER_FLOW1
    user = str(item['uid'])
    followers_count = item['user_fansnum']
    friends_count = item.get("user_friendsnum", 0)
    cluster_redis.hset(user, 'user_fansnum', followers_count)
    cluster_redis.hset(user, 'user_friendsnum', friends_count)

    retweeted_uid = str(item['root_uid'])
    retweeted_mid = str(item['root_mid'])

    message_type = int(item['message_type'])
    mid = str(item['mid'])
    timestamp = item['timestamp']
    text = item['text']

    if message_type == 1:
        cluster_redis.sadd('user_set', user)
        cluster_redis.hset(user, mid, mid)  #
        cluster_redis.sadd(user + '_origin_weibo', mid)
        #cluster_redis.hset(user, mid + '_origin_weibo_retweeted', 0)
        #cluster_redis.hset(user, mid + '_origin_weibo_comment', 0)
        #cluster_redis.hset(user, mid + '_origin_weibo_timestamp', timestamp) # origin weibo mid and timestamp

    elif message_type == 2:  # comment weibo
        cluster_redis.sadd('user_set', user)
        #if cluster_redis.sismember(user + '_comment_weibo', retweeted_mid):
        #    return
        cluster_redis.sadd(user + '_comment_weibo', mid)
        RE = re.compile(u'//@([a-zA-Z-_⺀-⺙⺛-⻳⼀-⿕々〇〡-〩〸-〺〻㐀-䶵一-鿃豈-鶴侮-頻並-龎]+):',
                        re.UNICODE)
        nicknames = RE.findall(text)
        queue_index = get_queue_index(timestamp)
        #cluster_redis.hincrby(user, 'comment_weibo', 1)

        cluster_redis.hincrby(retweeted_uid,
                              retweeted_mid + '_origin_weibo_comment', 1)
        cluster_redis.hincrby(
            retweeted_uid, 'origin_weibo_comment_timestamp_%s' % queue_index,
            1)
        cluster_redis.zincrby('influence_comment', retweeted_uid,
                              1)  # 用户在15分钟内的转发
        index_id = mapping_index(queue_index)
        cluster_redis.zincrby('influence_comment_%s' % index_id, retweeted_uid,
                              1)

        if nicknames:
            for nick_id in nicknames:
                _id = r_name.hget("weibo_user", nick_id)
                if _id:
                    cluster_redis.zincrby("influence_comment_%s" % index_id,
                                          _id, 1)  # 用户在15分钟内的转发
                    tmp_mid = cluster_redis.hget(_id, retweeted_mid)
                    if tmp_mid:
                        cluster_redis.hincrby(
                            str(_id), tmp_mid + '_retweeted_weibo_comment', 1)
                        cluster_redis.hincrby(
                            str(_id), 'retweeted_weibo_comment_timestamp_%s' %
                            queue_index, 1)
                #cluster_redis.hset(str(_id), retweeted_mid + '_retweeted_weibo_comment_timestamp', timestamp)

    elif message_type == 3:
        cluster_redis.sadd('user_set', user)
        #if cluster_redis.sismember(user + '_retweeted_weibo', retweeted_mid):
        #    return

        cluster_redis.hset(user, retweeted_mid, mid)  # uid, root_mid, mid
        cluster_redis.sadd(user + '_retweeted_weibo', mid)
        #cluster_redis.hset(user, retweeted_mid + '_retweeted_weibo_timestamp', timestamp)
        queue_index = get_queue_index(timestamp)
        index_id = mapping_index(queue_index)
        cluster_redis.hincrby(
            retweeted_uid, 'origin_weibo_retweeted_timestamp_%s' % queue_index,
            1)
        cluster_redis.hincrby(retweeted_uid,
                              retweeted_mid + '_origin_weibo_retweeted', 1)
        cluster_redis.zincrby('influence_retweeted', retweeted_uid,
                              1)  # 用户在15分钟内的转发
        cluster_redis.zincrby('influence_retweeted_%s' % index_id,
                              retweeted_uid, 1)  # 用户在15分钟内的转发
        #cluster_redis.hset(user, retweeted_mid + '_retweeted_weibo_retweeted', 0)
        #cluster_redis.hset(user, retweeted_mid + '_retweeted_weibo_comment', 0)
        RE = re.compile(u'//@([a-zA-Z-_⺀-⺙⺛-⻳⼀-⿕々〇〡-〩〸-〺〻㐀-䶵一-鿃豈-鶴侮-頻並-龎]+):',
                        re.UNICODE)
        nicknames = RE.findall(text)
        if len(nicknames) != 0:
            for nick_id in nicknames:
                _id = r_name.hget("weibo_user", nick_id)
                if _id:
                    cluster_redis.zincrby('influence_retweeted_%s' % index_id,
                                          _id, 1)  # 用户在15分钟内的转发
                    tmp_mid = cluster_redis.hget(_id, retweeted_mid)
                    if tmp_mid:
                        cluster_redis.hincrby(
                            str(_id), tmp_mid + '_retweeted_weibo_retweeted',
                            1)
                        #cluster_redis.hset(str(_id), retweeted_mid+'_retweeted_weibo_retweeted_timestamp', timestamp)
                        cluster_redis.hincrby(
                            str(_id),
                            'retweeted_weibo_retweeted_timestamp_%s' %
                            queue_index, 1)