def uname2uid(uname): try: uid = uname2uid_redis.hget(uname2uid_hash, uname) except: uid = None return uid
def uname2uid(uname): uid = uname2uid_redis.hget(uname2uid_hash, uname) if not uid: uid = '' return uid
def cal_propage_work(item): cluster_redis = R_CLUSTER_FLOW1 user = str(item['uid']) followers_count = item['user_fansnum'] friends_count = item.get("user_friendsnum", 0) cluster_redis.hset(user, 'user_fansnum', followers_count) cluster_redis.hset(user, 'user_friendsnum', friends_count) retweeted_uid = str(item['root_uid']) retweeted_mid = str(item['root_mid']) message_type = int(item['message_type']) mid = str(item['mid']) timestamp = item['timestamp'] text = item['text'] if message_type == 1: cluster_redis.sadd('user_set', user) cluster_redis.sadd(user + '_origin_weibo', mid) #cluster_redis.hset(user, mid + '_origin_weibo_retweeted', 0) #cluster_redis.hset(user, mid + '_origin_weibo_comment', 0) #cluster_redis.hset(user, mid + '_origin_weibo_timestamp', timestamp) # origin weibo mid and timestamp elif message_type == 2: # comment weibo cluster_redis.sadd('user_set', user) if cluster_redis.sismember(user + '_comment_weibo', retweeted_mid): return cluster_redis.sadd(user + '_comment_weibo', retweeted_mid) RE = re.compile(u'//@([a-zA-Z-_⺀-⺙⺛-⻳⼀-⿕々〇〡-〩〸-〺〻㐀-䶵一-鿃豈-鶴侮-頻並-龎]+):', re.UNICODE) nicknames = RE.findall(text) queue_index = get_queue_index(timestamp) #cluster_redis.hincrby(user, 'comment_weibo', 1) #if 1: if len(nicknames) == 0: cluster_redis.hincrby(retweeted_uid, retweeted_mid + '_origin_weibo_comment', 1) cluster_redis.hincrby(retweeted_uid, 'origin_weibo_comment_timestamp_%s' % queue_index, 1) #cluster_redis.hset(retweeted_uid, retweeted_mid + '_origin_weibo_comment_timestamp', timestamp) else: nick_id = nicknames[0] _id = r_name.hget("weibo_user", nick_id) if _id: cluster_redis.hincrby(str(_id), retweeted_mid + '_retweeted_weibo_comment', 1) cluster_redis.hincrby(str(_id), 'retweeted_weibo_comment_timestamp_%s' % queue_index, 1) #cluster_redis.hset(str(_id), retweeted_mid + '_retweeted_weibo_comment_timestamp', timestamp) elif message_type == 3: cluster_redis.sadd('user_set', user) if cluster_redis.sismember(user + '_retweeted_weibo', retweeted_mid): return cluster_redis.sadd(user + '_retweeted_weibo', retweeted_mid) #cluster_redis.hset(user, retweeted_mid + '_retweeted_weibo_timestamp', timestamp) queue_index = get_queue_index(timestamp) cluster_redis.hincrby(retweeted_uid, 'origin_weibo_retweeted_timestamp_%s' % queue_index, 1) cluster_redis.hincrby(retweeted_uid, retweeted_mid + '_origin_weibo_retweeted', 1) #cluster_redis.hset(user, retweeted_mid + '_retweeted_weibo_retweeted', 0) #cluster_redis.hset(user, retweeted_mid + '_retweeted_weibo_comment', 0) RE = re.compile(u'//@([a-zA-Z-_⺀-⺙⺛-⻳⼀-⿕々〇〡-〩〸-〺〻㐀-䶵一-鿃豈-鶴侮-頻並-龎]+):', re.UNICODE) nicknames = RE.findall(text) if len(nicknames) != 0: for nick_id in nicknames: _id = r_name.hget("weibo_user", nick_id) if _id: cluster_redis.hincrby(str(_id), retweeted_mid+'_retweeted_weibo_retweeted', 1) #cluster_redis.hset(str(_id), retweeted_mid+'_retweeted_weibo_retweeted_timestamp', timestamp) cluster_redis.hincrby(str(_id), 'retweeted_weibo_retweeted_timestamp_%s' % queue_index, 1)
def cal_propage_work(item): cluster_redis = R_CLUSTER_FLOW1 user = str(item['uid']) followers_count = item['user_fansnum'] friends_count = item.get("user_friendsnum", 0) cluster_redis.hset(user, 'user_fansnum', followers_count) cluster_redis.hset(user, 'user_friendsnum', friends_count) retweeted_uid = str(item['root_uid']) retweeted_mid = str(item['root_mid']) message_type = int(item['message_type']) mid = str(item['mid']) timestamp = item['timestamp'] text = item['text'] if message_type == 1: cluster_redis.sadd('user_set', user) cluster_redis.hset(user, mid, mid) # cluster_redis.sadd(user + '_origin_weibo', mid) #cluster_redis.hset(user, mid + '_origin_weibo_retweeted', 0) #cluster_redis.hset(user, mid + '_origin_weibo_comment', 0) #cluster_redis.hset(user, mid + '_origin_weibo_timestamp', timestamp) # origin weibo mid and timestamp elif message_type == 2: # comment weibo cluster_redis.sadd('user_set', user) #if cluster_redis.sismember(user + '_comment_weibo', retweeted_mid): # return cluster_redis.sadd(user + '_comment_weibo', mid) RE = re.compile(u'//@([a-zA-Z-_⺀-⺙⺛-⻳⼀-⿕々〇〡-〩〸-〺〻㐀-䶵一-鿃豈-鶴侮-頻並-龎]+):', re.UNICODE) nicknames = RE.findall(text) queue_index = get_queue_index(timestamp) #cluster_redis.hincrby(user, 'comment_weibo', 1) cluster_redis.hincrby(retweeted_uid, retweeted_mid + '_origin_weibo_comment', 1) cluster_redis.hincrby( retweeted_uid, 'origin_weibo_comment_timestamp_%s' % queue_index, 1) cluster_redis.zincrby('influence_comment', retweeted_uid, 1) # 用户在15分钟内的转发 index_id = mapping_index(queue_index) cluster_redis.zincrby('influence_comment_%s' % index_id, retweeted_uid, 1) if nicknames: for nick_id in nicknames: _id = r_name.hget("weibo_user", nick_id) if _id: cluster_redis.zincrby("influence_comment_%s" % index_id, _id, 1) # 用户在15分钟内的转发 tmp_mid = cluster_redis.hget(_id, retweeted_mid) if tmp_mid: cluster_redis.hincrby( str(_id), tmp_mid + '_retweeted_weibo_comment', 1) cluster_redis.hincrby( str(_id), 'retweeted_weibo_comment_timestamp_%s' % queue_index, 1) #cluster_redis.hset(str(_id), retweeted_mid + '_retweeted_weibo_comment_timestamp', timestamp) elif message_type == 3: cluster_redis.sadd('user_set', user) #if cluster_redis.sismember(user + '_retweeted_weibo', retweeted_mid): # return cluster_redis.hset(user, retweeted_mid, mid) # uid, root_mid, mid cluster_redis.sadd(user + '_retweeted_weibo', mid) #cluster_redis.hset(user, retweeted_mid + '_retweeted_weibo_timestamp', timestamp) queue_index = get_queue_index(timestamp) index_id = mapping_index(queue_index) cluster_redis.hincrby( retweeted_uid, 'origin_weibo_retweeted_timestamp_%s' % queue_index, 1) cluster_redis.hincrby(retweeted_uid, retweeted_mid + '_origin_weibo_retweeted', 1) cluster_redis.zincrby('influence_retweeted', retweeted_uid, 1) # 用户在15分钟内的转发 cluster_redis.zincrby('influence_retweeted_%s' % index_id, retweeted_uid, 1) # 用户在15分钟内的转发 #cluster_redis.hset(user, retweeted_mid + '_retweeted_weibo_retweeted', 0) #cluster_redis.hset(user, retweeted_mid + '_retweeted_weibo_comment', 0) RE = re.compile(u'//@([a-zA-Z-_⺀-⺙⺛-⻳⼀-⿕々〇〡-〩〸-〺〻㐀-䶵一-鿃豈-鶴侮-頻並-龎]+):', re.UNICODE) nicknames = RE.findall(text) if len(nicknames) != 0: for nick_id in nicknames: _id = r_name.hget("weibo_user", nick_id) if _id: cluster_redis.zincrby('influence_retweeted_%s' % index_id, _id, 1) # 用户在15分钟内的转发 tmp_mid = cluster_redis.hget(_id, retweeted_mid) if tmp_mid: cluster_redis.hincrby( str(_id), tmp_mid + '_retweeted_weibo_retweeted', 1) #cluster_redis.hset(str(_id), retweeted_mid+'_retweeted_weibo_retweeted_timestamp', timestamp) cluster_redis.hincrby( str(_id), 'retweeted_weibo_retweeted_timestamp_%s' % queue_index, 1)