def uname2uid(uname):
try:
uid = uname2uid_redis.hget(uname2uid_hash, uname)
except:
uid = None
return uid
def uname2uid(uname):
uid = uname2uid_redis.hget(uname2uid_hash, uname)
if not uid:
uid = ''
return uid
def cal_propage_work(item):
cluster_redis = R_CLUSTER_FLOW1
user = str(item['uid'])
followers_count = item['user_fansnum']
friends_count = item.get("user_friendsnum", 0)
cluster_redis.hset(user, 'user_fansnum', followers_count)
cluster_redis.hset(user, 'user_friendsnum', friends_count)
retweeted_uid = str(item['root_uid'])
retweeted_mid = str(item['root_mid'])
message_type = int(item['message_type'])
mid = str(item['mid'])
timestamp = item['timestamp']
text = item['text']
if message_type == 1:
cluster_redis.sadd('user_set', user)
cluster_redis.sadd(user + '_origin_weibo', mid)
#cluster_redis.hset(user, mid + '_origin_weibo_retweeted', 0)
#cluster_redis.hset(user, mid + '_origin_weibo_comment', 0)
#cluster_redis.hset(user, mid + '_origin_weibo_timestamp', timestamp) # origin weibo mid and timestamp
elif message_type == 2: # comment weibo
cluster_redis.sadd('user_set', user)
if cluster_redis.sismember(user + '_comment_weibo', retweeted_mid):
return
cluster_redis.sadd(user + '_comment_weibo', retweeted_mid)
RE = re.compile(u'//@([a-zA-Z-_⺀-⺙⺛-⻳⼀-⿕々〇〡-〩〸-〺〻㐀-䶵一-鿃豈-鶴侮-頻並-龎]+):', re.UNICODE)
nicknames = RE.findall(text)
queue_index = get_queue_index(timestamp)
#cluster_redis.hincrby(user, 'comment_weibo', 1)
#if 1:
if len(nicknames) == 0:
cluster_redis.hincrby(retweeted_uid, retweeted_mid + '_origin_weibo_comment', 1)
cluster_redis.hincrby(retweeted_uid, 'origin_weibo_comment_timestamp_%s' % queue_index, 1)
#cluster_redis.hset(retweeted_uid, retweeted_mid + '_origin_weibo_comment_timestamp', timestamp)
else:
nick_id = nicknames[0]
_id = r_name.hget("weibo_user", nick_id)
if _id:
cluster_redis.hincrby(str(_id), retweeted_mid + '_retweeted_weibo_comment', 1)
cluster_redis.hincrby(str(_id), 'retweeted_weibo_comment_timestamp_%s' % queue_index, 1)
#cluster_redis.hset(str(_id), retweeted_mid + '_retweeted_weibo_comment_timestamp', timestamp)
elif message_type == 3:
cluster_redis.sadd('user_set', user)
if cluster_redis.sismember(user + '_retweeted_weibo', retweeted_mid):
return
cluster_redis.sadd(user + '_retweeted_weibo', retweeted_mid)
#cluster_redis.hset(user, retweeted_mid + '_retweeted_weibo_timestamp', timestamp)
queue_index = get_queue_index(timestamp)
cluster_redis.hincrby(retweeted_uid, 'origin_weibo_retweeted_timestamp_%s' % queue_index, 1)
cluster_redis.hincrby(retweeted_uid, retweeted_mid + '_origin_weibo_retweeted', 1)
#cluster_redis.hset(user, retweeted_mid + '_retweeted_weibo_retweeted', 0)
#cluster_redis.hset(user, retweeted_mid + '_retweeted_weibo_comment', 0)
RE = re.compile(u'//@([a-zA-Z-_⺀-⺙⺛-⻳⼀-⿕々〇〡-〩〸-〺〻㐀-䶵一-鿃豈-鶴侮-頻並-龎]+):', re.UNICODE)
nicknames = RE.findall(text)
if len(nicknames) != 0:
for nick_id in nicknames:
_id = r_name.hget("weibo_user", nick_id)
if _id:
cluster_redis.hincrby(str(_id), retweeted_mid+'_retweeted_weibo_retweeted', 1)
#cluster_redis.hset(str(_id), retweeted_mid+'_retweeted_weibo_retweeted_timestamp', timestamp)
cluster_redis.hincrby(str(_id), 'retweeted_weibo_retweeted_timestamp_%s' % queue_index, 1)
def uname2uid(uname):
uid = uname2uid_redis.hget(uname2uid_hash, uname)
if not uid:
uid = ''
return uid
def uname2uid(uname):
try:
uid = uname2uid_redis.hget(uname2uid_hash, uname)
except:
uid = None
return uid
def cal_propage_work(item):
cluster_redis = R_CLUSTER_FLOW1
user = str(item['uid'])
followers_count = item['user_fansnum']
friends_count = item.get("user_friendsnum", 0)
cluster_redis.hset(user, 'user_fansnum', followers_count)
cluster_redis.hset(user, 'user_friendsnum', friends_count)
retweeted_uid = str(item['root_uid'])
retweeted_mid = str(item['root_mid'])
message_type = int(item['message_type'])
mid = str(item['mid'])
timestamp = item['timestamp']
text = item['text']
if message_type == 1:
cluster_redis.sadd('user_set', user)
cluster_redis.hset(user, mid, mid) #
cluster_redis.sadd(user + '_origin_weibo', mid)
#cluster_redis.hset(user, mid + '_origin_weibo_retweeted', 0)
#cluster_redis.hset(user, mid + '_origin_weibo_comment', 0)
#cluster_redis.hset(user, mid + '_origin_weibo_timestamp', timestamp) # origin weibo mid and timestamp
elif message_type == 2: # comment weibo
cluster_redis.sadd('user_set', user)
#if cluster_redis.sismember(user + '_comment_weibo', retweeted_mid):
# return
cluster_redis.sadd(user + '_comment_weibo', mid)
RE = re.compile(u'//@([a-zA-Z-_⺀-⺙⺛-⻳⼀-⿕々〇〡-〩〸-〺〻㐀-䶵一-鿃豈-鶴侮-頻並-龎]+):',
re.UNICODE)
nicknames = RE.findall(text)
queue_index = get_queue_index(timestamp)
#cluster_redis.hincrby(user, 'comment_weibo', 1)
cluster_redis.hincrby(retweeted_uid,
retweeted_mid + '_origin_weibo_comment', 1)
cluster_redis.hincrby(
retweeted_uid, 'origin_weibo_comment_timestamp_%s' % queue_index,
1)
cluster_redis.zincrby('influence_comment', retweeted_uid,
1) # 用户在15分钟内的转发
index_id = mapping_index(queue_index)
cluster_redis.zincrby('influence_comment_%s' % index_id, retweeted_uid,
1)
if nicknames:
for nick_id in nicknames:
_id = r_name.hget("weibo_user", nick_id)
if _id:
cluster_redis.zincrby("influence_comment_%s" % index_id,
_id, 1) # 用户在15分钟内的转发
tmp_mid = cluster_redis.hget(_id, retweeted_mid)
if tmp_mid:
cluster_redis.hincrby(
str(_id), tmp_mid + '_retweeted_weibo_comment', 1)
cluster_redis.hincrby(
str(_id), 'retweeted_weibo_comment_timestamp_%s' %
queue_index, 1)
#cluster_redis.hset(str(_id), retweeted_mid + '_retweeted_weibo_comment_timestamp', timestamp)
elif message_type == 3:
cluster_redis.sadd('user_set', user)
#if cluster_redis.sismember(user + '_retweeted_weibo', retweeted_mid):
# return
cluster_redis.hset(user, retweeted_mid, mid) # uid, root_mid, mid
cluster_redis.sadd(user + '_retweeted_weibo', mid)
#cluster_redis.hset(user, retweeted_mid + '_retweeted_weibo_timestamp', timestamp)
queue_index = get_queue_index(timestamp)
index_id = mapping_index(queue_index)
cluster_redis.hincrby(
retweeted_uid, 'origin_weibo_retweeted_timestamp_%s' % queue_index,
1)
cluster_redis.hincrby(retweeted_uid,
retweeted_mid + '_origin_weibo_retweeted', 1)
cluster_redis.zincrby('influence_retweeted', retweeted_uid,
1) # 用户在15分钟内的转发
cluster_redis.zincrby('influence_retweeted_%s' % index_id,
retweeted_uid, 1) # 用户在15分钟内的转发
#cluster_redis.hset(user, retweeted_mid + '_retweeted_weibo_retweeted', 0)
#cluster_redis.hset(user, retweeted_mid + '_retweeted_weibo_comment', 0)
RE = re.compile(u'//@([a-zA-Z-_⺀-⺙⺛-⻳⼀-⿕々〇〡-〩〸-〺〻㐀-䶵一-鿃豈-鶴侮-頻並-龎]+):',
re.UNICODE)
nicknames = RE.findall(text)
if len(nicknames) != 0:
for nick_id in nicknames:
_id = r_name.hget("weibo_user", nick_id)
if _id:
cluster_redis.zincrby('influence_retweeted_%s' % index_id,
_id, 1) # 用户在15分钟内的转发
tmp_mid = cluster_redis.hget(_id, retweeted_mid)
if tmp_mid:
cluster_redis.hincrby(
str(_id), tmp_mid + '_retweeted_weibo_retweeted',
1)
#cluster_redis.hset(str(_id), retweeted_mid+'_retweeted_weibo_retweeted_timestamp', timestamp)
cluster_redis.hincrby(
str(_id),
'retweeted_weibo_retweeted_timestamp_%s' %
queue_index, 1)
