class Test(unittest.TestCase):
consumer_key = ''
consumer_secret = ''
def __init__(self):
""" constructor """
def getAtt(self, key):
try:
return self.obj.__getattribute__(key)
except Exception as e:
print(e)
return ''
def setAccessToken(self, key, secret):
self.auth = OAuthHandler(self.consumer_key, self.consumer_secret)
self.auth.setAccessToken(key, secret)
self.api = API(self.auth)
def basicAuth(self, source, username, password):
self.auth = BasicAuthHandler(username, password)
self.api = API(self.auth, source=source)
def friends_timeline(self):
timeline = self.api.friends_timeline(count=2, page=1)
for line in timeline:
self.obj = line
mid = self.getAtt("id")
text = self.getAtt("text")
print("friends_timeline---" + str(mid) + ":" + text)
def comments_timeline(self):
timeline = self.api.comments_timeline(count=2, page=1)
for line in timeline:
self.obj = line
mid = self.getAtt("id")
text = self.getAtt("text")
print("comments_timeline---" + str(mid) + ":" + text)
def user_timeline(self):
timeline = self.api.user_timeline(count=5, page=1)
for line in timeline:
self.obj = line
mid = self.getAtt("id")
text = self.getAtt("text")
created_at = self.getAtt("created_at")
print("user_timeline---" + str(mid) + ":" + str(created_at) + ":" +
text)
timeline = self.api.user_timeline(count=20, page=2)
def public_timeline(self):
timeline = self.api.public_timeline(count=2, page=1)
for line in timeline:
self.obj = line
mid = self.getAtt("id")
text = self.getAtt("text")
print("public_timeline---" + str(mid) + ":" + text)
class Test(unittest.TestCase):
consumer_key=''
consumer_secret=''
def __init__(self):
""" constructor """
def getAtt(self, key):
try:
return self.obj.__getattribute__(key)
except Exception as e:
print(e)
return ''
def setAccessToken(self, key, secret):
self.auth = OAuthHandler(self.consumer_key, self.consumer_secret)
self.auth.setAccessToken(key, secret)
self.api = API(self.auth)
def basicAuth(self, source, username, password):
self.auth = BasicAuthHandler(username, password)
self.api = API(self.auth,source=source)
def friends_timeline(self):
timeline = self.api.friends_timeline(count=2, page=1)
for line in timeline:
self.obj = line
mid = self.getAtt("id")
text = self.getAtt("text")
print("friends_timeline---"+ str(mid) +":"+ text)
def comments_timeline(self):
timeline = self.api.comments_timeline(count=2, page=1)
for line in timeline:
self.obj = line
mid = self.getAtt("id")
text = self.getAtt("text")
print("comments_timeline---"+ str(mid) +":"+ text)
def user_timeline(self):
timeline = self.api.user_timeline(count=5, page=1)
for line in timeline:
self.obj = line
mid = self.getAtt("id")
text = self.getAtt("text")
created_at = self.getAtt("created_at")
print("user_timeline---"+ str(mid) +":"+ str(created_at)+":"+ text)
timeline = self.api.user_timeline(count=20, page=2)
def public_timeline(self):
timeline = self.api.public_timeline(count=2, page=1)
for line in timeline:
self.obj = line
mid = self.getAtt("id")
text = self.getAtt("text")
print("public_timeline---"+ str(mid) +":"+ text)
def mainPage(request):
session = get_current_session()
access_token_key = session['access_token_key']
access_token_secret = session['access_token_secret']
oauth_verifier = request.GET.get('oauth_verifier', '')
get_absolute_path(request)
if not access_token_key:
#params['test'] = reverse('sinaweibo.views.login', args=[], kwargs={})#, current_app=context.current_app)
login_url = reverse('sinaweibo.views.login', args=[], kwargs={})
#return shortcuts.render_to_response('test.html', params)
return http.HttpResponseRedirect(login_url)
else:
auth = OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.setToken(access_token_key, access_token_secret)
api = API(auth)
#myself = api.get_user(id=1894001933)
#screen_name = myself. __getattribute__('screen_name')
myweibo = []
myweibo_obj = api.user_timeline(count=20, page=1)
for weibo in myweibo_obj:
myweibo.append({'text': weibo.text,
'created_at': weibo.created_at,
'retweeted_status': hasattr(weibo, 'retweeted_status') and weibo.retweeted_status or None,
'source': weibo.source})
wrapper__at(myweibo)
params = {}
params['user'] = api.verify_credentials()
params['result'] = myweibo
template = get_template_uri(appContext, 'weibo.html')
return shortcuts.render_to_response(template, params)
def get(self):
from weibopy.auth import OAuthHandler
from weibopy.api import API
APP_KEY = "2567661656"
APP_SECRET = "ed0a7bd30de2cc4fbffbba61dc09e60b"
auth = OAuthHandler(APP_KEY, APP_SECRET, callback = 'http://localhost:8423/weibo')
auth_url = auth.get_authorization_url()
verifier = self.get_argument('oauth_verifier', None)
if verifier:
oauth_token = self.get_argument('oauth_token')
REQUEST_SECRET = self.get_secure_cookie('rs')
auth.set_request_token(oauth_token, REQUEST_SECRET)
auth.get_access_token(verifier)
api = API(auth)
res = []
for msg in api.user_timeline(count=10, user_id=1231453741):
txt = msg.__getattribute__('text')
res.append(txt)
self.write(json.dumps(res))
else:
self.set_secure_cookie('rs', auth.request_token.secret)
self.set_secure_cookie('ot', auth.request_token.key)
self.redirect(auth_url, permanent=True)
'''
Access token key: 6c21ec2ce87bf7b6c2955feee8663bc1
Access token secret: 900473fd7334c64fb7ee18f2ae6b9cf0
oauth_token_secret=900473fd7334c64fb7ee18f2ae6b9cf0&oauth_token=6c21ec2ce87bf7b6c2955feee8663bc1
api = API(auth)
status = api.update_status(status="Hello Fouland")
'''
pass
def getWeibo(self,name):
self.auth.setToken(self.key,self.secret)
api = API(self.auth)
start = time.time()
if self.weiboflag == False:
try:
timeline = api.user_timeline(screen_name=name,count=50)
except Exception ,e:
print e
for node in timeline:
lst = []
lst.append(node.text.encode('utf-8'))
tmp = self.getStatus(api,node.id)
if tmp == []:
lst.extend([0,0])
else:
lst.extend(tmp)
refer = getattr(node,'retweeted_status',None)
if refer:
lst.append(refer.text.encode('utf-8'))
#print refer.mid
#print refer.id
tmp = self.getStatus(api,refer.mid)
#print tmp
if tmp == []:
lst.extend[0,0]
else:
lst.extend(tmp)
else:
lst.extend(['None',0,0])
#print (len(lst))
self.updateWeibo(lst)
def get(self):
auth = BasicAuthHandler("user", "password")
api = API(auth, source="3990552168")
timeline = api.user_timeline(count=1, page=1)[0]
tweet = Tweet.all().get()
if not tweet:
tweet = Tweet(text=timeline.text, date=timeline.created_at)
tweet.text = timeline.text
tweet.date = timeline.created_at
tweet.put()
class WeiboBackup(object):
"""
新浪微博自动备份.
"""
def __init__(self):
self.hdl = OAuthHandler(config.APP_KEY, config.APP_SECRET)
self.api = None
self.writer = None
self.token = {}
self.auth()
def auth(self):
try:
with open("../" + config.TOKEN_FILE) as f:
self.token = pickle.load(f)
self.hdl.setToken(self.token["key"], self.token["secret"])
self.api = API(self.hdl)
except Exception as e:
print e
def get_auth_url(self):
return self.hdl.get_authorization_url()
def get_data(self, screen_name, page):
count = 200
while True:
try:
res = self.api.user_timeline(screen_name=screen_name, count=count, page=page)
if len(res) == 0:
return page
else:
for status in res:
text = status.text
retweet = getattr(status, "retweeted_status", False)
if retweet:
text = text + "//" + retweet.text
text = text.encode("utf-8")
self.writer.append(text)
page = page + 1
except Exception as e:
print e
def backup(self, screen_name, filename=""):
if filename:
self.writer = Writer(filename)
else:
self.writer = []
page, alert_num = 1, 0
while alert_num < ALERT_MAX_TIMES:
page = self.get_data(screen_name, page)
alert_num += 1
return self.writer
class Bot:
def __init__(self):
BACK_URL = ""
#验证开发者密钥.
auth = OAuthHandler(APP_KEY, APP_SECRET, BACK_URL)
auth.setToken(BOT_TOKEN_KEY, BOT_TOKEN_SECRET)
self.api = API(auth)
def send(self, message):
try:
return self.api.update_status(message)
except WeibopError, e:
return self.api.user_timeline(count=1)[0]
class Bot:
def __init__(self):
BACK_URL = ""
#验证开发者密钥.
auth = OAuthHandler( APP_KEY, APP_SECRET, BACK_URL )
auth.setToken( BOT_TOKEN_KEY, BOT_TOKEN_SECRET )
self.api = API(auth)
def send(self, message):
try:
return self.api.update_status(message)
except WeibopError, e:
return self.api.user_timeline(count=1)[0]
def handle(self, *args, **options):
print('Start classifying influence area\n')
YORO_CONSUMER_KEY = settings.SINA_CONSUMER_KEY
YORO_CONSUMER_SECRET = settings.SINA_CONSUMER_SECRET
auth = OAuthHandler(YORO_CONSUMER_KEY, YORO_CONSUMER_SECRET)
auth.setToken('128021658f2bfdae185d89bdffb3cede','1713185d5c8208e8f1ef27a1f484ebc9')
api = API(auth)
mid_list = []
rt_count = 0 #Count of status that retweet my status
ids_list = []
tried_count = 0
while True:
timeline = api.user_timeline(count=200,user_id=1275017594)
if len(timeline) == 0:
tried_count += 1
#print 'try again in getting timeline'
else:
break
if tried_count > 3:
raise Exception('weibo api error. No timeline got')
break
for line in timeline:
text = getAtt(line, 'text')
retweet = getAtt(line, 'retweet')
if retweet:
text += getAtt(retweet, 'text')
if is_fashion(text):
mid_list.append(str(getAtt(line, "id")))
if len(mid_list) == 20:
ids_list.append(','.join(mid_list))
mid_list = []
if mid_list: #append the remaining ids
ids_list.append(','.join(mid_list))
for ids in ids_list:
counts = api.counts(ids=ids)
for obj in counts:
rt_count += getAtt(obj, 'rt')
sample_size = (len(ids_list)-1)*20 + len(ids_list[-1]) if ids_list else 0
influence_count = long(rt_count * 50 * 3990 * 1.0/(sample_size+1)) #indirect influence
influence_count += 3899834 #direct influence
print influence_count
print('Finished classifying influence area\n')
try:
#ImagePath为图片在操作系统中的访问地址，其余同上.
api.upload( ImagePath, message, lat, long );
except WeibopError, e:
return e.reason;
# 五、获取微博消息列表
#设定用户令牌密钥.
auth.setToken( atKey, atSecret );
#绑定用户验证信息.
api = API(auth);
WeiboList = [];
#获取微博列表.
#count为每页消息数量，page为从1开始计数的页码.
try:
timeline = api.user_timeline( count = count,
page = page );
except:
return None;
#对微博列表中的微博信息进行逐个枚举.
for status in timeline:
weibo = {};
#微博id
weibo["id"] = status.id;
#微博创建时间.
weibo["created"] = status.created_at;
#微博发布用户.
weibo["user"] = status.user.name.encode('utf-8');
#微博文字.
class WeiboBackup(object):
"""
新浪微博自动备份.
"""
def __init__(self):
self.hdl = OAuthHandler(APP_KEY, APP_SECRET)
self.api = None
self.writer = None
self.token = {}
def auth(self, pin):
try:
token = self.hdl.get_access_token(pin)
"""
self.token = dict(parse_qsl(token))
self.hdl.setToken(
self.token['oauth_token'],
self.token['oauth_token_secret']
)
self.hdl.setToken(
ACCESS_TOKEN,
ACCESS_SECRET
)
"""
self.api = API(self.hdl)
except Exception as e:
print e
def get_auth_url(self):
return self.hdl.get_authorization_url()
def get_data(self, screen_name, page):
count = 200
while True:
try:
res = self.api.user_timeline(
screen_name=screen_name,
count=count,
page=page
)
if len(res)==0:
return page
else:
for status in res:
text = status.text
retweet = getattr(
status,
"retweeted_status",
False
)
if retweet:
text = text+"//"+retweet.text
text = text.encode("utf-8")
self.writer.append(text)
page = page+1
except Exception as e:
print e
def backup(self, screen_name, filename):
self.writer = Writer(filename)
page,alert_num = 1,0
while alert_num<ALERT_MAX_TIMES:
page = self.get_data(screen_name, page)
alert_num += 1
def compute(str_inf_pk):
try:
inf = Influence.objects.get(pk=str_inf_pk)
auth = OAuthHandler(settings.SINA_CONSUMER_KEY, settings.SINA_CONSUMER_SECRET)
#auth.setToken(inf.sina_key, inf.sina_secret)
auth.setToken('128021658f2bfdae185d89bdffb3cede', '1713185d5c8208e8f1ef27a1f484ebc9') #Calculating for celebrity
api = API(auth)
followed_dict = {}
if inf.follower_count > 1000: #only for popular users that have too many people retweet his status
followed, cursors = api.friends(count=200, cursor=-1)
while True:
for f in followed:
followed_dict[getAtt(f, 'screen_name')] = 1
if cursors[1] == 0:
break
followed, cursors = api.friends(count=200, cursor=cursors[1])
"""TODO:This part seems to be unstable;Sometimes it doens work"""
#Calculate the most influenced friends through mentions api
mentions = api.mentions(count=200)
friends_dict = {}
friends_image_dict = {}
for m in mentions:
user = getAtt(m, 'user')
screen_name = getAtt(user, 'screen_name')
if screen_name in friends_dict:
friends_dict[screen_name] += 1
else:
if screen_name == inf.screen_name:
continue
friends_dict[screen_name] = 1
friends_image_dict[screen_name] = getAtt(user, 'profile_image_url')
friends = sorted(friends_dict.iteritems(), key=operator.itemgetter(1), reverse=True)
if followed_dict:
friends_list = []
backup_list = []
for f in friends:
if f[0] in followed_dict:
friends_list.append(f[0])
else:
backup_list.append(f[0])
if len(friends_list) == 4:
break
friends_list.extend(backup_list[:4-len(friends_list)])
else:
friends_list = [f[0] for f in friends[:4]]
influence_friends = ','.join(friends_list)
if influence_friends:
inf.influence_friends = influence_friends
friends_images = [friends_image_dict[f] for f in friends_list]
else:
friends_images = []
#Calculates the quality of followers; Also get average followers count of sampled followers
total = 0
active_count = 0
counted = 0
followers, cursors = api.followers(user_id=inf.sina_account, cursor=-1, count=200)
while True: #next_cursor is not zero means not the end
for f in followers:
counted += 1
f_follower_count = getAtt(f, 'followers_count')
#f_friends_count = getAtt(f, 'friends_count')
f_status_count = getAtt(f, 'statuses_count')
if ((f_follower_count > 50 or f_status_count > 50) and f_follower_count + f_status_count > 75
or f_follower_count + f_status_count > 150):
active_count += 1
total += f_follower_count
break #200 samples should be enough
#if cursors[1] == 0:
# break
#followers, cursors = api.followers(user_id=inf.sina_account, cursor=cursors[1], count=200)
avg_follower_of_followers = total*1.0/counted if counted !=0 else 0
active_follower_ratio = active_count * 1.0 /counted if counted !=0 else 0
inf.active_follower_count = int(math.ceil(active_follower_ratio*inf.follower_count))
#Calculates the average rt_count of each tweet based on 200 samples
mid_list = []
rt_count = 0 #Count of status that retweet my status
ids_list = []
timeline = api.user_timeline(user_id=inf.sina_account, count=200)
for line in timeline:
mid_list.append(str(getAtt(line, "id")))
if len(mid_list) == 20:
ids_list.append(','.join(mid_list))
mid_list = []
if mid_list: #append the remaining ids
ids_list.append(','.join(mid_list))
if inf.status_count > 0 and not ids_list:
raise Exception('weibo api fails')
for ids in ids_list:
counts = api.counts(ids=ids)
for obj in counts:
rt_count += getAtt(obj, 'rt')
sample_size = (len(ids_list)-1)*20 + len(ids_list[-1]) if ids_list else 0
average_rt_count = rt_count*1.0/sample_size if sample_size != 0 else 0
inf.average_rt_count = average_rt_count
influence_count = long(average_rt_count * inf.status_count * avg_follower_of_followers)
influence_count += inf.status_count * inf.follower_count #plus this so that it won't be zero
inf.influence_count = influence_count
magic_constant = 4.07 #99/math.log(36613694013) for YangMi
inf.influence_score = magic_constant * math.log(inf.influence_count+0.1) if inf.influence_count > 0 else 0
inf.influence_score = int(math.ceil(math.sqrt(inf.influence_score)*10)) #To make the score larger
if inf.influence_score > 99:
inf.influence_score = 99
print inf.influence_count
if inf.influence_count > 250000000: #To avoid huge number exceeding the number of users on weibo
inf.influence_count = 250000000 + long(random.random()*1000000)
#img_url = settings.MEDIA_URL + 'apps/influence/sample.gif'
"""TODO: check whether the profile is complete; But this may be not necessary"""
inf.last_use_date = datetime.now()
inf.error = 'success' #Erase the error message last time
inf.save()
except Exception, e:
try:
inf.error = str(e)
inf.save()
except:
pass
print str(e)
def classify_influence(inf_id, other=False):
"""
TODO:
1, Calculate the most popular tags, so that we can assign idf score
2, Accumulate more meaningful tags
"""
inf_id = inf_id.decode('gbk')
print inf_id.encode('gbk')
try:
inf_id = int(inf_id)
inf = Influence.objects.get(pk=inf_id)
except:
inf, created = Influence.objects.get_or_create(screen_name=inf_id)
if created:
auth = OAuthHandler(settings.SINA_CONSUMER_KEY, settings.SINA_CONSUMER_SECRET)
auth.setToken('128021658f2bfdae185d89bdffb3cede','1713185d5c8208e8f1ef27a1f484ebc9')
api = API(auth)
user = api.get_user(screen_name=inf_id)
inf.sina_account = getAtt(user, 'id')
inf.verified = getAtt(user,'verified')
inf.screen_name = getAtt(user, 'screen_name')
inf.description = getAtt(user, 'description')
inf.follower_count = getAtt(user, 'followers_count')
inf.following_count = getAtt(user, 'friends_count')
inf.status_count = getAtt(user, 'statuses_count')
inf.favourites_count = getAtt(user, 'favourites_count')
inf.create_date = getAtt(user, 'created_at')
inf.save()
auth = OAuthHandler(settings.SINA_CONSUMER_KEY, settings.SINA_CONSUMER_SECRET)
if other:
auth.setToken('128021658f2bfdae185d89bdffb3cede', '1713185d5c8208e8f1ef27a1f484ebc9')
else:
auth.setToken(inf.sina_key, inf.sina_secret)
api = API(auth)
mmseg.dict_load_defaults()
"""Put this in db first"""
candidate_tags = KeyValue.objects.get(key='CANDIDATE_TAGS')
area_dict = {}
# id_list = api.followers_ids(user_id=inf.sina_account, count=100) #default to 500, maximum is 5000; This consumes a lot of api limit
# ids = id_list[0].ids #Weird that getAtt won't work
# for id in ids:
# tags = api.tags(user_id=id) #user_id is required!
# tag_list = []
# for tag in tags:
# tag_list.append(getAtt(tag, 'value').lower().encode('utf-8'))
# mmseg_text = mmseg.Algorithm(' '.join(tag_list))
# for token in mmseg_text:
# try:
# term = token.text.decode('utf-8').lower()
# #next_term = mmseg_text[i+1].text.decode('utf-8') if i < len_list - 1 else ''
# except:
# continue
# train_value = area_train_data.get(term, None)
# #if not train_value:
# # train_value = area_train_data.get(term + next_term, None)
# if train_value:
# print 'in dict'
# for index in train_value:
# if index in area_dict:
# area_dict[index] += 1
# else:
# area_dict[index] = 1
# else:
# candidate_tags.value += ' ' + term
candidate_tags.save()
area_distr_dict = {}
mid_list = []
ids_list = []
tweet_list = [] #Store the text of tweet and retweet
rt_count_list = []
tried_count = 0
while True:
timeline = api.user_timeline(user_id=inf.sina_account, count=200)
if len(timeline) == 0 and inf.status_count >0:
tried_count += 1
print 'try again in getting timeline'
else:
break
if tried_count > 3:
raise Exception('weibo api error. No timeline got')
break
for line in timeline:
text = getAtt(line, 'text')
retweet = getAtt(line, 'retweeted_status')
retweet_text = getAtt(retweet, 'text')
if retweet_text:
text += retweet_text
tweet_list.append(text)
mid_list.append(str(getAtt(line, "id")))
if len(mid_list) == 20:
ids_list.append(','.join(mid_list))
mid_list = []
if mid_list: #append the remaining ids
ids_list.append(','.join(mid_list))
if inf.status_count > 0 and not ids_list:
raise Exception('weibo api fails')
tweet_list_correct = []
correct_index = 20
for ids in ids_list:
counts = api.counts(ids=ids)
if len(counts) == 0:
print 'error in counts!'
correct_index += 20
continue
for obj in counts:
rt_count_list.append(getAtt(obj, 'rt'))
tweet_list_correct.extend(tweet_list[correct_index-20:correct_index])
correct_index += 20
if len(tweet_list_correct) == 0 or len(tweet_list_correct) != len(rt_count_list):
raise Exception('weibo api fails')
print 'length of tweet list and rt_count list', len(tweet_list_correct), len(rt_count_list)
#Remedy for those user who has posted less than 200 status
amplify_ratio = 1.0 if len(tweet_list_correct) == 200 else 200.0/len(tweet_list_correct)
for i in range(len(tweet_list_correct)):
print i
#This number 100 should be replaced by avg_follower_count
#Use math.sqrt to boost those tweet that has not been retweeted,
#and smooth the effect of famous people tweeting about things not related to them
added_count = (rt_count_list[i]*100 + math.sqrt(inf.follower_count)) * amplify_ratio
assigned_area = {}
try: #In Unix environment
from sinal import signal, SIGALRM, alarm #@UnresolvedImport
def handler(signum, frame):
#print 'Signal handler called with signal', signum
raise Exception("This code block runs for too long time!")
signal(SIGALRM, handler)
alarm(3)
mmseg_text = mmseg.Algorithm(tweet_list_correct[i].encode('utf-8'))
alarm(0) #cancel the alarm after finised
except ImportError: # In windows, SIGALRM, alarm is not available in signal module
mmseg_text = mmseg.Algorithm(tweet_list_correct[i].encode('utf-8'))
except: #mmseg halts for too long, process next tweet
continue
for token in mmseg_text:
try:
term = token.text.decode('utf-8').lower()
except:
continue
train_value = area_train_data.get(term, None)
if train_value:
print 'in dict'
for index in train_value:
if index not in assigned_area: #This tweet has already been assigned to one area
if index in area_dict:
area_dict[index] += added_count
else:
area_dict[index] = added_count
assigned_area[index] = True
if index in area_distr_dict:
area_distr_dict[index] += 1
else:
area_distr_dict[index] = 1
else:
area_distr_dict[index] += 1
candidate_tags.save()
sorted_tuple = sorted(area_dict.iteritems(), key=operator.itemgetter(1), reverse=True)
if inf.follower_count > 100000:
for i in range(1,len(sorted_tuple)): #Only normalize on secondary influence areas and belows
index = sorted_tuple[i][0]
model_follower_count = areas[index][4]
if inf.follower_count > model_follower_count:
area_dict[index] = area_dict[index]*1.0/inf.follower_count*model_follower_count
num_areas = len(area_distr_dict)
total_keyword_count = 0
for index in area_distr_dict:
total_keyword_count += area_distr_dict[index]
for k in area_dict:
area_distr_ratio = num_areas * area_distr_dict[k]*1.0/total_keyword_count
print k , area_distr_ratio, area_distr_dict[k]
area_dict[k] = 100.0/math.log(areas[k][3]) * math.log(area_dict[k]*area_distr_ratio)
if area_dict[k] > 100:
area_dict[k] = 100.0
sorted_tuple = sorted(area_dict.iteritems(), key=operator.itemgetter(1), reverse=True)
for st in sorted_tuple:
print areas[st[0]][1].encode('gbk'), st[1]
