def log_in():
session["logged_in"] = request.form["log_in"]
session["user"] = request.form["user"]
if "logged_in" in session and session["logged_in"]:
return json.dumps("Right")
else:
return json.dumps("Wrong")
def log_in():
session['logged_in'] = request.form['log_in']
session['user'] = request.form['user']
if 'logged_in' in session and session['logged_in']:
return json.dumps('Right')
else:
return json.dumps('Wrong')
def profile_person_json():
nickname = request.args.get('nickname', None)
if nickname:
uid = getUidByName(urllib2.unquote(nickname.strip()))
if uid:
return json.dumps({'status': 'success', 'uid': str(uid)})
return json.dumps({'status': 'failed'})
def profile_group_topic(fieldEnName):
if request.method == 'GET' and fieldEnName:
result_arr = []
sort = request.args.get('sort', 'prob')
topic_type = request.args.get('topic_type', 'freq')
limit = request.args.get('limit', 50)
if limit:
limit = int(limit)
window_size = request.args.get('window_size', 24 * 60 * 60)
if window_size:
window_size = int(window_size)
start_ts = request.args.get('start_ts', None)
end_ts = request.args.get('end_ts', None)
if not start_ts or not end_ts:
start_ts, end_ts = _time_zone(_utf_encode(default_timerange))
if start_ts:
start_ts = int(start_ts)
if end_ts:
end_ts = int(end_ts)
interval = (end_ts - start_ts) / (24 * 3600) + 1
datestr = ts2datetimestr(end_ts) # '20130907'
domainid = DOMAIN_LIST.index(fieldEnName)
date_list = last_week_to_date(datestr, interval)
keywords_dict = {}
for datestr in date_list:
_keywords_dict = getDomainKeywordsData(domainid, datestr)
for k, v in _keywords_dict.iteritems():
try:
keywords_dict[k] += v
except KeyError:
keywords_dict[k] = v
if topic_type == 'freq':
keywords_sorted = sorted(keywords_dict.iteritems(), key=lambda(k, v): v, reverse=False)
top_keywords = keywords_sorted[len(keywords_sorted)-limit:]
result_arr = [{'text': k, 'size': float(v)} for k, v in top_keywords]
return json.dumps({'status': 'current finished', 'data': result_arr})
else:
return json.dumps([])
return json.dumps(result_arr)
def profile_group_status_count(fieldEnName):
start_ts = request.args.get('start_ts', None)
end_ts = request.args.get('end_ts', None)
if not start_ts or not end_ts:
start_ts, end_ts = _time_zone(_utf_encode(default_timerange))
if start_ts:
start_ts = int(start_ts)
if end_ts:
end_ts = int(end_ts)
interval = (end_ts - start_ts) / (24 * 3600) + 1
datestr = ts2datetimestr(end_ts) # '20130907'
date_list = last_week_to_date(datestr, interval)
domainid = DOMAIN_LIST.index(fieldEnName)
time_arr = []
total_arr = []
repost_arr = []
fipost_arr = []
for datestr in date_list:
active, important, reposts, original = getDomainCountData(domainid, datestr)
sumcount = reposts + original
time_arr.append(ts2date(datetimestr2ts(datestr)).isoformat())
total_arr.append(sumcount)
repost_arr.append(reposts)
fipost_arr.append(original)
return json.dumps({'time': time_arr, 'count': total_arr, 'repost': repost_arr, 'fipost': fipost_arr})
def single_analysis(mid):
test = request.args.get('test', '')
mid = int(mid)
blog_info = calculate_single(mid, test)
blog_img_url = blog_info['user']['profile_image_url']
blog_date_list = blog_info['datelist']
bloger_name = blog_info['user']['name']
blog_reposts_count = blog_info['status']['repostsCount']
blog_comments_count = blog_info['status']['commentsCount']
blog_attitudes_count = blog_info['status']['attitudesCount']
blog_time = blog_info['status']['postDate']
result = {'mid': mid}
#'tar_profile_image_url': blog_img_url,
#'tar_screen_name': bloger_name,
#'tar_repost_count': blog_reposts_count,
#'tar_comments_count': blog_comments_count,
#'tar_attitudes_count': blog_attitudes_count,
#'tar_post_date': blog_time,
#'blog_date_list': blog_date_list}
return json.dumps(result)
'''
def single_ajax_spatial():
if request.method == "GET":
return render_template('propagate/ajax/single_spatial.html')
else:
mid = int(request.form.get('mid', ""))
blog_info = calculate_single(mid, 'None')
area_list = blog_info['geo']
return json.dumps({'map_data': area_list})
def burst_monitor():
request_method = request.method
if request_method == "POST":
form = request.form
current_time = time.time()
current_date = ts2datetime(current_time)
data = burst_caculate(current_date, 1, "active", 5)
return json.dumps(data)
else:
abort(404)
def remove_kd():
result = "Right"
new_id = request.form["f_id"]
old_items = db.session.query(KnowledgeList).filter(KnowledgeList.kID == new_id).all()
if len(old_items):
for old_item in old_items:
db.session.delete(old_item)
db.session.commit()
else:
result = "Wrong"
return json.dumps(result)
def single_ajax_trend():
if request.method == "GET":
return render_template('propagate/ajax/single_trend.html')
else:
mid = int(request.form.get('mid', ""))
blog_info = calculate_single(mid, 'None')
perday_repost_count = blog_info['perday_count']
blog_date_list = blog_info['datelist']
date_list = [int(time.mktime(d.timetuple()))*1000 for d in blog_date_list]
return json.dumps({'perday_blog_count': zip(date_list, perday_repost_count)})
def profile_person(uid):
if uid:
count, get_results = xapian_search_user.search(query={'_id': int(uid)}, fields=['profile_image_url', 'name', 'friends_count', \
'statuses_count', 'followers_count', 'gender', 'verified', 'created_at', 'location'])
if count > 0:
for r in get_results():
user = {'id': uid, 'profile_image_url': r['profile_image_url'], 'userName': unicode(r['name'], 'utf-8'), 'friends_count': r['friends_count'], \
'statuses_count': r['statuses_count'], 'followers_count': r['followers_count'], 'gender': r['gender'], \
'verified': r['verified'], 'created_at': r['created_at'], 'location': unicode(r['location'], "utf-8")}
else:
return 'no such user'
return json.dumps(user)
def add_trash():
result = "Right"
new_field = request.form["f_id"]
count, get_results = xapian_search_user.search(query={"_id": new_field}, fields=["_id", "name"])
if count > 0:
for get_result in get_results():
new_item = BlackList(blackID=get_result["_id"], blackName=get_result["name"])
db.session.add(new_item)
db.session.commit()
else:
result = "Wrong"
return json.dumps(result)
def personal_weibo_count_false(uid):
total_days = 89
today = datetime.datetime.today()
now_ts = time.mktime(datetime.datetime(today.year, today.month, today.day, 2, 0).timetuple())
now_ts = int(now_ts)
during = 24 * 3600
time_arr = []
post_arr = []
repost_arr = []
fipost_arr = []
m = request.args.get('m')
if request.args.get('interval'):
total_days = int(request.args.get('interval')) - 1
for i in range(total_days-1, -1, -1):
end_ts = now_ts - i * during
begin_ts = end_ts - during
repost_query_dict = {
'timestamp': {
'$gt': begin_ts,
'$lt': end_ts
},
'user': int(uid),
'retweeted_status': '1'
}
post_query_dict = {
'timestamp': {
'$gt': begin_ts,
'$lt': end_ts
},
'user': int(uid)
}
if m == 'test':
post_count = xapian_search_weibo_test.search(query=post_query_dict, count_only=True)
repost_count = xapian_search_weibo_test.search(query=repost_query_dict, count_only=True)
fipost_count = post_count - repost_count
else:
post_count = xapian_search_weibo.search(query=post_query_dict, count_only=True)
repost_count = xapian_search_weibo.search(query=repost_query_dict, count_only=True)
fipost_count = post_count - repost_count
post_arr.append(post_count)
fipost_arr.append(fipost_count)
repost_arr.append(repost_count)
time_arr.append(ts2date(end_ts).isoformat())
print 'sum count: ', sum(post_arr)
return json.dumps({'time': time_arr, 'count': post_arr, 'repost': repost_arr, 'fipost': fipost_arr})
def profile_group_location(fieldEnName):
domainid = DOMAIN_LIST.index(fieldEnName)
datestr = '20130901'
verified_count, unverified_count, province_dict = 0, 0, {}
_verified_count, _unverified_count, _province_dict = getDomainBasic(domainid, datestr)
verified_count = int(_verified_count)
unverified_count = int(_unverified_count)
province_dict = _province_dict
city_count = province_dict
results = province_color_map(city_count)
return json.dumps(results)
def add_kd():
result = "Right"
new_field = request.form["f_id"]
t = request.form["type"]
count, get_results = xapian_search_user.search(query={"_id": new_field}, fields=["_id", "name"])
if count > 0:
for get_result in get_results():
new_item = KnowledgeList(kID=get_result["_id"], kName=get_result["name"], domain=t)
db.session.add(new_item)
db.session.commit()
else:
result = "Wrong"
return json.dumps(result)
def topic():
if "logged_in" in session and session["logged_in"]:
if session["user"] == "admin":
request_method = request.method
if request_method == "GET":
args = request.args
topic = args.get("keyword", None)
time = args.get("time", None)
rank_method = args.get("rank_method", "pagerank")
page_num = args.get("page_num", 20)
top_n = args.get("top_n", 500)
if not topic or not time:
return redirect("/identify/")
dur_time = _utf_encode(time)
start_ts, end_ts = _time_zone(dur_time)
return render_template(
"identify/topic.html",
topic=topic,
start_ts=start_ts,
end_ts=end_ts,
rank_method=rank_method,
page_num=page_num,
top_n=top_n,
window_size=1,
dur_time=_utf_decode(dur_time),
)
if request_method == "POST":
form = request.form
topic = form.get("topic", None)
start_ts = form.get("start_ts", None)
end_ts = form.get("end_ts", None)
if start_ts:
start_ts = int(start_ts)
if end_ts:
end_ts = int(end_ts)
rank_method = form.get("rank_method", "pagerank")
page_num = int(form.get("page_num", 20))
top_n = int(form.get("top_n", 500))
current_date = ts2datetime(end_ts)
window_size = (end_ts - start_ts) / (24 * 3600)
data = read_topic_rank_results(topic, top_n, rank_method, current_date, window_size)
return json.dumps({"status": "current finished", "data": data})
else:
return redirect("/")
def personal_weibo_count(uid):
total_days = 89
today = datetime.datetime.today()
now_ts = time.mktime(datetime.datetime(today.year, today.month, today.day, 2, 0).timetuple())
now_ts = int(now_ts)
during = 24 * 3600
time_arr = []
post_arr = []
repost_arr = []
fipost_arr = []
m = request.args.get('m')
if request.args.get('interval'):
total_days = int(request.args.get('interval')) - 1
for i in range(total_days-1, -1, -1):
end_ts = now_ts - i * during
begin_ts = end_ts - during
query_dict = {
'timestamp': {
'$gt': begin_ts,
'$lt': end_ts
},
'user': int(uid),
}
if m == 'test':
count, get_results = xapian_search_weibo_test.search(query=query_dict, fields=['retweeted_status'])
else:
count, get_results = xapian_search_weibo.search(query=query_dict, fields=['retweeted_status'])
post_count = 0
fipost_count = 0
repost_count = 0
for r in get_results():
if r['retweeted_status']:
repost_count += 1
else:
fipost_count += 1
post_count += 1
post_arr.append(post_count)
fipost_arr.append(fipost_count)
repost_arr.append(repost_count)
time_arr.append(ts2date(end_ts).isoformat())
print 'sum count: ', sum(post_arr)
return json.dumps({'time': time_arr, 'count': post_arr, 'repost': repost_arr, 'fipost': fipost_arr})
def topic_ajax_spatial():
if request.method == "GET":
return render_template('propagate/ajax/topic_spatial.html')
else:
keyword = request.form.get('keyword', "")
keyuser = request.form.get('keyuser', "")
beg_time = int(request.form.get('beg_time', ""))
end_time = int(request.form.get('end_time', ""))
fields_list = ['text', 'timestamp','reposts_count','comments_count','user', 'terms', '_id','retweeted_status','bmiddle_pic','geo','source','attitudes_count']
count, get_results = search_weibo.search(query={'text': [u'%s'%keyword], 'timestamp': {'$gt': beg_time, '$lt': end_time}}, sort_by=['timestamp'], fields=fields_list)
topic_info = calculate(get_results())
topic_area_list = topic_info['geo']
return json.dumps({'map_data': topic_area_list})
def profile_group_verify(fieldEnName):
domainid = DOMAIN_LIST.index(fieldEnName)
datestr = '20130901'
verified_count, unverified_count, province_dict = 0, 0, {}
_verified_count, _unverified_count, _province_dict = getDomainBasic(domainid, datestr)
verified_count = int(_verified_count)
unverified_count = int(_unverified_count)
province_dict = _province_dict
result_list = ''
if verified_count + unverified_count > 0:
sumcount = verified_count + unverified_count
result_list = str(verified_count) + ',' + str(unverified_count) + ',' + str(int(verified_count * 100 / sumcount) / 100.00) + ',' + str(1 - int(verified_count * 100 / sumcount) / 100.00)
return json.dumps(result_list)
def topic_ajax_trend():
if request.method == "GET":
return render_template('propagate/ajax/topic_trend.html')
else:
keyword = request.form.get('keyword', "")
keyuser = request.form.get('keyuser', "")
beg_time = int(request.form.get('beg_time', ""))
end_time = int(request.form.get('end_time', ""))
fields_list = ['text', 'timestamp','reposts_count','comments_count','user', 'terms', '_id','retweeted_status','bmiddle_pic','geo','source','attitudes_count']
count, get_results = search_weibo.search(query={'text': [u'%s'%keyword], 'timestamp': {'$gt': beg_time, '$lt': end_time}}, sort_by=['timestamp'], fields=fields_list)
topic_info = calculate(get_results())
perday_blog_count = topic_info['perday_count_list']
date_list = topic_info['date_list']
date_list = [int(time.mktime(d.timetuple()))*1000 for d in date_list]
return json.dumps({'perday_blog_count': zip(date_list, perday_blog_count)})
def profile_person_topic(uid):
if request.method == 'GET' and uid:
result_arr = []
limit = 50
start_ts = request.args.get('start_ts', None)
end_ts = request.args.get('end_ts', None)
if not start_ts or not end_ts:
start_ts, end_ts = _time_zone(_utf_encode(default_timerange))
if start_ts:
start_ts = int(start_ts)
if end_ts:
end_ts = int(end_ts)
try:
interval = (end_ts - start_ts) / (24 * 3600) + 1
datestr = ts2datetimestr(end_ts) # '20130907'
except Exception, e:
print e
interval, datestr = _default_time()
date_list = last_week_to_date(datestr, interval)
active = important = reposts = original = emoticon = 0
keywords_dict = {}
for current_time in date_list:
_active, _important, _reposts, _original, _emoticon, _direct_interact, _retweeted_interact, _keywords_dict = getPersonData(uid, current_time)
active += _active
important += _important
reposts += _reposts
original += _original
emoticon += _emoticon
if _keywords_dict:
keywords_dict.update(_keywords_dict)
if keywords_dict:
sortedkeywords = sorted(keywords_dict.iteritems(), key=operator.itemgetter(1), reverse=True)
for k, v in sortedkeywords[:limit]:
result_arr.append({'text': k, 'size': float(v)})
return json.dumps({'status': 'current finished', 'data': result_arr})
def search_history():
if request.method == "GET":
keyword = request.args.get("keyword", None)
now1 = request.args.get("now1", None)
now2 = request.args.get("now2", None)
now = request.args.get("now", None)
timestamp_end = request.args.get("timestamp", None)
if timestamp_end:
timestamp_end = int(timestamp_end)
if now1:
now1 = int(now1)
if now2:
now2 = int(now2)
if now:
now = int(now)
histories1 = None
histories2 = None
histories = None
if keyword != None:
status, histories = _search_history(keyword)
else:
if now:
status, histories = _all_history(now)
if now1:
status, histories1 = _all_history(now1)
if now2 == 0:
status, histories2 = _all_history(now2)
histories_names = []
if histories1:
for history in histories1:
start = time.strftime("%m月 %d日, %Y", time.localtime(history.start))
end = time.strftime("%m月 %d日, %Y", time.localtime(history.end - 24 * 3600))
datestr = str(start) + " - " + str(end)
if timestamp_end:
timestamp_start = int(history.db_date)
time_pass = timestamp_end - timestamp_start
time_pass_hour = int(time_pass) / 3600
time_pass_day = int(time_pass) / (24 * 3600)
time_pass = time.strftime("%M分钟 %S秒 ", time.localtime(time_pass))
time_pass = str(time_pass_day) + "天" + str(time_pass_hour) + "小时" + time_pass
time_pass = "******" + str(time_pass)
db_date = time.strftime("%m月 %d日, %Y %H:%M:%S", time.localtime(history.db_date))
db_date = " 提交时间： " + str(db_date)
histories_names.append([history.topic, datestr, db_date, time_pass])
else:
histories_names.append([history.topic, datestr])
if histories2:
for history in histories2:
start = time.strftime("%m月 %d日, %Y", time.localtime(history.start))
end = time.strftime("%m月 %d日, %Y", time.localtime(history.end - 24 * 3600))
datestr = str(start) + " - " + str(end)
if timestamp_end:
timestamp_start = int(history.db_date)
time_pass = timestamp_end - timestamp_start
time_pass_hour = int(time_pass) / 3600
time_pass_day = int(time_pass) / (24 * 3600)
time_pass = time.strftime("%M分钟 %S秒 ", time.localtime(time_pass))
time_pass = str(time_pass_day) + "天" + str(time_pass_hour) + "小时" + time_pass
time_pass = "******" + str(time_pass)
db_date = time.strftime("%m月 %d日, %Y %H:%M:%S", time.localtime(history.db_date))
db_date = " 提交时间： " + str(db_date)
histories_names.append([history.topic, datestr, db_date, time_pass])
else:
histories_names.append([history.topic, datestr])
if histories:
for history in histories:
start = time.strftime("%m月 %d日, %Y", time.localtime(history.start))
end = time.strftime("%m月 %d日, %Y", time.localtime(history.end - 24 * 3600))
datestr = str(start) + " - " + str(end)
histories_names.append([history.topic, datestr])
return json.dumps(histories_names)
else:
operator = request.form.get("operator", "add")
keyword = request.form.get("keyword", "")
start = request.form.get("start", "")
end = request.form.get("end", "")
sentiment = request.form.get("sentiment", "")
if keyword != "" and start != "" and end != "" and sentiment != "":
if operator == "add":
status, item = _add_history(-1, keyword, start_ts, end_ts, timestamp)
item = item.topic + "\t" + item.start + "\t" + item.end + "\t" + item.range + "\t" + item.status
else:
status, item = "failed", "Null"
else:
status, item = "failed", "Null"
return json.dumps({"status": status, "item": item})
def profile_network(friendship, uid):
if request.method == 'GET':
start_ts = request.args.get('start_ts', None)
end_ts = request.args.get('end_ts', None)
if not start_ts or not end_ts:
start_ts, end_ts = _time_zone(_utf_encode(default_timerange))
if start_ts:
start_ts = int(start_ts)
if end_ts:
end_ts = int(end_ts)
limit = 10
fri_fol = getFriendship(uid, friendship)
fri_fol_interact_uid_count = {}
domain_count = {}
# 关注者、粉丝领域统计
for uid in fri_fol:
domain = user2domain(uid)
try:
domain_count[domain] += 1
except KeyError:
domain_count[domain] = 1
domain_count = [(k, v) for k, v in domain_count.iteritems()]
try:
interval = (end_ts - start_ts) / (24 * 3600) + 1
datestr = ts2datetimestr(end_ts) # '20130907'
except Exception, e:
print e
interval, datestr = _default_time()
date_list = last_week_to_date(datestr, interval)
for datestr in date_list:
active, important, reposts, original, emoticon, direct_interact, retweeted_interact, keywords_dict = getPersonData(uid, datestr)
for k, v in retweeted_interact.iteritems():
k = int(k)
v = int(v)
if k in set(fri_fol):
try:
fri_fol_interact_uid_count[k] += v
except KeyError:
fri_fol_interact_uid_count[k] = v
for k, v in direct_interact.iteritems():
v = int(v)
k = getUidByName(k)
if k and k in set(fri_fol):
try:
fri_fol_interact_uid_count[k] += v
except KeyError:
fri_fol_interact_uid_count[k] = v
fri_fol_interact_name_count = {}
for k, v in fri_fol_interact_uid_count.iteritems():
name = getUserNameById(k)
fri_fol_interact_name_count[name] = v
fri_fol_interact_name_sorted = sorted(fri_fol_interact_name_count.iteritems(), key=operator.itemgetter(1), reverse=True)
sorted_uid_interact_count = sorted(fri_fol_interact_uid_count.iteritems(), key=operator.itemgetter(1), reverse=True)
users = getUsersInfoByUidInteract(sorted_uid_interact_count)
return json.dumps({'status': 'finished', 'data': {'fri_fol_interact': fri_fol_interact_name_sorted, \
'users': users, 'domains': domain_count}})
def area():
default_timerange = get_default_timerange()
default_field_enname, default_field_zhname = get_default_field_name()
default_field_dict = get_default_field_dict()
if "logged_in" in session and session["logged_in"]:
if session["user"] == "admin":
request_method = request.method
if request_method == "GET":
return render_template(
"identify/area.html",
from_external=True,
timerange=default_timerange,
field_dict=default_field_dict,
default_field_zh=default_field_zhname,
default_field_en=default_field_enname,
)
elif request_method == "POST":
form = request.form
action = form.get("action", "run")
top_n = int(form.get("top_n", 500))
if top_n > 500:
top_n = 500
page_num = int(form.get("page_num", 20))
rank_method = form.get("rank_method", "important")
during_date = form.get("window_size", default_timerange)
during_date = _utf_encode(during_date)
start_ts, end_ts = _time_zone(during_date)
window_size = (end_ts - start_ts) / (24 * 3600)
field = form.get("field", default_field_enname)
field_id = DOMAIN_LIST.index(field)
if action == "previous_rank":
action = "rank"
if action == "rank":
current_date = ts2datetime(end_ts - 24 * 3600)
previous_date = ts2datetime(start_ts - 24 * 3600)
data = area_caculate(current_date, window_size, rank_method, top_n, field_id)
previous_data = area_caculate(previous_date, window_size, rank_method, top_n, field_id)
index = dict()
for i in range(0, len(data)): # 比较上期结果
flag = 0
for j in range(0, len(previous_data)):
if previous_data[j][1] == data[i][1]:
flag = 1
compare = previous_data[j][0] - data[i][0]
index[previous_data[j][1]] = j
break
if flag == 0:
compare = 0
data[i].append(compare)
pre_data = []
for i in range(0, len(previous_data)):
if index.has_key(previous_data[i][1]):
pass
else:
pre_data.append(previous_data[i])
return json.dumps({"status": "current finished", "data": data, "pre_data": pre_data})
elif action == "run":
during_date = _utf_decode(during_date.strip())
return render_template(
"identify/area.html",
rank_method=rank_method,
during_date=during_date,
top_n=top_n,
page_num=page_num,
field=field,
timerange=default_timerange,
field_dict=default_field_dict,
default_field_zh=default_field_zhname,
default_field_en=default_field_enname,
)
else:
return redirect("/")
else:
return redirect("/")
def burst():
default_burst_time = get_default_burst_time()
if "logged_in" in session and session["logged_in"]:
if session["user"] == "admin":
request_method = request.method
if request_method == "GET":
return render_template("identify/burst.html", from_external=True, time=default_burst_time)
elif request_method == "POST":
form = request.form
action = form.get("action", "run")
top_n = int(form.get("top_n", 500))
if top_n > 500:
top_n = 500
page_num = int(form.get("page_num", 20))
rank_method = form.get("rank_method", "important")
burst_time = form.get("burst_time", default_burst_time)
burst_time = _utf_encode(burst_time)
time_ts = burst2ts(burst_time)
if action == "rank":
current_date = ts2datetime(time_ts)
data = burst_caculate(current_date, 1, rank_method, top_n)
previous_date = ts2datetime(time_ts - 24 * 3600)
previous_data = burst_caculate(previous_date, 1, rank_method, top_n)
index = dict()
for i in range(0, len(data)): # 比较上期结果
flag = 0
for j in range(0, len(previous_data)):
if previous_data[j][1] == data[i][1]:
flag = 1
compare = previous_data[j][0] - data[i][0]
index[previous_data[j][1]] = j
break
if flag == 0:
compare = 0
data[i].append(compare)
pre_data = []
for i in range(0, len(previous_data)):
if index.has_key(previous_data[i][1]):
pass
else:
pre_data.append(previous_data[i])
return json.dumps(
{
"status": "current finished",
"data": data,
"pre_data": pre_data,
"method": rank_method,
"time": time_ts,
}
)
elif action == "run":
burst_time = _utf_decode(burst_time.strip())
return render_template(
"identify/burst.html",
rank_method=rank_method,
burst_time=burst_time,
top_n=top_n,
page_num=page_num,
time=default_burst_time,
)
else:
abort(404)
else:
abort(404)
else:
pas = db.session.query(UserList).filter(UserList.username == session["user"]).all()
if pas != []:
for pa in pas:
identy = pa.identify
if identy == 1:
request_method = request.method
if request_method == "GET":
return render_template("identify/burst.html", from_external=True, time=default_burst_time)
elif request_method == "POST":
form = request.form
action = form.get("action", "run")
top_n = int(form.get("top_n", 500))
if top_n > 500:
top_n = 500
page_num = int(form.get("page_num", 20))
rank_method = form.get("rank_method", "important")
burst_time = form.get("burst_time", default_burst_time)
burst_time = _utf_encode(burst_time)
time_ts = burst2ts(burst_time)
if action == "rank":
current_date = ts2datetime(time_ts)
data = burst_caculate(current_date, 1, rank_method, top_n)
previous_date = ts2datetime(time_ts - 24 * 3600)
previous_data = burst_caculate(previous_date, 1, rank_method, top_n)
index = dict()
for i in range(0, len(data)): # 比较上期结果
flag = 0
for j in range(0, len(previous_data)):
if previous_data[j][1] == data[i][1]:
flag = 1
compare = previous_data[j][0] - data[i][0]
index[previous_data[j][1]] = j
break
if flag == 0:
compare = 0
data[i].append(compare)
pre_data = []
for i in range(0, len(previous_data)):
if index.has_key(previous_data[i][1]):
pass
else:
pre_data.append(previous_data[i])
return json.dumps(
{
"status": "current finished",
"data": data,
"pre_data": pre_data,
"method": rank_method,
"time": time_ts,
}
)
elif action == "run":
burst_time = _utf_decode(burst_time)
return render_template(
"identify/burst.html",
rank_method=rank_method,
burst_time=burst_time,
top_n=top_n,
page_num=page_num,
time=default_burst_time,
)
else:
return redirect("/")
return redirect("/")
else:
return redirect("/")
def profile_interact_network(uid):
if request.method == 'GET':
start_ts = request.args.get('start_ts', None)
end_ts = request.args.get('end_ts', None)
if not start_ts or not end_ts:
start_ts, end_ts = _time_zone(_utf_encode(default_timerange))
if start_ts:
start_ts = int(start_ts)
if end_ts:
end_ts = int(end_ts)
uid = int(uid)
center_uid = uid
direct_uid_interact_count = {}
retweeted_uid_interact_count = {}
uid_interact_count = {}
try:
interval = (end_ts - start_ts) / (24 * 3600) + 1
datestr = ts2datetimestr(end_ts) # '20130907'
except Exception, e:
print e
interval, datestr = _default_time()
date_list = last_week_to_date(datestr, interval)
for datestr in date_list:
active, important, reposts, original, emoticon, direct_interact, retweeted_interact, keywords_dict = getPersonData(uid, datestr)
for k, v in retweeted_interact.iteritems():
k = int(k)
v = int(v)
try:
retweeted_uid_interact_count[k] += v
except KeyError:
retweeted_uid_interact_count[k] = v
for k, v in direct_interact.iteritems():
v = int(v)
try:
direct_uid_interact_count[k] += v
except KeyError:
direct_uid_interact_count[k] = v
direct_uid_sorted = sorted(direct_uid_interact_count.iteritems(), key=operator.itemgetter(1), reverse=False)
retweeted_uid_sorted = sorted(retweeted_uid_interact_count.iteritems(), key=operator.itemgetter(1), reverse=False)
retweeted_uid2name_dict = {}
for k, v in retweeted_uid_interact_count.iteritems():
retweeted_uid2name_dict[getUserNameById(k)] = v
uid_interact_count = merge(direct_uid_interact_count, retweeted_uid2name_dict, lambda x, y: x+y)
uid_sorted = sorted(uid_interact_count.iteritems(), key=operator.itemgetter(1), reverse=False)
second_circle_limit = 8
third_circle_limit = 36
second_circle_uid_counts = uid_sorted[-second_circle_limit:]
third_circle_uid_counts = uid_sorted[-second_circle_limit-third_circle_limit-1:-second_circle_limit-1]
third_circle_limit = len(third_circle_uid_counts) if len(third_circle_uid_counts) < third_circle_limit else third_circle_limit
def node(friendsCount, followersCount, statusesCount, gender, verified, profileImageUrl, count, uid, name):
return {
"children": [],
"data": {
"friendsCount": friendsCount,
"followersCount": followersCount,
"statusesCount": statusesCount,
"gender": gender,
"verified": verified,
"profileImageUrl": profileImageUrl,
"$color": "#AEA9F8",
"$angularWidth": 1000,
"count": count
},
"id": uid,
"name": name
}
def unode(uid, name, count):
return {
"children": [],
"data": {
"$color": "#AEA9F8", #FCD9A1 AEA9F8 B0AAF6 B2ABF4 B6AEEF E0C7C0 D2BFD0 ECCFB3 D4C0CE
"$angularWidth": 1000,
"count":count
},
"id": uid,
"name": name
}
def source_data_structure(uid, name):
return {
"children": [],
"data":{
"type": "none"
},
"id": uid,
"name": name
}
# first circle
first_cicle = source_data_structure(center_uid, getUserNameById(center_uid))
second_circle = []
third_circle = []
# calcuate weight list
second_circle_sum_counts = 0
for uid, count in second_circle_uid_counts:
second_circle_sum_counts += count
# adjust residual weight to index 0 of list
if second_circle_sum_counts:
weight_list = [int(count * third_circle_limit / second_circle_sum_counts) if int(count * third_circle_limit / second_circle_sum_counts) >= 1 else 1 for uid, count in second_circle_uid_counts]
weight_list[0] += second_circle_sum_counts - sum(weight_list)
# second circle
for uid, count in second_circle_uid_counts:
info = yymInfo(uid)
if not info:
second_circle.append(unode(uid, uid, count))
else:
second_circle.append(node(info['friendsCount'], info['followersCount'], \
info['statusesCount'], info['gender'], \
info['verified'], info['profileImageUrl'], \
count, info['id'], info['userName']))
# set second circle color
for i in range(0, len(second_circle)):
second_circle[i]['data']['$color']="#B2ABF4"
# third circle
for uid, count in third_circle_uid_counts:
info = yymInfo(uid)
if not info:
third_circle.append(unode(uid, uid, count))
else:
third_circle.append(node(info['friendsCount'], info['followersCount'], \
info['statusesCount'], info['gender'], \
info['verified'], info['profileImageUrl'], \
count, info['id'], info['userName']))
# assign third circle to second circle, then second circle to first circle
start_idx = 0
if len(third_circle):
for i in range(0, len(second_circle)):
#for k in range(0, weight_list[i]):
if i < len(third_circle):
second_circle[i]['children'].append(third_circle[i])
first_cicle['children'] = second_circle
return json.dumps({'status': 'finished', 'data': first_cicle})
important += _important
reposts += _reposts
original += _original
emoticon += _emoticon
if _keywords_dict:
keywords_dict.update(_keywords_dict)
if keywords_dict:
sortedkeywords = sorted(keywords_dict.iteritems(), key=operator.itemgetter(1), reverse=True)
for k, v in sortedkeywords[:limit]:
result_arr.append({'text': k, 'size': float(v)})
return json.dumps({'status': 'current finished', 'data': result_arr})
else:
return json.dumps([])
return json.dumps(result_arr)
@mod.route('/person_interact_count', methods=['GET', 'POST'])
def profile_interact_count():
if request.method == 'GET':
limit = 10
start_ts = request.args.get('start_ts', None)
end_ts = request.args.get('end_ts', None)
if not start_ts or not end_ts:
start_ts, end_ts = _time_zone(_utf_encode(default_timerange))
if start_ts:
start_ts = int(start_ts)
def profile_search(model='hotest'):
default_search_time = get_default_search_time()
statuses_count_upBound = default_statuses_range
friends_count_upBound = default_friends_range
followers_count_upBound = default_followers_range
local_statuses_range = statuses_count_upBound[0]
local_followers_range = followers_count_upBound[0]
local_friends_range = friends_count_upBound[0]
if 'logged_in' in session and session['logged_in']:
if session['user'] == 'admin':
if request.method == 'GET':
province = default_province
province_str = ','.join(province)
if model == 'person':
nickname = urllib2.unquote(request.args.get('nickname'))
return render_template('profile/profile_search.html', statusescount_up=statuses_count_upBound, \
friendscount_up=friends_count_upBound, followerscount_up=followers_count_upBound, \
location=province_str, model=model, result=None, nickname=nickname, \
field_dict=default_field_dict, time=default_search_time, \
local_statusescount_up=local_statuses_range, local_friendscount_up=local_friends_range, \
local_followerscount_up=local_followers_range)
elif model == 'find':
if request.args.get('statuses_count_upBound', None):
local_statuses_range = request.args.get('statuses_count_upBound', None)
local_statuses_range = (local_statuses_range, default_statuses_dict[local_statuses_range])
if request.args.get('friends_count_upBound',None):
local_friends_range = request.args.get('friends_count_upBound',None)
local_friends_range = (local_friends_range, default_friends_dict[local_friends_range])
if request.args.get('followers_count_upBound',None):
local_followers_range = request.args.get('followers_count_upBound',None)
local_followers_range = (local_followers_range, default_followers_dict[local_followers_range])
top_n = request.args.get('search_top_n', None)
province_str = request.args.get('province_str', None)
rank_str = request.args.get('rankcount', None)
rank_str_zh = ''
if rank_str == 'followers_count':
rank_str_zh = u'粉丝数降序'
elif rank_str == 'statuses_count':
rank_str_zh = u'微博数降序'
elif rank_str == 'friends_count':
rank_str_zh = u'关注数降序'
elif rank_str == 'created_at':
rank_str_zh = u'注册时间降序'
if(top_n):
result_count = int(top_n)
else:
result_count = 100
return render_template('profile/profile_search.html',result_count=result_count, statusescount_up=statuses_count_upBound, \
friendscount_up=friends_count_upBound, followerscount_up=followers_count_upBound, \
location=province_str, model=model, rankcount=rank_str, field_dict=default_field_dict, \
time=default_search_time, sort=rank_str_zh, \
local_statusescount_up=local_statuses_range, local_friendscount_up=local_friends_range, \
local_followerscount_up=local_followers_range)
else:
return render_template('profile/profile_search.html', statusescount_up=statuses_count_upBound, \
friendscount_up=friends_count_upBound, followerscount_up=followers_count_upBound, \
location=province_str, model=model, result=None, \
field_dict=default_field_dict, time=default_search_time, \
local_statusescount_up=local_statuses_range, local_friendscount_up=local_friends_range, \
local_followerscount_up=local_followers_range)
if request.method == 'POST' and request.form['page']:
if model == 'newest':
top_n = 1000
page = int(request.form['page'])
if page == 1:
startoffset = 0
else:
startoffset = (page - 1) * COUNT_PER_PAGE
endoffset = startoffset + COUNT_PER_PAGE - 1
users = _newest_users(top_n)
return json.dumps(users[startoffset:endoffset])
elif model == 'hotest':
top_n = 1000
latest = get_default_hotest_time()
page = int(request.form['page'])
if page == 1:
startoffset = 0
else:
startoffset = (page - 1) * COUNT_PER_PAGE
endoffset = startoffset + COUNT_PER_PAGE - 1
users = _hotest_users(top_n, latest)
return json.dumps(users[startoffset:endoffset])
elif model == 'oversea':
page = int(request.form['page'])
if page == 1:
startoffset = 0
else:
startoffset = (page - 1) * COUNT_PER_PAGE
endoffset = startoffset + COUNT_PER_PAGE - 1
province = '海外'
count, get_results = xapian_search_user.search(query={'location': province}, sort_by=['followers_count'], max_offset=10000, fields=['_id', 'name', 'statuses_count', 'friends_count', 'followers_count', 'profile_image_url', 'description'])
users = []
offset = 0
for r in get_results():
if offset >= startoffset and offset <= endoffset:
users.append({'id': r['_id'], 'profileImageUrl': r['profile_image_url'], 'userName': r['name'], 'statusesCount': r['statuses_count'], 'friendsCount': r['friends_count'], 'followersCount': r['followers_count'], 'description': r['description']})
if offset >= endoffset:
break
offset += 1
return json.dumps(users)
elif model in DOMAIN_LIST:
page = int(request.form['page'])
if page == 1:
startoffset = 0
else:
startoffset = (page - 1) * COUNT_PER_PAGE
endoffset = startoffset + COUNT_PER_PAGE - 1
users = _domain_users(model)
return json.dumps(users[startoffset:endoffset])
elif model == 'person':
sharding = True
nickname = urllib2.unquote(request.form['nickname'])
uid = getUidByName(nickname)
users = []
if uid:
status, user = _search_person_basic(uid, sharding)
if status == 'success':
users.append(user._to_dict())
return json.dumps(users)
elif model == 'find':
read_from_xapian = 0
sharding = True
page = int(request.form['page'])
if page == 1:
startoffset = 0
else:
startoffset = (page - 1) * COUNT_PER_PAGE
endoffset = startoffset + COUNT_PER_PAGE - 1
result_count = int(request.form['result_count'])
statusescount_low, statusescount_up = parseRangePara(request.form['statusescount_up'])
friendscount_low, friendscount_up = parseRangePara(request.form['friendscount_up'])
followerscount_low, followerscount_up = parseRangePara(request.form['followerscount_up'])
province_str = request.form['province']
rankcount = request.form['rankcount']
province = province_str.split(',')
users = []
query_dict = {}
query_dict['$and'] = []
query_dict['$and'].append({'statuses_count':{'$gt': statusescount_low, '$lt': statusescount_up }})
query_dict['$and'].append({'followers_count':{'$gt': followerscount_low, '$lt': followerscount_up}})
query_dict['$and'].append({'friends_count':{'$gt': friendscount_low, '$lt': friendscount_up}})
or_dict = {}
or_dict['$or'] = []
for pro in province:
or_dict['$or'].append({'location': pro})
query_dict['$and'].append(or_dict)
if read_from_xapian:
count, get_results = xapian_search_user.search(query=query_dict, max_offset=result_count,
fields=[ '_id', 'name', 'statuses_count', 'followers_count', 'friends_count', 'description', 'profile_image_url'],
sort_by=[rankcount])
for r in get_results():
statusesCount = r['statuses_count']
followersCount = r['followers_count']
friendsCount = r['friends_count']
userName = r['name']
description = r['description']
uid = r['_id']
profileImageUrl = r['profile_image_url']
users.append({'id': uid, 'userName': userName, 'statusesCount': statusesCount, 'followersCount': followersCount, 'friendsCount': friendsCount,
'description': description, 'profileImageUrl': profileImageUrl})
else:
users = _multi_search(query_dict, rankcount, result_count, sharding)
return json.dumps(users[startoffset:endoffset])
else:
return redirect('/')
else:
return redirect('/')
def showresult_by_topic():
# get the input context
keyword = request.args.get('keyword', "")
keyuser = request.args.get('keyuser', "")
beg_time = request.args.get('beg_time', "")
end_time = request.args.get('end_time', "")
test = request.args.get('test', 'None')
keyword = keyword.strip('@\r\n\t')
keyuser = keyuser.strip('@\r\n\t')
beg_time = beg_time.strip('@\r\n\t')
end_time = end_time.strip('@\r\n\t')
return_beg_str = beg_time
return_end_str = end_time
keyuser_str = keyuser
if keyword == "":
flash(u'关键字（词）不能为空')
field_topics = getFieldTopics()
status_hot = getHotStatus()
return render_template('propagate/search.html',field_topics = field_topics,status_hot = status_hot)
if keyuser == "":
keyuser_str = u'无'
if beg_time == "":
beg_time = calendar.timegm(datetime(2013, 1, 1).timetuple())
return_beg_str = '2013-01-01'
else:
beg_time = datetime.strptime(beg_time,"%Y-%m-%d")
beg_time_year = int(beg_time.year)
beg_time_month = int(beg_time.month)
beg_time_day = int(beg_time.day)
beg_time = calendar.timegm(datetime(beg_time_year,beg_time_month,beg_time_day).timetuple())
if end_time == "":
end_time = calendar.timegm(datetime(2013, 1, 7).timetuple())
return_end_str = '2013-01-07'
else:
end_time = datetime.strptime(end_time,"%Y-%m-%d")
end_time_year = int(end_time.year)
end_time_month = int(end_time.month)
end_time_day = int(end_time.day)
end_time = calendar.timegm(datetime(end_time_year,end_time_month,end_time_day).timetuple())
fields_list = ['text', 'timestamp','reposts_count','comments_count','user', 'terms', '_id','retweeted_status','bmiddle_pic','geo','source','attitudes_count']
if test == 'None':
count, get_results = search_weibo.search(query={'text': [u'%s'%keyword], 'timestamp': {'$gt': beg_time, '$lt': end_time}}, sort_by=['timestamp'], fields=fields_list)
else:
count, get_results = search_weibo_test.search(query={'text': [u'%s'%keyword], 'timestamp': {'$gt': beg_time, '$lt': end_time}}, sort_by=['timestamp'], fields=fields_list)
if count == 0:
flash(u'您搜索的话题结果为空')
field_topics = getFieldTopics()
status_hot = getHotStatus()
return render_template('propagate/search.html',field_topics = field_topics,status_hot = status_hot)
else:
print count
topic_info = calculate(get_results())
print 'ok'
topic_ori_screen_name = topic_info['topic_poster']
topic_img_url = topic_info['topic_url']
topic_blog_count = topic_info['blogs_sum']
topic_blog_ori_count = topic_info['topic_ori_blog_count']
blog_ori_account = '%10.2f'%(float(topic_blog_ori_count)/topic_blog_count)
topic_leader_count = topic_info['topic_index']['leader_index']
topic_ori_date = topic_info['topic_post_date']
return json.dumps({'topic_profile_image_url': topic_img_url[0]})
'''
def profile_interact_count():
if request.method == 'GET':
limit = 10
start_ts = request.args.get('start_ts', None)
end_ts = request.args.get('end_ts', None)
if not start_ts or not end_ts:
start_ts, end_ts = _time_zone(_utf_encode(default_timerange))
if start_ts:
start_ts = int(start_ts)
if end_ts:
end_ts = int(end_ts)
direct_uid_interact_count = {}
retweeted_uid_interact_count = {}
uid_interact_count = {}
domain_count = {}
try:
interval = (end_ts - start_ts) / (24 * 3600) + 1
datestr = ts2datetimestr(end_ts) # '20130907'
except Exception, e:
print e
interval, datestr = _default_time()
date_list = last_week_to_date(datestr, interval)
for datestr in date_list:
active, important, reposts, original, emoticon, direct_interact, retweeted_interact, keywords_dict = getPersonData(uid, datestr)
for k, v in retweeted_interact.iteritems():
k = int(k)
v = int(v)
try:
retweeted_uid_interact_count[k] += v
except KeyError:
retweeted_uid_interact_count[k] = v
for k, v in direct_interact.iteritems():
v = int(v)
try:
direct_uid_interact_count[k] += v
except KeyError:
direct_uid_interact_count[k] = v
direct_uid_sorted = sorted(direct_uid_interact_count.iteritems(), key=operator.itemgetter(1), reverse=False)
retweeted_uid_sorted = sorted(retweeted_uid_interact_count.iteritems(), key=operator.itemgetter(1), reverse=False)
retweeted_uid2name_dict = {}
for k, v in retweeted_uid_interact_count.iteritems():
retweeted_uid2name_dict[getUserNameById(k)] = v
uid_interact_count = merge(direct_uid_interact_count, retweeted_uid2name_dict, lambda x, y: x+y)
uid_sorted = sorted(uid_interact_count.iteritems(), key=operator.itemgetter(1), reverse=True)
direct_uid_sorted = direct_uid_sorted[-limit:]
retweeted_uid_sorted = retweeted_uid_sorted[-limit:]
retweeted_name_sorted = []
for uid, count in retweeted_uid_sorted:
retweeted_name_sorted.append((getUserNameById(uid), count))
return json.dumps({'status': 'finished', 'data': {'direct_uid': direct_uid_sorted, \
'retweeted_uid': retweeted_name_sorted}})