def fetch_people_page(conn, username, page = 1):
url = "/people/{}/answers".format(username)
url_page = "{}?page={:d}".format(url, page)
print("\n{}\t".format(url_page), end='')
sys.stdout.flush()
timer.timer()
try:
conn.request("GET", url_page)
except socket.timeout as e:
print('wow! timeout')
raise e
response = conn.getresponse()
t = timer.timer()
avg = int(get_average(t, 'user page'))
code = response.status
print("[{}]\t{} ms\tAvg: {} ms".format(code, t, avg))
if code == 404:
slog("user username fetch fail, code code")
dbhelper.update_user_by_name(username, {'fetch': dbhelper.FETCH_FAIL})
print( "没有这个用户", username)
return None
if code != 200:
slog("user username fetch fail, code code")
dbhelper.update_user_by_name(username, {'fetch': dbhelper.FETCH_FAIL})
print( "奇奇怪怪的返回码", code)
return None
content = response.read()
return content
def fetch_proc(username):
with dblock:
dbhelper.update_user_by_name(username, {'fetch': dbhelper.FETCH_ING})
conn = zhihu.get_conn()
content = zhihu.fetch_people_page(conn, username)
if content is None:
conn.close()
return
src = zhihu.get_avatar_src(content)
with dblock:
dbhelper.update_user_by_name(username, {'avatar': src})
link_list = zhihu.get_answer_link_list(content)
rs = zhihu.saveAnswer(conn, username, link_list, dblock)
num = zhihu.get_page_num(content)
if num > 1:
for i in range(2, num):
content = zhihu.fetch_people_page(conn, username, i)
if content is None:
continue
link_list = zhihu.get_answer_link_list(content)
zhihu.saveAnswer(conn, username, link_list, dblock)
with dblock:
dbhelper.update_user_by_name(username, {'fetch': dbhelper.FETCH_OK})
conn.close()
zhihu.slog('### after saveAnswer ###')
s.release()
