def fetch_people_page(conn, username, page = 1): url = "/people/{}/answers".format(username) url_page = "{}?page={:d}".format(url, page) print("\n{}\t".format(url_page), end='') sys.stdout.flush() timer.timer() try: conn.request("GET", url_page) except socket.timeout as e: print('wow! timeout') raise e response = conn.getresponse() t = timer.timer() avg = int(get_average(t, 'user page')) code = response.status print("[{}]\t{} ms\tAvg: {} ms".format(code, t, avg)) if code == 404: slog("user username fetch fail, code code") dbhelper.update_user_by_name(username, {'fetch': dbhelper.FETCH_FAIL}) print( "没有这个用户", username) return None if code != 200: slog("user username fetch fail, code code") dbhelper.update_user_by_name(username, {'fetch': dbhelper.FETCH_FAIL}) print( "奇奇怪怪的返回码", code) return None content = response.read() return content
def fetch_proc(username): with dblock: dbhelper.update_user_by_name(username, {'fetch': dbhelper.FETCH_ING}) conn = zhihu.get_conn() content = zhihu.fetch_people_page(conn, username) if content is None: conn.close() return src = zhihu.get_avatar_src(content) with dblock: dbhelper.update_user_by_name(username, {'avatar': src}) link_list = zhihu.get_answer_link_list(content) rs = zhihu.saveAnswer(conn, username, link_list, dblock) num = zhihu.get_page_num(content) if num > 1: for i in range(2, num): content = zhihu.fetch_people_page(conn, username, i) if content is None: continue link_list = zhihu.get_answer_link_list(content) zhihu.saveAnswer(conn, username, link_list, dblock) with dblock: dbhelper.update_user_by_name(username, {'fetch': dbhelper.FETCH_OK}) conn.close() zhihu.slog('### after saveAnswer ###') s.release()