def scan_forum(db, url, sess): try: con = sess.get(url).content except: print "Get Page %s Failed!"%url return [] lst = web_parse.forum_list(con) return [i for i in lst if not (db.is_exist_forum(web_parse.url_to_Mark(i["link"])) or not i["name"]) ]
def Image_fetcher(db, P_que, F_que, sleep_len, sess): time.sleep(sleep_len) while ((not F_que.empty()) or (not P_que.empty())) : try: dic = F_que.get(timeout = 2) except: continue print "Catch %s, last %s"%(dic["link"] , F_que.qsize()) try: con = sess.get(dic["link"]).content except: continue lst = web_parse.img_list(con) Lock.acquire() try: db.data_in(lst, dic["name"], web_parse.url_to_Mark(urlparse.urlsplit(dic["link"]).path[1:]), dic["tag"]) except: raise print "Insert Data Failed @ %s"%dic["link"] print lst, dic["name"], web_parse.url_to_Mark(urlparse.urlsplit(dic["link"]).path[1:]), dic["tag"] finally: Lock.release() return