Python url_to_Mark Beispiele

Beispiel #1

def scan_forum(db, url, sess):
	try:
		con = sess.get(url).content
	except:
		print "Get Page %s Failed!"%url
		return []
	lst = web_parse.forum_list(con)
	return [i for i in lst if not (db.is_exist_forum(web_parse.url_to_Mark(i["link"])) or not i["name"]) ]

Beispiel #2

def Image_fetcher(db, P_que, F_que, sleep_len, sess):
	time.sleep(sleep_len)
	while ((not F_que.empty()) or (not P_que.empty())) :
		try:
			dic = F_que.get(timeout = 2)
		except:
			continue
		print "Catch %s, last %s"%(dic["link"] , F_que.qsize())
		try:
			con = sess.get(dic["link"]).content
		except:
			continue
		lst = web_parse.img_list(con)
		Lock.acquire()
		try:
			db.data_in(lst, dic["name"], web_parse.url_to_Mark(urlparse.urlsplit(dic["link"]).path[1:]), dic["tag"])
		except:
			raise
			print "Insert Data Failed @ %s"%dic["link"]
			print lst, dic["name"], web_parse.url_to_Mark(urlparse.urlsplit(dic["link"]).path[1:]), dic["tag"]
		finally:
			Lock.release()
	return