def Queue_Check_Push_Front(page):
"""Check the url and push into queue.
If check is not needed, push into queue directly.
:param page: {
url: the url of each page,
depth: the depth of each page, i.e., its minimum distance from one of the 10 start pages
}
"""
global number_collected_url
global pagesNumber
if len(queue) + number_collected_url > pagesNumber * 1.5:
return
href = page["url"]
global hash_table
global number_visited_url
href = CheckUrl.checkUrl(href)
if href != -1:
if CheckSite.checkSite_Visitable(href) == 1:
if not hash_table.has_key(href):
print "queue push front: " + str(len(queue)) + " " + href
queue.append(page)
hash_table[href] = number_visited_url
number_visited_url += 1
