Example #1
0
def spider(beginurl,pages,downFolder):

    priQueue = PriorityQueue()            
    downlist = PriorityQueue()            
    priQueue.push((1,beginurl))
    i = 0
    while not priQueue.empty() and i <pages:
        k,url = priQueue.pop()
        if downloadUrl(i+1,url,priQueue,downlist,downFolder,beginurl):
            i+=1

    print '\nDownload',i,'pages.'