Esempio n. 1
0
 def get_label_a(self):
     try:
         soup = BeautifulSoup(self.data, 'lxml')
         alist = soup.find_all('a')
         mange = Mange(self.new_urls, self.old_urls)
         for i in alist:
             if 'http://' in i['href']:
                 mange.add_new_urls(i['href'])
     except Exception,e:  # no href
         pass
Esempio n. 2
0
def main():

    # page thread url
    if not args.url:
        return

    mange = Mange(new_urls, old_urls)
    mange.add_new_urls(args.url)
    while mange.has_new_urls():
        if len(old_urls) >= args.page:
            break
        try:
            url = mange.pop()
            myt = SpiderThread(url, args.page, args.thread, new_urls, old_urls)
            myt.start()
            mange.add_old_urls(url)
        except Exception,e:
            print e
        time.sleep(1)