Пример #1
0
 def needs_static(self):
     while len(spider.todo())>0:
         if not spider.isAlive():
             logger.info("starting spider")
             spider.start()
         logger.info("spider todo %s", spider.todo())
         sleep(.5)
Пример #2
0
__author__ = 'zice'
from spider.spider import extractor,downloader,todo
import multiprocessing

if __name__ == "__main__":

    todo(url='http://finance.sina.com.cn/',func='a')

    process2 = multiprocessing.Process(target=extractor)
    process1 = multiprocessing.Process(target=downloader)

    process1.start()
    process2.start()

    process1.join()
    process2.join()
Пример #3
0
 def needs_static(self):
     while len(spider.todo()) > 0:
         print "spider todo", spider.todo()
         sleep(.5)
Пример #4
0
 def needs_static(self):
     while len(spider.todo())>0:
         print "spider todo", spider.todo()
         sleep(.5)