__author__ = 'gongxingfa'
from pony.orm import db_session
from bo.bo import Stock
from spider import BrowserSpider
from multiprocessing import Queue
from parsers import sessions_parser
from handlers import sessions_handler
if __name__ == '__main__':
url_queue = Queue(1024)
bs = BrowserSpider(url_queue, (('.*', sessions_parser, sessions_handler),), browser_nums=6)
bs.start()
with db_session:
stocks = Stock.select_by_sql('select * from stock')[:]
for s in stocks:
url_queue.put(s.url)
bs.wait_finish()
__author__ = "gongxingfa"
from pyquery import PyQuery
from pony.orm import db_session, commit
from bo.bo import Stock, Simple_Sessions
from spider import BrowserSpider
from multiprocessing import Queue
from parsers import simple_session_parser
from handlers import simple_sessions_handler
if __name__ == "__main__":
url_queue = Queue(1024)
bs = BrowserSpider(url_queue, ((".*", simple_session_parser, simple_sessions_handler),))
bs.start()
with db_session:
stocks = Stock.select_by_sql("select * from stock")[:]
print("......len:%d" % len(stocks))
for s in stocks:
url_queue.put(s.url)
import time
time.sleep(100)