def worker():
db = BaseDB()
while q.qsize():
novel = q.get()
try:
if ready_work(novel, db):
notify_page(novel[0], "start")
do_work(novel, db)
except Exception, e:
log.error("%s: %s" % (novel[0], traceback.format_exc()))
db.modify_status(novel[0], 4)
db.commit()
q.task_done()
def render_GET(self, request):
# 传入一个id，直接开始该id的抓取工作
id = request.args.get("id", [""])[0]
if id.isdigit():
db = BaseDB()
info = db.get_novel_by_id(id)
if info:
# 准备工作
if ready_work(info, db):
mb.new_msg(id, 'start')
deferToThread(do_work, info,
db).addErrback(self.get_error, id, db)
return "ok"
def setUp(self):
"""
var setups
"""
self.master_config = '../../docs/config/my_master.json'
self.slave_config = '../../docs/config/my_slave.json'
self.base = None
self.conn = None
self.ins = None
self.base = BaseDB()
self.base.load_config(self.slave_config)
self.conn, db, t = self.base.connect_server()
self.ins = Slave(self.conn)
def worker():
db = BaseDB()
while q.qsize():
novel = q.get()
try:
if ready_work(novel, db):
notify_page(novel[0], "start")
do_work(novel, db)
except Exception, e:
log.error("%s: %s" % (novel[0], traceback.format_exc()))
db.modify_status(novel[0], 4)
db.commit()
q.task_done()
# 插入任务
db = BaseDB()
novel_list = db.get_novel_list()
log.info("novel count %s", len(novel_list))
for novel in novel_list:
# db.modify_status(novel[0], 1)
q.put(novel)
# db.commit()
db.close()
del db
for i in range(min(q.qsize(), THREADS_COUNT)):
t = threading.Thread(target=worker)
t.setDaemon(True)
t.start()
q.join()
def test_connection(self):
ins = BaseDB()
ins.load_config(self.master_config)
conn, db, t = ins.connect_server()
self.assertTrue(t)
def test_slave(self):
ins = BaseDB()
ins.load_config(self.slave_config)
conn, db, t = ins.connect_server()
slave = Slave(conn)
self.assertGreater(len(slave.list_symbols()), 0)
def test_master(self):
ins = BaseDB()
ins.load_config(self.config)
conn, db, t = ins.connect_server()
mast = Master(conn)
self.assertGreater(len(mast.list_libraries()), 0)
