def __init__(self, filename=None):
"""Create a new SharedKeyDB.
@type filename: str
@param filename: Filename for an on-disk database, or None for
an in-memory database. If the filename already exists, follow
this with a call to open(). To create a new on-disk database,
follow this with a call to create().
"""
BaseDB.__init__(self, filename, "shared key")
def __init__(self, filename=None):
"""Create a new SharedKeyDB.
@type filename: str
@param filename: Filename for an on-disk database, or None for
an in-memory database. If the filename already exists, follow
this with a call to open(). To create a new on-disk database,
follow this with a call to create().
"""
BaseDB.__init__(self, filename, "shared key")
def __init__(self, filename=None):
"""Create a new VerifierDB instance.
@type filename: str
@param filename: Filename for an on-disk database, or None for
an in-memory database. If the filename already exists, follow
this with a call to open(). To create a new on-disk database,
follow this with a call to create().
"""
BaseDB.__init__(self, filename, "verifier")
def __init__(self, filename=None):
"""Create a new VerifierDB instance.
@type filename: str
@param filename: Filename for an on-disk database, or None for
an in-memory database. If the filename already exists, follow
this with a call to open(). To create a new on-disk database,
follow this with a call to create().
"""
BaseDB.__init__(self, filename, "verifier")
def render_GET(self, request):
# 传入一个id，直接开始该id的抓取工作
id = request.args.get("id", [""])[0]
if id.isdigit():
db = BaseDB()
info = db.get_novel_by_id(id)
if info:
# 准备工作
if ready_work(info, db):
mb.new_msg(id, 'start')
deferToThread(do_work, info, db).addErrback(self.get_error, id, db)
return "ok"
def render_GET(self, request):
# 传入一个id，直接开始该id的抓取工作
id = request.args.get("id", [""])[0]
if id.isdigit():
db = BaseDB()
info = db.get_novel_by_id(id)
if info:
# 准备工作
if ready_work(info, db):
mb.new_msg(id, 'start')
deferToThread(do_work, info,
db).addErrback(self.get_error, id, db)
return "ok"
def worker():
db = BaseDB()
while q.qsize():
novel = q.get()
try:
if ready_work(novel, db):
notify_page(novel[0], "start")
do_work(novel, db)
except Exception, e:
log.error("%s: %s" % (novel[0], traceback.format_exc()))
db.modify_status(novel[0], 4)
db.commit()
q.task_done()
def __setitem__(self, username, sharedKey):
"""Add a shared key to the database.
@type username: str
@param username: The username to associate the shared key with.
Must be less than or equal to 16 characters in length, and must
not already be in the database.
@type sharedKey: str
@param sharedKey: The shared key to add. Must be less than 48
characters in length.
"""
BaseDB.__setitem__(self, username, sharedKey)
def __setitem__(self, username, sharedKey):
"""Add a shared key to the database.
@type username: str
@param username: The username to associate the shared key with.
Must be less than or equal to 16 characters in length, and must
not already be in the database.
@type sharedKey: str
@param sharedKey: The shared key to add. Must be less than 48
characters in length.
"""
BaseDB.__setitem__(self, username, sharedKey)
def __setitem__(self, username, verifierEntry):
"""Add a verifier entry to the database.
@type username: str
@param username: The username to associate the verifier with.
Must be less than 256 characters in length. Must not already
be in the database.
@type verifierEntry: tuple
@param verifierEntry: The verifier entry to add. Use
L{tlslite.verifierdb.VerifierDB.makeVerifier} to create a
verifier entry.
"""
BaseDB.__setitem__(self, username, verifierEntry)
def __setitem__(self, username, verifierEntry):
"""Add a verifier entry to the database.
@type username: str
@param username: The username to associate the verifier with.
Must be less than 256 characters in length. Must not already
be in the database.
@type verifierEntry: tuple
@param verifierEntry: The verifier entry to add. Use
L{tlslite.VerifierDB.VerifierDB.makeVerifier} to create a
verifier entry.
"""
BaseDB.__setitem__(self, username, verifierEntry)
def setUp(self):
"""
var setups
"""
self.master_config = '../../docs/config/my_master.json'
self.slave_config = '../../docs/config/my_slave.json'
self.base = None
self.conn = None
self.ins = None
self.base = BaseDB()
self.base.load_config(self.slave_config)
self.conn, db, t = self.base.connect_server()
self.ins = Slave(self.conn)
class TestSlave(unittest.TestCase):
"""
Tests for class Slave, Jez I'm getting tired of writting
this shit
"""
def setUp(self):
"""
var setups
"""
self.master_config = '../../docs/config/my_master.json'
self.slave_config = '../../docs/config/my_slave.json'
self.base = None
self.conn = None
self.ins = None
self.base = BaseDB()
self.base.load_config(self.slave_config)
self.conn, db, t = self.base.connect_server()
self.ins = Slave(self.conn)
def test_slave(self):
self.assertIsNotNone(self.ins.get_job())
class TestMaster(unittest.TestCase):
"""
Tests for class Master, Jez I'm getting tired of writting
this shit
"""
def setUp(self):
"""
var setups
"""
self.master_config = '../../docs/config/my_master.json'
self.slave_config = '../../docs/config/my_slave.json'
self.base = None
self.conn = None
self.mast = None
self.base = BaseDB()
self.base.load_config(self.master_config)
self.conn, db, t = self.base.connect_server()
self.mast = Master(self.conn)
def test_master(self):
self.assertGreater(len(self.mast.list_libraries()), 0)
def test_add_job(self):
self.mast.clear_jobs()
self.assertEqual(self.mast.count_jobs(), 0)
jobs = [{
"symbol": "AMD",
'time_start': 'N/A',
'time_end': 'N/A',
'interval': '15min',
'field': 'price'
}]
self.mast.add_jobs(jobs)
self.assertEqual(self.mast.count_jobs(), 1)
def worker():
db = BaseDB()
while q.qsize():
novel = q.get()
try:
if ready_work(novel, db):
notify_page(novel[0], "start")
do_work(novel, db)
except Exception, e:
log.error("%s: %s" % (novel[0], traceback.format_exc()))
db.modify_status(novel[0], 4)
db.commit()
q.task_done()
def test_master(self):
ins = BaseDB()
ins.load_config(self.config)
conn, db, t = ins.connect_server()
mast = Master(conn)
self.assertGreater(len(mast.list_libraries()), 0)
def worker():
db = BaseDB()
while q.qsize():
novel = q.get()
try:
if ready_work(novel, db):
notify_page(novel[0], "start")
do_work(novel, db)
except Exception, e:
log.error("%s: %s" % (novel[0], traceback.format_exc()))
db.modify_status(novel[0], 4)
db.commit()
q.task_done()
# 插入任务
db = BaseDB()
novel_list = db.get_novel_list()
log.info("novel count %s", len(novel_list))
for novel in novel_list:
# db.modify_status(novel[0], 1)
q.put(novel)
# db.commit()
db.close()
del db
for i in range(min(q.qsize(), THREADS_COUNT)):
t = threading.Thread(target=worker)
t.setDaemon(True)
t.start()
q.join()
def test_slave(self):
ins = BaseDB()
ins.load_config(self.slave_config)
conn, db, t = ins.connect_server()
slave = Slave(conn)
self.assertGreater(len(slave.list_symbols()), 0)
def test_connection(self):
ins = BaseDB()
ins.load_config(self.master_config)
conn, db, t = ins.connect_server()
self.assertTrue(t)
def worker():
db = BaseDB()
while q.qsize():
novel = q.get()
try:
if ready_work(novel, db):
notify_page(novel[0], "start")
do_work(novel, db)
except Exception, e:
log.error("%s: %s" % (novel[0], traceback.format_exc()))
db.modify_status(novel[0], 4)
db.commit()
q.task_done()
# 插入任务
db = BaseDB()
novel_list = db.get_novel_list()
log.info("novel count %s", len(novel_list))
for novel in novel_list:
# db.modify_status(novel[0], 1)
q.put(novel)
# db.commit()
db.close()
del db
for i in range(min(q.qsize(), THREADS_COUNT)):
t = threading.Thread(target=worker)
t.setDaemon(True)
t.start()
q.join()
