class QueueCheck:
def __init__(self):
self.redis = Redis()
self.redisStore = RedisStore()
self.mongoCall = MongoClient('localhost')['icraw']['company']
def maincheck(self):
while True:
for i in range(11):
queue = ["tmp_seed_queue_%d" %i]
print 'queue'
print queue
premin = self.redisStore.getlimittime(20)
print premin
seeds = self.redis.zrangebyscore("queue",0,premin)
for seed in seeds:
print "xxxxxxxxxxx"
x = self.redis.hget(queue,seed.get("uid", None)[0])
self.add_job('save_testseed1', deepcopy(x))
# 将缓冲队列中的种子移入下一队列并从当前缓冲队列中删除
# print self.redis.zrange("tmp_seed_queue_0", 0, -1)
self.redis.zadd("tmp_seed_queue_%d" %(i+1), seed.get("uid", None)[0], self.onlyint(datetime.datetime.now()))
# print seed.get("uid", None)[0]
self.redis.zrem(queue, seed.get("uid", None)[0])
def setUp(self):
self.db = 1
self.store = RedisStore()
self.store.connect(self.db)
class TestRedisStore(unittest.TestCase):
def setUp(self):
self.db = 1
self.store = RedisStore()
self.store.connect(self.db)
def tearDown(self):
self.store._r.flushdb()
self.store.disconnect()
def test_put(self):
obj = ObjClass(1, 1, 2)
self.assertTrue(self.store.put(obj.as_doc()))
self.assertRaises(TypeError, self.store.put, 4)
self.assertRaises(TypeError, self.store.put, [1, 2])
def test_get_id(self):
res = self.store.get_by_id("123")
self.assertTrue(res is None)
obj = ObjClass(1, 1, 2)
doc = obj.as_doc()
mid = self.store.put(doc)
res = self.store.get_by_id(mid)
self.assertEquals(res['x'], doc['x'])
self.assertEquals(res['y'], doc['y'])
def test_get(self):
obj = ObjClass(1, 1, 2)
doc = obj.as_doc()
self.store.put(doc)
res = self.store.get('x', 1)
self.assertEquals(res['id'], 1)
res = self.store.get('x', 4)
self.assertTrue(res is None)
def test_get_all(self):
self.assertEquals(len([x for x in self.store.get_all()]), 0)
obj1 = ObjClass(1, 1, 2)
obj2 = ObjClass(2, 2, 4)
mid1 = self.store.put(obj1.as_doc())
mid2 = self.store.put(obj2.as_doc())
self.assertEquals(len([x for x in self.store.get_all()]), 2)
def test_filter(self):
obj1 = ObjClass(1, 1, 2)
obj2 = ObjClass(2, 2, 4)
obj3 = ObjClass(3, 2, 5)
mid1 = self.store.put(obj1.as_doc())
mid2 = self.store.put(obj2.as_doc())
mid3 = self.store.put(obj3.as_doc())
res = self.store.filter(x=2)
self.assertEquals(len([x for x in res]), 2)
for r in res:
self.assertEquals(r['x'], 2)
res = [x for x in self.store.filter(x=1)]
self.assertEquals(len(res), 1)
self.assertEquals(res[0]['x'], 1)
res = [x for x in self.store.filter(x=2, y=4)]
self.assertEquals(len(res), 1)
self.assertEquals(res[0]['x'], 2)
self.assertEquals(res[0]['y'], 4)
res = [x for x in self.store.filter(x=2, y=6)]
self.assertEquals(len(res), 0)
def setUp(self):
self.db = 1
self.store = RedisStore()
self.store.connect(self.db)
class TestRedisStore(unittest.TestCase):
def setUp(self):
self.db = 1
self.store = RedisStore()
self.store.connect(self.db)
def tearDown(self):
self.store._r.flushdb()
self.store.disconnect()
def test_put(self):
obj = ObjClass(1, 1, 2)
self.assertTrue(self.store.put(obj.as_doc()))
self.assertRaises(TypeError, self.store.put, 4)
self.assertRaises(TypeError, self.store.put, [1, 2])
def test_get_id(self):
res = self.store.get_by_id("123")
self.assertTrue(res is None)
obj = ObjClass(1, 1, 2)
doc = obj.as_doc()
mid = self.store.put(doc)
res = self.store.get_by_id(mid)
self.assertEquals(res['x'], doc['x'])
self.assertEquals(res['y'], doc['y'])
def test_get(self):
obj = ObjClass(1, 1, 2)
doc = obj.as_doc()
self.store.put(doc)
res = self.store.get('x', 1)
self.assertEquals(res['id'], 1)
res = self.store.get('x', 4)
self.assertTrue(res is None)
def test_get_all(self):
self.assertEquals(len([x for x in self.store.get_all()]), 0)
obj1 = ObjClass(1, 1, 2)
obj2 = ObjClass(2, 2, 4)
mid1 = self.store.put(obj1.as_doc())
mid2 = self.store.put(obj2.as_doc())
self.assertEquals(len([x for x in self.store.get_all()]), 2)
def test_filter(self):
obj1 = ObjClass(1, 1, 2)
obj2 = ObjClass(2, 2, 4)
obj3 = ObjClass(3, 2, 5)
mid1 = self.store.put(obj1.as_doc())
mid2 = self.store.put(obj2.as_doc())
mid3 = self.store.put(obj3.as_doc())
res = self.store.filter(x=2)
self.assertEquals(len([x for x in res]), 2)
for r in res:
self.assertEquals(r['x'], 2)
res = [x for x in self.store.filter(x=1)]
self.assertEquals(len(res), 1)
self.assertEquals(res[0]['x'], 1)
res = [x for x in self.store.filter(x=2, y=4)]
self.assertEquals(len(res), 1)
self.assertEquals(res[0]['x'], 2)
self.assertEquals(res[0]['y'], 4)
res = [x for x in self.store.filter(x=2, y=6)]
self.assertEquals(len(res), 0)
def __init__(self):
self.redis = Redis()
self.redisStore = RedisStore()
self.mongoCall = MongoClient('localhost')['icraw']['company']