def init_seed_queues(self):
logging.info("Initializing queues...")
if(SEED_QUEUE_ID == AGGREGATE_QUEUE_ID):
raise Exception("aggregate_queue and seed_queue cannot share the same id. Check app_config.json")
seed_queue = Queue(domain=SEED_QUEUE_DOMAIN,queue_id=SEED_QUEUE_ID)
agg_queue = Queue(domain=AGGREGATE_QUEUE_DOMAIN, queue_id=AGGREGATE_QUEUE_ID)
query = "SELECT queue_id from queues WHERE domain = %(domain)s"
db_seed = self.do_query(query, {'domain':SEED_QUEUE_DOMAIN})
db_agg = self.do_query(query, {'domain': AGGREGATE_QUEUE_DOMAIN})
if len(db_seed) == 0:
self.insert_queue(seed_queue)
elif db_seed[0]['queue_id'] != SEED_QUEUE_ID:
raise Exception("seed_queue id mismatch. seed_queue should be set to %s Check app_config.json" % db_seed[0]['queue_id'])
if len(db_agg) == 0:
self.insert_queue(agg_queue)
elif(db_agg[0]['queue_id'] != AGGREGATE_QUEUE_ID):
raise Exception("aggregate queue_id mismatch. aggregate_queue should be set to %s Check app_config.json" % db_agg[0]['queue_id'])
cursor = self.cursor()
query = """
BEGIN;
LOCK TABLE queues IN EXCLUSIVE MODE;
SELECT setval('queues_queue_id_seq', COALESCE((SELECT MAX(queue_id)+1 FROM queues),1), false);
COMMIT;
"""
cursor.execute(query)
cursor.close()
logging.info("Finished initializing queue.")
def create_queue(self,url):
logging.info("creating queue for %s" % url)
domain = parse_domain(url)
all_queues_by_domain = {queue.domain: queue for queue in self.redis_mgr.get_all_queues()}
if domain in all_queues_by_domain:
logging.warn("Trying to create a queue that already exists.")
return all_queues_by_domain[domain]
return self.redis_mgr.register_queue(Queue(domain=domain))
def sync_to_db(self):
new_queues = [Queue(**self.redis_mgr.redis.hgetall(q)) for q in self.redis_mgr.redis.keys("qt_*")]
new_proxies = [Proxy(**self.redis_mgr.redis.hgetall(p)) for p in self.redis_mgr.redis.keys("pt_*")]
new_detail_keys = set(self.redis_mgr.redis.keys('d_qt*') + self.redis_mgr.redis.keys('d_*pt*'))
for ndk in new_detail_keys:
self.redis_mgr.redis.sadd('new_details', ndk)
new_details = [Detail(**self.redis_mgr.redis.hgetall(d)) for d in list(new_detail_keys)]
cursor = self.db_mgr.cursor()
queue_keys_to_id = {}
proxy_keys_to_id = {}
for q in new_queues:
self.db_mgr.insert_queue(q,cursor)
queue_id = cursor.fetchone()[0]
queue_keys_to_id[q.queue_key] = queue_id
for p in new_proxies:
self.db_mgr.insert_proxy(p,cursor)
proxy_id = cursor.fetchone()[0]
proxy_keys_to_id[p.proxy_key] = proxy_id
for d in new_details:
if d.proxy_id is None:
d.proxy_id = proxy_keys_to_id[d.proxy_key]
if d.queue_id is None:
d.queue_id = queue_keys_to_id[d.queue_key]
self.db_mgr.insert_detail(d,cursor)
changed_detail_keys = self.redis_mgr.redis.sdiff('changed_details','new_details')
changed_details = [Detail(**self.redis_mgr.redis.hgetall(d)) for d in self.redis_mgr.redis.sdiff('changed_details','new_details')]
for changed in changed_details:
if(changed.queue_id is None or changed.proxy_id is None):
raise Exception("Unable to get a queue_id or proxy_id for an existing detail")
self.db_mgr.update_detail(changed)
logging.info("synced redis cache to database, resetting cache.")
cursor.close()
self.redis_mgr.redis.flushall()
return True
def get_queue_by_id(self,qid):
lookup_key = "%s_%s" % (QUEUE_PREFIX,qid)
if not self.redis.exists(lookup_key):
raise Exception("No such queue with id %s" % qid)
return Queue(**self.redis.hgetall(lookup_key))
def get_queue_by_domain(self,domain):
queue_dict = {q.domain: q for q in self.get_all_queues()}
if domain in queue_dict:
return queue_dict[domain]
return self.register_queue(Queue(domain=domain))
def get_all_queues(self):
return [Queue(**self.redis.hgetall(q)) for q in self.redis.keys('q*')]
def register_queue(self,queue):
logging.info("registering queue")
queue_key = self.register_object(QUEUE_PREFIX,queue)
self.redis.hmset(queue_key, {'queue_key': queue_key})
return Queue(**self.redis.hgetall(queue_key))
def get_queues(self):
self.init_seed_queues()
return [Queue(**r) for r in self.do_query("SELECT * FROM queues;")]