def register_detail(self,detail):
logging.info("registering detail")
if detail.proxy_key is None or detail.queue_key is None:
raise Exception('detail object must have a proxy and queue key')
if not self.redis.exists(detail.proxy_key) or not self.redis.exists(detail.queue_key):
raise Exception("Unable to locate queue or proxy for detail")
detail_key = detail.detail_key
if self.redis.exists(detail.detail_key):
logging.warn("Detail already exists")
return Detail(**self.redis.hgetall(detail_key))
else:
self.redis.hmset(detail_key, detail.to_dict())
relational_keys = {'proxy_key': detail.proxy_key, 'queue_key': detail.queue_key}
self.redis.hmset(detail_key, relational_keys)
active_rdq = RedisDetailQueue(detail.queue_key,active=True)
inactive_rdq = RedisDetailQueue(detail.queue_key,active=False)
if detail.active:
active_rdq.enqueue(detail)
else:
inactive_rdq.enqueue(detail)
return Detail(**self.redis.hgetall(detail_key))
def clone_detail(self,detail,new_queue):
new_queue_key = new_queue.queue_key
if detail.queue_id != SEED_QUEUE_ID:
raise Exception("can only clone details from seed queue")
if not self.redis_mgr.redis.exists(new_queue_key):
raise Exception("Invalid queue key while cloning detail")
new_queue_id = self.redis_mgr.redis.hget(new_queue_key,'queue_id')
proxy_id = detail.proxy_id
proxy_key = detail.proxy_key
cloned = Detail(proxy_id=proxy_id,queue_id=new_queue_id,queue_key=new_queue_key, proxy_key=proxy_key)
new_detail_key = cloned.detail_key
if self.redis_mgr.redis.exists(new_detail_key):
logging.warn("trying to clone a detail into a queue where it already exists.")
return Detail(**self.redis_mgr.redis.hgetall(new_detail_key))
if new_queue_id is not None and proxy_id is not None:
db_detail = self.db_mgr.get_detail_by_queue_and_proxy(new_queue_id,proxy_id)
if db_detail is not None:
logging.warn("Attempting to clone a detail that already exists")
return self.redis_mgr.register_detail(db_detail)
return self.redis_mgr.register_detail(cloned)
def get_non_seed_details(self):
query= """
SELECT * FROM details
WHERE queue_id != %(seed_queue_id)s
AND queue_id != %(agg_queue_id)s
AND active=%(active)s
ORDER BY last_used ASC
LIMIT %(limit)s;
"""
active_params = {
'seed_queue_id': SEED_QUEUE_ID,
'agg_queue_id': AGGREGATE_QUEUE_ID,
'active': True,
'limit': ACTIVE_LIMIT
}
inactive_params = {
'seed_queue_id': SEED_QUEUE_ID,
'agg_queue_id': AGGREGATE_QUEUE_ID,
'active': False,
'limit': INACTIVE_LIMIT
}
active = [Detail(**d) for d in self.do_query(query, active_params)]
inactive = [Detail(**d) for d in self.do_query(query, inactive_params)]
return active + inactive
def dequeue(self,requeue=True):
if self.is_empty():
raise RedisDetailQueueEmpty("No proxies available for queue key %s" % self.queue_key)
detail = Detail(**self.redis.hgetall(self.redis.lpop(self.redis_key)))
if requeue:
self.enqueue(detail)
return detail
def get_seed_details(self):
self.init_seed_details()
params = {'seed_queue_id': SEED_QUEUE_ID}
query= """
SELECT * FROM details
WHERE queue_id=%(queue_id)s
AND active=%(active)s
ORDER BY last_used ASC
LIMIT %(limit)s;
"""
a_params = {"queue_id":SEED_QUEUE_ID, "active":True,"limit": ACTIVE_LIMIT}
ia_params = {"queue_id":SEED_QUEUE_ID, "active":False,"limit": INACTIVE_LIMIT}
active = [Detail(**d) for d in self.do_query(query,a_params)]
inactive = [Detail(**d) for d in self.do_query(query,ia_params)]
return active + inactive
def sync_to_db(self):
new_queues = [Queue(**self.redis_mgr.redis.hgetall(q)) for q in self.redis_mgr.redis.keys("qt_*")]
new_proxies = [Proxy(**self.redis_mgr.redis.hgetall(p)) for p in self.redis_mgr.redis.keys("pt_*")]
new_detail_keys = set(self.redis_mgr.redis.keys('d_qt*') + self.redis_mgr.redis.keys('d_*pt*'))
for ndk in new_detail_keys:
self.redis_mgr.redis.sadd('new_details', ndk)
new_details = [Detail(**self.redis_mgr.redis.hgetall(d)) for d in list(new_detail_keys)]
cursor = self.db_mgr.cursor()
queue_keys_to_id = {}
proxy_keys_to_id = {}
for q in new_queues:
self.db_mgr.insert_queue(q,cursor)
queue_id = cursor.fetchone()[0]
queue_keys_to_id[q.queue_key] = queue_id
for p in new_proxies:
self.db_mgr.insert_proxy(p,cursor)
proxy_id = cursor.fetchone()[0]
proxy_keys_to_id[p.proxy_key] = proxy_id
for d in new_details:
if d.proxy_id is None:
d.proxy_id = proxy_keys_to_id[d.proxy_key]
if d.queue_id is None:
d.queue_id = queue_keys_to_id[d.queue_key]
self.db_mgr.insert_detail(d,cursor)
changed_detail_keys = self.redis_mgr.redis.sdiff('changed_details','new_details')
changed_details = [Detail(**self.redis_mgr.redis.hgetall(d)) for d in self.redis_mgr.redis.sdiff('changed_details','new_details')]
for changed in changed_details:
if(changed.queue_id is None or changed.proxy_id is None):
raise Exception("Unable to get a queue_id or proxy_id for an existing detail")
self.db_mgr.update_detail(changed)
logging.info("synced redis cache to database, resetting cache.")
cursor.close()
self.redis_mgr.redis.flushall()
return True
def get_detail_by_queue_and_proxy(self,queue_id,proxy_id):
query = "SELECT * FROM details WHERE proxy_id=%(proxy_id)s AND queue_id=%(queue_id)s"
params = {'queue_id': queue_id, 'proxy_id':proxy_id}
cursor = self.cursor()
cursor.execute(query,params)
detail_data = cursor.fetchone()
if detail_data is None:
cursor.close()
return None
detail = Detail(**detail_data)
cursor.close()
return detail
def create_proxy(self, address, port, protocol):
proxy_keys = self.redis_mgr.redis.keys('p*')
for pkey in proxy_keys:
if self.redis_mgr.redis.hget(pkey,'address') == address and self.redis_mgr.redis.hget(pkey,'port'):
logging.warn("Trying to create a proxy that already exists")
return Proxy(**self.redis_mgr.redis.hgetall(pkey))
proxy = Proxy(address=address,port=port,protocol=protocol)
proxy = self.redis_mgr.register_proxy(proxy)
proxy_key = proxy.proxy_key
queue_key = "%s_%s" % (QUEUE_PREFIX,SEED_QUEUE_ID)
detail = Detail(proxy_key=proxy_key,queue_id=SEED_QUEUE_ID,queue_key=queue_key)
self.redis_mgr.register_detail(detail)
return proxy
def init_seed_details(self):
seed_count = self.do_query("SELECT COUNT(*) as c FROM details WHERE queue_id=%(queue_id)s", {'queue_id':SEED_QUEUE_ID})[0]['c']
logging.info("initializing seed proxies")
cursor = self.cursor()
if seed_count == 0:
seed_details = [Detail(proxy_id=p['proxy_id'], queue_id=SEED_QUEUE_ID) for p in self.do_query("SELECT proxy_id FROM proxies")]
for sd in seed_details:
self.insert_detail(sd,cursor)
query = """
BEGIN;
LOCK TABLE details IN EXCLUSIVE MODE;
SELECT setval('details_detail_id_seq', COALESCE((SELECT MAX(detail_id)+1 FROM details),1), false);
COMMIT;
"""
cursor.execute(query)
cursor.close()
logging.info("done initializing seeds")
def get_detail(self,redis_detail_key):
return Detail(**self.redis.hgetall(redis_detail_key))