def register_detail(self,detail): logging.info("registering detail") if detail.proxy_key is None or detail.queue_key is None: raise Exception('detail object must have a proxy and queue key') if not self.redis.exists(detail.proxy_key) or not self.redis.exists(detail.queue_key): raise Exception("Unable to locate queue or proxy for detail") detail_key = detail.detail_key if self.redis.exists(detail.detail_key): logging.warn("Detail already exists") return Detail(**self.redis.hgetall(detail_key)) else: self.redis.hmset(detail_key, detail.to_dict()) relational_keys = {'proxy_key': detail.proxy_key, 'queue_key': detail.queue_key} self.redis.hmset(detail_key, relational_keys) active_rdq = RedisDetailQueue(detail.queue_key,active=True) inactive_rdq = RedisDetailQueue(detail.queue_key,active=False) if detail.active: active_rdq.enqueue(detail) else: inactive_rdq.enqueue(detail) return Detail(**self.redis.hgetall(detail_key))
def clone_detail(self,detail,new_queue): new_queue_key = new_queue.queue_key if detail.queue_id != SEED_QUEUE_ID: raise Exception("can only clone details from seed queue") if not self.redis_mgr.redis.exists(new_queue_key): raise Exception("Invalid queue key while cloning detail") new_queue_id = self.redis_mgr.redis.hget(new_queue_key,'queue_id') proxy_id = detail.proxy_id proxy_key = detail.proxy_key cloned = Detail(proxy_id=proxy_id,queue_id=new_queue_id,queue_key=new_queue_key, proxy_key=proxy_key) new_detail_key = cloned.detail_key if self.redis_mgr.redis.exists(new_detail_key): logging.warn("trying to clone a detail into a queue where it already exists.") return Detail(**self.redis_mgr.redis.hgetall(new_detail_key)) if new_queue_id is not None and proxy_id is not None: db_detail = self.db_mgr.get_detail_by_queue_and_proxy(new_queue_id,proxy_id) if db_detail is not None: logging.warn("Attempting to clone a detail that already exists") return self.redis_mgr.register_detail(db_detail) return self.redis_mgr.register_detail(cloned)
def get_non_seed_details(self): query= """ SELECT * FROM details WHERE queue_id != %(seed_queue_id)s AND queue_id != %(agg_queue_id)s AND active=%(active)s ORDER BY last_used ASC LIMIT %(limit)s; """ active_params = { 'seed_queue_id': SEED_QUEUE_ID, 'agg_queue_id': AGGREGATE_QUEUE_ID, 'active': True, 'limit': ACTIVE_LIMIT } inactive_params = { 'seed_queue_id': SEED_QUEUE_ID, 'agg_queue_id': AGGREGATE_QUEUE_ID, 'active': False, 'limit': INACTIVE_LIMIT } active = [Detail(**d) for d in self.do_query(query, active_params)] inactive = [Detail(**d) for d in self.do_query(query, inactive_params)] return active + inactive
def dequeue(self,requeue=True): if self.is_empty(): raise RedisDetailQueueEmpty("No proxies available for queue key %s" % self.queue_key) detail = Detail(**self.redis.hgetall(self.redis.lpop(self.redis_key))) if requeue: self.enqueue(detail) return detail
def get_seed_details(self): self.init_seed_details() params = {'seed_queue_id': SEED_QUEUE_ID} query= """ SELECT * FROM details WHERE queue_id=%(queue_id)s AND active=%(active)s ORDER BY last_used ASC LIMIT %(limit)s; """ a_params = {"queue_id":SEED_QUEUE_ID, "active":True,"limit": ACTIVE_LIMIT} ia_params = {"queue_id":SEED_QUEUE_ID, "active":False,"limit": INACTIVE_LIMIT} active = [Detail(**d) for d in self.do_query(query,a_params)] inactive = [Detail(**d) for d in self.do_query(query,ia_params)] return active + inactive
def sync_to_db(self): new_queues = [Queue(**self.redis_mgr.redis.hgetall(q)) for q in self.redis_mgr.redis.keys("qt_*")] new_proxies = [Proxy(**self.redis_mgr.redis.hgetall(p)) for p in self.redis_mgr.redis.keys("pt_*")] new_detail_keys = set(self.redis_mgr.redis.keys('d_qt*') + self.redis_mgr.redis.keys('d_*pt*')) for ndk in new_detail_keys: self.redis_mgr.redis.sadd('new_details', ndk) new_details = [Detail(**self.redis_mgr.redis.hgetall(d)) for d in list(new_detail_keys)] cursor = self.db_mgr.cursor() queue_keys_to_id = {} proxy_keys_to_id = {} for q in new_queues: self.db_mgr.insert_queue(q,cursor) queue_id = cursor.fetchone()[0] queue_keys_to_id[q.queue_key] = queue_id for p in new_proxies: self.db_mgr.insert_proxy(p,cursor) proxy_id = cursor.fetchone()[0] proxy_keys_to_id[p.proxy_key] = proxy_id for d in new_details: if d.proxy_id is None: d.proxy_id = proxy_keys_to_id[d.proxy_key] if d.queue_id is None: d.queue_id = queue_keys_to_id[d.queue_key] self.db_mgr.insert_detail(d,cursor) changed_detail_keys = self.redis_mgr.redis.sdiff('changed_details','new_details') changed_details = [Detail(**self.redis_mgr.redis.hgetall(d)) for d in self.redis_mgr.redis.sdiff('changed_details','new_details')] for changed in changed_details: if(changed.queue_id is None or changed.proxy_id is None): raise Exception("Unable to get a queue_id or proxy_id for an existing detail") self.db_mgr.update_detail(changed) logging.info("synced redis cache to database, resetting cache.") cursor.close() self.redis_mgr.redis.flushall() return True
def get_detail_by_queue_and_proxy(self,queue_id,proxy_id): query = "SELECT * FROM details WHERE proxy_id=%(proxy_id)s AND queue_id=%(queue_id)s" params = {'queue_id': queue_id, 'proxy_id':proxy_id} cursor = self.cursor() cursor.execute(query,params) detail_data = cursor.fetchone() if detail_data is None: cursor.close() return None detail = Detail(**detail_data) cursor.close() return detail
def create_proxy(self, address, port, protocol): proxy_keys = self.redis_mgr.redis.keys('p*') for pkey in proxy_keys: if self.redis_mgr.redis.hget(pkey,'address') == address and self.redis_mgr.redis.hget(pkey,'port'): logging.warn("Trying to create a proxy that already exists") return Proxy(**self.redis_mgr.redis.hgetall(pkey)) proxy = Proxy(address=address,port=port,protocol=protocol) proxy = self.redis_mgr.register_proxy(proxy) proxy_key = proxy.proxy_key queue_key = "%s_%s" % (QUEUE_PREFIX,SEED_QUEUE_ID) detail = Detail(proxy_key=proxy_key,queue_id=SEED_QUEUE_ID,queue_key=queue_key) self.redis_mgr.register_detail(detail) return proxy
def init_seed_details(self): seed_count = self.do_query("SELECT COUNT(*) as c FROM details WHERE queue_id=%(queue_id)s", {'queue_id':SEED_QUEUE_ID})[0]['c'] logging.info("initializing seed proxies") cursor = self.cursor() if seed_count == 0: seed_details = [Detail(proxy_id=p['proxy_id'], queue_id=SEED_QUEUE_ID) for p in self.do_query("SELECT proxy_id FROM proxies")] for sd in seed_details: self.insert_detail(sd,cursor) query = """ BEGIN; LOCK TABLE details IN EXCLUSIVE MODE; SELECT setval('details_detail_id_seq', COALESCE((SELECT MAX(detail_id)+1 FROM details),1), false); COMMIT; """ cursor.execute(query) cursor.close() logging.info("done initializing seeds")
def get_detail(self,redis_detail_key): return Detail(**self.redis.hgetall(redis_detail_key))