class RedisCache(Cache): def __init__(self, application): super(RedisCache, self).__init__(application) if self.application.config.REDIS_HOST is not None: self.redis = redis.StrictRedis( host=self.application.config.REDIS_HOST, port=self.application.config.REDIS_PORT, db=self.application.config.REDIS_DB_COUNT, password=self.application.config.REDIS_PASSWORD ) def get(self, key): return self.redis.get(key) def incr(self, key): return self.redis.incr(key) def set(self, key, expiration): self.lock = RedisLock(self.redis, lock_key='cyclops:lock:%s' % key, lock_timeout=5*60) if not self.lock.acquire(): return try: self.redis.setex( key, expiration, 0 ) finally: self.lock.release()
def __init__(self, redisSettings=globalRedisSettings): self.redisHostname = redisSettings['redisHostname'] self.redisPort = redisSettings['redisPort'] self.redisDb = redisSettings['redisDb'] self.prefixInRedis = redisSettings['prefixInRedis'] self.checkInterval = redisSettings['checkInterval'] self.redisConnection = redis.Redis(host=self.redisHostname, port=self.redisPort, db=self.redisDb) self.lock = RedisLock(self.redisConnection, lock_key='LOCK:WebSiteStatusService')
def set(self, key, expiration): self.lock = RedisLock(self.redis, lock_key='cyclops:lock:%s' % key, lock_timeout=5*60) if not self.lock.acquire(): return try: self.redis.setex( key, expiration, 0 ) finally: self.lock.release()
def __init__(self,redisSettings=globalRedisSettings): self.redisHostname=redisSettings['redisHostname'] self.redisPort=redisSettings['redisPort'] self.redisDb=redisSettings['redisDb'] self.prefixInRedis=redisSettings['prefixInRedis'] self.checkInterval=redisSettings['checkInterval'] self.redisConnection=redis.Redis(host=self.redisHostname,port=self.redisPort,db=self.redisDb) self.lock=RedisLock(self.redisConnection,lock_key='LOCK:WebSiteStatusService')
def _do_execute_query(parameterized_query, wlm_queue=None): # This method should always be getting executed within a Lambda context # Distributed dog pile lock pattern # From: https://pypi.python.org/pypi/python-redis-lock log.info("About to attempt acquiring lock...") redis_client = redshift.get_redshift_cache_redis_client() with RedisLock(redis_client, parameterized_query.cache_key, expire=300): # Get a lock with a 5-minute lifetime since that's the maximum duration of a Lambda # to ensure the lock is held for as long as the Python process / Lambda is running. log.info("Lock acquired.") return _do_execute_query_work(parameterized_query, wlm_queue)
class RedisCache(Cache): def __init__(self, application): super(RedisCache, self).__init__(application) if self.application.config.REDIS_HOST is not None: self.redis = redis.StrictRedis( host=self.application.config.REDIS_HOST, port=self.application.config.REDIS_PORT, db=self.application.config.REDIS_DB_COUNT, password=self.application.config.REDIS_PASSWORD ) def get(self, key): value = self.redis.get(key) if value is None: return None return int(value) def incr(self, key): return self.redis.incr(key) def set(self, key, expiration): self.lock = RedisLock(self.redis, lock_key='cyclops:lock:%s' % key, lock_timeout=5*60) if not self.lock.acquire(): return try: self.redis.setex( key, expiration, 0 ) finally: self.lock.release() def flushdb(self): self.redis.flushdb()
class WebSiteStatusService(object): def __init__(self, redisSettings=globalRedisSettings): self.redisHostname = redisSettings['redisHostname'] self.redisPort = redisSettings['redisPort'] self.redisDb = redisSettings['redisDb'] self.prefixInRedis = redisSettings['prefixInRedis'] self.checkInterval = redisSettings['checkInterval'] self.redisConnection = redis.Redis(host=self.redisHostname, port=self.redisPort, db=self.redisDb) self.lock = RedisLock(self.redisConnection, lock_key='LOCK:WebSiteStatusService') def __addOneUrl(self, oneUrl=None): if oneUrl is None: return if not isinstance(oneUrl, str): return self.redisConnection.sadd('SITES', oneUrl) def addUrlList(self, urlList=[]): if not isinstance(urlList, list): return self.lock.acquire() for oneUrl in urlList: self.__addOneUrl(oneUrl) self.lock.release() def __removeUrl(self, oneUrl): if oneUrl is None: return if not isinstance(oneUrl, str): return self.redisConnection.srem('SITES', oneUrl) def removeUrlList(self, urlList=[]): if not isinstance(urlList, list): return self.lock.acquire() for oneUrl in urlList: self.__removeUrl(oneUrl) self.lock.release() def __removePrefix(self, string): stringArray = string.split(':') newStringArray = stringArray[1:] #remove the first part this is URL newString = ':'.join(newStringArray) #reassemble the left things return newString def startService(self): tMain = threading.Thread(target=self.runServiceInThread, args=()) tMain.daemon = True tSlave1 = threading.Thread(target=self.updateTimestampInThread, args=()) tSlave1.daemon = True tSlave2 = threading.Thread(target=self.updateKeepAliveStatusInThead, args=()) tSlave2.daemon = True tMain.start() tSlave1.start() tSlave2.start() def updateKeepAliveStatusInThead(self): while True: debug('keep alive') runStatus = self.redisConnection.setex('KEEPALIVE', 'TRUE', 2) time.sleep(1) def updateTimestampInThread(self): while True: runStatus = self.redisConnection.get('STATUS') if (runStatus == 'STOP') or (runStatus is None): #check and do nothing debug('no timestamp') elif runStatus == 'RUN': self.redisConnection.set('TIMESTAMP', int(time.time())) debug('do timestamp') time.sleep(self.checkInterval) else: pass time.sleep(self.checkInterval) def runServiceInThread(self): while True: #this thread will loop forever runStatus = self.redisConnection.get('STATUS') if (runStatus == 'STOP') or (runStatus is None): #check and do nothing debug('no check') elif runStatus == 'RUN': debug('do check') webSiteStatus = WebSiteStatus() for oneUrl in self.redisConnection.smembers('SITES'): webSiteStatus.addSiteUrl(oneUrl) debug('happen this 1') #webSiteStatus.addSiteUrl(oneUrl) webSiteStatus.checkAll() (urlList, stateList) = webSiteStatus.getStatusList() debug('happen this 2') #append the site state into redis for oneUrl, oneState in zip(urlList, stateList): key = self.prefixInRedis + ':' + oneUrl self.redisConnection.set(key, oneState) #remove the url not in urlList keyPattern = self.prefixInRedis + ':*' #'URL:*' oldUrlListWithPrefix = self.redisConnection.keys( pattern=keyPattern) oldUrlList = [ self.__removePrefix(item) for item in oldUrlListWithPrefix ] newUrlList = self.redisConnection.smembers('SITES') self.lock.acquire() for oneUrl in oldUrlList: if oneUrl not in newUrlList: self.redisConnection.delete(self.prefixInRedis + ':' + oneUrl) self.lock.release() else: pass time.sleep(self.checkInterval) def stopService(self): self.lock.acquire() if self.redisConnection.get('STATUS') == 'RUN': self.redisConnection.set('STATUS', 'STOP') else: pass self.lock.release() def getStatus(self): returnStr = "" returnStr += 'HOST:%s PORT:%s DB:%s' % (self.redisHostname, self.redisPort, self.redisDb) returnStr += 'SITE LIST:%s' % (self.redisConnection.smembers('SITES')) returnStr += 'This is the content of begin' return returnStr def getUrlList(self): urlList = [] listSet = self.redisConnection.smembers('SITES') for item in listSet: urlList.append(item) return urlList
class ServiceMonitor(object): def __init__(self,redisSettings=globalRedisSettings): self.redisHostname=redisSettings['redisHostname'] self.redisPort=redisSettings['redisPort'] self.redisDb=redisSettings['redisDb'] self.prefixInRedis=redisSettings['prefixInRedis'] self.checkInterval=redisSettings['checkInterval'] self.redisConnection=redis.Redis(host=self.redisHostname,port=self.redisPort,db=self.redisDb) self.lock=RedisLock(self.redisConnection,lock_key='LOCK:WebSiteStatusService') def __removePrefix(self,string): stringArray=string.split(':') newStringArray=stringArray[1:] #remove the first part this is URL newString=':'.join(newStringArray) #reassemble the left things return newString def isServiceProcessDown(self): keepaliveStatus=self.redisConnection.get('KEEPALIVE') if keepaliveStatus=='TRUE': return False else: return True def getServiceStatus(self): #statusDict={} runStatus=self.redisConnection.get('STATUS') if (runStatus is None)or(runStatus=='STOP'): runStatus='stop' if runStatus=='RUN': runStatus='run' #statusDict['service_status']=runStatus return runStatus def __addOneUrl(self,oneUrl=None): if oneUrl is None: return if not isinstance(oneUrl, str): return self.redisConnection.sadd('SITES',oneUrl) def addUrlList(self,urlList=[]): if not isinstance(urlList, list): return self.lock.acquire() for oneUrl in urlList: self.__addOneUrl(oneUrl) self.lock.release() def __removeUrl(self,oneUrl): if oneUrl is None: return if not isinstance(oneUrl, str): return self.redisConnection.srem('SITES',oneUrl) def removeUrlList(self,urlList=[]): if not isinstance(urlList, list): return self.lock.acquire() for oneUrl in urlList: self.__removeUrl(oneUrl) self.lock.release() def changeServiceToRun(self): statusDict={} self.redisConnection.set('STATUS', 'RUN') statusDict['status']='success' return statusDict def changeServiceToStop(self): statusDict={} self.redisConnection.set('STATUS', 'STOP') statusDict['status']='success' return statusDict def getUrlList(self): urlListSet=self.redisConnection.smembers('SITES') urlList=[oneUrl for oneUrl in urlListSet] #change the set into list of python returnDict={} #print urlList returnDict['urls']=urlList return returnDict def getURLStatus(self): keyPattern=self.prefixInRedis+':*' #'URL:*' urlListWithPrefix=self.redisConnection.keys(pattern=keyPattern) urlList=[self.__removePrefix(item) for item in urlListWithPrefix ] statusList=[] for oneUrlWithPrefix in urlListWithPrefix: oneStatus=self.redisConnection.get(oneUrlWithPrefix) statusList.append(oneStatus) urlDict={} for url,status in zip(urlList,statusList): urlDict[url]=status returnDict={} returnDict['urls']=urlDict timeStamp=self.redisConnection.get('TIMESTAMP') if timeStamp is None: timeStamp='' returnDict['timestamp']=timeStamp return returnDict
class ServiceMonitor(object): def __init__(self, redisSettings=globalRedisSettings): self.redisHostname = redisSettings['redisHostname'] self.redisPort = redisSettings['redisPort'] self.redisDb = redisSettings['redisDb'] self.prefixInRedis = redisSettings['prefixInRedis'] self.checkInterval = redisSettings['checkInterval'] self.redisConnection = redis.Redis(host=self.redisHostname, port=self.redisPort, db=self.redisDb) self.lock = RedisLock(self.redisConnection, lock_key='LOCK:WebSiteStatusService') def __removePrefix(self, string): stringArray = string.split(':') newStringArray = stringArray[1:] #remove the first part this is URL newString = ':'.join(newStringArray) #reassemble the left things return newString def isServiceProcessDown(self): keepaliveStatus = self.redisConnection.get('KEEPALIVE') if keepaliveStatus == 'TRUE': return False else: return True def getServiceStatus(self): #statusDict={} runStatus = self.redisConnection.get('STATUS') if (runStatus is None) or (runStatus == 'STOP'): runStatus = 'stop' if runStatus == 'RUN': runStatus = 'run' #statusDict['service_status']=runStatus return runStatus def __addOneUrl(self, oneUrl=None): if oneUrl is None: return if not isinstance(oneUrl, str): return self.redisConnection.sadd('SITES', oneUrl) def addUrlList(self, urlList=[]): if not isinstance(urlList, list): return self.lock.acquire() for oneUrl in urlList: self.__addOneUrl(oneUrl) self.lock.release() def __removeUrl(self, oneUrl): if oneUrl is None: return if not isinstance(oneUrl, str): return self.redisConnection.srem('SITES', oneUrl) def removeUrlList(self, urlList=[]): if not isinstance(urlList, list): return self.lock.acquire() for oneUrl in urlList: self.__removeUrl(oneUrl) self.lock.release() def changeServiceToRun(self): statusDict = {} self.redisConnection.set('STATUS', 'RUN') statusDict['status'] = 'success' return statusDict def changeServiceToStop(self): statusDict = {} self.redisConnection.set('STATUS', 'STOP') statusDict['status'] = 'success' return statusDict def getUrlList(self): urlListSet = self.redisConnection.smembers('SITES') urlList = [oneUrl for oneUrl in urlListSet ] #change the set into list of python returnDict = {} #print urlList returnDict['urls'] = urlList return returnDict def getURLStatus(self): keyPattern = self.prefixInRedis + ':*' #'URL:*' urlListWithPrefix = self.redisConnection.keys(pattern=keyPattern) urlList = [self.__removePrefix(item) for item in urlListWithPrefix] statusList = [] for oneUrlWithPrefix in urlListWithPrefix: oneStatus = self.redisConnection.get(oneUrlWithPrefix) statusList.append(oneStatus) urlDict = {} for url, status in zip(urlList, statusList): urlDict[url] = status returnDict = {} returnDict['urls'] = urlDict timeStamp = self.redisConnection.get('TIMESTAMP') if timeStamp is None: timeStamp = '' returnDict['timestamp'] = timeStamp return returnDict