class RedisCache(Cache):
def __init__(self, application):
super(RedisCache, self).__init__(application)
if self.application.config.REDIS_HOST is not None:
self.redis = redis.StrictRedis(
host=self.application.config.REDIS_HOST,
port=self.application.config.REDIS_PORT,
db=self.application.config.REDIS_DB_COUNT,
password=self.application.config.REDIS_PASSWORD
)
def get(self, key):
return self.redis.get(key)
def incr(self, key):
return self.redis.incr(key)
def set(self, key, expiration):
self.lock = RedisLock(self.redis, lock_key='cyclops:lock:%s' % key, lock_timeout=5*60)
if not self.lock.acquire():
return
try:
self.redis.setex(
key,
expiration,
0
)
finally:
self.lock.release()
def __init__(self, redisSettings=globalRedisSettings):
self.redisHostname = redisSettings['redisHostname']
self.redisPort = redisSettings['redisPort']
self.redisDb = redisSettings['redisDb']
self.prefixInRedis = redisSettings['prefixInRedis']
self.checkInterval = redisSettings['checkInterval']
self.redisConnection = redis.Redis(host=self.redisHostname,
port=self.redisPort,
db=self.redisDb)
self.lock = RedisLock(self.redisConnection,
lock_key='LOCK:WebSiteStatusService')
def set(self, key, expiration):
self.lock = RedisLock(self.redis, lock_key='cyclops:lock:%s' % key, lock_timeout=5*60)
if not self.lock.acquire():
return
try:
self.redis.setex(
key,
expiration,
0
)
finally:
self.lock.release()
def __init__(self,redisSettings=globalRedisSettings):
self.redisHostname=redisSettings['redisHostname']
self.redisPort=redisSettings['redisPort']
self.redisDb=redisSettings['redisDb']
self.prefixInRedis=redisSettings['prefixInRedis']
self.checkInterval=redisSettings['checkInterval']
self.redisConnection=redis.Redis(host=self.redisHostname,port=self.redisPort,db=self.redisDb)
self.lock=RedisLock(self.redisConnection,lock_key='LOCK:WebSiteStatusService')
def _do_execute_query(parameterized_query, wlm_queue=None):
# This method should always be getting executed within a Lambda context
# Distributed dog pile lock pattern
# From: https://pypi.python.org/pypi/python-redis-lock
log.info("About to attempt acquiring lock...")
redis_client = redshift.get_redshift_cache_redis_client()
with RedisLock(redis_client, parameterized_query.cache_key, expire=300):
# Get a lock with a 5-minute lifetime since that's the maximum duration of a Lambda
# to ensure the lock is held for as long as the Python process / Lambda is running.
log.info("Lock acquired.")
return _do_execute_query_work(parameterized_query, wlm_queue)
def set(self, key, expiration):
self.lock = RedisLock(self.redis, lock_key='cyclops:lock:%s' % key, lock_timeout=5*60)
if not self.lock.acquire():
return
try:
self.redis.setex(
key,
expiration,
0
)
finally:
self.lock.release()
class RedisCache(Cache):
def __init__(self, application):
super(RedisCache, self).__init__(application)
if self.application.config.REDIS_HOST is not None:
self.redis = redis.StrictRedis(
host=self.application.config.REDIS_HOST,
port=self.application.config.REDIS_PORT,
db=self.application.config.REDIS_DB_COUNT,
password=self.application.config.REDIS_PASSWORD
)
def get(self, key):
value = self.redis.get(key)
if value is None:
return None
return int(value)
def incr(self, key):
return self.redis.incr(key)
def set(self, key, expiration):
self.lock = RedisLock(self.redis, lock_key='cyclops:lock:%s' % key, lock_timeout=5*60)
if not self.lock.acquire():
return
try:
self.redis.setex(
key,
expiration,
0
)
finally:
self.lock.release()
def flushdb(self):
self.redis.flushdb()
class WebSiteStatusService(object):
def __init__(self, redisSettings=globalRedisSettings):
self.redisHostname = redisSettings['redisHostname']
self.redisPort = redisSettings['redisPort']
self.redisDb = redisSettings['redisDb']
self.prefixInRedis = redisSettings['prefixInRedis']
self.checkInterval = redisSettings['checkInterval']
self.redisConnection = redis.Redis(host=self.redisHostname,
port=self.redisPort,
db=self.redisDb)
self.lock = RedisLock(self.redisConnection,
lock_key='LOCK:WebSiteStatusService')
def __addOneUrl(self, oneUrl=None):
if oneUrl is None:
return
if not isinstance(oneUrl, str):
return
self.redisConnection.sadd('SITES', oneUrl)
def addUrlList(self, urlList=[]):
if not isinstance(urlList, list):
return
self.lock.acquire()
for oneUrl in urlList:
self.__addOneUrl(oneUrl)
self.lock.release()
def __removeUrl(self, oneUrl):
if oneUrl is None:
return
if not isinstance(oneUrl, str):
return
self.redisConnection.srem('SITES', oneUrl)
def removeUrlList(self, urlList=[]):
if not isinstance(urlList, list):
return
self.lock.acquire()
for oneUrl in urlList:
self.__removeUrl(oneUrl)
self.lock.release()
def __removePrefix(self, string):
stringArray = string.split(':')
newStringArray = stringArray[1:] #remove the first part this is URL
newString = ':'.join(newStringArray) #reassemble the left things
return newString
def startService(self):
tMain = threading.Thread(target=self.runServiceInThread, args=())
tMain.daemon = True
tSlave1 = threading.Thread(target=self.updateTimestampInThread,
args=())
tSlave1.daemon = True
tSlave2 = threading.Thread(target=self.updateKeepAliveStatusInThead,
args=())
tSlave2.daemon = True
tMain.start()
tSlave1.start()
tSlave2.start()
def updateKeepAliveStatusInThead(self):
while True:
debug('keep alive')
runStatus = self.redisConnection.setex('KEEPALIVE', 'TRUE', 2)
time.sleep(1)
def updateTimestampInThread(self):
while True:
runStatus = self.redisConnection.get('STATUS')
if (runStatus
== 'STOP') or (runStatus is None): #check and do nothing
debug('no timestamp')
elif runStatus == 'RUN':
self.redisConnection.set('TIMESTAMP', int(time.time()))
debug('do timestamp')
time.sleep(self.checkInterval)
else:
pass
time.sleep(self.checkInterval)
def runServiceInThread(self):
while True: #this thread will loop forever
runStatus = self.redisConnection.get('STATUS')
if (runStatus
== 'STOP') or (runStatus is None): #check and do nothing
debug('no check')
elif runStatus == 'RUN':
debug('do check')
webSiteStatus = WebSiteStatus()
for oneUrl in self.redisConnection.smembers('SITES'):
webSiteStatus.addSiteUrl(oneUrl)
debug('happen this 1')
#webSiteStatus.addSiteUrl(oneUrl)
webSiteStatus.checkAll()
(urlList, stateList) = webSiteStatus.getStatusList()
debug('happen this 2')
#append the site state into redis
for oneUrl, oneState in zip(urlList, stateList):
key = self.prefixInRedis + ':' + oneUrl
self.redisConnection.set(key, oneState)
#remove the url not in urlList
keyPattern = self.prefixInRedis + ':*' #'URL:*'
oldUrlListWithPrefix = self.redisConnection.keys(
pattern=keyPattern)
oldUrlList = [
self.__removePrefix(item) for item in oldUrlListWithPrefix
]
newUrlList = self.redisConnection.smembers('SITES')
self.lock.acquire()
for oneUrl in oldUrlList:
if oneUrl not in newUrlList:
self.redisConnection.delete(self.prefixInRedis + ':' +
oneUrl)
self.lock.release()
else:
pass
time.sleep(self.checkInterval)
def stopService(self):
self.lock.acquire()
if self.redisConnection.get('STATUS') == 'RUN':
self.redisConnection.set('STATUS', 'STOP')
else:
pass
self.lock.release()
def getStatus(self):
returnStr = ""
returnStr += 'HOST:%s PORT:%s DB:%s' % (self.redisHostname,
self.redisPort, self.redisDb)
returnStr += 'SITE LIST:%s' % (self.redisConnection.smembers('SITES'))
returnStr += 'This is the content of begin'
return returnStr
def getUrlList(self):
urlList = []
listSet = self.redisConnection.smembers('SITES')
for item in listSet:
urlList.append(item)
return urlList
class ServiceMonitor(object):
def __init__(self,redisSettings=globalRedisSettings):
self.redisHostname=redisSettings['redisHostname']
self.redisPort=redisSettings['redisPort']
self.redisDb=redisSettings['redisDb']
self.prefixInRedis=redisSettings['prefixInRedis']
self.checkInterval=redisSettings['checkInterval']
self.redisConnection=redis.Redis(host=self.redisHostname,port=self.redisPort,db=self.redisDb)
self.lock=RedisLock(self.redisConnection,lock_key='LOCK:WebSiteStatusService')
def __removePrefix(self,string):
stringArray=string.split(':')
newStringArray=stringArray[1:] #remove the first part this is URL
newString=':'.join(newStringArray) #reassemble the left things
return newString
def isServiceProcessDown(self):
keepaliveStatus=self.redisConnection.get('KEEPALIVE')
if keepaliveStatus=='TRUE':
return False
else:
return True
def getServiceStatus(self):
#statusDict={}
runStatus=self.redisConnection.get('STATUS')
if (runStatus is None)or(runStatus=='STOP'):
runStatus='stop'
if runStatus=='RUN':
runStatus='run'
#statusDict['service_status']=runStatus
return runStatus
def __addOneUrl(self,oneUrl=None):
if oneUrl is None:
return
if not isinstance(oneUrl, str):
return
self.redisConnection.sadd('SITES',oneUrl)
def addUrlList(self,urlList=[]):
if not isinstance(urlList, list):
return
self.lock.acquire()
for oneUrl in urlList:
self.__addOneUrl(oneUrl)
self.lock.release()
def __removeUrl(self,oneUrl):
if oneUrl is None:
return
if not isinstance(oneUrl, str):
return
self.redisConnection.srem('SITES',oneUrl)
def removeUrlList(self,urlList=[]):
if not isinstance(urlList, list):
return
self.lock.acquire()
for oneUrl in urlList:
self.__removeUrl(oneUrl)
self.lock.release()
def changeServiceToRun(self):
statusDict={}
self.redisConnection.set('STATUS', 'RUN')
statusDict['status']='success'
return statusDict
def changeServiceToStop(self):
statusDict={}
self.redisConnection.set('STATUS', 'STOP')
statusDict['status']='success'
return statusDict
def getUrlList(self):
urlListSet=self.redisConnection.smembers('SITES')
urlList=[oneUrl for oneUrl in urlListSet] #change the set into list of python
returnDict={}
#print urlList
returnDict['urls']=urlList
return returnDict
def getURLStatus(self):
keyPattern=self.prefixInRedis+':*' #'URL:*'
urlListWithPrefix=self.redisConnection.keys(pattern=keyPattern)
urlList=[self.__removePrefix(item) for item in urlListWithPrefix ]
statusList=[]
for oneUrlWithPrefix in urlListWithPrefix:
oneStatus=self.redisConnection.get(oneUrlWithPrefix)
statusList.append(oneStatus)
urlDict={}
for url,status in zip(urlList,statusList):
urlDict[url]=status
returnDict={}
returnDict['urls']=urlDict
timeStamp=self.redisConnection.get('TIMESTAMP')
if timeStamp is None:
timeStamp=''
returnDict['timestamp']=timeStamp
return returnDict
class ServiceMonitor(object):
def __init__(self, redisSettings=globalRedisSettings):
self.redisHostname = redisSettings['redisHostname']
self.redisPort = redisSettings['redisPort']
self.redisDb = redisSettings['redisDb']
self.prefixInRedis = redisSettings['prefixInRedis']
self.checkInterval = redisSettings['checkInterval']
self.redisConnection = redis.Redis(host=self.redisHostname,
port=self.redisPort,
db=self.redisDb)
self.lock = RedisLock(self.redisConnection,
lock_key='LOCK:WebSiteStatusService')
def __removePrefix(self, string):
stringArray = string.split(':')
newStringArray = stringArray[1:] #remove the first part this is URL
newString = ':'.join(newStringArray) #reassemble the left things
return newString
def isServiceProcessDown(self):
keepaliveStatus = self.redisConnection.get('KEEPALIVE')
if keepaliveStatus == 'TRUE':
return False
else:
return True
def getServiceStatus(self):
#statusDict={}
runStatus = self.redisConnection.get('STATUS')
if (runStatus is None) or (runStatus == 'STOP'):
runStatus = 'stop'
if runStatus == 'RUN':
runStatus = 'run'
#statusDict['service_status']=runStatus
return runStatus
def __addOneUrl(self, oneUrl=None):
if oneUrl is None:
return
if not isinstance(oneUrl, str):
return
self.redisConnection.sadd('SITES', oneUrl)
def addUrlList(self, urlList=[]):
if not isinstance(urlList, list):
return
self.lock.acquire()
for oneUrl in urlList:
self.__addOneUrl(oneUrl)
self.lock.release()
def __removeUrl(self, oneUrl):
if oneUrl is None:
return
if not isinstance(oneUrl, str):
return
self.redisConnection.srem('SITES', oneUrl)
def removeUrlList(self, urlList=[]):
if not isinstance(urlList, list):
return
self.lock.acquire()
for oneUrl in urlList:
self.__removeUrl(oneUrl)
self.lock.release()
def changeServiceToRun(self):
statusDict = {}
self.redisConnection.set('STATUS', 'RUN')
statusDict['status'] = 'success'
return statusDict
def changeServiceToStop(self):
statusDict = {}
self.redisConnection.set('STATUS', 'STOP')
statusDict['status'] = 'success'
return statusDict
def getUrlList(self):
urlListSet = self.redisConnection.smembers('SITES')
urlList = [oneUrl for oneUrl in urlListSet
] #change the set into list of python
returnDict = {}
#print urlList
returnDict['urls'] = urlList
return returnDict
def getURLStatus(self):
keyPattern = self.prefixInRedis + ':*' #'URL:*'
urlListWithPrefix = self.redisConnection.keys(pattern=keyPattern)
urlList = [self.__removePrefix(item) for item in urlListWithPrefix]
statusList = []
for oneUrlWithPrefix in urlListWithPrefix:
oneStatus = self.redisConnection.get(oneUrlWithPrefix)
statusList.append(oneStatus)
urlDict = {}
for url, status in zip(urlList, statusList):
urlDict[url] = status
returnDict = {}
returnDict['urls'] = urlDict
timeStamp = self.redisConnection.get('TIMESTAMP')
if timeStamp is None:
timeStamp = ''
returnDict['timestamp'] = timeStamp
return returnDict
