def redischeck(instance, url_list):
try:
redisclient = redis.StrictRedis()
except Exception as e:
logging.error('Redis server connect error')
raise e
url_not_cached = []
for url in url_list:
post_id = get_post_id(url)
if redisclient.get(post_id):
logging.debug('Cache %s hit' % post_id)
redisclient.expire(post_id, 600)
else:
logging.debug('Caching %s' % post_id)
redisclient.set(post_id, 'True')
redisclient.expire(post_id, 600)
url_not_cached.append(url)
query_count = len(url_list)
cached = len(url_list) - len(url_not_cached)
logging.info("{0} QUERY {1} CACHED HIT RATE {2}".format(query_count, cached, cached / len(url_list)))
return func(instance, url_not_cached)
def get_content(self, url):
"""
Get content of url with cookie
:param url:
:return String if success
None if failed:
"""
try:
response = self.session_worker.get(url, timeout=10)
logging.debug('Get {0} SUCCESS'.format(common.get_post_id(url)))
return response
except requests.Timeout as e:
logging.warning('Get {0} TIMEOUT'.format(common.get_post_id(url), e))
except Exception as e:
logging.warning('Get {0} FAILED :{1} '.format(common.get_post_id(url), e))
return None
def get_content(self, url):
"""
Get content of url with cookie
:param url:
:return String if success
None if failed:
"""
try:
response = self.session_worker.get(url, timeout=10)
logging.info('Get {0} succeed'.format(url))
return response
except requests.Timeout as e:
logging.warning('Get {0} failed : {1}'.format(common.get_post_id(url), e))
except Exception as e:
logging.warning('Get {0} failed : unexpected :{1} '.format(common.get_post_id(url), e))
return None
def cache_check(instance, url_list):
url_not_cached = []
for url in url_list:
post_id = get_post_id(url)
if self.redisclient.get(post_id):
logging.info('Cache %s hit' % post_id)
self.redisclient.expire(post_id, 600)
else:
logging.info('Caching %s' % post_id)
self.redisclient.set(post_id, 'True')
self.redisclient.expire(post_id, 600)
url_not_cached.append(url)
return get_content_method(instance, url_not_cached)
def cache_check(instance, url_list):
url_not_cached = []
for url in url_list:
post_id = get_post_id(url)
if self.redisclient.get(post_id):
logging.info('Cache %s hit' % post_id)
self.redisclient.expire(post_id, 600)
else:
logging.info('Caching %s' % post_id)
self.redisclient.set(post_id, 'True')
self.redisclient.expire(post_id, 600)
url_not_cached.append(url)
return get_content_method(instance, url_not_cached)
def redischeck(instance, url_list):
redisclient = redis.StrictRedis()
url_not_cached = []
for url in url_list:
post_id = get_post_id(url)
if redisclient.get(post_id):
logging.info('Cache %s hit' % post_id)
redisclient.expire(post_id, 600)
else:
logging.info('Caching %s' % post_id)
redisclient.set(post_id, 'True')
redisclient.expire(post_id, 600)
url_not_cached.append(url)
return func(instance, url_not_cached)
def get_content(self, url):
"""Get content of url with cookie
:param url:
:return String if success
None if failed:
"""
while True:
try:
response = self.session_worker.get(url, cookies=self.cookie, timeout=10)
logging.info('Get {0} succeed'.format(url))
return response
except requests.Timeout as e:
logging.error('Get {0} failed : {1}'.format(common.get_post_id(url), e))
logging.error('Trying again')
def __call__(self, url_list):
print(self)
url_not_cached = []
for url in url_list:
post_id = get_post_id(url)
if self.redisclient.get(post_id):
logging.info('Cache %s hit' % post_id)
self.redisclient.expire(post_id, 600)
else:
logging.info('Caching %s' % post_id)
self.redisclient.set(post_id, 'True')
self.redisclient.expire(post_id, 600)
url_not_cached.append(url)
self.search_count = len(url_list)
self.hit_count = len(url_list) - len(url_not_cached)
return self.func(self, url_not_cached)
def redischeck(instance, url_list):
try:
redisclient = redis.StrictRedis()
except Exception as e:
logging.error('Redis server connect error')
raise e
url_not_cached = []
for url in url_list:
post_id = get_post_id(url)
if redisclient.get(post_id):
logging.info('Cache %s hit' % post_id)
redisclient.expire(post_id, 600)
else:
logging.info('Caching %s' % post_id)
redisclient.set(post_id, 'True')
redisclient.expire(post_id, 600)
url_not_cached.append(url)
return func(instance, url_not_cached)
def redischeck(instance, url_list):
try:
redisclient = redis.StrictRedis()
except Exception as e:
logging.error('Redis server connect error')
raise e
url_not_cached = []
for url in url_list:
post_id = get_post_id(url)
if redisclient.get(post_id):
logging.info('Cache %s hit' % post_id)
redisclient.expire(post_id, 600)
else:
logging.info('Caching %s' % post_id)
redisclient.set(post_id, 'True')
redisclient.expire(post_id, 600)
url_not_cached.append(url)
return func(instance, url_not_cached)