class MemCacheWrapper(object):
"""
Memcache client wrapper.
No exception raise and add some useful function.
"""
def __init__(self, servers, logerr=None):
self.cache = MCClient(servers=servers, debug=False)
self.logerr = logerr
def add(self, key, val=1, time=0):
try:
return self.cache.add(key, val, time)
except Exception as e:
_logger.warning("Exception during `add`: %s", e)
return None
def count(self, key, expires=0, delta=1):
try:
result = self.cache.incr(key, delta)
if result is None:
if not self.cache.add(key, delta, expires):
result = self.cache.incr(key, delta)
else:
return delta
return result
except Exception as e:
_logger.warning("Exception during `count`: %s", e)
return None
def get(self, key):
result = None
try:
result = self.cache.get(str(key))
except Exception as e:
_logger.warning("Exception during `get`: %s", e)
return result
def set(self, key, value, expires):
result = False
try:
result = self.cache.set(str(key), value, expires)
except Exception as e:
_logger.warning("Exception during `set`: %s", e)
return result
def delete(self, key):
result = False
try:
result = self.cache.delete(key)
except Exception as e:
_logger.warning("Exception during `del`: %s", e)
return result
def whereyoulive(addr):
conn = Client([gethostname()+':11211'])
if conn.get(addr_prefix+addr) != None:
conn.incr(addr_prefix+addr)
else:
conn.set(addr_prefix+addr, 1)
if not addresses.__contains__(addr):
addresses.append(addr)
def whereyoulive(addr):
conn = Client([gethostname() + ':11211'])
if conn.get(addr_prefix + addr) != None:
conn.incr(addr_prefix + addr)
else:
conn.set(addr_prefix + addr, 1)
if not addresses.__contains__(addr):
addresses.append(addr)
class TestMemcache(unittest.TestCase):
def setUp(self):
# TODO(): unix socket server stuff
servers = ["127.0.0.1:11211"]
self.mc = Client(servers, debug=1)
def tearDown(self):
self.mc.disconnect_all()
def check_setget(self, key, val, noreply=False):
self.mc.set(key, val, noreply=noreply)
newval = self.mc.get(key)
self.assertEqual(newval, val)
def test_setget(self):
self.check_setget("a_string", "some random string")
self.check_setget("a_string_2", "some random string", noreply=True)
self.check_setget("an_integer", 42)
self.check_setget("an_integer_2", 42, noreply=True)
def test_delete(self):
self.check_setget("long", int(1 << 30))
result = self.mc.delete("long")
self.assertEqual(result, True)
self.assertEqual(self.mc.get("long"), None)
def test_get_multi(self):
self.check_setget("gm_a_string", "some random string")
self.check_setget("gm_an_integer", 42)
self.assertEqual(self.mc.get_multi(["gm_a_string", "gm_an_integer"]), {
"gm_an_integer": 42,
"gm_a_string": "some random string"
})
def test_get_unknown_value(self):
self.mc.delete("unknown_value")
self.assertEqual(self.mc.get("unknown_value"), None)
def test_setget_foostruct(self):
f = FooStruct()
self.check_setget("foostruct", f)
self.check_setget("foostruct_2", f, noreply=True)
def test_incr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.incr("i_an_integer", 1), 43)
def test_incr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.incr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 43)
def test_decr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.decr("i_an_integer", 1), 41)
def test_decr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.decr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 41)
def test_sending_spaces(self):
try:
self.mc.set("this has spaces", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_control_characters(self):
try:
self.mc.set("this\x10has\x11control characters\x02", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_key_too_long(self):
try:
self.mc.set('a' * SERVER_MAX_KEY_LENGTH + 'a', 1)
except Client.MemcachedKeyLengthError as err:
self.assertTrue("length is >" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyLengthError, nothing raised")
# These should work.
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1)
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1, noreply=True)
def test_setget_boolean(self):
"""GitHub issue #75. Set/get with boolean values."""
self.check_setget("bool", True)
def test_unicode_key(self):
s = six.u('\u4f1a')
maxlen = SERVER_MAX_KEY_LENGTH // len(s.encode('utf-8'))
key = s * maxlen
self.mc.set(key, 5)
value = self.mc.get(key)
self.assertEqual(value, 5)
def test_ignore_too_large_value(self):
# NOTE: "MemCached: while expecting[...]" is normal...
key = 'keyhere'
value = 'a' * (SERVER_MAX_VALUE_LENGTH // 2)
self.assertTrue(self.mc.set(key, value))
self.assertEqual(self.mc.get(key), value)
value = 'a' * SERVER_MAX_VALUE_LENGTH
self.assertFalse(self.mc.set(key, value))
# This test fails if the -I option is used on the memcached server
self.assertTrue(self.mc.get(key) is None)
def test_get_set_multi_key_prefix(self):
"""Testing set_multi() with no memcacheds running."""
prefix = 'pfx_'
values = {'key1': 'a', 'key2': 'b'}
errors = self.mc.set_multi(values, key_prefix=prefix)
self.assertEqual(errors, [])
keys = list(values)
self.assertEqual(self.mc.get_multi(keys, key_prefix=prefix), values)
def test_set_multi_dead_servers(self):
"""Testing set_multi() with no memcacheds running."""
self.mc.disconnect_all()
for server in self.mc.servers:
server.mark_dead('test')
errors = self.mc.set_multi({'key1': 'a', 'key2': 'b'})
self.assertEqual(sorted(errors), ['key1', 'key2'])
def test_disconnect_all_delete_multi(self):
"""Testing delete_multi() with no memcacheds running."""
self.mc.disconnect_all()
ret = self.mc.delete_multi({'keyhere': 'a', 'keythere': 'b'})
self.assertEqual(ret, 1)
redis_port = '9250'
password = ''
hostname = 'ec2-54-183-15-168.us-west-1.compute.amazonaws.com'
mc = Client([hostname + ':' + memcached_port], debug=0)
#mc = Redis(host=hostname, port=redis_port, password=password)
start_time = datetime.now()
mc.set("first_key", "first value")
value = mc.get("first_key")
print(value)
mc.set("second_key", 3)
mc.delete("second_key")
mc.set("key", "1") # note that the key used for incr/decr must be
# a string.
value = mc.get('key')
print(value)
mc.incr("key")
value = mc.get('key')
print(value)
mc.decr("key")
value = mc.get('key')
print(value)
end_time = datetime.now()
elapsed_time = end_time - start_time
print('Total Time = ', elapsed_time.microseconds / 1000, ' ms')
class TestMemcache(unittest.TestCase):
def setUp(self):
# TODO(): unix socket server stuff
servers = ["127.0.0.1:11211"]
self.mc = Client(servers, debug=1)
def tearDown(self):
self.mc.flush_all()
self.mc.disconnect_all()
def check_setget(self, key, val, noreply=False):
self.mc.set(key, val, noreply=noreply)
newval = self.mc.get(key)
self.assertEqual(newval, val)
def test_setget(self):
self.check_setget("a_string", "some random string")
self.check_setget("a_string_2", "some random string", noreply=True)
self.check_setget("an_integer", 42)
self.check_setget("an_integer_2", 42, noreply=True)
def test_delete(self):
self.check_setget("long", int(1 << 30))
result = self.mc.delete("long")
self.assertEqual(result, True)
self.assertEqual(self.mc.get("long"), None)
@mock.patch.object(_Host, 'send_cmd')
@mock.patch.object(_Host, 'readline')
def test_touch(self, mock_readline, mock_send_cmd):
with captured_stderr():
self.mc.touch('key')
mock_send_cmd.assert_called_with(b'touch key 0')
def test_get_multi(self):
self.check_setget("gm_a_string", "some random string")
self.check_setget("gm_an_integer", 42)
self.assertEqual(self.mc.get_multi(["gm_a_string", "gm_an_integer"]), {
"gm_an_integer": 42,
"gm_a_string": "some random string"
})
def test_get_unknown_value(self):
self.mc.delete("unknown_value")
self.assertEqual(self.mc.get("unknown_value"), None)
def test_setget_foostruct(self):
f = FooStruct()
self.check_setget("foostruct", f)
self.check_setget("foostruct_2", f, noreply=True)
def test_incr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.incr("i_an_integer", 1), 43)
def test_incr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.incr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 43)
def test_decr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.decr("i_an_integer", 1), 41)
def test_decr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.decr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 41)
def test_sending_spaces(self):
try:
self.mc.set("this has spaces", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_control_characters(self):
try:
self.mc.set("this\x10has\x11control characters\x02", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_key_too_long(self):
try:
self.mc.set('a' * SERVER_MAX_KEY_LENGTH + 'a', 1)
except Client.MemcachedKeyLengthError as err:
self.assertTrue("length is >" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyLengthError, nothing raised")
# These should work.
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1)
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1, noreply=True)
def test_setget_boolean(self):
"""GitHub issue #75. Set/get with boolean values."""
self.check_setget("bool", True)
def test_unicode_key(self):
s = u'\u4f1a'
maxlen = SERVER_MAX_KEY_LENGTH // len(s.encode('utf-8'))
key = s * maxlen
self.mc.set(key, 5)
value = self.mc.get(key)
self.assertEqual(value, 5)
def test_unicode_value(self):
key = 'key'
value = u'Iñtërnâtiônàlizætiøn2'
self.mc.set(key, value)
cached_value = self.mc.get(key)
self.assertEqual(value, cached_value)
def test_binary_string(self):
value = 'value_to_be_compressed'
compressed_value = zlib.compress(value.encode())
self.mc.set('binary1', compressed_value)
compressed_result = self.mc.get('binary1')
self.assertEqual(compressed_value, compressed_result)
self.assertEqual(value, zlib.decompress(compressed_result).decode())
self.mc.add('binary1-add', compressed_value)
compressed_result = self.mc.get('binary1-add')
self.assertEqual(compressed_value, compressed_result)
self.assertEqual(value, zlib.decompress(compressed_result).decode())
self.mc.set_multi({'binary1-set_many': compressed_value})
compressed_result = self.mc.get('binary1-set_many')
self.assertEqual(compressed_value, compressed_result)
self.assertEqual(value, zlib.decompress(compressed_result).decode())
def test_ignore_too_large_value(self):
# NOTE: "MemCached: while expecting[...]" is normal...
key = 'keyhere'
value = 'a' * (SERVER_MAX_VALUE_LENGTH // 2)
self.assertTrue(self.mc.set(key, value))
self.assertEqual(self.mc.get(key), value)
value = 'a' * SERVER_MAX_VALUE_LENGTH
with captured_stderr() as log:
self.assertIs(self.mc.set(key, value), False)
self.assertEqual(
log.getvalue(),
"MemCached: while expecting 'STORED', got unexpected response "
"'SERVER_ERROR object too large for cache'\n")
# This test fails if the -I option is used on the memcached server
self.assertTrue(self.mc.get(key) is None)
def test_get_set_multi_key_prefix(self):
"""Testing set_multi() with no memcacheds running."""
prefix = 'pfx_'
values = {'key1': 'a', 'key2': 'b'}
errors = self.mc.set_multi(values, key_prefix=prefix)
self.assertEqual(errors, [])
keys = list(values)
self.assertEqual(self.mc.get_multi(keys, key_prefix=prefix), values)
def test_set_multi_dead_servers(self):
"""Testing set_multi() with no memcacheds running."""
self.mc.disconnect_all()
with captured_stderr() as log:
for server in self.mc.servers:
server.mark_dead('test')
self.assertIn('Marking dead.', log.getvalue())
errors = self.mc.set_multi({'key1': 'a', 'key2': 'b'})
self.assertEqual(sorted(errors), ['key1', 'key2'])
def test_disconnect_all_delete_multi(self):
"""Testing delete_multi() with no memcacheds running."""
self.mc.disconnect_all()
with captured_stderr() as output:
ret = self.mc.delete_multi(('keyhere', 'keythere'))
self.assertEqual(ret, 1)
self.assertEqual(
output.getvalue(),
"MemCached: while expecting 'DELETED', got unexpected response "
"'NOT_FOUND'\n"
"MemCached: while expecting 'DELETED', got unexpected response "
"'NOT_FOUND'\n")
@mock.patch.object(_Host, 'send_cmd') # Don't send any commands.
@mock.patch.object(_Host, 'readline')
def test_touch_unexpected_reply(self, mock_readline, mock_send_cmd):
"""touch() logs an error upon receiving an unexpected reply."""
mock_readline.return_value = 'SET' # the unexpected reply
with captured_stderr() as output:
self.mc.touch('key')
self.assertEqual(
output.getvalue(),
"MemCached: touch expected %s, got: 'SET'\n" % b'TOUCHED')
class PrintFavicon(BaseHandler):
def __init__(self):
super(PrintFavicon, self).__init__()
default_icon_data = self.open(DEFAULT_FAVICON_LOC, time()).read()
self.default_icon = Icon(data=default_icon_data,
location=DEFAULT_FAVICON_LOC,
type=DEFAULT_FAVICON_TYPE)
self.env = Environment(loader=FileSystemLoader(
os.path.join(cherrypy.config['favicon.root'], 'templates')))
self.mc = Client(
['%(memcache.host)s:%(memcache.port)d' % cherrypy.config], debug=2)
# Initialize counters
for counter in ['requests', 'hits', 'defaults']:
self.mc.add('counter-%s' % counter, '0')
def open(self, url, start, headers=None):
time_spent = int(time() - start)
if time_spent >= TIMEOUT:
raise TimeoutError(time_spent)
if not headers:
headers = dict()
headers.update({
'User-Agent':
'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; ' +
'rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13'
})
opener = build_opener(HTTPRedirectHandler(), HTTPCookieProcessor())
return opener.open(Request(url, headers=headers),
timeout=min(CONNECTION_TIMEOUT,
TIMEOUT - time_spent))
def validateIconResponse(self, iconResponse):
if iconResponse.getcode() != 200:
cherrypy.log('Non-success response:%d fetching url:%s' % \
(iconResponse.getcode(), iconResponse.geturl()),
severity=INFO)
return None
iconContentType = iconResponse.info().gettype()
if iconContentType in ICON_MIMETYPE_BLACKLIST:
cherrypy.log('Url:%s favicon content-Type:%s blacklisted' % \
(iconResponse.geturl(), iconContentType),
severity=INFO)
return None
icon = iconResponse.read()
iconLength = len(icon)
if iconLength == 0:
cherrypy.log('Url:%s null content length' % iconResponse.geturl(),
severity=INFO)
return None
if iconLength < MIN_ICON_LENGTH or iconLength > MAX_ICON_LENGTH:
# Issue warning, but accept nonetheless!
cherrypy.log('Warning: url:%s favicon size:%d out of bounds' % \
(iconResponse.geturl(), iconLength),
severity=INFO)
return Icon(data=icon, type=iconContentType)
# Icon at [domain]/favicon.ico?
def iconAtRoot(self, targetDomain, start):
cherrypy.log('Attempting to locate favicon for domain:%s at root' % \
targetDomain,
severity=INFO)
rootIconPath = targetDomain + '/favicon.ico'
try:
rootDomainFaviconResult = self.open(rootIconPath, start)
rootIcon = self.validateIconResponse(rootDomainFaviconResult)
if rootIcon:
cherrypy.log('Found favicon for domain:%s at root' %
targetDomain,
severity=INFO)
self.cacheIcon(targetDomain, rootIcon.data, rootIconPath)
rootIcon.location = rootIconPath
return rootIcon
except:
cherrypy.log('Error fetching favicon at domain root:%s, err:%s, msg:%s' % \
(targetDomain, sys.exc_info()[0], sys.exc_info()[1]),
severity=INFO)
# Icon specified in page?
def iconInPage(self, targetDomain, targetPath, start, refresh=True):
cherrypy.log('Attempting to locate embedded favicon link in page:%s' % \
targetPath,
severity=INFO)
try:
rootDomainPageResult = self.open(targetPath, start)
if rootDomainPageResult.getcode() == 200:
pageSoup = BeautifulSoup(rootDomainPageResult.read())
pageSoupIcon = pageSoup.find(
'link',
rel=compile('^(shortcut|icon|shortcut icon)$', IGNORECASE))
if pageSoupIcon:
pageIconHref = pageSoupIcon.get('href')
if pageIconHref:
pageIconPath = urljoin(targetPath, pageIconHref)
cherrypy.log('Found embedded favicon link:%s for domain:%s' % \
(pageIconPath, targetDomain),
severity=INFO)
cookies = rootDomainPageResult.headers.getheaders(
"Set-Cookie")
headers = None
if cookies:
headers = {'Cookie': ';'.join(cookies)}
pagePathFaviconResult = self.open(pageIconPath,
start,
headers=headers)
pageIcon = self.validateIconResponse(
pagePathFaviconResult)
if pageIcon:
cherrypy.log('Found favicon at:%s for domain:%s' % \
(pageIconPath, targetDomain),
severity=INFO)
self.cacheIcon(targetDomain, pageIcon.data,
pageIconPath)
pageIcon.location = pageIconPath
return pageIcon
else:
if refresh:
for meta in pageSoup.findAll('meta'):
if meta.get('http-equiv', '').lower() == 'refresh':
match = search('url=([^;]+)',
meta.get('content', ''),
flags=IGNORECASE)
if match:
refreshPath = urljoin(
rootDomainPageResult.geturl(),
match.group(1))
cherrypy.log('Processing refresh directive:%s for domain:%s' % \
(refreshPath, targetDomain),
severity=INFO)
return self.iconInPage(targetDomain,
refreshPath,
start,
refresh=False)
cherrypy.log('No link tag found:%s' % targetPath,
severity=INFO)
else:
cherrypy.log('Non-success response:%d for url:%s' % \
(rootDomainPageResult.getcode(), targetPath),
severity=INFO)
except:
cherrypy.log('Error extracting favicon from page:%s, err:%s, msg:%s' % \
(targetPath, sys.exc_info()[0], sys.exc_info()[1]),
severity=WARNING)
def cacheIcon(self, domain, icon, loc):
cherrypy.log('Caching icon at location:%s for domain:%s' %
(loc, domain),
severity=INFO)
if not self.mc.set('icon-%s' % domain, icon, time=MC_CACHE_TIME):
cherrypy.log('Could not cache icon for domain:%s' % domain,
severity=ERROR)
def iconInCache(self, targetDomain, start):
icon = self.mc.get('icon-%s' % targetDomain)
if icon:
self.mc.incr('counter-hits')
cherrypy.log('Cache hit:%s' % targetDomain, severity=INFO)
cherrypy.response.headers['X-Cache'] = 'Hit'
if icon == 'DEFAULT':
self.mc.incr('counter-defaults')
cherrypy.response.headers['X-Cache'] = 'Hit'
return self.default_icon
else:
return Icon(data=icon)
def writeIcon(self, icon):
self.writeHeaders(icon)
return icon.data
def writeHeaders(self, icon, fmt='%a, %d %b %Y %H:%M:%S %z'):
# MIME Type
cherrypy.response.headers['Content-Type'] = icon.type or 'image/x-icon'
# Set caching headers
cherrypy.response.headers['Cache-Control'] = 'public, max-age=2592000'
cherrypy.response.headers['Expires'] = \
(datetime.now() + timedelta(days=30)).strftime(fmt)
def parse(self, url):
# Get page path
targetPath = self.urldecode(url)
if not targetPath.startswith('http'):
targetPath = 'http://%s' % targetPath
cherrypy.log('Decoded URL:%s' % targetPath, severity=INFO)
# Split path to get domain
targetURL = urlparse(targetPath)
if not targetURL or not targetURL.scheme or not targetURL.netloc:
raise cherrypy.HTTPError(400, 'Malformed URL:%s' % url)
targetDomain = '%s://%s' % (targetURL.scheme, targetURL.netloc)
cherrypy.log('URL:%s, domain:%s' % (targetPath, targetDomain),
severity=INFO)
return (targetPath, targetDomain)
@cherrypy.expose
def index(self):
status = {'status': 'ok', 'counters': dict()}
for counter in ['requests', 'hits', 'defaults']:
status['counters'][counter] = self.mc.get('counter-%s' % counter)
return json.dumps(status)
@cherrypy.expose
def test(self):
topSites = open(
os.path.join(cherrypy.config['favicon.root'], 'topsites.txt'),
'r').read().split()
template = self.env.get_template('test.html')
return template.render(topSites=topSites)
@cherrypy.expose
def clear(self, url):
cherrypy.log('Incoming cache invalidation request:%s' % url,
severity=INFO)
targetPath, targetDomain = self.parse(str(url))
self.mc.delete('icon_loc-%s' % targetDomain)
cherrypy.log('Evicted cache entry for %s' % targetDomain,
severity=INFO)
@cherrypy.expose
def s(self, url, skipCache='false'):
start = time()
if skipCache.lower() == 'true':
skipCache = True
else:
skipCache = False
cherrypy.log('Incoming request:%s (skipCache=%s)' % (url, skipCache),
severity=INFO)
self.mc.incr('counter-requests')
targetPath, targetDomain = self.parse(str(url))
icon = (not skipCache and self.iconInCache(targetDomain, start)) or \
self.iconInPage(targetDomain, targetPath, start) or \
self.iconAtRoot(targetDomain, start)
if not icon:
cherrypy.log('Falling back to default icon for:%s' % targetDomain,
severity=INFO)
self.cacheIcon(targetDomain, 'DEFAULT', 'DEFAULT_LOC')
self.mc.incr('counter-defaults')
icon = self.default_icon
cherrypy.log('Time taken to process domain:%s %f' % \
(targetDomain, time() - start),
severity=INFO)
return self.writeIcon(icon)
class PrintFavicon(BaseHandler):
def __init__(self):
super(PrintFavicon, self).__init__()
default_icon_data = self.open(DEFAULT_FAVICON_LOC, time()).read()
self.default_icon = Icon(data=default_icon_data, location=DEFAULT_FAVICON_LOC, type=DEFAULT_FAVICON_TYPE)
self.env = Environment(loader=FileSystemLoader(os.path.join(cherrypy.config["favicon.root"], "templates")))
self.mc = Client(["%(memcache.host)s:%(memcache.port)d" % cherrypy.config], debug=2)
# Initialize counters
for counter in ["requests", "hits", "defaults"]:
self.mc.add("counter-%s" % counter, "0")
def open(self, url, start, headers=None):
time_spent = int(time() - start)
if time_spent >= TIMEOUT:
raise TimeoutError(time_spent)
if not headers:
headers = dict()
headers.update(
{
"User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; "
+ "rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13"
}
)
opener = build_opener(HTTPRedirectHandler(), HTTPCookieProcessor())
return opener.open(Request(url, headers=headers), timeout=min(CONNECTION_TIMEOUT, TIMEOUT - time_spent))
def validateIconResponse(self, iconResponse):
if iconResponse.getcode() != 200:
cherrypy.log(
"Non-success response:%d fetching url:%s" % (iconResponse.getcode(), iconResponse.geturl()),
severity=INFO,
)
return None
iconContentType = iconResponse.info().gettype()
if iconContentType in ICON_MIMETYPE_BLACKLIST:
cherrypy.log(
"Url:%s favicon content-Type:%s blacklisted" % (iconResponse.geturl(), iconContentType), severity=INFO
)
return None
icon = iconResponse.read()
iconLength = len(icon)
if iconLength == 0:
cherrypy.log("Url:%s null content length" % iconResponse.geturl(), severity=INFO)
return None
if iconLength < MIN_ICON_LENGTH or iconLength > MAX_ICON_LENGTH:
# Issue warning, but accept nonetheless!
cherrypy.log(
"Warning: url:%s favicon size:%d out of bounds" % (iconResponse.geturl(), iconLength), severity=INFO
)
return Icon(data=icon, type=iconContentType)
# Icon at [domain]/favicon.ico?
def iconAtRoot(self, targetDomain, start):
cherrypy.log("Attempting to locate favicon for domain:%s at root" % targetDomain, severity=INFO)
rootIconPath = targetDomain + "/favicon.ico"
try:
rootDomainFaviconResult = self.open(rootIconPath, start)
rootIcon = self.validateIconResponse(rootDomainFaviconResult)
if rootIcon:
cherrypy.log("Found favicon for domain:%s at root" % targetDomain, severity=INFO)
self.cacheIcon(targetDomain, rootIcon.data, rootIconPath)
rootIcon.location = rootIconPath
return rootIcon
except:
cherrypy.log(
"Error fetching favicon at domain root:%s, err:%s, msg:%s"
% (targetDomain, sys.exc_info()[0], sys.exc_info()[1]),
severity=INFO,
)
# Icon specified in page?
def iconInPage(self, targetDomain, targetPath, start, refresh=True):
cherrypy.log("Attempting to locate embedded favicon link in page:%s" % targetPath, severity=INFO)
try:
rootDomainPageResult = self.open(targetPath, start)
if rootDomainPageResult.getcode() == 200:
pageSoup = BeautifulSoup(rootDomainPageResult.read())
pageSoupIcon = pageSoup.find("link", rel=compile("^(shortcut|icon|shortcut icon)$", IGNORECASE))
if pageSoupIcon:
pageIconHref = pageSoupIcon.get("href")
if pageIconHref:
pageIconPath = urljoin(targetPath, pageIconHref)
cherrypy.log(
"Found embedded favicon link:%s for domain:%s" % (pageIconPath, targetDomain), severity=INFO
)
cookies = rootDomainPageResult.headers.getheaders("Set-Cookie")
headers = None
if cookies:
headers = {"Cookie": ";".join(cookies)}
pagePathFaviconResult = self.open(pageIconPath, start, headers=headers)
pageIcon = self.validateIconResponse(pagePathFaviconResult)
if pageIcon:
cherrypy.log(
"Found favicon at:%s for domain:%s" % (pageIconPath, targetDomain), severity=INFO
)
self.cacheIcon(targetDomain, pageIcon.data, pageIconPath)
pageIcon.location = pageIconPath
return pageIcon
else:
if refresh:
for meta in pageSoup.findAll("meta"):
if meta.get("http-equiv", "").lower() == "refresh":
match = search("url=([^;]+)", meta.get("content", ""), flags=IGNORECASE)
if match:
refreshPath = urljoin(rootDomainPageResult.geturl(), match.group(1))
cherrypy.log(
"Processing refresh directive:%s for domain:%s" % (refreshPath, targetDomain),
severity=INFO,
)
return self.iconInPage(targetDomain, refreshPath, start, refresh=False)
cherrypy.log("No link tag found:%s" % targetPath, severity=INFO)
else:
cherrypy.log(
"Non-success response:%d for url:%s" % (rootDomainPageResult.getcode(), targetPath), severity=INFO
)
except:
cherrypy.log(
"Error extracting favicon from page:%s, err:%s, msg:%s"
% (targetPath, sys.exc_info()[0], sys.exc_info()[1]),
severity=WARNING,
)
def cacheIcon(self, domain, icon, loc):
cherrypy.log("Caching icon at location:%s for domain:%s" % (loc, domain), severity=INFO)
if not self.mc.set("icon-%s" % domain, icon, time=MC_CACHE_TIME):
cherrypy.log("Could not cache icon for domain:%s" % domain, severity=ERROR)
def iconInCache(self, targetDomain, start):
icon = self.mc.get("icon-%s" % targetDomain)
if icon:
self.mc.incr("counter-hits")
cherrypy.log("Cache hit:%s" % targetDomain, severity=INFO)
cherrypy.response.headers["X-Cache"] = "Hit"
if icon == "DEFAULT":
self.mc.incr("counter-defaults")
cherrypy.response.headers["X-Cache"] = "Hit"
return self.default_icon
else:
return Icon(data=icon)
def writeIcon(self, icon):
self.writeHeaders(icon)
return icon.data
def writeHeaders(self, icon, fmt="%a, %d %b %Y %H:%M:%S %z"):
# MIME Type
cherrypy.response.headers["Content-Type"] = icon.type or "image/x-icon"
# Set caching headers
cherrypy.response.headers["Cache-Control"] = "public, max-age=2592000"
cherrypy.response.headers["Expires"] = (datetime.now() + timedelta(days=30)).strftime(fmt)
def parse(self, url):
# Get page path
targetPath = self.urldecode(url)
if not targetPath.startswith("http"):
targetPath = "http://%s" % targetPath
cherrypy.log("Decoded URL:%s" % targetPath, severity=INFO)
# Split path to get domain
targetURL = urlparse(targetPath)
if not targetURL or not targetURL.scheme or not targetURL.netloc:
raise cherrypy.HTTPError(400, "Malformed URL:%s" % url)
targetDomain = "%s://%s" % (targetURL.scheme, targetURL.netloc)
cherrypy.log("URL:%s, domain:%s" % (targetPath, targetDomain), severity=INFO)
return (targetPath, targetDomain)
@cherrypy.expose
def index(self):
status = {"status": "ok", "counters": dict()}
for counter in ["requests", "hits", "defaults"]:
status["counters"][counter] = self.mc.get("counter-%s" % counter)
return json.dumps(status)
@cherrypy.expose
def test(self):
topSites = open(os.path.join(cherrypy.config["favicon.root"], "topsites.txt"), "r").read().split()
template = self.env.get_template("test.html")
return template.render(topSites=topSites)
@cherrypy.expose
def clear(self, url):
cherrypy.log("Incoming cache invalidation request:%s" % url, severity=INFO)
targetPath, targetDomain = self.parse(str(url))
self.mc.delete("icon_loc-%s" % targetDomain)
cherrypy.log("Evicted cache entry for %s" % targetDomain, severity=INFO)
@cherrypy.expose
def s(self, url, skipCache="false"):
start = time()
if skipCache.lower() == "true":
skipCache = True
else:
skipCache = False
cherrypy.log("Incoming request:%s (skipCache=%s)" % (url, skipCache), severity=INFO)
self.mc.incr("counter-requests")
targetPath, targetDomain = self.parse(str(url))
icon = (
(not skipCache and self.iconInCache(targetDomain, start))
or self.iconInPage(targetDomain, targetPath, start)
or self.iconAtRoot(targetDomain, start)
)
if not icon:
cherrypy.log("Falling back to default icon for:%s" % targetDomain, severity=INFO)
self.cacheIcon(targetDomain, "DEFAULT", "DEFAULT_LOC")
self.mc.incr("counter-defaults")
icon = self.default_icon
cherrypy.log("Time taken to process domain:%s %f" % (targetDomain, time() - start), severity=INFO)
return self.writeIcon(icon)
class TestMemcache(TestCase):
def setUp(self):
# TODO: unix socket server stuff
servers = ["127.0.0.1:11211"]
self.mc = Client(servers, debug=1)
pass
def check_setget(self, key, val, noreply=False):
self.mc.set(key, val, noreply=noreply)
newval = self.mc.get(key)
self.assertEqual(newval, val)
def test_setget(self):
self.check_setget("a_string", "some random string")
self.check_setget("a_string_2", "some random string", noreply=True)
self.check_setget("an_integer", 42)
self.check_setget("an_integer_2", 42, noreply=True)
def test_delete(self):
self.check_setget("long", int(1 << 30))
result = self.mc.delete("long")
self.assertEqual(result, True)
self.assertEqual(self.mc.get("long"), None)
def test_get_multi(self):
self.check_setget("gm_a_string", "some random string")
self.check_setget("gm_an_integer", 42)
self.assertEqual(
self.mc.get_multi(["gm_a_string", "gm_an_integer"]),
{"gm_an_integer": 42, "gm_a_string": "some random string"})
def test_get_unknown_value(self):
self.assertEqual(self.mc.get("unknown_value"), None)
def test_setget_foostruct(self):
f = FooStruct()
self.check_setget("foostruct", f)
self.check_setget("foostruct_2", f, noreply=True)
def test_incr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.incr("i_an_integer", 1), 43)
def test_incr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.incr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 43)
def test_decr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.decr("i_an_integer", 1), 41)
def test_decr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.decr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 41)
def test_sending_spaces(self):
try:
self.mc.set("this has spaces", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_control_characters(self):
try:
self.mc.set("this\x10has\x11control characters\x02", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_key_too_long(self):
try:
self.mc.set('a' * SERVER_MAX_KEY_LENGTH + 'a', 1)
except Client.MemcachedKeyLengthError as err:
self.assertTrue("length is >" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyLengthError, nothing raised")
# These should work.
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1)
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1, noreply=True)
class TestMemcache(unittest.TestCase):
def setUp(self):
# TODO(): unix socket server stuff
servers = ["127.0.0.1:11211"]
self.mc = Client(servers, debug=1)
def tearDown(self):
self.mc.flush_all()
self.mc.disconnect_all()
def check_setget(self, key, val, noreply=False):
self.mc.set(key, val, noreply=noreply)
newval = self.mc.get(key)
self.assertEqual(newval, val)
def test_setget(self):
self.check_setget("a_string", "some random string")
self.check_setget("a_string_2", "some random string", noreply=True)
self.check_setget("an_integer", 42)
self.check_setget("an_integer_2", 42, noreply=True)
def test_delete(self):
self.check_setget("long", int(1 << 30))
result = self.mc.delete("long")
self.assertEqual(result, True)
self.assertEqual(self.mc.get("long"), None)
@mock.patch.object(_Host, 'send_cmd')
@mock.patch.object(_Host, 'readline')
def test_touch(self, mock_readline, mock_send_cmd):
with captured_stderr():
self.mc.touch('key')
mock_send_cmd.assert_called_with(b'touch key 0')
def test_get_multi(self):
self.check_setget("gm_a_string", "some random string")
self.check_setget("gm_an_integer", 42)
self.assertEqual(
self.mc.get_multi(["gm_a_string", "gm_an_integer"]),
{"gm_an_integer": 42, "gm_a_string": "some random string"})
def test_get_unknown_value(self):
self.mc.delete("unknown_value")
self.assertEqual(self.mc.get("unknown_value"), None)
def test_setget_foostruct(self):
f = FooStruct()
self.check_setget("foostruct", f)
self.check_setget("foostruct_2", f, noreply=True)
def test_incr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.incr("i_an_integer", 1), 43)
def test_incr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.incr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 43)
def test_decr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.decr("i_an_integer", 1), 41)
def test_decr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.decr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 41)
def test_sending_spaces(self):
try:
self.mc.set("this has spaces", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_control_characters(self):
try:
self.mc.set("this\x10has\x11control characters\x02", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_key_too_long(self):
try:
self.mc.set('a' * SERVER_MAX_KEY_LENGTH + 'a', 1)
except Client.MemcachedKeyLengthError as err:
self.assertTrue("length is >" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyLengthError, nothing raised")
# These should work.
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1)
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1, noreply=True)
def test_setget_boolean(self):
"""GitHub issue #75. Set/get with boolean values."""
self.check_setget("bool", True)
def test_unicode_key(self):
s = u'\u4f1a'
maxlen = SERVER_MAX_KEY_LENGTH // len(s.encode('utf-8'))
key = s * maxlen
self.mc.set(key, 5)
value = self.mc.get(key)
self.assertEqual(value, 5)
def test_unicode_value(self):
key = 'key'
value = u'Iñtërnâtiônàlizætiøn2'
self.mc.set(key, value)
cached_value = self.mc.get(key)
self.assertEqual(value, cached_value)
def test_binary_string(self):
value = 'value_to_be_compressed'
compressed_value = zlib.compress(value.encode())
self.mc.set('binary1', compressed_value)
compressed_result = self.mc.get('binary1')
self.assertEqual(compressed_value, compressed_result)
self.assertEqual(value, zlib.decompress(compressed_result).decode())
self.mc.add('binary1-add', compressed_value)
compressed_result = self.mc.get('binary1-add')
self.assertEqual(compressed_value, compressed_result)
self.assertEqual(value, zlib.decompress(compressed_result).decode())
self.mc.set_multi({'binary1-set_many': compressed_value})
compressed_result = self.mc.get('binary1-set_many')
self.assertEqual(compressed_value, compressed_result)
self.assertEqual(value, zlib.decompress(compressed_result).decode())
def test_ignore_too_large_value(self):
# NOTE: "MemCached: while expecting[...]" is normal...
key = 'keyhere'
value = 'a' * (SERVER_MAX_VALUE_LENGTH // 2)
self.assertTrue(self.mc.set(key, value))
self.assertEqual(self.mc.get(key), value)
value = 'a' * SERVER_MAX_VALUE_LENGTH
with captured_stderr() as log:
self.assertIs(self.mc.set(key, value), False)
self.assertEqual(
log.getvalue(),
"MemCached: while expecting 'STORED', got unexpected response "
"'SERVER_ERROR object too large for cache'\n"
)
# This test fails if the -I option is used on the memcached server
self.assertTrue(self.mc.get(key) is None)
def test_get_set_multi_key_prefix(self):
"""Testing set_multi() with no memcacheds running."""
prefix = 'pfx_'
values = {'key1': 'a', 'key2': 'b'}
errors = self.mc.set_multi(values, key_prefix=prefix)
self.assertEqual(errors, [])
keys = list(values)
self.assertEqual(self.mc.get_multi(keys, key_prefix=prefix),
values)
def test_set_multi_dead_servers(self):
"""Testing set_multi() with no memcacheds running."""
self.mc.disconnect_all()
with captured_stderr() as log:
for server in self.mc.servers:
server.mark_dead('test')
self.assertIn('Marking dead.', log.getvalue())
errors = self.mc.set_multi({'key1': 'a', 'key2': 'b'})
self.assertEqual(sorted(errors), ['key1', 'key2'])
def test_disconnect_all_delete_multi(self):
"""Testing delete_multi() with no memcacheds running."""
self.mc.disconnect_all()
with captured_stderr() as output:
ret = self.mc.delete_multi(('keyhere', 'keythere'))
self.assertEqual(ret, 1)
self.assertEqual(
output.getvalue(),
"MemCached: while expecting 'DELETED', got unexpected response "
"'NOT_FOUND'\n"
"MemCached: while expecting 'DELETED', got unexpected response "
"'NOT_FOUND'\n"
)
@mock.patch.object(_Host, 'send_cmd') # Don't send any commands.
@mock.patch.object(_Host, 'readline')
def test_touch_unexpected_reply(self, mock_readline, mock_send_cmd):
"""touch() logs an error upon receiving an unexpected reply."""
mock_readline.return_value = 'SET' # the unexpected reply
with captured_stderr() as output:
self.mc.touch('key')
self.assertEqual(
output.getvalue(),
"MemCached: touch expected %s, got: 'SET'\n" % b'TOUCHED'
)
class TestMemcache(unittest.TestCase):
def setUp(self):
# TODO: unix socket server stuff
servers = ["127.0.0.1:11211"]
self.mc = Client(servers, debug=1)
def tearDown(self):
self.mc.disconnect_all()
def check_setget(self, key, val, noreply=False):
self.mc.set(key, val, noreply=noreply)
newval = self.mc.get(key)
self.assertEqual(newval, val)
def test_setget(self):
self.check_setget("a_string", "some random string")
self.check_setget("a_string_2", "some random string", noreply=True)
self.check_setget("an_integer", 42)
self.check_setget("an_integer_2", 42, noreply=True)
def test_delete(self):
self.check_setget("long", int(1 << 30))
result = self.mc.delete("long")
self.assertEqual(result, True)
self.assertEqual(self.mc.get("long"), None)
def test_get_multi(self):
self.check_setget("gm_a_string", "some random string")
self.check_setget("gm_an_integer", 42)
self.assertEqual(
self.mc.get_multi(["gm_a_string", "gm_an_integer"]),
{"gm_an_integer": 42, "gm_a_string": "some random string"})
def test_get_unknown_value(self):
self.mc.delete("unknown_value")
self.assertEqual(self.mc.get("unknown_value"), None)
def test_setget_foostruct(self):
f = FooStruct()
self.check_setget("foostruct", f)
self.check_setget("foostruct_2", f, noreply=True)
def test_incr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.incr("i_an_integer", 1), 43)
def test_incr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.incr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 43)
def test_decr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.decr("i_an_integer", 1), 41)
def test_decr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.decr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 41)
def test_sending_spaces(self):
try:
self.mc.set("this has spaces", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_control_characters(self):
try:
self.mc.set("this\x10has\x11control characters\x02", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_key_too_long(self):
try:
self.mc.set('a' * SERVER_MAX_KEY_LENGTH + 'a', 1)
except Client.MemcachedKeyLengthError as err:
self.assertTrue("length is >" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyLengthError, nothing raised")
# These should work.
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1)
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1, noreply=True)
def test_setget_boolean(self):
"""GitHub issue #75. Set/get with boolean values."""
self.check_setget("bool", True)
def test_unicode_key(self):
s = six.u('\u4f1a')
maxlen = SERVER_MAX_KEY_LENGTH // len(s.encode('utf-8'))
key = s * maxlen
self.mc.set(key, 5)
value = self.mc.get(key)
self.assertEqual(value, 5)
def test_ignore_too_large_value(self):
# NOTE: "MemCached: while expecting[...]" is normal...
key = 'keyhere'
value = 'a' * (SERVER_MAX_VALUE_LENGTH // 2)
self.assertTrue(self.mc.set(key, value))
self.assertEqual(self.mc.get(key), value)
value = 'a' * SERVER_MAX_VALUE_LENGTH
self.assertFalse(self.mc.set(key, value))
# This test fails if the -I option is used on the memcached server
self.assertTrue(self.mc.get(key) is None)
def test_get_set_multi_key_prefix(self):
"""Testing set_multi() with no memcacheds running."""
prefix = 'pfx_'
values = {'key1': 'a', 'key2': 'b'}
errors = self.mc.set_multi(values, key_prefix=prefix)
self.assertEqual(errors, [])
keys = list(values)
self.assertEqual(self.mc.get_multi(keys, key_prefix=prefix),
values)
def test_set_multi_dead_servers(self):
"""Testing set_multi() with no memcacheds running."""
self.mc.disconnect_all()
for server in self.mc.servers:
server.mark_dead('test')
errors = self.mc.set_multi({'key1': 'a', 'key2': 'b'})
self.assertEqual(sorted(errors), ['key1', 'key2'])
def test_disconnect_all_delete_multi(self):
"""Testing delete_multi() with no memcacheds running."""
self.mc.disconnect_all()
ret = self.mc.delete_multi({'keyhere': 'a', 'keythere': 'b'})
self.assertEqual(ret, 1)
def test_tags_set(self):
self.mc.disconnect_all()
self.mc.set('key', 'val', tags=['t1'])
self.mc.set('key1', 'val2', tags=['t2'])
self.mc.delete_by_tag('t1')
self.assertIsNone(self.mc.get('key'))
self.assertEqual(self.mc.get('key1'), 'val2')
def test_tags_set_multi(self):
self.mc.disconnect_all()
mapping = {
'k': 1,
'k2': 2,
}
self.mc.set_multi(mapping, tags=['t1'])
self.mc.delete_by_tag('t1')
self.assertIsNone(self.mc.get('k'))
self.assertIsNone(self.mc.get('k1'))
