class MemcacheStore(DictProxy):
def __init__(self, host, port):
from memcache import Client
self.mc= Client(['%s:%s' % (host, port)])
def update(self, updates):
for update in updates:
(processId, data)= update
self.mc.set(processId, data)
def get(self, processId, default= None):
data= self.mc.get(processId)
if data == None:
return default
return data
def pop(self, processId):
data= self.mc.get(processId)
self.mc.delete(processId)
return data
if data == None:
return default
def __len__(self):
return int(self.mc.get_stats()[0][1].get('curr_items'))
def save(self, body, layer, coord, format):
""" Save a cached tile.
"""
mem = Client(self.servers)
key = tile_key(layer, coord, format, self.revision)
mem.set(key, body, layer.cache_lifespan or 0)
def save(self, body, layer, coord, format):
""" Save a cached tile.
"""
mem = Client(self.servers)
key = tile_key(layer, coord, format, self.revision)
mem.set(key, body, layer.cache_lifespan or 0)
class MemcachedCache(CacheBase):
def __init__(self, config, section):
from memcache import Client
servers = config.get(section, 'memcache-server').split()
self.conn = Client(servers)
def key(self, raw):
if sys.version_info >= (3, ):
return self.prefix(hashlib.md5(bytes(raw, 'utf-8')).hexdigest())
else:
return self.prefix(hashlib.md5(raw).hexdigest()).encode('utf-8')
def check_password(self, user, password):
cached = self.conn.get(self.key('%s-pass' % user))
if cached is None:
return cached
return cached == self.hash(password, cached)
def set_password(self, user, password):
self.conn.set(self.key('%s-pass' % user), self.hash(password, None), self.expire)
def in_groups(self, user, groups):
cached = self.conn.get(self.key('%s-groups' % user))
if cached is None:
return None
return not cached.isdisjoint(groups)
def set_groups(self, user, groups):
self.conn.set(self.key('%s-groups' % user), groups, self.expire)
class MemcachedAdapter(OWMCache):
"""
Needs: python-memcached bindings (https://pypi.python.org/pypi/python-memcached)
"""
# Defaults:
# cache item expiration time in seconds
__ITEM_LIFETIME_SECONDS = 4000 # Ten minutes
# hostname and port
__HOSTNAME = "127.0.0.1"
__PORT = "11211"
def __init__(self,
hostname=__HOSTNAME,
port=__PORT,
item_lifetime=__ITEM_LIFETIME_SECONDS):
from memcache import Client
self._memcached = Client([hostname + ":" + port])
self._item_lifetime = item_lifetime
def get(self, request_url):
return self._memcached.get(request_url)
def set(self, request_url, response_json):
self._memcached.set(request_url, response_json, self._item_lifetime)
class MemcachedCache(CacheBase):
def __init__(self, config, section):
from memcache import Client
servers = config.get(section, 'memcache-server').split()
self.conn = Client(servers)
def key(self, raw):
if sys.version_info >= (3, ):
return self.prefix(hashlib.md5(bytes(raw, 'utf-8')).hexdigest())
else:
return self.prefix(hashlib.md5(raw).hexdigest()).encode('utf-8')
def check_password(self, user, password):
cached = self.conn.get(self.key('%s-pass' % user))
if cached is None:
return cached
return cached == self.hash(password, cached)
def set_password(self, user, password):
self.conn.set(self.key('%s-pass' % user), self.hash(password, None),
self.expire)
def in_groups(self, user, groups):
cached = self.conn.get(self.key('%s-groups' % user))
if cached is None:
return None
return not cached.isdisjoint(groups)
def set_groups(self, user, groups):
self.conn.set(self.key('%s-groups' % user), groups, self.expire)
def update_mailinglists(self):
lists = []
for pl in PUBLIC_LISTS:
cur_list = {'name': pl, 'members': []}
query = "select comment_subscriber,user_subscriber, reception_subscriber from subscriber_table ' \
'where list_subscriber = '%s' and reception_subscriber != 'nomail';" % pl
db_con = psycopg2.connect(
database=SYMPA_DB_NAME, host=SYMPA_DB_HOST, port=SYMPA_DB_PORT,
user=SYMPA_DB_USER, password=SYMPA_DB_PASSWD
)
cursor = db_con.cursor()
cursor.execute(query)
rows = cursor.fetchall()
cursor.close()
db_con.close()
for row in rows:
comment_subscriber, user_subscriber, reception_subscriber = row
member = {}
member['name'] = comment_subscriber
member['subscriber'] = reception_subscriber
member['email'] = user_subscriber
cur_list['members'].append(member)
lists.append(cur_list)
mc = Client(["127.0.0.1:11211"], debug=0)
mc.set("sympa_lists", lists)
class FoursquareAPI:
url_base = "https://api.foursquare.com/v2/%s"
payload = {}
mc = False
mcprefix = False
def __init__(self, lifestream):
OAUTH_FILENAME = lifestream.config.get("foursquare", "secrets_file")
CONSUMER_KEY = lifestream.config.get("foursquare", "client_id")
CONSUMER_SECRET = lifestream.config.get("foursquare", "secret")
MEMCACHE_HOST = lifestream.config.get("memcache", "host")
MEMCACHE_PORT = lifestream.config.get("memcache", "port")
self.mcprefix = lifestream.config.get("memcache", "prefix")
servers = ["%s:%s" % (MEMCACHE_HOST, MEMCACHE_PORT)]
self.mc = Client(servers, debug=1)
if not os.path.exists(OAUTH_FILENAME):
logger.error("No OAUTH found at %s" % OAUTH_FILENAME)
raise Exception("You need to run foursquare_oauth.py to generate the oauth key")
oauth_token, oauth_token_secret = read_token_file(OAUTH_FILENAME)
self.payload = {
'v': "20170801",
'oauth_token' : oauth_token
}
def cache_get(self, url, params):
m = hashlib.sha224()
m.update(url)
m.update(str(params))
key = m.hexdigest()
res = self.mc.get(key)
if(res):
return json.loads(res)
r = requests.get(self.url_base % "users/self/checkins", params=self.payload)
self.mc.set(key, json.dumps(r.json()))
return r.json()
def my_checkins(self):
return self.cache_get(self.url_base % "users/self/checkins", params=self.payload)
def search_near(self, lat, lng, intent="checkin", radius=50, limit=10):
payload = self.payload
payload['ll'] = "%s,%s" % (lat, lng)
payload['intent'] = intent
payload['radius'] = radius
payload['limit'] = limit
return self.cache_get(self.url_base % "venues/search", params=self.payload)
class OODictMongoMemcache(OODictMongo):
"""Add memcache caching capabilities to a OODictMongo."""
def __init__(self,
memcache_host,
mongo_uri,
mongo_db,
mongo_coll,
mongo_id="_id",
mongo_client=None,
memcache_client=None,
_dict=None):
super(OODictMongoMemcache,
self).__init__(mongo_uri, mongo_db, mongo_coll, mongo_id,
mongo_client, _dict)
self._memcache_host = memcache_host
## self._memcache_lock = memache_lock
if memcache_client is None:
self._memcache = MemcacheClient(memcache_host)
else:
self._memcache = memcache_client
def _memcache_key(self, mongo_id=None):
return str("%s:%s:%s" % (self._mongo_db, self._mongo_coll, mongo_id
or self._dict.get(self._mongo_id, '')))
def get_from_field(self, key, value, flush=False):
"""Get user by a key:value pair from mongo or memcache."""
# if searching by id key, then we can find it in memcache
if not flush and key == self._mongo_id:
item = self._memcache.get(self._memcache_key(value))
if item:
log.info("Cache hit.")
return self._reinit(item)
log.info("Cache miss.")
# didn't find it in memcache, search in mongo and update cache
super(OODictMongoMemcache, self).get_from_field(key, value)
item = self._dict
if item:
self._memcache.set(self._memcache_key(), item)
return self._reinit(item)
def refresh(self, flush=False):
"""Refresh self data from memcache. If flush is True, then
flush memcache entry and force a refresh from mongo.
"""
self.get_from_field(self._mongo_id, self._dict[self._mongo_id], flush)
def save(self):
"""Save user data to storage."""
self._memcache.set(self._memcache_key(), self._dict)
super(OODictMongoMemcache, self).save()
def delete(self):
"""Delete user from storage."""
self._memcache.delete(self._memcache_key())
super(OODictMongoMemcache, self).delete()
def save(self, body, layer, coord, format):
""" Save a cached tile.
"""
mem = Client(self.servers)
key = tile_key(layer, coord, format, self.revision, self.key_prefix)
mem.set(key, body, layer.cache_lifespan or 0)
mem.disconnect_all()
def save(self, body, layer, coord, format):
""" Save a cached tile.
"""
mem = Client(self.servers)
key = tile_key(layer, coord, format, self.revision, self.key_prefix)
mem.set(key, body, layer.cache_lifespan or 0)
mem.disconnect_all()
class MemcachedCacheStorage:
def __init__(self, settings):
self.expiration_secs = settings.getint("HTTPCACHE_EXPIRATION_SECS")
self.client = Client((settings["MEMCACHED_LOCATION"], ))
def key_for(self, spider, request, type_):
"""Creates unique identifiers to be used as cache keys in Memcached."""
key = request_fingerprint(request)
return "{}-{}-{}".format(spider.name, key, type_)
def store_response(self, spider, request, response):
"""Store the given response in the cache."""
meta_data = {
"url": request.url,
"method": request.method,
"status": response.status,
"response_url": response.url,
"timestamp": time(),
}
data = (
("meta_data", json.dumps(meta_data)),
("response_headers", headers_dict_to_raw(response.headers)),
("response_body", response.body),
("request_headers", headers_dict_to_raw(request.headers)),
("request_body", request.body),
)
for type_, value in data:
key = self.key_for(spider, request, type_)
self.client.set(key, value, self.expiration_secs)
def retrieve_response(self, spider, request):
"""Return response if present in cache, or None otherwise."""
meta_data_key = self.key_for(spider, request, "meta_data")
raw_meta_data = self.client.get(meta_data_key)
meta_data = json.loads(raw_meta_data) if raw_meta_data else None
if not meta_data:
return
body, raw_headers = (self.client.get(
self.key_for(spider, request, type_))
for type_ in ("response_body",
"response_headers"))
if not all((body, raw_headers)):
return
kwargs = {
"url": meta_data.get("response_url"),
"headers": Headers(headers_raw_to_dict(raw_headers)),
}
response_class = responsetypes.from_args(**kwargs)
kwargs.update({"status": meta_data.get("status"), "body": body})
return response_class(**kwargs)
def open_spider(self, spider):
logger.debug("Using memcached cache in", extra={"spider": spider})
def close_spider(self, spider):
pass
def preset(sample):
global addresses
conn = Client([gethostname()+':11211'])
if conn.get_stats()[0][1]['curr_items'] == 0:
for k in addresses:
conn.set(addr_prefix+k, 0)
def preset(sample):
global addresses
conn = Client([gethostname() + ':11211'])
if conn.get_stats()[0][1]['curr_items'] == 0:
for k in addresses:
conn.set(addr_prefix + k, 0)
class OODictMongoMemcache(OODictMongo):
"""Add memcache caching capabilities to a OODictMongo."""
def __init__(self, memcache_host, mongo_uri, mongo_db, mongo_coll,
mongo_id="_id", mongo_client=None, memcache_client=None,
_dict=None):
super(OODictMongoMemcache, self).__init__(
mongo_uri, mongo_db, mongo_coll, mongo_id,
mongo_client, _dict
)
self._memcache_host = memcache_host
## self._memcache_lock = memache_lock
if memcache_client is None:
self._memcache = MemcacheClient(memcache_host)
else:
self._memcache = memcache_client
def _memcache_key(self, mongo_id=None):
return str("%s:%s:%s" % (self._mongo_db, self._mongo_coll,
mongo_id or self._dict.get(self._mongo_id, '')))
def get_from_field(self, key, value, flush=False):
"""Get user by a key:value pair from mongo or memcache."""
# if searching by id key, then we can find it in memcache
if not flush and key == self._mongo_id:
item = self._memcache.get(self._memcache_key(value))
if item:
log.info("Cache hit.")
return self._reinit(item)
log.info("Cache miss.")
# didn't find it in memcache, search in mongo and update cache
super(OODictMongoMemcache, self).get_from_field(key, value)
item = self._dict
if item:
self._memcache.set(self._memcache_key(), item)
return self._reinit(item)
def refresh(self, flush=False):
"""Refresh self data from memcache. If flush is True, then
flush memcache entry and force a refresh from mongo.
"""
self.get_from_field(self._mongo_id, self._dict[self._mongo_id], flush)
def save(self):
"""Save user data to storage."""
self._memcache.set(self._memcache_key(), self._dict)
super(OODictMongoMemcache, self).save()
def delete(self):
"""Delete user from storage."""
self._memcache.delete(self._memcache_key())
super(OODictMongoMemcache, self).delete()
def save(self, body, layer, coord, format):
""" Save a cached tile.
"""
mem = Client(self.servers)
key = tile_key(layer, coord, format, self.revision, self.key_prefix)
if body is not None:
body = b64encode(body).decode('ascii')
mem.set(key, body, layer.cache_lifespan or 0)
mem.disconnect_all()
def save(self, body, layer, coord, format):
""" Save a cached tile.
"""
mem = Client(self.servers)
key = tile_key(layer, coord, format, self.revision, self.key_prefix)
if body is not None:
body = b64encode(body).decode('ascii')
mem.set(key, body, layer.cache_lifespan or 0)
mem.disconnect_all()
def whereyoulive(addr):
conn = Client([gethostname() + ':11211'])
if conn.get(addr_prefix + addr) != None:
conn.incr(addr_prefix + addr)
else:
conn.set(addr_prefix + addr, 1)
if not addresses.__contains__(addr):
addresses.append(addr)
def whereyoulive(addr):
conn = Client([gethostname()+':11211'])
if conn.get(addr_prefix+addr) != None:
conn.incr(addr_prefix+addr)
else:
conn.set(addr_prefix+addr, 1)
if not addresses.__contains__(addr):
addresses.append(addr)
class MemcachedManager():
class Prefix():
Session = 'SESSION'
SysCache = 'ORDER_SYS_CACHE'
def __init__(self):
# if settings.ENVIRONMENT['dev']:
# pass
# else:
self.conn = Client(settings.CACHE['clients'])
def get(self, *arg):
if len(arg) ==0:
return None
arg = [str(i) for i in arg]
key = PRODUCT_PREFIX+ '|'.join(arg)
try:
# if settings.ENVIRONMENT['dev']:
# return _localCache.get(key)
# else:
return self.conn.get(key)
# self.conn.
except:# Client.MemcachedKeyNoneError:
return None
def set(self, value, timeout, *arg):
if len(arg) ==0:
return None
arg = [str(i) for i in arg]
key = PRODUCT_PREFIX+'|'.join(arg)
# if settings.ENVIRONMENT['dev']:
# _localCache[key] = value
# else:
self.conn.set(key, value, timeout)
def delete(self, *arg):
if len(arg) ==0:
return None
arg = [str(i) for i in arg]
key = PRODUCT_PREFIX+'|'.join(arg)
try:
# if settings.ENVIRONMENT['dev']:
# if _localCache.has_key(key):
# _localCache.pop(key)
# else:
self.conn.delete(key)
except: # Client.MemcachedKeyNoneError:
return None
class SessionHandler:
inst = None
def __init__(self):
SessionHandler.inst = self
self.conn = Client(["127.0.0.1:11211"])
def setuid(self, uid):
key = uuid4().get_hex()
self.conn.set(key, uid, 3600)
return key
def getuid(self, key):
return self.conn.get(key)
def deluid(self, key):
self.conn.delete(key)
class MemcachedZipState(object):
"""get or set state of an zip job.
"""
def __init__(self, uid):
self._uid = uid
if ZIPSTATE_MEMCACHEDSERVER not in os.environ:
raise ValueError(
'Expect environment variable "{0}: pointing a memcached '
'server in order to share state '
'information.'.format(ZIPSTATE_MEMCACHEDSERVER)
)
self._client = Client([os.environ[ZIPSTATE_MEMCACHEDSERVER]])
def _combined_key(self, key):
return '{0}-{1}'.format(self._uid, key)
def __getitem__(self, key):
"""get state of zip job
"""
return self._client.get(self._combined_key(key))
def __setitem__(self, key, value):
"""set state of zip job
"""
return self._client.set(self._combined_key(key), value)
def test_memcached_connection(self):
# python-memcached
print('MEMCACHED_PORT_11211_TCP_ADDR')
print(os.environ.get('MEMCACHED_PORT_11211_TCP_ADDR'))
print('MEMCACHED_PORT_11211_TCP_PORT')
print(os.environ.get('MEMCACHED_PORT_11211_TCP_PORT'))
mc = Client([
os.environ.get('MEMCACHED_PORT_11211_TCP_ADDR')
], debug=1)
mc.set('a_string', 'some random string')
newval = mc.get('a_string')
self.assertEqual(newval, 'some random string')
mc.disconnect_all()
class CacheMgr(object):
def __init__(self, servers=["127.0.0.1:11211"]):
self.__mc = Client(servers)
pass
def tearDown(self):
self.__mc.flush_all()
self.__mc.disconnect_all()
def get(self, key):
val = self.__mc.get(key)
return val
def set(self, key, val):
# TODO: noreply setting
self.__mc.set(key, val)
def insert_appsinstalled(
memcache_client: memcache.Client,
appsinstalled: AppsInstalled,
dry_run: bool = False,
) -> bool:
ua = appsinstalled_pb2.UserApps()
ua.lat = appsinstalled.lat
ua.lon = appsinstalled.lon
key = "%s:%s" % (appsinstalled.dev_type, appsinstalled.dev_id)
ua.apps.extend(appsinstalled.apps)
packed = ua.SerializeToString()
if dry_run:
logging.debug("%s -> %s" % (key, str(ua).replace("\n", " ")))
return True
for _ in range(MEMCACHE_RETRY_NUMBER):
try:
# Use a tuple as key to write to specific Memcached server
# https://github.com/linsomniac/python-memcached/blob/bad41222379102e3f18f6f2f7be3ee608de6fbff/memcache.py#L698
success: bool = memcache_client.set(
(appsinstalled.dev_type.value, key), packed)
except Exception as e:
logging.exception(f"Cannot write to Memcache: {e}")
return False
if success:
return True
time.sleep(MEMCACHE_RETRY_TIMEOUT_SECONDS)
logging.error("Cannot write to Memcache. Server is down")
return False
class ResetCodeMemcache(ResetCode):
""" Implements the reset code methods for auth backends.
"""
def __init__(self,
product='auth',
nodes=None,
debug=0,
expiration=_6HOURS,
**kw):
if nodes is None:
nodes = ['127.0.0.1:11211']
self._engine = Client(nodes, debug)
self.product = product
self.expiration = expiration
#
# Private methods
#
def _get_reset_code(self, user_id):
return self._engine.get(self._generate_key(user_id))
def _generate_key(self, user_id):
return "reset:%s:%s" % (user_id, self.product)
def _set_reset_code(self, user_id):
code = self._generate_reset_code()
key = self._generate_key(user_id)
if not self._engine.set(key, code, self.expiration):
raise BackendError()
return code
#
# Public methods
#
def generate_reset_code(self, user, overwrite=False):
user_id = self._get_user_id(user)
if not overwrite:
stored_code = self._get_reset_code(user_id)
if stored_code is not None:
return stored_code
return self._set_reset_code(user_id)
def verify_reset_code(self, user, code):
user_id = self._get_user_id(user)
if not self._check_reset_code(code):
return False
stored_code = self._get_reset_code(user_id)
if stored_code is None:
return False
return stored_code == code
def clear_reset_code(self, user):
user_id = self._get_user_id(user)
return self._engine.delete(self._generate_key(user_id))
class MemCacheWrapper(object):
"""
Memcache client wrapper.
No exception raise and add some useful function.
"""
def __init__(self, servers, logerr=None):
self.cache = MCClient(servers=servers, debug=False)
self.logerr = logerr
def add(self, key, val=1, time=0):
try:
return self.cache.add(key, val, time)
except Exception as e:
_logger.warning("Exception during `add`: %s", e)
return None
def count(self, key, expires=0, delta=1):
try:
result = self.cache.incr(key, delta)
if result is None:
if not self.cache.add(key, delta, expires):
result = self.cache.incr(key, delta)
else:
return delta
return result
except Exception as e:
_logger.warning("Exception during `count`: %s", e)
return None
def get(self, key):
result = None
try:
result = self.cache.get(str(key))
except Exception as e:
_logger.warning("Exception during `get`: %s", e)
return result
def set(self, key, value, expires):
result = False
try:
result = self.cache.set(str(key), value, expires)
except Exception as e:
_logger.warning("Exception during `set`: %s", e)
return result
def delete(self, key):
result = False
try:
result = self.cache.delete(key)
except Exception as e:
_logger.warning("Exception during `del`: %s", e)
return result
class Memcache(object):
"""
address=127.0.0.1:11211
# TODO 链接断开后没有自动重连机制
"""
def __init__(self, address):
from memcache import Client
self.conn = Client([address])
def get(self, key):
return self.conn.get(key)
def set(self, key, value, timeout):
self.conn.set(key, value, timeout)
def delete(self, key):
self.conn.delete(key)
def lock(self, layer, coord, format):
""" Acquire a cache lock for this tile.
Returns nothing, but blocks until the lock has been acquired.
"""
mem = Client(self.servers)
key = tile_key(layer, coord, format, self.revision, self.key_prefix)
due = _time() + layer.stale_lock_timeout
try:
while _time() < due:
if mem.add(key + '-lock', 'locked.', layer.stale_lock_timeout):
return
_sleep(.2)
mem.set(key + '-lock', 'locked.', layer.stale_lock_timeout)
return
finally:
mem.disconnect_all()
def lock(self, layer, coord, format):
""" Acquire a cache lock for this tile.
Returns nothing, but blocks until the lock has been acquired.
"""
mem = Client(self.servers)
key = tile_key(layer, coord, format, self.revision, self.key_prefix)
due = _time() + layer.stale_lock_timeout
try:
while _time() < due:
if mem.add(key + "-lock", "locked.", layer.stale_lock_timeout):
return
_sleep(0.2)
mem.set(key + "-lock", "locked.", layer.stale_lock_timeout)
return
finally:
mem.disconnect_all()
class MemcacheMemory(Memory):
"""
Memory gateway to a Memcache server
"""
def __init__(self, servers=["127.0.0.1:11211"], expire=0, debug=False):
"""
:param servers: List of servers to use. Please, read
memcache.Client help.
"""
self._client = MemcacheClient(servers)
self._expire = expire
logging.basicConfig(level=logging.WARNING)
self.log = logging.getLogger("Memcache-Gateway")
if debug:
self.log.setLevel(logging.DEBUG)
def __getitem__(self, key):
self.log.debug("Accessing key %s", key)
value = self._client.get(key)
if isinstance(value, NotSet):
value = None
elif value is None:
raise KeyError
return value
def __setitem__(self, key, value):
self.log.debug("Setting key")
if value is None:
value = NotSet()
self._client.set(key, value, self._expire)
def __delitem__(self, key):
self.log.debug("Deleting key %s", key)
if self._client.delete(key) == 0:
raise KeyError
def open(self, key):
return KeyFile(self, key)
class MemcachedCacheStore(AbstractCacheStore):
servers = ("127.0.0.1:11211")
def __init__(self, servers=None, debug=False):
if servers is None:
servers = self.servers
from memcache import Client as MemcachedClient
self._client = MemcachedClient(servers, debug)
def set(self, key, val, time=0):
self._client.set(key, val, time)
def add(self, key, val, time=0):
res = self._client.add(key, val, time)
if not res:
raise Error("a value for key %r is already in the cache" % key)
self._data[key] = (val, time)
def replace(self, key, val, time=0):
res = self._client.replace(key, val, time)
if not res:
raise Error("a value for key %r is already in the cache" % key)
self._data[key] = (val, time)
def delete(self, key):
res = self._client.delete(key, time=0)
if not res:
raise KeyError(key)
def get(self, key):
val = self._client.get(key)
if val is None:
raise KeyError(key)
return val
def clear(self):
self._client.flush_all()
class MemcachedCacheStore(AbstractCacheStore):
servers = ('127.0.0.1:11211')
def __init__(self, servers=None, debug=False):
if servers is None:
servers = self.servers
from memcache import Client as MemcachedClient
self._client = MemcachedClient(servers, debug)
def set(self, key, val, time=0):
self._client.set(key, val, time)
def add(self, key, val, time=0):
res = self._client.add(key, val, time)
if not res:
raise Error('a value for key %r is already in the cache'%key)
self._data[key] = (val, time)
def replace(self, key, val, time=0):
res = self._client.replace(key, val, time)
if not res:
raise Error('a value for key %r is already in the cache'%key)
self._data[key] = (val, time)
def delete(self, key):
res = self._client.delete(key, time=0)
if not res:
raise KeyError(key)
def get(self, key):
val = self._client.get(key)
if val is None:
raise KeyError(key)
else:
return val
def clear(self):
self._client.flush_all()
class MemcachedAdapter(OWMCache):
"""
Needs: python-memcached bindings (https://pypi.python.org/pypi/python-memcached)
"""
# Defaults:
# cache item expiration time in seconds
__ITEM_LIFETIME_SECONDS = 60*10 # Ten minutes
# hostname and port
__HOSTNAME = "127.0.0.1"
__PORT = "11211"
def __init__(self, hostname=__HOSTNAME, port=__PORT, item_lifetime=__ITEM_LIFETIME_SECONDS):
from memcache import Client
self._memcached = Client([hostname+":"+port])
self._item_lifetime = item_lifetime
def get(self, request_url):
return self._memcached.get(request_url)
def set(self, request_url, response_json):
self._memcached.set(request_url, response_json, self._item_lifetime)
from os.path import exists
from memcache import Client
from sekrit import MEMCACHE_SOCKET
if exists(MEMCACHE_SOCKET):
CACHE = Client(['unix:' + MEMCACHE_SOCKET], debug=True)
else:
CACHE = Client(['127.0.0.1:11213'], debug=True)
S = lambda key, value: CACHE.set(str(key), value)
G = lambda key: CACHE.get(str(key))
def store_dec(s):
def inner(tag, url):
s(tag, url)
S(tag, url)
return inner
def retrieve_dec(r):
def inner(tag):
url = G(tag)
if url is None:
url = r(tag)
S(tag, url)
return url
return inner
class TestMemcache(unittest.TestCase):
def setUp(self):
# TODO(): unix socket server stuff
servers = ["127.0.0.1:11211"]
self.mc = Client(servers, debug=1)
def tearDown(self):
self.mc.flush_all()
self.mc.disconnect_all()
def check_setget(self, key, val, noreply=False):
self.mc.set(key, val, noreply=noreply)
newval = self.mc.get(key)
self.assertEqual(newval, val)
def test_setget(self):
self.check_setget("a_string", "some random string")
self.check_setget("a_string_2", "some random string", noreply=True)
self.check_setget("an_integer", 42)
self.check_setget("an_integer_2", 42, noreply=True)
def test_delete(self):
self.check_setget("long", int(1 << 30))
result = self.mc.delete("long")
self.assertEqual(result, True)
self.assertEqual(self.mc.get("long"), None)
@mock.patch.object(_Host, 'send_cmd')
@mock.patch.object(_Host, 'readline')
def test_touch(self, mock_readline, mock_send_cmd):
with captured_stderr():
self.mc.touch('key')
mock_send_cmd.assert_called_with(b'touch key 0')
def test_get_multi(self):
self.check_setget("gm_a_string", "some random string")
self.check_setget("gm_an_integer", 42)
self.assertEqual(
self.mc.get_multi(["gm_a_string", "gm_an_integer"]),
{"gm_an_integer": 42, "gm_a_string": "some random string"})
def test_get_unknown_value(self):
self.mc.delete("unknown_value")
self.assertEqual(self.mc.get("unknown_value"), None)
def test_setget_foostruct(self):
f = FooStruct()
self.check_setget("foostruct", f)
self.check_setget("foostruct_2", f, noreply=True)
def test_incr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.incr("i_an_integer", 1), 43)
def test_incr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.incr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 43)
def test_decr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.decr("i_an_integer", 1), 41)
def test_decr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.decr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 41)
def test_sending_spaces(self):
try:
self.mc.set("this has spaces", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_control_characters(self):
try:
self.mc.set("this\x10has\x11control characters\x02", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_key_too_long(self):
try:
self.mc.set('a' * SERVER_MAX_KEY_LENGTH + 'a', 1)
except Client.MemcachedKeyLengthError as err:
self.assertTrue("length is >" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyLengthError, nothing raised")
# These should work.
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1)
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1, noreply=True)
def test_setget_boolean(self):
"""GitHub issue #75. Set/get with boolean values."""
self.check_setget("bool", True)
def test_unicode_key(self):
s = u'\u4f1a'
maxlen = SERVER_MAX_KEY_LENGTH // len(s.encode('utf-8'))
key = s * maxlen
self.mc.set(key, 5)
value = self.mc.get(key)
self.assertEqual(value, 5)
def test_unicode_value(self):
key = 'key'
value = u'Iñtërnâtiônàlizætiøn2'
self.mc.set(key, value)
cached_value = self.mc.get(key)
self.assertEqual(value, cached_value)
def test_binary_string(self):
value = 'value_to_be_compressed'
compressed_value = zlib.compress(value.encode())
self.mc.set('binary1', compressed_value)
compressed_result = self.mc.get('binary1')
self.assertEqual(compressed_value, compressed_result)
self.assertEqual(value, zlib.decompress(compressed_result).decode())
self.mc.add('binary1-add', compressed_value)
compressed_result = self.mc.get('binary1-add')
self.assertEqual(compressed_value, compressed_result)
self.assertEqual(value, zlib.decompress(compressed_result).decode())
self.mc.set_multi({'binary1-set_many': compressed_value})
compressed_result = self.mc.get('binary1-set_many')
self.assertEqual(compressed_value, compressed_result)
self.assertEqual(value, zlib.decompress(compressed_result).decode())
def test_ignore_too_large_value(self):
# NOTE: "MemCached: while expecting[...]" is normal...
key = 'keyhere'
value = 'a' * (SERVER_MAX_VALUE_LENGTH // 2)
self.assertTrue(self.mc.set(key, value))
self.assertEqual(self.mc.get(key), value)
value = 'a' * SERVER_MAX_VALUE_LENGTH
with captured_stderr() as log:
self.assertIs(self.mc.set(key, value), False)
self.assertEqual(
log.getvalue(),
"MemCached: while expecting 'STORED', got unexpected response "
"'SERVER_ERROR object too large for cache'\n"
)
# This test fails if the -I option is used on the memcached server
self.assertTrue(self.mc.get(key) is None)
def test_get_set_multi_key_prefix(self):
"""Testing set_multi() with no memcacheds running."""
prefix = 'pfx_'
values = {'key1': 'a', 'key2': 'b'}
errors = self.mc.set_multi(values, key_prefix=prefix)
self.assertEqual(errors, [])
keys = list(values)
self.assertEqual(self.mc.get_multi(keys, key_prefix=prefix),
values)
def test_set_multi_dead_servers(self):
"""Testing set_multi() with no memcacheds running."""
self.mc.disconnect_all()
with captured_stderr() as log:
for server in self.mc.servers:
server.mark_dead('test')
self.assertIn('Marking dead.', log.getvalue())
errors = self.mc.set_multi({'key1': 'a', 'key2': 'b'})
self.assertEqual(sorted(errors), ['key1', 'key2'])
def test_disconnect_all_delete_multi(self):
"""Testing delete_multi() with no memcacheds running."""
self.mc.disconnect_all()
with captured_stderr() as output:
ret = self.mc.delete_multi(('keyhere', 'keythere'))
self.assertEqual(ret, 1)
self.assertEqual(
output.getvalue(),
"MemCached: while expecting 'DELETED', got unexpected response "
"'NOT_FOUND'\n"
"MemCached: while expecting 'DELETED', got unexpected response "
"'NOT_FOUND'\n"
)
@mock.patch.object(_Host, 'send_cmd') # Don't send any commands.
@mock.patch.object(_Host, 'readline')
def test_touch_unexpected_reply(self, mock_readline, mock_send_cmd):
"""touch() logs an error upon receiving an unexpected reply."""
mock_readline.return_value = 'SET' # the unexpected reply
with captured_stderr() as output:
self.mc.touch('key')
self.assertEqual(
output.getvalue(),
"MemCached: touch expected %s, got: 'SET'\n" % b'TOUCHED'
)
"""
This is a simple app to generate random numbers for keys stored in memcached.
The keys are upper case letters starting from 'A' through to 'Z' (inclusive).
The key value will only be updated if a random number is less than 10 or greater than 100.
"""
from memcache import Client
import string, random, time
servers = ["127.0.0.1:11211"]
mc = Client(servers, debug=1)
if __name__ == '__main__':
for c in string.ascii_uppercase:
mc.set(c, 0, time=0)
while True:
for c in string.ascii_uppercase:
r = random.randint(0, 100)
if r < 10 or r > 90:
print('Setting value for {}'.format(c), end='')
v = random.randint(0, 100)
mc.set(c, v, time=0)
print(' -> test: v={}'.format(mc.get(c)))
print('*')
time.sleep(1)
# EOF
from memcache import Client
from datetime import datetime
# port = '11211'
#host = 'localhost'
memcached_port = '9150'
redis_port = '9250'
password = ''
hostname = 'ec2-54-183-15-168.us-west-1.compute.amazonaws.com'
mc = Client([hostname + ':' + memcached_port], debug=0)
#mc = Redis(host=hostname, port=redis_port, password=password)
start_time = datetime.now()
mc.set("first_key", "first value")
value = mc.get("first_key")
print(value)
mc.set("second_key", 3)
mc.delete("second_key")
mc.set("key", "1") # note that the key used for incr/decr must be
# a string.
value = mc.get('key')
print(value)
mc.incr("key")
value = mc.get('key')
print(value)
mc.decr("key")
value = mc.get('key')
from os.path import exists
from memcache import Client
from sekrit import MEMCACHE_SOCKET
if exists(MEMCACHE_SOCKET):
CACHE = Client(['unix:' + MEMCACHE_SOCKET], debug=True)
else:
CACHE = Client(['127.0.0.1:11213'], debug=True)
S = lambda key, value: CACHE.set(str(key), value)
G = lambda key: CACHE.get(str(key))
def store_dec(s):
def inner(tag, url):
s(tag, url)
S(tag, url)
return inner
def retrieve_dec(r):
def inner(tag):
url = G(tag)
if url is None:
url = r(tag)
S(tag, url)
return url
return inner
from memcache import Client
"""
安装memcache时，遇到的问题，提示libevent
解决 yum install libevent
yum install libevent-devel
"""
# 这里是个list，可以吧memcache集群这么搞
MC_SERVERS = ['192.168.52.3:11211', '192.168.52.3:11212']
CONN = Client(MC_SERVERS)
status = CONN.set('key1', 'val2', 0)
print(status)
status = CONN.delete('key')
print(status)
status = CONN.add('key', 'val', 20)
print(status)
status = CONN.replace('key', 'val1', 0)
print(status)
status = CONN.append('key', ',val2')
print(status)
data = CONN.get('key')
print(data)
class TestMemcache(unittest.TestCase):
def setUp(self):
# TODO(): unix socket server stuff
servers = ["127.0.0.1:11211"]
self.mc = Client(servers, debug=1)
def tearDown(self):
self.mc.flush_all()
self.mc.disconnect_all()
def check_setget(self, key, val, noreply=False):
self.mc.set(key, val, noreply=noreply)
newval = self.mc.get(key)
self.assertEqual(newval, val)
def test_setget(self):
self.check_setget("a_string", "some random string")
self.check_setget("a_string_2", "some random string", noreply=True)
self.check_setget("an_integer", 42)
self.check_setget("an_integer_2", 42, noreply=True)
def test_delete(self):
self.check_setget("long", int(1 << 30))
result = self.mc.delete("long")
self.assertEqual(result, True)
self.assertEqual(self.mc.get("long"), None)
@mock.patch.object(_Host, 'send_cmd')
@mock.patch.object(_Host, 'readline')
def test_touch(self, mock_readline, mock_send_cmd):
with captured_stderr():
self.mc.touch('key')
mock_send_cmd.assert_called_with(b'touch key 0')
def test_get_multi(self):
self.check_setget("gm_a_string", "some random string")
self.check_setget("gm_an_integer", 42)
self.assertEqual(self.mc.get_multi(["gm_a_string", "gm_an_integer"]), {
"gm_an_integer": 42,
"gm_a_string": "some random string"
})
def test_get_unknown_value(self):
self.mc.delete("unknown_value")
self.assertEqual(self.mc.get("unknown_value"), None)
def test_setget_foostruct(self):
f = FooStruct()
self.check_setget("foostruct", f)
self.check_setget("foostruct_2", f, noreply=True)
def test_incr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.incr("i_an_integer", 1), 43)
def test_incr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.incr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 43)
def test_decr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.decr("i_an_integer", 1), 41)
def test_decr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.decr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 41)
def test_sending_spaces(self):
try:
self.mc.set("this has spaces", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_control_characters(self):
try:
self.mc.set("this\x10has\x11control characters\x02", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_key_too_long(self):
try:
self.mc.set('a' * SERVER_MAX_KEY_LENGTH + 'a', 1)
except Client.MemcachedKeyLengthError as err:
self.assertTrue("length is >" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyLengthError, nothing raised")
# These should work.
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1)
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1, noreply=True)
def test_setget_boolean(self):
"""GitHub issue #75. Set/get with boolean values."""
self.check_setget("bool", True)
def test_unicode_key(self):
s = u'\u4f1a'
maxlen = SERVER_MAX_KEY_LENGTH // len(s.encode('utf-8'))
key = s * maxlen
self.mc.set(key, 5)
value = self.mc.get(key)
self.assertEqual(value, 5)
def test_unicode_value(self):
key = 'key'
value = u'Iñtërnâtiônàlizætiøn2'
self.mc.set(key, value)
cached_value = self.mc.get(key)
self.assertEqual(value, cached_value)
def test_binary_string(self):
value = 'value_to_be_compressed'
compressed_value = zlib.compress(value.encode())
self.mc.set('binary1', compressed_value)
compressed_result = self.mc.get('binary1')
self.assertEqual(compressed_value, compressed_result)
self.assertEqual(value, zlib.decompress(compressed_result).decode())
self.mc.add('binary1-add', compressed_value)
compressed_result = self.mc.get('binary1-add')
self.assertEqual(compressed_value, compressed_result)
self.assertEqual(value, zlib.decompress(compressed_result).decode())
self.mc.set_multi({'binary1-set_many': compressed_value})
compressed_result = self.mc.get('binary1-set_many')
self.assertEqual(compressed_value, compressed_result)
self.assertEqual(value, zlib.decompress(compressed_result).decode())
def test_ignore_too_large_value(self):
# NOTE: "MemCached: while expecting[...]" is normal...
key = 'keyhere'
value = 'a' * (SERVER_MAX_VALUE_LENGTH // 2)
self.assertTrue(self.mc.set(key, value))
self.assertEqual(self.mc.get(key), value)
value = 'a' * SERVER_MAX_VALUE_LENGTH
with captured_stderr() as log:
self.assertIs(self.mc.set(key, value), False)
self.assertEqual(
log.getvalue(),
"MemCached: while expecting 'STORED', got unexpected response "
"'SERVER_ERROR object too large for cache'\n")
# This test fails if the -I option is used on the memcached server
self.assertTrue(self.mc.get(key) is None)
def test_get_set_multi_key_prefix(self):
"""Testing set_multi() with no memcacheds running."""
prefix = 'pfx_'
values = {'key1': 'a', 'key2': 'b'}
errors = self.mc.set_multi(values, key_prefix=prefix)
self.assertEqual(errors, [])
keys = list(values)
self.assertEqual(self.mc.get_multi(keys, key_prefix=prefix), values)
def test_set_multi_dead_servers(self):
"""Testing set_multi() with no memcacheds running."""
self.mc.disconnect_all()
with captured_stderr() as log:
for server in self.mc.servers:
server.mark_dead('test')
self.assertIn('Marking dead.', log.getvalue())
errors = self.mc.set_multi({'key1': 'a', 'key2': 'b'})
self.assertEqual(sorted(errors), ['key1', 'key2'])
def test_disconnect_all_delete_multi(self):
"""Testing delete_multi() with no memcacheds running."""
self.mc.disconnect_all()
with captured_stderr() as output:
ret = self.mc.delete_multi(('keyhere', 'keythere'))
self.assertEqual(ret, 1)
self.assertEqual(
output.getvalue(),
"MemCached: while expecting 'DELETED', got unexpected response "
"'NOT_FOUND'\n"
"MemCached: while expecting 'DELETED', got unexpected response "
"'NOT_FOUND'\n")
@mock.patch.object(_Host, 'send_cmd') # Don't send any commands.
@mock.patch.object(_Host, 'readline')
def test_touch_unexpected_reply(self, mock_readline, mock_send_cmd):
"""touch() logs an error upon receiving an unexpected reply."""
mock_readline.return_value = 'SET' # the unexpected reply
with captured_stderr() as output:
self.mc.touch('key')
self.assertEqual(
output.getvalue(),
"MemCached: touch expected %s, got: 'SET'\n" % b'TOUCHED')
class MemcachedCacheClient(CacheClient):
"""Memcached cache client implementation."""
def __init__(self, config):
super(MemcachedCacheClient, self).__init__(config["host"], config["port"], config["cache"])
self.config = config
if self.cache_name and self.cache_name != "" and self.cache_name != DEFAULT_MEMCACHED_CACHE_NAME:
print "WARNING: memcached client doesn't support named caches. cache_name config value will be ignored and the cache name configured on the server will be used instead."
self.memcached_client = Client([self.host + ':' + self.port], debug=0)
return
def put(self, key, value, version=None, lifespan=None, max_idle=None, put_if_absent=False):
time = 0
if lifespan != None:
if lifespan > MEMCACHED_LIFESPAN_MAX_SECONDS:
self._error("Memcached cache client supports lifespan values only up to %s seconds (30 days)." % MEMCACHED_LIFESPAN_MAX_SECONDS)
time = lifespan
if max_idle != None:
self._error("Memcached cache client doesn't support max idle time setting.")
try:
if (version == None):
if (put_if_absent):
if not self.memcached_client.add(key, value, time, 0):
# current python-memcached doesn't recoginze these states
# if self.memcached_client.last_set_status == "NOT_STORED":
# raise ConflictError
# else:
# self._error("Operation unsuccessful. " + self.memcached_client.last_set_status)
self._error("Operation unsuccessful. Possibly CONFLICT.")
else:
if not self.memcached_client.set(key, value, time, 0):
# self._error("Operation unsuccessful. " + self.memcached_client.last_set_status)
self._error("Operation unsuccessful.")
else:
try:
self.memcached_client.cas_ids[key] = int(version)
except ValueError:
self._error("Please provide an integer version.")
if not self.memcached_client.cas(key, value, time, 0):
# if self.memcached_client.last_set_status == "EXISTS":
# raise ConflictError
# if self.memcached_client.last_set_status == "NOT_FOUND":
# raise NotFoundError
# else:
# self._error("Operation unsuccessful. " + self.memcached_client.last_set_status)
self._error("Operation unsuccessful. Possibly CONFLICT, NOT_FOUND.")
except CacheClientError as e:
raise e #rethrow
except Exception as e:
self._error(e)
def get(self, key, get_version=False):
try:
if get_version:
val = self.memcached_client.gets(key)
if val == None:
raise NotFoundError
version = self.memcached_client.cas_ids[key]
if version == None:
self._error("Couldn't obtain version info from memcached server.")
return version, val
else:
val = self.memcached_client.get(key)
if val == None:
raise NotFoundError
return val
except CacheClientError as e:
raise e #rethrow
except Exception as e:
self._error(e.args)
def delete(self, key, version=None):
try:
if version:
self._error("versioned delete operation not available for memcached client")
if self.memcached_client.delete(key, 0):
if self.memcached_client.last_set_status == "NOT_FOUND":
raise NotFoundError
else:
self._error("Operation unsuccessful. " + self.memcached_client.last_set_status)
except CacheClientError as e:
raise e #rethrow
except Exception as e:
self._error(e.args)
def clear(self):
try:
self.memcached_client.flush_all()
except CacheClientError as e:
raise e #rethrow
except Exception as e:
self._error(e.args)
import time
from random import randint
from sys import exit
import uwsgi
from flask import Flask
from memcache import Client
app = Flask(__name__)
# NOTE: ugly way to manage the connection.
mclient = Client(['127.0.0.1:11311'], debug=0)
mclient.set("sleep", None)
@app.route("/")
def hello():
try:
# NOTE: simulate a failure.
if randint(1, 5) == 1:
raise Exception("this is the end")
mkey = mclient.get("sleep")
if mkey:
return "Hello World from cache :: {} !".format(mkey)
# NOTE: simulate a long operation.
print("sleep my friend !")
class TestMemcache(unittest.TestCase):
def setUp(self):
# TODO: unix socket server stuff
servers = ["127.0.0.1:11211"]
self.mc = Client(servers, debug=1)
def tearDown(self):
self.mc.disconnect_all()
def check_setget(self, key, val, noreply=False):
self.mc.set(key, val, noreply=noreply)
newval = self.mc.get(key)
self.assertEqual(newval, val)
def test_setget(self):
self.check_setget("a_string", "some random string")
self.check_setget("a_string_2", "some random string", noreply=True)
self.check_setget("an_integer", 42)
self.check_setget("an_integer_2", 42, noreply=True)
def test_delete(self):
self.check_setget("long", int(1 << 30))
result = self.mc.delete("long")
self.assertEqual(result, True)
self.assertEqual(self.mc.get("long"), None)
def test_get_multi(self):
self.check_setget("gm_a_string", "some random string")
self.check_setget("gm_an_integer", 42)
self.assertEqual(
self.mc.get_multi(["gm_a_string", "gm_an_integer"]),
{"gm_an_integer": 42, "gm_a_string": "some random string"})
def test_get_unknown_value(self):
self.mc.delete("unknown_value")
self.assertEqual(self.mc.get("unknown_value"), None)
def test_setget_foostruct(self):
f = FooStruct()
self.check_setget("foostruct", f)
self.check_setget("foostruct_2", f, noreply=True)
def test_incr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.incr("i_an_integer", 1), 43)
def test_incr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.incr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 43)
def test_decr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.decr("i_an_integer", 1), 41)
def test_decr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.decr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 41)
def test_sending_spaces(self):
try:
self.mc.set("this has spaces", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_control_characters(self):
try:
self.mc.set("this\x10has\x11control characters\x02", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_key_too_long(self):
try:
self.mc.set('a' * SERVER_MAX_KEY_LENGTH + 'a', 1)
except Client.MemcachedKeyLengthError as err:
self.assertTrue("length is >" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyLengthError, nothing raised")
# These should work.
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1)
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1, noreply=True)
def test_setget_boolean(self):
"""GitHub issue #75. Set/get with boolean values."""
self.check_setget("bool", True)
def test_unicode_key(self):
s = six.u('\u4f1a')
maxlen = SERVER_MAX_KEY_LENGTH // len(s.encode('utf-8'))
key = s * maxlen
self.mc.set(key, 5)
value = self.mc.get(key)
self.assertEqual(value, 5)
def test_ignore_too_large_value(self):
# NOTE: "MemCached: while expecting[...]" is normal...
key = 'keyhere'
value = 'a' * (SERVER_MAX_VALUE_LENGTH // 2)
self.assertTrue(self.mc.set(key, value))
self.assertEqual(self.mc.get(key), value)
value = 'a' * SERVER_MAX_VALUE_LENGTH
self.assertFalse(self.mc.set(key, value))
# This test fails if the -I option is used on the memcached server
self.assertTrue(self.mc.get(key) is None)
def test_get_set_multi_key_prefix(self):
"""Testing set_multi() with no memcacheds running."""
prefix = 'pfx_'
values = {'key1': 'a', 'key2': 'b'}
errors = self.mc.set_multi(values, key_prefix=prefix)
self.assertEqual(errors, [])
keys = list(values)
self.assertEqual(self.mc.get_multi(keys, key_prefix=prefix),
values)
def test_set_multi_dead_servers(self):
"""Testing set_multi() with no memcacheds running."""
self.mc.disconnect_all()
for server in self.mc.servers:
server.mark_dead('test')
errors = self.mc.set_multi({'key1': 'a', 'key2': 'b'})
self.assertEqual(sorted(errors), ['key1', 'key2'])
def test_disconnect_all_delete_multi(self):
"""Testing delete_multi() with no memcacheds running."""
self.mc.disconnect_all()
ret = self.mc.delete_multi({'keyhere': 'a', 'keythere': 'b'})
self.assertEqual(ret, 1)
def test_tags_set(self):
self.mc.disconnect_all()
self.mc.set('key', 'val', tags=['t1'])
self.mc.set('key1', 'val2', tags=['t2'])
self.mc.delete_by_tag('t1')
self.assertIsNone(self.mc.get('key'))
self.assertEqual(self.mc.get('key1'), 'val2')
def test_tags_set_multi(self):
self.mc.disconnect_all()
mapping = {
'k': 1,
'k2': 2,
}
self.mc.set_multi(mapping, tags=['t1'])
self.mc.delete_by_tag('t1')
self.assertIsNone(self.mc.get('k'))
self.assertIsNone(self.mc.get('k1'))
from werkzeug.contrib.cache import MemcachedCache
from memcache import Client
import sys
cache = Client(["127.0.0.1:12000"], debug=True)
print sys.argv
#cache.set("foo", "sss")
if sys.argv[1] == "1":
cache.set("foo", "asdsadasdsadas", 4, 1)
else:
print cache.get("foo")
#
def set(self, key, val, **kwargs):
return Client.set(self, self.uniqeKey + str(key), val, **kwargs)
class TestMemcache(TestCase):
def setUp(self):
# TODO: unix socket server stuff
servers = ["127.0.0.1:11211"]
self.mc = Client(servers, debug=1)
pass
def check_setget(self, key, val, noreply=False):
self.mc.set(key, val, noreply=noreply)
newval = self.mc.get(key)
self.assertEqual(newval, val)
def test_setget(self):
self.check_setget("a_string", "some random string")
self.check_setget("a_string_2", "some random string", noreply=True)
self.check_setget("an_integer", 42)
self.check_setget("an_integer_2", 42, noreply=True)
def test_delete(self):
self.check_setget("long", int(1 << 30))
result = self.mc.delete("long")
self.assertEqual(result, True)
self.assertEqual(self.mc.get("long"), None)
def test_get_multi(self):
self.check_setget("gm_a_string", "some random string")
self.check_setget("gm_an_integer", 42)
self.assertEqual(
self.mc.get_multi(["gm_a_string", "gm_an_integer"]),
{"gm_an_integer": 42, "gm_a_string": "some random string"})
def test_get_unknown_value(self):
self.assertEqual(self.mc.get("unknown_value"), None)
def test_setget_foostruct(self):
f = FooStruct()
self.check_setget("foostruct", f)
self.check_setget("foostruct_2", f, noreply=True)
def test_incr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.incr("i_an_integer", 1), 43)
def test_incr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.incr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 43)
def test_decr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.decr("i_an_integer", 1), 41)
def test_decr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.decr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 41)
def test_sending_spaces(self):
try:
self.mc.set("this has spaces", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_control_characters(self):
try:
self.mc.set("this\x10has\x11control characters\x02", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_key_too_long(self):
try:
self.mc.set('a' * SERVER_MAX_KEY_LENGTH + 'a', 1)
except Client.MemcachedKeyLengthError as err:
self.assertTrue("length is >" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyLengthError, nothing raised")
# These should work.
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1)
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1, noreply=True)
class MemcachedCacheClient(CacheClient):
"""Memcached cache client implementation."""
def __init__(self, config):
super(MemcachedCacheClient, self).__init__(config["host"], config["port"], config["cache"])
self.config = config
if self.cache_name != DEFAULT_CACHE_NAME:
print "WARNING: memcached client doesn't support named caches. cache_name config value will be ignored and default cache will be used instead."
self.memcached_client = Client([self.host + ':' + self.port], debug=0)
return
def put(self, key, value, version=None, lifespan=None, max_idle=None, put_if_absent=False):
time = 0
if lifespan != None:
if lifespan > MEMCACHED_LIFESPAN_MAX_SECONDS:
self._error("Memcached cache client supports lifespan values only up to %s seconds (30 days)." % MEMCACHED_LIFESPAN_MAX_SECONDS)
time = lifespan
if max_idle != None:
self._error("Memcached cache client doesn't support max idle time setting.")
try:
if (version == None):
if (put_if_absent):
if not self.memcached_client.add(key, value, time, 0):
# current python-memcached doesn't recoginze these states
# if self.memcached_client.last_set_status == "NOT_STORED":
# raise ConflictError
# else:
# self._error("Operation unsuccessful. " + self.memcached_client.last_set_status)
self._error("Operation unsuccessful. Possibly CONFLICT.")
else:
if not self.memcached_client.set(key, value, time, 0):
# self._error("Operation unsuccessful. " + self.memcached_client.last_set_status)
self._error("Operation unsuccessful.")
else:
try:
self.memcached_client.cas_ids[key] = int(version)
except ValueError:
self._error("Please provide an integer version.")
if not self.memcached_client.cas(key, value, time, 0):
# if self.memcached_client.last_set_status == "EXISTS":
# raise ConflictError
# if self.memcached_client.last_set_status == "NOT_FOUND":
# raise NotFoundError
# else:
# self._error("Operation unsuccessful. " + self.memcached_client.last_set_status)
self._error("Operation unsuccessful. Possibly CONFLICT, NOT_FOUND.")
except CacheClientError as e:
raise e #rethrow
except Exception as e:
self._error(e)
def get(self, key, get_version=False):
try:
if get_version:
val = self.memcached_client.gets(key)
if val == None:
raise NotFoundError
version = self.memcached_client.cas_ids[key]
if version == None:
self._error("Couldn't obtain version info from memcached server.")
return version, val
else:
val = self.memcached_client.get(key)
if val == None:
raise NotFoundError
return val
except CacheClientError as e:
raise e #rethrow
except Exception as e:
self._error(e.args)
def delete(self, key, version=None):
try:
if version:
self._error("versioned delete operation not available for memcached client")
if self.memcached_client.delete(key, 0):
if self.memcached_client.last_set_status == "NOT_FOUND":
raise NotFoundError
else:
self._error("Operation unsuccessful. " + self.memcached_client.last_set_status)
except CacheClientError as e:
raise e #rethrow
except Exception as e:
self._error(e.args)
def clear(self):
try:
self.memcached_client.flush_all()
except CacheClientError as e:
raise e #rethrow
except Exception as e:
self._error(e.args)
def cmd_set(key: bin, client: Client) -> None:
client.set(key, b"Some value")
class PrintFavicon(BaseHandler):
def __init__(self):
super(PrintFavicon, self).__init__()
default_icon_data = self.open(DEFAULT_FAVICON_LOC, time()).read()
self.default_icon = Icon(data=default_icon_data, location=DEFAULT_FAVICON_LOC, type=DEFAULT_FAVICON_TYPE)
self.env = Environment(loader=FileSystemLoader(os.path.join(cherrypy.config["favicon.root"], "templates")))
self.mc = Client(["%(memcache.host)s:%(memcache.port)d" % cherrypy.config], debug=2)
# Initialize counters
for counter in ["requests", "hits", "defaults"]:
self.mc.add("counter-%s" % counter, "0")
def open(self, url, start, headers=None):
time_spent = int(time() - start)
if time_spent >= TIMEOUT:
raise TimeoutError(time_spent)
if not headers:
headers = dict()
headers.update(
{
"User-Agent": "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; "
+ "rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13"
}
)
opener = build_opener(HTTPRedirectHandler(), HTTPCookieProcessor())
return opener.open(Request(url, headers=headers), timeout=min(CONNECTION_TIMEOUT, TIMEOUT - time_spent))
def validateIconResponse(self, iconResponse):
if iconResponse.getcode() != 200:
cherrypy.log(
"Non-success response:%d fetching url:%s" % (iconResponse.getcode(), iconResponse.geturl()),
severity=INFO,
)
return None
iconContentType = iconResponse.info().gettype()
if iconContentType in ICON_MIMETYPE_BLACKLIST:
cherrypy.log(
"Url:%s favicon content-Type:%s blacklisted" % (iconResponse.geturl(), iconContentType), severity=INFO
)
return None
icon = iconResponse.read()
iconLength = len(icon)
if iconLength == 0:
cherrypy.log("Url:%s null content length" % iconResponse.geturl(), severity=INFO)
return None
if iconLength < MIN_ICON_LENGTH or iconLength > MAX_ICON_LENGTH:
# Issue warning, but accept nonetheless!
cherrypy.log(
"Warning: url:%s favicon size:%d out of bounds" % (iconResponse.geturl(), iconLength), severity=INFO
)
return Icon(data=icon, type=iconContentType)
# Icon at [domain]/favicon.ico?
def iconAtRoot(self, targetDomain, start):
cherrypy.log("Attempting to locate favicon for domain:%s at root" % targetDomain, severity=INFO)
rootIconPath = targetDomain + "/favicon.ico"
try:
rootDomainFaviconResult = self.open(rootIconPath, start)
rootIcon = self.validateIconResponse(rootDomainFaviconResult)
if rootIcon:
cherrypy.log("Found favicon for domain:%s at root" % targetDomain, severity=INFO)
self.cacheIcon(targetDomain, rootIcon.data, rootIconPath)
rootIcon.location = rootIconPath
return rootIcon
except:
cherrypy.log(
"Error fetching favicon at domain root:%s, err:%s, msg:%s"
% (targetDomain, sys.exc_info()[0], sys.exc_info()[1]),
severity=INFO,
)
# Icon specified in page?
def iconInPage(self, targetDomain, targetPath, start, refresh=True):
cherrypy.log("Attempting to locate embedded favicon link in page:%s" % targetPath, severity=INFO)
try:
rootDomainPageResult = self.open(targetPath, start)
if rootDomainPageResult.getcode() == 200:
pageSoup = BeautifulSoup(rootDomainPageResult.read())
pageSoupIcon = pageSoup.find("link", rel=compile("^(shortcut|icon|shortcut icon)$", IGNORECASE))
if pageSoupIcon:
pageIconHref = pageSoupIcon.get("href")
if pageIconHref:
pageIconPath = urljoin(targetPath, pageIconHref)
cherrypy.log(
"Found embedded favicon link:%s for domain:%s" % (pageIconPath, targetDomain), severity=INFO
)
cookies = rootDomainPageResult.headers.getheaders("Set-Cookie")
headers = None
if cookies:
headers = {"Cookie": ";".join(cookies)}
pagePathFaviconResult = self.open(pageIconPath, start, headers=headers)
pageIcon = self.validateIconResponse(pagePathFaviconResult)
if pageIcon:
cherrypy.log(
"Found favicon at:%s for domain:%s" % (pageIconPath, targetDomain), severity=INFO
)
self.cacheIcon(targetDomain, pageIcon.data, pageIconPath)
pageIcon.location = pageIconPath
return pageIcon
else:
if refresh:
for meta in pageSoup.findAll("meta"):
if meta.get("http-equiv", "").lower() == "refresh":
match = search("url=([^;]+)", meta.get("content", ""), flags=IGNORECASE)
if match:
refreshPath = urljoin(rootDomainPageResult.geturl(), match.group(1))
cherrypy.log(
"Processing refresh directive:%s for domain:%s" % (refreshPath, targetDomain),
severity=INFO,
)
return self.iconInPage(targetDomain, refreshPath, start, refresh=False)
cherrypy.log("No link tag found:%s" % targetPath, severity=INFO)
else:
cherrypy.log(
"Non-success response:%d for url:%s" % (rootDomainPageResult.getcode(), targetPath), severity=INFO
)
except:
cherrypy.log(
"Error extracting favicon from page:%s, err:%s, msg:%s"
% (targetPath, sys.exc_info()[0], sys.exc_info()[1]),
severity=WARNING,
)
def cacheIcon(self, domain, icon, loc):
cherrypy.log("Caching icon at location:%s for domain:%s" % (loc, domain), severity=INFO)
if not self.mc.set("icon-%s" % domain, icon, time=MC_CACHE_TIME):
cherrypy.log("Could not cache icon for domain:%s" % domain, severity=ERROR)
def iconInCache(self, targetDomain, start):
icon = self.mc.get("icon-%s" % targetDomain)
if icon:
self.mc.incr("counter-hits")
cherrypy.log("Cache hit:%s" % targetDomain, severity=INFO)
cherrypy.response.headers["X-Cache"] = "Hit"
if icon == "DEFAULT":
self.mc.incr("counter-defaults")
cherrypy.response.headers["X-Cache"] = "Hit"
return self.default_icon
else:
return Icon(data=icon)
def writeIcon(self, icon):
self.writeHeaders(icon)
return icon.data
def writeHeaders(self, icon, fmt="%a, %d %b %Y %H:%M:%S %z"):
# MIME Type
cherrypy.response.headers["Content-Type"] = icon.type or "image/x-icon"
# Set caching headers
cherrypy.response.headers["Cache-Control"] = "public, max-age=2592000"
cherrypy.response.headers["Expires"] = (datetime.now() + timedelta(days=30)).strftime(fmt)
def parse(self, url):
# Get page path
targetPath = self.urldecode(url)
if not targetPath.startswith("http"):
targetPath = "http://%s" % targetPath
cherrypy.log("Decoded URL:%s" % targetPath, severity=INFO)
# Split path to get domain
targetURL = urlparse(targetPath)
if not targetURL or not targetURL.scheme or not targetURL.netloc:
raise cherrypy.HTTPError(400, "Malformed URL:%s" % url)
targetDomain = "%s://%s" % (targetURL.scheme, targetURL.netloc)
cherrypy.log("URL:%s, domain:%s" % (targetPath, targetDomain), severity=INFO)
return (targetPath, targetDomain)
@cherrypy.expose
def index(self):
status = {"status": "ok", "counters": dict()}
for counter in ["requests", "hits", "defaults"]:
status["counters"][counter] = self.mc.get("counter-%s" % counter)
return json.dumps(status)
@cherrypy.expose
def test(self):
topSites = open(os.path.join(cherrypy.config["favicon.root"], "topsites.txt"), "r").read().split()
template = self.env.get_template("test.html")
return template.render(topSites=topSites)
@cherrypy.expose
def clear(self, url):
cherrypy.log("Incoming cache invalidation request:%s" % url, severity=INFO)
targetPath, targetDomain = self.parse(str(url))
self.mc.delete("icon_loc-%s" % targetDomain)
cherrypy.log("Evicted cache entry for %s" % targetDomain, severity=INFO)
@cherrypy.expose
def s(self, url, skipCache="false"):
start = time()
if skipCache.lower() == "true":
skipCache = True
else:
skipCache = False
cherrypy.log("Incoming request:%s (skipCache=%s)" % (url, skipCache), severity=INFO)
self.mc.incr("counter-requests")
targetPath, targetDomain = self.parse(str(url))
icon = (
(not skipCache and self.iconInCache(targetDomain, start))
or self.iconInPage(targetDomain, targetPath, start)
or self.iconAtRoot(targetDomain, start)
)
if not icon:
cherrypy.log("Falling back to default icon for:%s" % targetDomain, severity=INFO)
self.cacheIcon(targetDomain, "DEFAULT", "DEFAULT_LOC")
self.mc.incr("counter-defaults")
icon = self.default_icon
cherrypy.log("Time taken to process domain:%s %f" % (targetDomain, time() - start), severity=INFO)
return self.writeIcon(icon)
from time import time
from memcache import Client
client = Client(['127.0.0.1:11211'])
client.set("my_key", "val " + str(999))
print client.get("my_key")
n = 100000
start_time = time()
for i in xrange(0, n):
client.set(str(i), "val " + str(i))
print "set time: %s ms" % ((time()-start_time)*1000)
start_time = time()
for i in xrange(0, n):
client.get(str(i))
print client.get("555")
print "get time: %s ms" % ((time()-start_time)*1000)
#print client.stats
class PrintFavicon(BaseHandler):
def __init__(self):
super(PrintFavicon, self).__init__()
default_icon_data = self.open(DEFAULT_FAVICON_LOC, time()).read()
self.default_icon = Icon(data=default_icon_data,
location=DEFAULT_FAVICON_LOC,
type=DEFAULT_FAVICON_TYPE)
self.env = Environment(loader=FileSystemLoader(
os.path.join(cherrypy.config['favicon.root'], 'templates')))
self.mc = Client(
['%(memcache.host)s:%(memcache.port)d' % cherrypy.config], debug=2)
# Initialize counters
for counter in ['requests', 'hits', 'defaults']:
self.mc.add('counter-%s' % counter, '0')
def open(self, url, start, headers=None):
time_spent = int(time() - start)
if time_spent >= TIMEOUT:
raise TimeoutError(time_spent)
if not headers:
headers = dict()
headers.update({
'User-Agent':
'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; ' +
'rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13'
})
opener = build_opener(HTTPRedirectHandler(), HTTPCookieProcessor())
return opener.open(Request(url, headers=headers),
timeout=min(CONNECTION_TIMEOUT,
TIMEOUT - time_spent))
def validateIconResponse(self, iconResponse):
if iconResponse.getcode() != 200:
cherrypy.log('Non-success response:%d fetching url:%s' % \
(iconResponse.getcode(), iconResponse.geturl()),
severity=INFO)
return None
iconContentType = iconResponse.info().gettype()
if iconContentType in ICON_MIMETYPE_BLACKLIST:
cherrypy.log('Url:%s favicon content-Type:%s blacklisted' % \
(iconResponse.geturl(), iconContentType),
severity=INFO)
return None
icon = iconResponse.read()
iconLength = len(icon)
if iconLength == 0:
cherrypy.log('Url:%s null content length' % iconResponse.geturl(),
severity=INFO)
return None
if iconLength < MIN_ICON_LENGTH or iconLength > MAX_ICON_LENGTH:
# Issue warning, but accept nonetheless!
cherrypy.log('Warning: url:%s favicon size:%d out of bounds' % \
(iconResponse.geturl(), iconLength),
severity=INFO)
return Icon(data=icon, type=iconContentType)
# Icon at [domain]/favicon.ico?
def iconAtRoot(self, targetDomain, start):
cherrypy.log('Attempting to locate favicon for domain:%s at root' % \
targetDomain,
severity=INFO)
rootIconPath = targetDomain + '/favicon.ico'
try:
rootDomainFaviconResult = self.open(rootIconPath, start)
rootIcon = self.validateIconResponse(rootDomainFaviconResult)
if rootIcon:
cherrypy.log('Found favicon for domain:%s at root' %
targetDomain,
severity=INFO)
self.cacheIcon(targetDomain, rootIcon.data, rootIconPath)
rootIcon.location = rootIconPath
return rootIcon
except:
cherrypy.log('Error fetching favicon at domain root:%s, err:%s, msg:%s' % \
(targetDomain, sys.exc_info()[0], sys.exc_info()[1]),
severity=INFO)
# Icon specified in page?
def iconInPage(self, targetDomain, targetPath, start, refresh=True):
cherrypy.log('Attempting to locate embedded favicon link in page:%s' % \
targetPath,
severity=INFO)
try:
rootDomainPageResult = self.open(targetPath, start)
if rootDomainPageResult.getcode() == 200:
pageSoup = BeautifulSoup(rootDomainPageResult.read())
pageSoupIcon = pageSoup.find(
'link',
rel=compile('^(shortcut|icon|shortcut icon)$', IGNORECASE))
if pageSoupIcon:
pageIconHref = pageSoupIcon.get('href')
if pageIconHref:
pageIconPath = urljoin(targetPath, pageIconHref)
cherrypy.log('Found embedded favicon link:%s for domain:%s' % \
(pageIconPath, targetDomain),
severity=INFO)
cookies = rootDomainPageResult.headers.getheaders(
"Set-Cookie")
headers = None
if cookies:
headers = {'Cookie': ';'.join(cookies)}
pagePathFaviconResult = self.open(pageIconPath,
start,
headers=headers)
pageIcon = self.validateIconResponse(
pagePathFaviconResult)
if pageIcon:
cherrypy.log('Found favicon at:%s for domain:%s' % \
(pageIconPath, targetDomain),
severity=INFO)
self.cacheIcon(targetDomain, pageIcon.data,
pageIconPath)
pageIcon.location = pageIconPath
return pageIcon
else:
if refresh:
for meta in pageSoup.findAll('meta'):
if meta.get('http-equiv', '').lower() == 'refresh':
match = search('url=([^;]+)',
meta.get('content', ''),
flags=IGNORECASE)
if match:
refreshPath = urljoin(
rootDomainPageResult.geturl(),
match.group(1))
cherrypy.log('Processing refresh directive:%s for domain:%s' % \
(refreshPath, targetDomain),
severity=INFO)
return self.iconInPage(targetDomain,
refreshPath,
start,
refresh=False)
cherrypy.log('No link tag found:%s' % targetPath,
severity=INFO)
else:
cherrypy.log('Non-success response:%d for url:%s' % \
(rootDomainPageResult.getcode(), targetPath),
severity=INFO)
except:
cherrypy.log('Error extracting favicon from page:%s, err:%s, msg:%s' % \
(targetPath, sys.exc_info()[0], sys.exc_info()[1]),
severity=WARNING)
def cacheIcon(self, domain, icon, loc):
cherrypy.log('Caching icon at location:%s for domain:%s' %
(loc, domain),
severity=INFO)
if not self.mc.set('icon-%s' % domain, icon, time=MC_CACHE_TIME):
cherrypy.log('Could not cache icon for domain:%s' % domain,
severity=ERROR)
def iconInCache(self, targetDomain, start):
icon = self.mc.get('icon-%s' % targetDomain)
if icon:
self.mc.incr('counter-hits')
cherrypy.log('Cache hit:%s' % targetDomain, severity=INFO)
cherrypy.response.headers['X-Cache'] = 'Hit'
if icon == 'DEFAULT':
self.mc.incr('counter-defaults')
cherrypy.response.headers['X-Cache'] = 'Hit'
return self.default_icon
else:
return Icon(data=icon)
def writeIcon(self, icon):
self.writeHeaders(icon)
return icon.data
def writeHeaders(self, icon, fmt='%a, %d %b %Y %H:%M:%S %z'):
# MIME Type
cherrypy.response.headers['Content-Type'] = icon.type or 'image/x-icon'
# Set caching headers
cherrypy.response.headers['Cache-Control'] = 'public, max-age=2592000'
cherrypy.response.headers['Expires'] = \
(datetime.now() + timedelta(days=30)).strftime(fmt)
def parse(self, url):
# Get page path
targetPath = self.urldecode(url)
if not targetPath.startswith('http'):
targetPath = 'http://%s' % targetPath
cherrypy.log('Decoded URL:%s' % targetPath, severity=INFO)
# Split path to get domain
targetURL = urlparse(targetPath)
if not targetURL or not targetURL.scheme or not targetURL.netloc:
raise cherrypy.HTTPError(400, 'Malformed URL:%s' % url)
targetDomain = '%s://%s' % (targetURL.scheme, targetURL.netloc)
cherrypy.log('URL:%s, domain:%s' % (targetPath, targetDomain),
severity=INFO)
return (targetPath, targetDomain)
@cherrypy.expose
def index(self):
status = {'status': 'ok', 'counters': dict()}
for counter in ['requests', 'hits', 'defaults']:
status['counters'][counter] = self.mc.get('counter-%s' % counter)
return json.dumps(status)
@cherrypy.expose
def test(self):
topSites = open(
os.path.join(cherrypy.config['favicon.root'], 'topsites.txt'),
'r').read().split()
template = self.env.get_template('test.html')
return template.render(topSites=topSites)
@cherrypy.expose
def clear(self, url):
cherrypy.log('Incoming cache invalidation request:%s' % url,
severity=INFO)
targetPath, targetDomain = self.parse(str(url))
self.mc.delete('icon_loc-%s' % targetDomain)
cherrypy.log('Evicted cache entry for %s' % targetDomain,
severity=INFO)
@cherrypy.expose
def s(self, url, skipCache='false'):
start = time()
if skipCache.lower() == 'true':
skipCache = True
else:
skipCache = False
cherrypy.log('Incoming request:%s (skipCache=%s)' % (url, skipCache),
severity=INFO)
self.mc.incr('counter-requests')
targetPath, targetDomain = self.parse(str(url))
icon = (not skipCache and self.iconInCache(targetDomain, start)) or \
self.iconInPage(targetDomain, targetPath, start) or \
self.iconAtRoot(targetDomain, start)
if not icon:
cherrypy.log('Falling back to default icon for:%s' % targetDomain,
severity=INFO)
self.cacheIcon(targetDomain, 'DEFAULT', 'DEFAULT_LOC')
self.mc.incr('counter-defaults')
icon = self.default_icon
cherrypy.log('Time taken to process domain:%s %f' % \
(targetDomain, time() - start),
severity=INFO)
return self.writeIcon(icon)
class TestMemcache(unittest.TestCase):
def setUp(self):
# TODO(): unix socket server stuff
servers = ["127.0.0.1:11211"]
self.mc = Client(servers, debug=1)
def tearDown(self):
self.mc.disconnect_all()
def check_setget(self, key, val, noreply=False):
self.mc.set(key, val, noreply=noreply)
newval = self.mc.get(key)
self.assertEqual(newval, val)
def test_setget(self):
self.check_setget("a_string", "some random string")
self.check_setget("a_string_2", "some random string", noreply=True)
self.check_setget("an_integer", 42)
self.check_setget("an_integer_2", 42, noreply=True)
def test_delete(self):
self.check_setget("long", int(1 << 30))
result = self.mc.delete("long")
self.assertEqual(result, True)
self.assertEqual(self.mc.get("long"), None)
def test_get_multi(self):
self.check_setget("gm_a_string", "some random string")
self.check_setget("gm_an_integer", 42)
self.assertEqual(self.mc.get_multi(["gm_a_string", "gm_an_integer"]), {
"gm_an_integer": 42,
"gm_a_string": "some random string"
})
def test_get_unknown_value(self):
self.mc.delete("unknown_value")
self.assertEqual(self.mc.get("unknown_value"), None)
def test_setget_foostruct(self):
f = FooStruct()
self.check_setget("foostruct", f)
self.check_setget("foostruct_2", f, noreply=True)
def test_incr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.incr("i_an_integer", 1), 43)
def test_incr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.incr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 43)
def test_decr(self):
self.check_setget("i_an_integer", 42)
self.assertEqual(self.mc.decr("i_an_integer", 1), 41)
def test_decr_noreply(self):
self.check_setget("i_an_integer_2", 42)
self.assertEqual(self.mc.decr("i_an_integer_2", 1, noreply=True), None)
self.assertEqual(self.mc.get("i_an_integer_2"), 41)
def test_sending_spaces(self):
try:
self.mc.set("this has spaces", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_control_characters(self):
try:
self.mc.set("this\x10has\x11control characters\x02", 1)
except Client.MemcachedKeyCharacterError as err:
self.assertTrue("characters not allowed" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyCharacterError, nothing raised")
def test_sending_key_too_long(self):
try:
self.mc.set('a' * SERVER_MAX_KEY_LENGTH + 'a', 1)
except Client.MemcachedKeyLengthError as err:
self.assertTrue("length is >" in err.args[0])
else:
self.fail(
"Expected Client.MemcachedKeyLengthError, nothing raised")
# These should work.
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1)
self.mc.set('a' * SERVER_MAX_KEY_LENGTH, 1, noreply=True)
def test_setget_boolean(self):
"""GitHub issue #75. Set/get with boolean values."""
self.check_setget("bool", True)
def test_unicode_key(self):
s = six.u('\u4f1a')
maxlen = SERVER_MAX_KEY_LENGTH // len(s.encode('utf-8'))
key = s * maxlen
self.mc.set(key, 5)
value = self.mc.get(key)
self.assertEqual(value, 5)
def test_ignore_too_large_value(self):
# NOTE: "MemCached: while expecting[...]" is normal...
key = 'keyhere'
value = 'a' * (SERVER_MAX_VALUE_LENGTH // 2)
self.assertTrue(self.mc.set(key, value))
self.assertEqual(self.mc.get(key), value)
value = 'a' * SERVER_MAX_VALUE_LENGTH
self.assertFalse(self.mc.set(key, value))
# This test fails if the -I option is used on the memcached server
self.assertTrue(self.mc.get(key) is None)
def test_get_set_multi_key_prefix(self):
"""Testing set_multi() with no memcacheds running."""
prefix = 'pfx_'
values = {'key1': 'a', 'key2': 'b'}
errors = self.mc.set_multi(values, key_prefix=prefix)
self.assertEqual(errors, [])
keys = list(values)
self.assertEqual(self.mc.get_multi(keys, key_prefix=prefix), values)
def test_set_multi_dead_servers(self):
"""Testing set_multi() with no memcacheds running."""
self.mc.disconnect_all()
for server in self.mc.servers:
server.mark_dead('test')
errors = self.mc.set_multi({'key1': 'a', 'key2': 'b'})
self.assertEqual(sorted(errors), ['key1', 'key2'])
def test_disconnect_all_delete_multi(self):
"""Testing delete_multi() with no memcacheds running."""
self.mc.disconnect_all()
ret = self.mc.delete_multi({'keyhere': 'a', 'keythere': 'b'})
self.assertEqual(ret, 1)
from memcache import Client
import burnerconf, local_cache
mc = Client([burnerconf.CACHE_URL], debug=True)
with open('local_cache.txt') as data:
for tag, url in local_cache.LocalCache.read(data):
mc.set(tag, url)
print tag, url
from memcache import Client
import pandas as pd
memcached_port = '9150'
hostname = 'ec2-54-183-15-168.us-west-1.compute.amazonaws.com'
mc = Client([hostname + ':' + memcached_port],
debug=0) #defining memcached client
df = pd.read_csv("SO2 Emissions Key Value Cache.csv"
) # reading csv file to a pandas dataframe
for key in range(len(df)):
mc.set(str(key) + 'Ali', df.loc[key, ]
) #writing each row of a dataframe as key-value pair in memcached
read_df = pd.DataFrame(mc.get('0Ali')).transpose(
) #reading first item of database in a new pandas dataframe
for key in range(1, len(df)):
read_df = read_df.append(
mc.get(str(key) +
'Ali')) #reading all remaining items in respective dataframe
read_df.to_csv("New_SO2 Emissions Key Value Cache.csv"
) #writling new dataframe into a new csv file
class MemcachedLoaderMixin(object):
"""
Uses a memcached server to cache the templates.
Requires the memcache library from tummy__.
__ http://www.tummy.com/Community/software/python-memcached/
"""
def __init__(self,
use_memcache,
memcache_time=60 * 60 * 24 * 7,
memcache_host=None,
item_prefix='template/'):
try:
from memcache import Client
except ImportError:
raise RuntimeError('the %r loader requires an installed '
'memcache module' % self.__class__.__name__)
if memcache_host is None:
memcache_host = ['127.0.0.1:11211']
if use_memcache:
self.__memcache = Client(list(memcache_host))
self.__memcache_time = memcache_time
else:
self.__memcache = None
self.__item_prefix = item_prefix
self.__lock = Lock()
def load(self, environment, name, translator):
"""
Load and translate a template. First we check if there is a
cached version of this template in the memory cache. If this is
not the cache check for a compiled template in the disk cache
folder. And if none of this is the case we translate the template,
cache and return it.
"""
self.__lock.acquire()
try:
# caching is only possible for the python translator. skip
# all other translators
if translator is not PythonTranslator:
return super(MemcachedLoaderMixin,
self).load(environment, name, translator)
tmpl = None
push_to_memory = False
# check if we have something in the memory cache and the
# memory cache is enabled.
if self.__memcache is not None:
bytecode = self.__memcache.get(self.__item_prefix + name)
if bytecode:
tmpl = Template.load(environment, bytecode)
else:
push_to_memory = True
# if we still have no template we load, parse and translate it.
if tmpl is None:
tmpl = super(MemcachedLoaderMixin,
self).load(environment, name, translator)
# if memcaching is enabled and the template not loaded
# we add that there.
if push_to_memory:
self.__memcache.set(self.__item_prefix + name, tmpl.dump(),
self.__memcache_time)
return tmpl
finally:
self.__lock.release()
