def testlru(self):
l = LRUCache(10, threshold=.2)
for id in range(1,20):
l[id] = item(id)
# first couple of items should be gone
self.assert_(not l.has_key(1))
self.assert_(not l.has_key(2))
# next batch over the threshold of 10 should be present
for id in range(11,20):
self.assert_(l.has_key(id))
l[12]
l[15]
l[23] = item(23)
l[24] = item(24)
l[25] = item(25)
l[26] = item(26)
l[27] = item(27)
self.assert_(not l.has_key(11))
self.assert_(not l.has_key(13))
for id in (25, 24, 23, 14, 12, 19, 18, 17, 16, 15):
self.assert_(l.has_key(id))
def testlru(self):
l = LRUCache(10, threshold=.2)
for id in range(1,20):
l[id] = item(id)
# first couple of items should be gone
assert 1 not in l
assert 2 not in l
# next batch over the threshold of 10 should be present
for id in range(11,20):
assert id in l
l[12]
l[15]
l[23] = item(23)
l[24] = item(24)
l[25] = item(25)
l[26] = item(26)
l[27] = item(27)
assert 11 not in l
assert 13 not in l
for id in (25, 24, 23, 14, 12, 19, 18, 17, 16, 15):
assert id in l
def reset_cache(self):
limit = config.get("smisk.mvc.template.cache_limit", -1)
if limit == -1:
self.instances = {}
self._uri_cache = {}
else:
self.instances = LRUCache(limit)
self._uri_cache = LRUCache(limit)
def func_wrapper(*args, **kwargs):
store = __cache.setdefault(rpc.session.db, {}).setdefault(func_name, LRUCache(limit))
key = cPickle.dumps((args, kwargs))
if key not in store:
store[key] = func(*args, **kwargs)
return copy.deepcopy(store[key])
def testlru(self):
l = LRUCache(10, threshold=.2)
for id in range(1, 20):
l[id] = item(id)
# first couple of items should be gone
self.assert_(not l.has_key(1))
self.assert_(not l.has_key(2))
# next batch over the threshold of 10 should be present
for id in range(11, 20):
self.assert_(l.has_key(id))
l[12]
l[15]
l[23] = item(23)
l[24] = item(24)
l[25] = item(25)
l[26] = item(26)
l[27] = item(27)
self.assert_(not l.has_key(11))
self.assert_(not l.has_key(13))
for id in (25, 24, 23, 14, 12, 19, 18, 17, 16, 15):
self.assert_(l.has_key(id))
def _disabled_test_threaded(self):
size = 100
threshold = .5
all_elems = 2000
hot_zone = list(range(30,40))
cache = LRUCache(size, threshold)
# element to store
class Element(object):
def __init__(self, id):
self.id = id
self.regets = 0
# return an element. we will favor ids in the relatively small
# "hot zone" 25% of the time.
def get_elem():
if random.randint(1,4) == 1:
return hot_zone[random.randint(0, len(hot_zone) - 1)]
else:
return random.randint(1, all_elems)
total = [0]
# request thread.
def request_elem():
while True:
total[0] += 1
id = get_elem()
try:
elem = cache[id]
elem.regets += 1
except KeyError:
e = Element(id)
cache[id] = e
time.sleep(random.random() / 1000)
for x in range(0,20):
_thread.start_new_thread(request_elem, ())
# assert size doesn't grow unbounded, doesnt shrink well below size
for x in range(0,5):
time.sleep(1)
print("size:", len(cache))
assert len(cache) < size + size * threshold * 2
assert len(cache) > size - (size * .1)
# computs the average number of times a range of elements were "reused",
# i.e. without being removed from the cache.
def average_regets_in_range(start, end):
elem = [e for e in list(cache.values()) if e.id >= start and e.id <= end]
if len(elem) == 0:
return 0
avg = sum([e.regets for e in elem]) / len(elem)
return avg
hotzone_avg = average_regets_in_range(30, 40)
control_avg = average_regets_in_range(450,760)
total_avg = average_regets_in_range(0, 2000)
# hotzone should be way above the others
print("total fetches", total[0], "hotzone", \
hotzone_avg, "control", \
control_avg, "total", total_avg)
assert hotzone_avg > total_avg * 5 > control_avg * 5
class Templates(object):
"""Templates.
"""
imports = [
"import os, sys, time, logging",
"from smisk.core import app, request, response",
"from smisk.mvc.template.filters import j",
"log = logging.getLogger('template:' + _template_uri)",
]
""":type: list"""
directories = None
"""Directories in which to find templates.
:type: list
"""
is_useable = Template is not None
"""True if mako is installed and templating can be used"""
def __init__(self):
self.directories = []
self.get_template = self.template_for_uri # for compat with mako
self.reset_cache()
def reset_cache(self):
limit = config.get("smisk.mvc.template.cache_limit", -1)
if limit == -1:
self.instances = {}
self._uri_cache = {}
else:
self.instances = LRUCache(limit)
self._uri_cache = LRUCache(limit)
def template_for_uri(self, uri, exc_if_not_found=True):
"""
:return: template for the uri provided
:rtype: Template
"""
try:
template = self.instances[uri]
if config.get("smisk.mvc.template.autoreload", config.get("smisk.autoreload")):
if template is not None:
template = self._check(uri, template)
else:
raise KeyError("check again")
if exc_if_not_found and template is None:
raise exceptions.TopLevelLookupException("Failed to locate template for uri '%s'" % uri)
return template
except KeyError:
u = re.sub(r"^\/+", "", uri)
for dn in self.directories:
srcfile = posixpath.normpath(posixpath.join(dn, u))
if os.access(srcfile, os.F_OK):
return self._load(srcfile, uri)
else:
self.instances[uri] = None
if exc_if_not_found:
raise exceptions.TopLevelLookupException("Failed to locate template for uri '%s'" % uri)
return None
def adjust_uri(self, uri, relativeto):
"""adjust the given uri based on the calling filename."""
if uri[0] != "/":
if relativeto is not None:
return posixpath.join(posixpath.dirname(relativeto), uri)
else:
return "/" + uri
else:
return uri
def filename_to_uri(self, filename):
try:
return self._uri_cache[filename]
except KeyError:
value = self._relativeize(filename)
self._uri_cache[filename] = value
return value
def render_error(self, status, params={}, format="html"):
# Compile body from template
errors = config.get("smisk.mvc.template.errors", {})
if status.code in errors:
template = self.template_for_uri("%s.%s" % (errors[status.code], format), False)
elif status in errors:
template = self.template_for_uri("%s.%s" % (errors[status], format), False)
elif 0 in errors:
template = self.template_for_uri("%s.%s" % (errors[0], format), False)
else:
template = None
# We can't render this error because we did not find a suiting template.
if template is None:
return None
# Render template
return template.render(**params)
def _relativeize(self, filename):
"""return the portion of a filename that is 'relative' to the directories in this lookup."""
filename = posixpath.normpath(filename)
for dn in self.directories:
if filename[0 : len(dn)] == dn:
return filename[len(dn) :]
else:
return None
def _load(self, filename, uri, text=None):
try:
if filename is not None:
filename = posixpath.normpath(filename)
encoding_errors = "replace"
if len(uri) > 4 and (uri[-5:].lower() == ".html" or uri[-4:].lower() == ".xml"):
encoding_errors = "htmlentityreplace"
cache_type = config.get("smisk.mvc.template.cache_type", "memory")
self.instances[uri] = Template(
uri=uri,
filename=filename,
text=text,
lookup=self,
module_filename=None,
format_exceptions=config.get("smisk.mvc.template.format_exceptions", True),
input_encoding=config.get("smisk.mvc.template.input_encoding", "utf-8"),
output_encoding=smisk.mvc.Response.charset,
encoding_errors=encoding_errors,
cache_type=cache_type,
default_filters=config.get("smisk.mvc.template.default_filters", ["unicode"]),
imports=self.imports,
)
if log.level <= logging.DEBUG and cache_type != "file":
code = self.instances[uri].code
log.debug("Compiled %s into %d bytes of python code:\n%s", uri, len(code), code)
return self.instances[uri]
except:
self.instances.pop(uri, None)
raise
def _check(self, uri, template):
if template.filename is None:
return template
if not os.access(template.filename, os.F_OK):
self.instances.pop(uri, None)
raise exceptions.TemplateLookupException("Can't locate template for uri '%s'" % uri)
elif template.module._modified_time < os.stat(template.filename)[stat.ST_MTIME]:
self.instances.pop(uri, None)
return self._load(template.filename, uri)
else:
return template
def put_string(self, uri, text):
raise NotImplementedError
def put_template(self, uri, template):
self.instances[uri] = template
# in start-up page and the software header, even if the application
# source code has been changed or updated or code has been added.
#
# You can see the MPL licence at: http://www.mozilla.org/MPL/MPL-1.1.html
#
###############################################################################
import copy
import cPickle
import functools
import cherrypy
from mako.util import LRUCache
import rpc
__cache = LRUCache(cherrypy.config.get('server.db_cache_size', 8))
def clear():
__cache.pop(rpc.session.db, None)
def memoize(limit=100, force=False):
def memoize_wrapper(func):
# Don't use cache for development environment
if not force and cherrypy.config.get('server.environment') == 'development':
return func
func_name = "%s.%s.%s" % (func.__module__, func.func_code.co_firstlineno, func.func_name)
@functools.wraps(func)
def func_wrapper(*args, **kwargs):