Пример #1
0
    def testlru(self):                
        l = LRUCache(10, threshold=.2)
        
        for id in range(1,20):
            l[id] = item(id)
        
        # first couple of items should be gone
        self.assert_(not l.has_key(1))    
        self.assert_(not l.has_key(2))
        
        # next batch over the threshold of 10 should be present
        for id in range(11,20):
            self.assert_(l.has_key(id))

        l[12]
        l[15]
        l[23] = item(23)
        l[24] = item(24)
        l[25] = item(25)
        l[26] = item(26)
        l[27] = item(27)

        self.assert_(not l.has_key(11))
        self.assert_(not l.has_key(13))
        
        for id in (25, 24, 23, 14, 12, 19, 18, 17, 16, 15):
            self.assert_(l.has_key(id))    
Пример #2
0
    def testlru(self):
        l = LRUCache(10, threshold=.2)

        for id in range(1,20):
            l[id] = item(id)

        # first couple of items should be gone
        assert 1 not in l
        assert 2 not in l

        # next batch over the threshold of 10 should be present
        for id in range(11,20):
            assert id in l

        l[12]
        l[15]
        l[23] = item(23)
        l[24] = item(24)
        l[25] = item(25)
        l[26] = item(26)
        l[27] = item(27)

        assert 11 not in l
        assert 13 not in l

        for id in (25, 24, 23, 14, 12, 19, 18, 17, 16, 15):
            assert id in l
Пример #3
0
 def reset_cache(self):
     limit = config.get("smisk.mvc.template.cache_limit", -1)
     if limit == -1:
         self.instances = {}
         self._uri_cache = {}
     else:
         self.instances = LRUCache(limit)
         self._uri_cache = LRUCache(limit)
Пример #4
0
        def func_wrapper(*args, **kwargs):

            store = __cache.setdefault(rpc.session.db, {}).setdefault(func_name, LRUCache(limit))

            key = cPickle.dumps((args, kwargs))
            if key not in store:
                store[key] = func(*args, **kwargs)
            return copy.deepcopy(store[key])
Пример #5
0
    def testlru(self):
        l = LRUCache(10, threshold=.2)

        for id in range(1, 20):
            l[id] = item(id)

        # first couple of items should be gone
        self.assert_(not l.has_key(1))
        self.assert_(not l.has_key(2))

        # next batch over the threshold of 10 should be present
        for id in range(11, 20):
            self.assert_(l.has_key(id))

        l[12]
        l[15]
        l[23] = item(23)
        l[24] = item(24)
        l[25] = item(25)
        l[26] = item(26)
        l[27] = item(27)

        self.assert_(not l.has_key(11))
        self.assert_(not l.has_key(13))

        for id in (25, 24, 23, 14, 12, 19, 18, 17, 16, 15):
            self.assert_(l.has_key(id))
Пример #6
0
    def _disabled_test_threaded(self):
        size = 100
        threshold = .5
        all_elems = 2000
        hot_zone = list(range(30,40))
        cache = LRUCache(size, threshold)

        # element to store
        class Element(object):
            def __init__(self, id):
                self.id = id
                self.regets = 0

        # return an element.  we will favor ids in the relatively small
        # "hot zone" 25% of  the time.
        def get_elem():
            if random.randint(1,4) == 1:
                return hot_zone[random.randint(0, len(hot_zone) - 1)]
            else:
                return random.randint(1, all_elems)

        total = [0]
        # request thread.
        def request_elem():
            while True:
                total[0] += 1
                id = get_elem()
                try:
                    elem = cache[id]
                    elem.regets += 1
                except KeyError:
                    e = Element(id)
                    cache[id] = e

                time.sleep(random.random() / 1000)

        for x in range(0,20):
            _thread.start_new_thread(request_elem, ())

        # assert size doesn't grow unbounded, doesnt shrink well below size
        for x in range(0,5):
            time.sleep(1)
            print("size:", len(cache))
            assert len(cache) < size + size * threshold * 2
            assert len(cache) > size - (size * .1)

        # computs the average number of times a range of elements were "reused",
        # i.e. without being removed from the cache.
        def average_regets_in_range(start, end):
            elem = [e for e in list(cache.values()) if e.id >= start and e.id <= end]
            if len(elem) == 0:
                return 0
            avg = sum([e.regets for e in elem]) / len(elem)
            return avg

        hotzone_avg = average_regets_in_range(30, 40)
        control_avg = average_regets_in_range(450,760)
        total_avg = average_regets_in_range(0, 2000)

        # hotzone should be way above the others
        print("total fetches", total[0], "hotzone", \
                                hotzone_avg, "control", \
                                control_avg, "total", total_avg)

        assert hotzone_avg > total_avg * 5 > control_avg * 5
Пример #7
0
class Templates(object):
    """Templates.
  """

    imports = [
        "import os, sys, time, logging",
        "from smisk.core import app, request, response",
        "from smisk.mvc.template.filters import j",
        "log = logging.getLogger('template:' + _template_uri)",
    ]
    """:type: list"""

    directories = None
    """Directories in which to find templates.
  
  :type: list
  """

    is_useable = Template is not None
    """True if mako is installed and templating can be used"""

    def __init__(self):
        self.directories = []
        self.get_template = self.template_for_uri  # for compat with mako
        self.reset_cache()

    def reset_cache(self):
        limit = config.get("smisk.mvc.template.cache_limit", -1)
        if limit == -1:
            self.instances = {}
            self._uri_cache = {}
        else:
            self.instances = LRUCache(limit)
            self._uri_cache = LRUCache(limit)

    def template_for_uri(self, uri, exc_if_not_found=True):
        """
    :return: template for the uri provided 
    :rtype:  Template
    """
        try:
            template = self.instances[uri]
            if config.get("smisk.mvc.template.autoreload", config.get("smisk.autoreload")):
                if template is not None:
                    template = self._check(uri, template)
                else:
                    raise KeyError("check again")
            if exc_if_not_found and template is None:
                raise exceptions.TopLevelLookupException("Failed to locate template for uri '%s'" % uri)
            return template
        except KeyError:
            u = re.sub(r"^\/+", "", uri)
            for dn in self.directories:
                srcfile = posixpath.normpath(posixpath.join(dn, u))
                if os.access(srcfile, os.F_OK):
                    return self._load(srcfile, uri)
            else:
                self.instances[uri] = None
                if exc_if_not_found:
                    raise exceptions.TopLevelLookupException("Failed to locate template for uri '%s'" % uri)
                return None

    def adjust_uri(self, uri, relativeto):
        """adjust the given uri based on the calling filename."""
        if uri[0] != "/":
            if relativeto is not None:
                return posixpath.join(posixpath.dirname(relativeto), uri)
            else:
                return "/" + uri
        else:
            return uri

    def filename_to_uri(self, filename):
        try:
            return self._uri_cache[filename]
        except KeyError:
            value = self._relativeize(filename)
            self._uri_cache[filename] = value
            return value

    def render_error(self, status, params={}, format="html"):
        # Compile body from template
        errors = config.get("smisk.mvc.template.errors", {})
        if status.code in errors:
            template = self.template_for_uri("%s.%s" % (errors[status.code], format), False)
        elif status in errors:
            template = self.template_for_uri("%s.%s" % (errors[status], format), False)
        elif 0 in errors:
            template = self.template_for_uri("%s.%s" % (errors[0], format), False)
        else:
            template = None

        # We can't render this error because we did not find a suiting template.
        if template is None:
            return None

        # Render template
        return template.render(**params)

    def _relativeize(self, filename):
        """return the portion of a filename that is 'relative' to the directories in this lookup."""
        filename = posixpath.normpath(filename)
        for dn in self.directories:
            if filename[0 : len(dn)] == dn:
                return filename[len(dn) :]
        else:
            return None

    def _load(self, filename, uri, text=None):
        try:
            if filename is not None:
                filename = posixpath.normpath(filename)

            encoding_errors = "replace"
            if len(uri) > 4 and (uri[-5:].lower() == ".html" or uri[-4:].lower() == ".xml"):
                encoding_errors = "htmlentityreplace"

            cache_type = config.get("smisk.mvc.template.cache_type", "memory")

            self.instances[uri] = Template(
                uri=uri,
                filename=filename,
                text=text,
                lookup=self,
                module_filename=None,
                format_exceptions=config.get("smisk.mvc.template.format_exceptions", True),
                input_encoding=config.get("smisk.mvc.template.input_encoding", "utf-8"),
                output_encoding=smisk.mvc.Response.charset,
                encoding_errors=encoding_errors,
                cache_type=cache_type,
                default_filters=config.get("smisk.mvc.template.default_filters", ["unicode"]),
                imports=self.imports,
            )
            if log.level <= logging.DEBUG and cache_type != "file":
                code = self.instances[uri].code
                log.debug("Compiled %s into %d bytes of python code:\n%s", uri, len(code), code)
            return self.instances[uri]
        except:
            self.instances.pop(uri, None)
            raise

    def _check(self, uri, template):
        if template.filename is None:
            return template
        if not os.access(template.filename, os.F_OK):
            self.instances.pop(uri, None)
            raise exceptions.TemplateLookupException("Can't locate template for uri '%s'" % uri)
        elif template.module._modified_time < os.stat(template.filename)[stat.ST_MTIME]:
            self.instances.pop(uri, None)
            return self._load(template.filename, uri)
        else:
            return template

    def put_string(self, uri, text):
        raise NotImplementedError

    def put_template(self, uri, template):
        self.instances[uri] = template
Пример #8
0
#      in start-up page and the software header, even if the application
#      source code has been changed or updated or code has been added.
#
#  You can see the MPL licence at: http://www.mozilla.org/MPL/MPL-1.1.html
#
###############################################################################
import copy
import cPickle
import functools

import cherrypy
from mako.util import LRUCache

import rpc

__cache = LRUCache(cherrypy.config.get('server.db_cache_size', 8))

def clear():
    __cache.pop(rpc.session.db, None)

def memoize(limit=100, force=False):

    def memoize_wrapper(func):

        # Don't use cache for development environment
        if not force and cherrypy.config.get('server.environment') == 'development':
            return func

        func_name = "%s.%s.%s" % (func.__module__, func.func_code.co_firstlineno, func.func_name)
        @functools.wraps(func)
        def func_wrapper(*args, **kwargs):