class QueryCacheBackend(object):
"""This class is the engine behind the query cache. It reads the queries
going through the django Query and returns from the cache using
the generation keys, or on a miss from the database and caches the results.
Each time a model is updated the table keys for that model are re-created,
invalidating all cached querysets for that model.
There are different QueryCacheBackend's for different versions of django;
call ``johnny.cache.get_backend`` to automatically get the proper class.
"""
__shared_state = {}
def __init__(self, cache_backend=None, keyhandler=None, keygen=None):
self.__dict__ = self.__shared_state
self.prefix = settings.MIDDLEWARE_KEY_PREFIX
if keyhandler:
self.kh_class = keyhandler
if keygen:
self.kg_class = keygen
if not cache_backend and not hasattr(self, 'cache_backend'):
cache_backend = settings._get_backend()
if not keygen and not hasattr(self, 'kg_class'):
self.kg_class = KeyGen
if keyhandler is None and not hasattr(self, 'kh_class'):
self.kh_class = KeyHandler
if cache_backend:
self.cache_backend = TransactionManager(cache_backend,
self.kg_class)
self.keyhandler = self.kh_class(self.cache_backend,
self.kg_class, self.prefix)
self._patched = getattr(self, '_patched', False)
def _monkey_select(self, original):
from django.db.models.sql.constants import MULTI
from django.db.models.sql.datastructures import EmptyResultSet
@wraps(original, assigned=available_attrs(original))
def newfun(cls, *args, **kwargs):
if args:
result_type = args[0]
else:
result_type = kwargs.get('result_type', MULTI)
if any([isinstance(cls, c) for c in self._write_compilers]):
return original(cls, *args, **kwargs)
try:
sql, params = cls.as_sql()
if not sql:
raise EmptyResultSet
except EmptyResultSet:
if result_type == MULTI:
# this was moved in 1.2 to compiler
return empty_iter()
else:
return
db = getattr(cls, 'using', 'default')
key, val = None, NotInCache()
# check the blacklist for any of the involved tables; if it's not
# there, then look for the value in the cache.
tables = get_tables_for_query(cls.query)
# if the tables are blacklisted, send a qc_skip signal
blacklisted = disallowed_table(*tables)
if blacklisted:
signals.qc_skip.send(sender=cls, tables=tables,
query=(sql, params, cls.ordering_aliases),
key=key)
if tables and not blacklisted:
gen_key = self.keyhandler.get_generation(*tables, **{'db': db})
key = self.keyhandler.sql_key(gen_key, sql, params,
cls.get_ordering(),
result_type, db)
val = self.cache_backend.get(key, NotInCache(), db)
if not isinstance(val, NotInCache):
if val == no_result_sentinel:
val = []
signals.qc_hit.send(sender=cls, tables=tables,
query=(sql, params, cls.ordering_aliases),
size=len(val), key=key)
return val
if not blacklisted:
signals.qc_miss.send(sender=cls, tables=tables,
query=(sql, params, cls.ordering_aliases),
key=key)
val = original(cls, *args, **kwargs)
if hasattr(val, '__iter__'):
#Can't permanently cache lazy iterables without creating
#a cacheable data structure. Note that this makes them
#no longer lazy...
#todo - create a smart iterable wrapper
val = list(val)
if key is not None:
if not val:
self.cache_backend.set(key, no_result_sentinel, settings.MIDDLEWARE_SECONDS, db)
else:
self.cache_backend.set(key, val, settings.MIDDLEWARE_SECONDS, db)
return val
return newfun
def _monkey_write(self, original):
@wraps(original, assigned=available_attrs(original))
def newfun(cls, *args, **kwargs):
db = getattr(cls, 'using', 'default')
from django.db.models.sql import compiler
# we have to do this before we check the tables, since the tables
# are actually being set in the original function
ret = original(cls, *args, **kwargs)
if isinstance(cls, compiler.SQLInsertCompiler):
#Inserts are a special case where cls.tables
#are not populated.
tables = [cls.query.model._meta.db_table]
else:
#if cls.query.tables != list(cls.query.table_map):
# pass
#tables = list(cls.query.table_map)
tables = cls.query.tables
for table in tables:
if not disallowed_table(table):
self.keyhandler.invalidate_table(table, db)
return ret
return newfun
def patch(self):
"""
monkey patches django.db.models.sql.compiler.SQL*Compiler series
"""
from django.db.models.sql import compiler
self._read_compilers = (
compiler.SQLCompiler,
compiler.SQLAggregateCompiler,
compiler.SQLDateCompiler,
)
self._write_compilers = (
compiler.SQLInsertCompiler,
compiler.SQLDeleteCompiler,
compiler.SQLUpdateCompiler,
)
if not self._patched:
self._original = {}
for reader in self._read_compilers:
self._original[reader] = reader.execute_sql
reader.execute_sql = self._monkey_select(reader.execute_sql)
for updater in self._write_compilers:
self._original[updater] = updater.execute_sql
updater.execute_sql = self._monkey_write(updater.execute_sql)
self._patched = True
self.cache_backend.patch()
self._handle_signals()
def unpatch(self):
"""un-applies this patch."""
if not self._patched:
return
for func in self._read_compilers + self._write_compilers:
func.execute_sql = self._original[func]
self.cache_backend.unpatch()
self._patched = False
def invalidate_m2m(self, instance, **kwargs):
if self._patched:
table = resolve_table(instance)
if not disallowed_table(table):
self.keyhandler.invalidate_table(instance)
def invalidate(self, instance, **kwargs):
if self._patched:
table = resolve_table(instance)
if not disallowed_table(table):
self.keyhandler.invalidate_table(table)
tables = set()
tables.add(table)
try:
instance._meta._related_objects_cache
except AttributeError:
instance._meta._fill_related_objects_cache()
for obj in instance._meta._related_objects_cache.keys():
obj_table = obj.model._meta.db_table
if obj_table not in tables:
tables.add(obj_table)
if not disallowed_table(obj_table):
self.keyhandler.invalidate_table(obj_table)
def _handle_signals(self):
post_save.connect(self.invalidate, sender=None)
post_delete.connect(self.invalidate, sender=None)
# FIXME: only needed in 1.1?
signals.qc_m2m_change.connect(self.invalidate_m2m, sender=None)
def flush_query_cache(self):
from django.db import connection
tables = connection.introspection.table_names()
#seen_models = connection.introspection.installed_models(tables)
for table in tables:
# we want this to just work, so invalidate even things in blacklist
self.keyhandler.invalidate_table(table)
class QueryCacheBackend(object):
"""This class is the engine behind the query cache. It reads the queries
going through the django Query and returns from the cache using
the generation keys, or on a miss from the database and caches the results.
Each time a model is updated the table keys for that model are re-created,
invalidating all cached querysets for that model.
There are different QueryCacheBackend's for different versions of django;
call ``johnny.cache.get_backend`` to automatically get the proper class.
"""
__shared_state = {}
def __init__(self, cache_backend=None, keyhandler=None, keygen=None):
self.__dict__ = self.__shared_state
self.prefix = settings.MIDDLEWARE_KEY_PREFIX
if keyhandler:
self.kh_class = keyhandler
if keygen:
self.kg_class = keygen
if not cache_backend and not hasattr(self, 'cache_backend'):
cache_backend = settings._get_backend()
if not keygen and not hasattr(self, 'kg_class'):
self.kg_class = KeyGen
if keyhandler is None and not hasattr(self, 'kh_class'):
self.kh_class = KeyHandler
if cache_backend:
self.cache_backend = TransactionManager(cache_backend,
self.kg_class)
self.keyhandler = self.kh_class(self.cache_backend, self.kg_class,
self.prefix)
self._patched = getattr(self, '_patched', False)
def _monkey_select(self, original):
from django.db.models.sql.constants import MULTI
from django.db.models.sql.datastructures import EmptyResultSet
@wraps(original, assigned=available_attrs(original))
def newfun(cls, *args, **kwargs):
if args:
result_type = args[0]
else:
result_type = kwargs.get('result_type', MULTI)
if any([isinstance(cls, c) for c in self._write_compilers]):
return original(cls, *args, **kwargs)
try:
sql, params = cls.as_sql()
if not sql:
raise EmptyResultSet
except EmptyResultSet:
if result_type == MULTI:
# this was moved in 1.2 to compiler
return empty_iter()
else:
return
db = getattr(cls, 'using', 'default')
key, val = None, NotInCache()
# check the blacklist for any of the involved tables; if it's not
# there, then look for the value in the cache.
tables = get_tables_for_query(cls.query)
# if the tables are blacklisted, send a qc_skip signal
blacklisted = disallowed_table(*tables)
if blacklisted:
signals.qc_skip.send(sender=cls,
tables=tables,
query=(sql, params,
cls.query.ordering_aliases),
key=key)
if tables and not blacklisted and not is_query_random(
cls.as_sql()[0]):
gen_key = self.keyhandler.get_generation(*tables, **{'db': db})
key = self.keyhandler.sql_key(gen_key, sql, params,
cls.get_ordering(), result_type,
db)
val = self.cache_backend.get(key, NotInCache(), db)
if not isinstance(val, NotInCache):
if val == no_result_sentinel:
val = []
signals.qc_hit.send(sender=cls,
tables=tables,
query=(sql, params,
cls.query.ordering_aliases),
size=len(val),
key=key)
return val
if not blacklisted:
signals.qc_miss.send(sender=cls,
tables=tables,
query=(sql, params,
cls.query.ordering_aliases),
key=key)
val = original(cls, *args, **kwargs)
if hasattr(val, '__iter__'):
#Can't permanently cache lazy iterables without creating
#a cacheable data structure. Note that this makes them
#no longer lazy...
#todo - create a smart iterable wrapper
val = list(val)
if key is not None:
if not val:
self.cache_backend.set(key, no_result_sentinel,
settings.MIDDLEWARE_SECONDS, db)
else:
self.cache_backend.set(key, val,
settings.MIDDLEWARE_SECONDS, db)
return val
return newfun
def _monkey_write(self, original):
@wraps(original, assigned=available_attrs(original))
def newfun(cls, *args, **kwargs):
db = getattr(cls, 'using', 'default')
from django.db.models.sql import compiler
# we have to do this before we check the tables, since the tables
# are actually being set in the original function
ret = original(cls, *args, **kwargs)
if isinstance(cls, compiler.SQLInsertCompiler):
#Inserts are a special case where cls.tables
#are not populated.
tables = [cls.query.model._meta.db_table]
else:
#if cls.query.tables != list(cls.query.table_map):
# pass
#tables = list(cls.query.table_map)
tables = cls.query.tables
for table in tables:
if not disallowed_table(table):
self.keyhandler.invalidate_table(table, db)
return ret
return newfun
def patch(self):
"""
monkey patches django.db.models.sql.compiler.SQL*Compiler series
"""
from django.db.models.sql import compiler
self._read_compilers = (
compiler.SQLCompiler,
compiler.SQLAggregateCompiler,
compiler.SQLDateCompiler,
)
self._write_compilers = (
compiler.SQLInsertCompiler,
compiler.SQLDeleteCompiler,
compiler.SQLUpdateCompiler,
)
if not self._patched:
self._original = {}
for reader in self._read_compilers:
self._original[reader] = reader.execute_sql
reader.execute_sql = self._monkey_select(reader.execute_sql)
for updater in self._write_compilers:
self._original[updater] = updater.execute_sql
updater.execute_sql = self._monkey_write(updater.execute_sql)
self._patched = True
self.cache_backend.patch()
self._handle_signals()
def unpatch(self):
"""un-applies this patch."""
if not self._patched:
return
for func in self._read_compilers + self._write_compilers:
func.execute_sql = self._original[func]
self.cache_backend.unpatch()
self._patched = False
def invalidate_m2m(self, instance, **kwargs):
if self._patched:
table = resolve_table(instance)
if not disallowed_table(table):
self.keyhandler.invalidate_table(instance)
def invalidate(self, instance, **kwargs):
if self._patched:
table = resolve_table(instance)
if not disallowed_table(table):
self.keyhandler.invalidate_table(table)
tables = set()
tables.add(table)
try:
instance._meta._related_objects_cache
except AttributeError:
instance._meta._fill_related_objects_cache()
for obj in instance._meta._related_objects_cache.keys():
obj_table = obj.model._meta.db_table
if obj_table not in tables:
tables.add(obj_table)
if not disallowed_table(obj_table):
self.keyhandler.invalidate_table(obj_table)
def _handle_signals(self):
post_save.connect(self.invalidate, sender=None)
post_delete.connect(self.invalidate, sender=None)
# FIXME: only needed in 1.1?
signals.qc_m2m_change.connect(self.invalidate_m2m, sender=None)
def flush_query_cache(self):
from django.db import connection
tables = connection.introspection.table_names()
#seen_models = connection.introspection.installed_models(tables)
for table in tables:
# we want this to just work, so invalidate even things in blacklist
self.keyhandler.invalidate_table(table)
