def celery_task_wrapper(f):
"""
Provides a task wrapper for celery that sets up cache and ensures
that the local store is cleared after completion
"""
from celery.utils import fun_takes_kwargs
@wraps(f, assigned=available_attrs(f))
def newf(*args, **kwargs):
backend = get_backend()
was_patched = backend._patched
get_backend().patch()
# since this function takes all keyword arguments,
# we will pass only the ones the function below accepts,
# just as celery does
supported_keys = fun_takes_kwargs(f, kwargs)
new_kwargs = dict((key, val) for key, val in kwargs.items()
if key in supported_keys)
try:
ret = f(*args, **new_kwargs)
finally:
local.clear()
if not was_patched:
get_backend().unpatch()
return ret
return newf
def _monkey_execute_sql(self, original):
from django.db.models.sql import query
from django.db.models.sql.constants import MULTI
from django.db.models.sql.datastructures import EmptyResultSet
@wraps(original, assigned=available_attrs(original))
def newfun(cls, result_type=MULTI):
try:
sql, params = cls.as_sql()
if not sql:
raise EmptyResultSet
except EmptyResultSet:
if result_type == MULTI:
return query.empty_iter()
else:
return
val, key = None, None
tables = get_tables_for_query11(cls)
# check the blacklist for any of the involved tables; if it's not
# there, then look for the value in the cache.
if tables and not disallowed_table(*tables):
gen_key = self.keyhandler.get_generation(*tables)
key = self.keyhandler.sql_key(gen_key, sql, params,
cls.ordering_aliases,
result_type)
val = self.cache_backend.get(key, None)
if val is not None:
signals.qc_hit.send(sender=cls,
tables=tables,
query=(sql, params,
cls.ordering_aliases),
size=len(val),
key=key)
return val
# we didn't find the value in the cache, so execute the query
result = original(cls, result_type)
if (tables and not sql.startswith('UPDATE')
and not sql.startswith('DELETE')):
# I think we should always be sending a signal here if we
# miss..
signals.qc_miss.send(sender=cls,
tables=tables,
query=(sql, params, cls.ordering_aliases),
key=key)
if hasattr(result, '__iter__'):
result = list(result)
# 'key' will be None here if any of these tables were
# blacklisted, in which case we just don't care.
if key is not None:
self.cache_backend.set(key, result)
elif tables and sql.startswith('UPDATE'):
# issue #1 in bitbucket, not invalidating on update
for table in tables:
self.keyhandler.invalidate_table(table)
return result
return newfun
def _monkey_select(self, original):
from django.db.models.sql import compiler
from django.db.models.sql.constants import MULTI
from django.db.models.sql.datastructures import EmptyResultSet
@wraps(original, assigned=available_attrs(original))
def newfun(cls, *args, **kwargs):
if args:
result_type = args[0]
else:
result_type = kwargs.get('result_type', MULTI)
if any([isinstance(cls, c) for c in self._write_compilers]):
return original(cls, *args, **kwargs)
try:
sql, params = cls.as_sql()
if not sql:
raise EmptyResultSet
except EmptyResultSet:
if result_type == MULTI:
# this was moved in 1.2 to compiler
return compiler.empty_iter()
else:
return
db = getattr(cls, 'using', 'default')
key, val = None, None
# check the blacklist for any of the involved tables; if it's not
# there, then look for the value in the cache.
tables = get_tables_for_query(cls.query)
if tables and not disallowed_table(*tables):
gen_key = self.keyhandler.get_generation(*tables,
**{'db': db})
key = self.keyhandler.sql_key(gen_key, sql, params,
cls.get_ordering(),
result_type, db)
val = self.cache_backend.get(key, None, db)
if val is not None:
signals.qc_hit.send(sender=cls, tables=tables,
query=(sql, params, cls.query.ordering_aliases),
size=len(val), key=key)
return val
signals.qc_miss.send(sender=cls, tables=tables,
query=(sql, params, cls.query.ordering_aliases),
key=key)
val = original(cls, *args, **kwargs)
if hasattr(val, '__iter__'):
#Can't permanently cache lazy iterables without creating
#a cacheable data structure. Note that this makes them
#no longer lazy...
#todo - create a smart iterable wrapper
val = list(val)
if key is not None:
self.cache_backend.set(key, val, settings.MIDDLEWARE_SECONDS, db)
return val
return newfun
def _monkey_execute_sql(self, original):
from django.db.models.sql.constants import MULTI
from django.db.models.sql.datastructures import EmptyResultSet
@wraps(original, assigned=available_attrs(original))
def newfun(cls, result_type=MULTI):
try:
sql, params = cls.as_sql()
if not sql:
raise EmptyResultSet
except EmptyResultSet:
if result_type == MULTI:
return iter([])
else:
return
val, key = None, None
tables = get_tables_for_query11(cls)
blacklisted = disallowed_table(*tables)
if blacklisted:
signals.qc_skip.send(sender=cls, tables=tables,
query=(sql, params, cls.ordering_aliases),
key=key)
if tables and not blacklisted and not is_query_random(cls.as_sql()[0]):
gen_key = self.keyhandler.get_generation(*tables)
key = self.keyhandler.sql_key(gen_key, sql, params,
cls.ordering_aliases, result_type)
val = self.cache_backend.get(key, None)
if val is not None:
signals.qc_hit.send(sender=cls, tables=tables,
query=(sql, params, cls.ordering_aliases),
size=len(val), key=key)
return val
# we didn't find the value in the cache, so execute the query
result = original(cls, result_type)
if (tables and not sql.startswith('UPDATE') and
not sql.startswith('DELETE')):
if not blacklisted:
# don't send a miss out on blacklist hits, since we never
# looked in the first place, so it wasn't a "miss"
signals.qc_miss.send(sender=cls, tables=tables,
query=(sql, params, cls.ordering_aliases),
key=key)
if hasattr(result, '__iter__'):
result = list(result)
# 'key' will be None here if any of these tables were
# blacklisted, in which case we just don't care.
if key is not None:
self.cache_backend.set(key, result)
elif tables and sql.startswith('UPDATE'):
# issue #1 in bitbucket, not invalidating on update
for table in tables:
if not disallowed_table(table):
self.keyhandler.invalidate_table(table)
return result
return newfun
def _monkey_write(self, original):
@wraps(original, assigned=available_attrs(original))
def newfun(cls, *args, **kwargs):
db = getattr(cls, "using", "default")
from django.db.models.sql import compiler
# we have to do this before we check the tables, since the tables
# are actually being set in the original function
ret = original(cls, *args, **kwargs)
if isinstance(cls, compiler.SQLInsertCompiler):
# Inserts are a special case where cls.tables
# are not populated.
tables = [cls.query.model._meta.db_table]
else:
# if cls.query.tables != list(cls.query.table_map):
# pass
# tables = list(cls.query.table_map)
tables = cls.query.tables
for table in tables:
if not disallowed_table(table):
self.keyhandler.invalidate_table(table, db)
return ret
return newfun
def _patched(self, original, commit=True):
@wraps(original, assigned=available_attrs(original))
def newfun(using=None):
#1.2 version
original(using=using)
self._flush(commit=commit, using=using)
return newfun
def _patched(self, original, commit=True):
@wraps(original, assigned=available_attrs(original))
def newfun(using=None):
#1.2 version
original(using=using)
self._flush(commit=commit, using=using)
return newfun
def _patched(self, original, commit=True):
@wraps(original, assigned=available_attrs(original))
def newfun(using=None):
#1.2 version
original(using=using)
self._flush(commit=commit, using=using)
@wraps(original, assigned=available_attrs(original))
def newfun11():
#1.1 version
original()
self._flush(commit=commit)
if django.VERSION[:2] == (1, 1):
return newfun11
elif django.VERSION[:2] > (1, 1):
return newfun
return original
def _patched(self, original, commit=True):
@wraps(original, assigned=available_attrs(original))
def newfun(using=None):
#1.2 version
original(using=using)
self._flush(commit=commit, using=using)
@wraps(original, assigned=available_attrs(original))
def newfun11():
#1.1 version
original()
self._flush(commit=commit)
if django.VERSION[:2] == (1, 1):
return newfun11
elif django.VERSION[:2] > (1, 1):
return newfun
return original
def _patched(self, original, commit=True, unless_managed=False):
@wraps(original, assigned=available_attrs(original))
def newfun(using=None):
original(using=using)
# copying behavior of original func
# if it is an 'unless_managed' version we should do nothing if transaction is managed
if not unless_managed or not self.is_managed(using=using):
self._flush(commit=commit, using=using)
return newfun
def _patched(self, original, commit=True, unless_managed=False):
@wraps(original, assigned=available_attrs(original))
def newfun(using=None):
original(using=using)
# copying behavior of original func
# if it is an 'unless_managed' version we should do nothing if transaction is managed
if not unless_managed or not self.is_managed(using=using):
self._flush(commit=commit, using=using)
return newfun
def _savepoint(self, original):
@wraps(original, assigned=available_attrs(original))
def newfun(using=None):
if using != None:
sid = original(using=using)
else:
sid = original()
if self._uses_savepoints():
self._create_savepoint(sid, using)
return sid
return newfun
def _savepoint(self, original):
@wraps(original, assigned=available_attrs(original))
def newfun(using=None):
if using != None:
sid = original(using=using)
else:
sid = original()
if self._uses_savepoints():
self._create_savepoint(sid, using)
return sid
return newfun
def _monkey_execute_sql(self, original):
from django.db.models.sql import query
from django.db.models.sql.constants import MULTI
from django.db.models.sql.datastructures import EmptyResultSet
@wraps(original, assigned=available_attrs(original))
def newfun(cls, result_type=MULTI):
try:
sql, params = cls.as_sql()
if not sql:
raise EmptyResultSet
except EmptyResultSet:
if result_type == MULTI:
return query.empty_iter()
else:
return
val, key = None, None
tables = get_tables_for_query11(cls)
# check the blacklist for any of the involved tables; if it's not
# there, then look for the value in the cache.
if tables and not disallowed_table(*tables):
gen_key = self.keyhandler.get_generation(*tables)
key = self.keyhandler.sql_key(gen_key, sql, params,
cls.ordering_aliases, result_type)
val = self.cache_backend.get(key, None)
if val is not None:
signals.qc_hit.send(sender=cls, tables=tables,
query=(sql, params, cls.ordering_aliases),
size=len(val), key=key)
return val
# we didn't find the value in the cache, so execute the query
result = original(cls, result_type)
if (tables and not sql.startswith('UPDATE') and
not sql.startswith('DELETE')):
# I think we should always be sending a signal here if we
# miss..
signals.qc_miss.send(sender=cls, tables=tables,
query=(sql, params, cls.ordering_aliases),
key=key)
if hasattr(result, '__iter__'):
result = list(result)
# 'key' will be None here if any of these tables were
# blacklisted, in which case we just don't care.
if key is not None:
self.cache_backend.set(key, result)
elif tables and sql.startswith('UPDATE'):
# issue #1 in bitbucket, not invalidating on update
for table in tables:
self.keyhandler.invalidate_table(table)
return result
return newfun
def timer(func):
times = []
@wraps(func, assigned=available_attrs(func))
def foo(*args, **kwargs):
t0 = time.time()
ret = func(*args, **kwargs)
times.append(time.time() - t0)
print ("%d runs, %0.6f avg" %
(len(times), sum(times) / float(len(times))))
return ret
return foo
def timer(func):
times = []
@wraps(func, assigned=available_attrs(func))
def foo(*args, **kwargs):
t0 = time.time()
ret = func(*args, **kwargs)
times.append(time.time() - t0)
print ("%d runs, %0.6f avg" %
(len(times), sum(times) / float(len(times))))
return ret
return foo
def deco(func):
@wraps(func, assigned=available_attrs(func))
def wrapped(*args, **kwargs):
qc_hit.connect(hit)
qc_miss.connect(miss)
try:
ret = func(*args, **kwargs)
finally:
qc_hit.disconnect(hit)
qc_miss.disconnect(miss)
return ret
return wrapped
def deco(func):
@wraps(func, assigned=available_attrs(func))
def wrapped(*args, **kwargs):
qc_hit.connect(hit)
qc_miss.connect(miss)
try:
ret = func(*args, **kwargs)
finally:
qc_hit.disconnect(hit)
qc_miss.disconnect(miss)
return ret
return wrapped
def _monkey_write(self, original):
@wraps(original, assigned=available_attrs(original))
def newfun(cls, *args, **kwargs):
db = getattr(cls, 'using', 'default')
from django.db.models.sql import compiler
# we have to do this before we check the tables, since the tables
# are actually being set in the original function
ret = original(cls, *args, **kwargs)
if isinstance(cls, compiler.SQLInsertCompiler):
#Inserts are a special case where cls.tables
#are not populated.
tables = [cls.query.model._meta.db_table]
else:
tables = cls.query.tables
for table in tables:
self.keyhandler.invalidate_table(table, db)
return ret
return newfun
def _monkey_select(self, original):
from django.db.models.sql.constants import MULTI
from django.db.models.sql.datastructures import EmptyResultSet
@wraps(original, assigned=available_attrs(original))
def newfun(cls, *args, **kwargs):
if args:
result_type = args[0]
else:
result_type = kwargs.get('result_type', MULTI)
if any([isinstance(cls, c) for c in self._write_compilers]):
return original(cls, *args, **kwargs)
try:
sql, params = cls.as_sql()
if not sql:
raise EmptyResultSet
except EmptyResultSet:
if result_type == MULTI:
# this was moved in 1.2 to compiler
return empty_iter()
else:
return
db = getattr(cls, 'using', 'default')
key, val = None, NotInCache()
# check the blacklist for any of the involved tables; if it's not
# there, then look for the value in the cache.
tables = get_tables_for_query(cls.query)
# if the tables are blacklisted, send a qc_skip signal
blacklisted = disallowed_table(*tables)
if blacklisted:
signals.qc_skip.send(sender=cls,
tables=tables,
query=(sql, params,
cls.query.ordering_aliases),
key=key)
if tables and not blacklisted and not is_query_random(
cls.as_sql()[0]):
gen_key = self.keyhandler.get_generation(*tables, **{'db': db})
key = self.keyhandler.sql_key(gen_key, sql, params,
cls.get_ordering(), result_type,
db)
val = self.cache_backend.get(key, NotInCache(), db)
if not isinstance(val, NotInCache):
if val == no_result_sentinel:
val = []
signals.qc_hit.send(sender=cls,
tables=tables,
query=(sql, params,
cls.query.ordering_aliases),
size=len(val),
key=key)
return val
if not blacklisted:
signals.qc_miss.send(sender=cls,
tables=tables,
query=(sql, params,
cls.query.ordering_aliases),
key=key)
val = original(cls, *args, **kwargs)
if hasattr(val, '__iter__'):
#Can't permanently cache lazy iterables without creating
#a cacheable data structure. Note that this makes them
#no longer lazy...
#todo - create a smart iterable wrapper
val = list(val)
if key is not None:
if not val:
self.cache_backend.set(key, no_result_sentinel,
settings.MIDDLEWARE_SECONDS, db)
else:
self.cache_backend.set(key, val,
settings.MIDDLEWARE_SECONDS, db)
return val
return newfun
