def load_menu_settings():
tlm, fm = get_from_cache()
if tlm and fm:
return tlm, fm
with menu_lock:
tlm, fm = get_from_cache()
if tlm and fm:
return tlm, fm
tlm = set()
fm = set()
pages = Page.objects.filter(
Q(menuextension__show_on_top_menu=True) |
Q(menuextension__show_on_footer_menu=True))\
.select_related('menuextension')\
.all()
for page in pages:
if page.menuextension.show_on_top_menu:
tlm.add(page.id)
if page.menuextension.show_on_footer_menu:
fm.add(page.id)
cache.set_many({
'top_level_menu_ids': tlm,
'footer_menu_ids': fm,
})
return tlm, fm
def fetch_by_id(self):
"""
Run two queries to get objects: one for the ids, one for id__in=ids.
After getting ids from the first query we can try cache.get_many to
reuse objects we've already seen. Then we fetch the remaining items
from the db, and put those in the cache. This prevents cache
duplication.
"""
# Include columns from extra since they could be used in the query's
# order_by.
vals = self.values_list('pk', *self.query.extra.keys())
pks = [val[0] for val in vals]
keys = dict((byid(self.model._cache_key(pk)), pk) for pk in pks)
cached = dict((k, v) for k, v in cache.get_many(keys).items()
if v is not None)
# Pick up the objects we missed.
missed = [pk for key, pk in keys.items() if key not in cached]
if missed:
others = self.fetch_missed(missed)
# Put the fetched objects back in cache.
new = dict((byid(o), o) for o in others)
cache.set_many(new)
else:
new = {}
# Use pks to return the objects in the correct order.
objects = dict((o.pk, o) for o in cached.values() + new.values())
for pk in pks:
yield objects[pk]
def get_context_data(self, *args, **kwargs):
context = super(InboxView, self).get_context_data(*args, **kwargs)
object_list = []
object_id_list = []
for email in context["page_obj"].object_list:
object_list.append(email)
object_id_list.append(email.id)
if len(object_id_list) == 0:
return context
headers = cache.get_many(object_id_list, version="email-header")
missing_list = set(object_id_list) - set(headers.keys())
if len(missing_list) > 0:
missing_headers = models.Header.objects.filter(part__parent=None, part__email__in=missing_list)
missing_headers = missing_headers.get_many("Subject", "From", group_by="part__email_id")
headers.update(missing_headers)
cache.set_many(missing_headers, version="email-header", timeout=None)
for email in object_list:
header_set = headers[email.id]
email.subject = header_set.get("Subject")
email.sender = header_set["From"]
inbox = getattr(self, 'inbox_obj', None)
if inbox is not None:
inbox = inbox.id
deal_with_flags.delay(object_id_list, self.request.user.id, inbox)
return context
def load_cache_signals(sender, **kwargs):
"""On startup, sync signals with registered models"""
from django.db import connection
if not cache_signals.ready:
# Have to load directly from db, because CacheBotSignals is not prepared yet
cursor = connection.cursor()
try:
cursor.execute("SELECT * FROM %s" % CacheBotSignals._meta.db_table)
except Exception, ex:
# This should only happen on syncdb when CacheBot tables haven't been created yet,
# but there's not really a good way to catch this error
sql, references = connection.creation.sql_create_model(CacheBotSignals, no_style())
cursor.execute(sql[0])
cursor.execute("SELECT * FROM %s" % CacheBotSignals._meta.db_table)
results = cursor.fetchall()
tables = [r[1] for r in results]
mapping = cache.get_many(tables)
for r in results:
key = version_key(".".join(("cachesignals", r[1])))
accessor_set = mapping.get(key)
if accessor_set is None:
accessor_set = set()
accessor_set.add(r[2:5])
mapping[key] = accessor_set
cache.set_many(mapping, CACHE_SECONDS)
cache_signals.ready = True
def snippet_update_lastmod(sender, instance, **kwargs):
"""On a change to a snippet, bump its cached lastmod timestamp"""
now = mktime(gmtime())
lastmods = {"%s%s" % (CACHE_SNIPPET_LASTMOD_PREFIX, instance.id): now}
for rule in instance.client_match_rules.all():
lastmods["%s%s" % (CACHE_RULE_LASTMOD_PREFIX, rule.id)] = now
cache.set_many(lastmods, CACHE_TIMEOUT)
def check_token():
"""
{ codigo token expirado
"code": 108,
"message": "api.error.token.expired"}
"""
url = URL + 'signin'
dados = {"email": settings.EMAIL, "password": settings.PASS}
headers = {
'Access-Control-Allow-Origin': '*',
'Content-Type': 'application/json',
'Accept-Language': "en",
'Date': time.strftime('%a, %d %b %Y %H:%M:%S GMT'),
'X-Api-Key': settings.API_STRANS,
}
vality = cache.get('vality', None)
token = cache.get('token', None)
if vality is None \
or token is None \
or vality < datetime.now():
r = requests.post(url, headers=headers, data=json.dumps(dados))
resp = json.loads(r.text)
dt = datetime.now() + timedelta(minutes=9)
cache.set_many({'vality': dt, 'token': resp['token']})
def get_avatars(user_ids, size=AVATAR_DEFAULT_SIZE):
# Bulk -get for avatars. First try to get the avatars from the cache
user_ids = set(user_ids)
cache_keys = [bulk_cache_key(user_id, size) for user_id in user_ids]
avatar_dict = cache.get_many(cache_keys)
avatars = {}
for key, avatar in avatar_dict.iteritems():
avatars[avatar.user_id] = avatar
user_ids.remove(avatar.user_id)
# Fill in any avatars we don't yet have cached
new_avatars = {}
from avatar.models import Avatar
for avatar in Avatar.objects.select_related('user').only('avatar', 'user__id', 'user__username').filter(primary=True, user_id__in=user_ids):
if not avatar.thumbnail_exists(size):
avatar.create_thumbnail(size)
avatars[avatar.user_id] = avatar
new_avatars[bulk_cache_key(avatar.user_id, size)] = avatar
if new_avatars:
# Bulk-cache the new avatars
cache.set_many(new_avatars, 60*60*24)
return avatars
def cached_state_set(self, state):
"""Sets both the whole state, or just a partial of it...
Very useful, but ambiguous."""
cached_state = self.cached_state_get()
cached_state.update(state)
cache.set_many(cached_state)
def test_unicode(self):
# Unicode values can be cached
stuff = {
'ascii': 'ascii_value',
'unicode_ascii': 'Iñtërnâtiônàlizætiøn1',
'Iñtërnâtiônàlizætiøn': 'Iñtërnâtiônàlizætiøn2',
'ascii2': {'x': 1}
}
# Test `set`
for (key, value) in stuff.items():
cache.set(key, value)
assert cache.get(key) == value
# Test `add`
for (key, value) in stuff.items():
cache.delete(key)
cache.add(key, value)
assert cache.get(key) == value
# Test `set_many`
for (key, value) in stuff.items():
cache.delete(key)
cache.set_many(stuff)
for (key, value) in stuff.items():
assert cache.get(key) == value
def count(self, func_name, request, ip=True, field=None, period=60):
counters = dict((key, 0) for key in self._keys(
func_name, request, ip, field, period))
counters.update(cache.get_many(counters.keys()))
for key in counters:
counters[key] += 1
cache.set_many(counters, timeout=period)
def test_unicode(self):
# Unicode values can be cached
stuff = {
"ascii": "ascii_value",
"unicode_ascii": "Iñtërnâtiônàlizætiøn1",
"Iñtërnâtiônàlizætiøn": "Iñtërnâtiônàlizætiøn2",
"ascii2": {"x": 1},
}
# Test `set`
for (key, value) in stuff.items():
cache.set(key, value)
self.assertEqual(cache.get(key), value)
# Test `add`
for (key, value) in stuff.items():
cache.delete(key)
cache.add(key, value)
self.assertEqual(cache.get(key), value)
# Test `set_many`
for (key, value) in stuff.items():
cache.delete(key)
cache.set_many(stuff)
for (key, value) in stuff.items():
self.assertEqual(cache.get(key), value)
def get_many_for_update(namespaces, keys):
"""Get cached data, full keys and create namespace counter where necessary.
Arguments namespaces and keys should be a lists of the same size.
Returns a pair (cached_data, full_keys),
where cached_data is a dictionary {full_keys: data}
and full_keys is a list of full keys, in the same order as
given namespaces and keys.
Use cache.set_many({full_key: value}) to finish updating.
It is assumed namespace counters won't change in between. (if they do,
maybe it is even better to immediately 'mark' data as invalid...)
"""
counters = cache.get_many(namespaces)
# Create new counters
new_counters = {x: 1 for x in namespaces if x not in counters}
cache.set_many(new_counters)
# Generate list of full_keys and list of keys to read from cache.
full_keys = []
to_get = []
for namespace, key in zip(namespaces, keys):
counter = counters.get(namespace, None)
if counter:
full_key = make_full_key(namespace, counter, key)
full_keys.append(full_key)
to_get.append(full_key)
else:
full_keys.append(make_full_key(namespace, 1, key))
return cache.get_many(to_get), full_keys
def store_project_layers_info(project_key, publish, extent, resolutions, projection):
prefix = "{0}:{1}:".format(project_key, publish)
cache.set_many({
prefix+'extent': ','.join(map(str, extent)),
prefix+'resolutions': ','.join(map(str, resolutions)),
prefix+'projection': projection
})
def invalidate_cache(model_class, objects, **extra_keys):
"""
Flushes the cache of any cached objects associated with this instance.
Explicitly set a None value instead of just deleting so we don't have any race
conditions where:
Thread 1 -> Cache miss, get object from DB
Thread 2 -> Object saved, deleted from cache
Thread 1 -> Store (stale) object fetched from DB in cache
Five second should be more than enough time to prevent this from happening for
a web app.
"""
invalidation_dict = {}
accessor_set = cache_signals.get_global_signals(model_class)
for obj in objects:
for (accessor_path, lookup_type, negate) in accessor_set:
for value in get_values(obj, accessor_path):
invalidation_key = get_invalidation_key(
model_class._meta.db_table, accessor_path=accessor_path, negate=negate, value=value, save=False
)
invalidation_dict[invalidation_key] = None
invalidation_dict.update(cache.get_many(invalidation_dict.keys()))
cache_keys = set()
for obj_key, cache_key_list in invalidation_dict.iteritems():
if cache_key_list:
cache_keys.update(cache_key_list.split(","))
keys_to_invalidate = dict([(key, None) for key in chain(cache_keys, invalidation_dict.keys())])
keys_to_invalidate.update(extra_keys)
cache.set_many(keys_to_invalidate, 5)
cache.delete_many(keys_to_invalidate.keys())
def cache_nodes(nodes):
"""
Takes a uri:content dict and caches all at once.
Encode node content together with plugin ext, persist as native string in cache to prevent pickling.
"""
data = dict(_prepare_node(uri, content) for uri, content in nodes.iteritems())
cache.set_many(data, timeout=None)
def put(obj):
if type(obj) == list:
cache.set_many(dict(zip([_cache_key(o) for o in obj], [o for o in obj])))
for o in obj:
_process_cache[_cache_key(o)] = obj
else:
cache.set(_cache_key(obj), obj)
_process_cache[_cache_key(obj)] = obj
def uncache_criteria(**kwargs):
criteria = kwargs.get('instance')
cache.set_many({
CRITERIA_KEY % criteria.name: None,
CRITERIA_FLAGS_KEY % criteria.name: None,
CRITERIA_USERS_KEY % criteria.name: None,
CRITERIA_GROUPS_KEY % criteria.name: None,
ALL_CRITERIA_KEY: None}, 5)
def test_get_many(self):
data = dict(zip(self.KEY_LIST, self.VALUE_LIST))
cache.set_many(data)
vals = cache.get_many(self.KEY_LIST)
self.assertEqual(data, vals)
def count(self, func_name, request, ip=True, field=None, period=60):
"""Increment counters for all relevant cache_keys given a request."""
counters = dict((key, 1) for key in self._keys(
func_name, request, ip, field, period))
counters.update(cache.get_many(counters.keys()))
for key in counters:
counters[key] += 1
cache.set_many(counters, timeout=period)
def generate_access_codes(user):
ts = _get_timestamp()
salt = AUTH_CODE_SALT
codes = [formatter(user, salt, ts) for formatter in AUTH_CODE_FORMATTERS]
formatted_codes = dict([(_gen_auth_key(code), user.id) for code in codes])
cache.set_many(formatted_codes, AUTH_CODE_TIMEOUT)
cache.set(_gen_user_key(user.id), codes, AUTH_CODE_TIMEOUT)
return codes
def test_set_many(self):
data = dict(zip(self.KEY_LIST, self.VALUE_LIST))
cache.set_many(data)
for idx, key in enumerate(self.KEY_LIST):
val = cache.get(key)
self.assertEqual(val, self.VALUE_LIST[idx])
def diff(self, request, new_imgid, old_imgid):
new_img = Image.objects.select_related("container_repo").prefetch_related("repo", "container_repo__platform").get(pk = new_imgid)
old_img = Image.objects.select_related("container_repo").prefetch_related("repo", "container_repo__platform").get(pk = old_imgid)
live_diff = (False, False)
new_repo = new_img.container_repo
old_repo = old_img.container_repo
trace_reqs = _get_trace(old_repo, new_repo)
issue_ref = []
names = []
for repo in new_img.repo.all():
for i in repo.platform.issuetracker_set.all():
if not i.name in names:
issue_ref.append({'name' : i.name, 're' : i.re, 'url' : i.url})
names.append(i.name)
filter_repos = set(request.GET.getlist("repo"))
filter_meta = _get_filter_meta(request.GET)
cachekey = "%s%s%s" % ("repodiff", new_repo.id, old_repo.id)
cached = cache.get_many([cachekey, cachekey + 'ts'])
diff = cached.get(cachekey)
diffts = cached.get(cachekey + 'ts')
if diff is None:
diff = _diff_sacks(new_repo, old_repo)
diffts = datetime.datetime.now()
cachelife = (60 * 3) if (live_diff[0] or live_diff[1]) else (60 * 60 * 24)
cache.set_many({cachekey: diff, cachekey + 'ts' : diffts}, cachelife)
diff = _sort_filter_diff(diff, pkgs=list(set(new_img.packages) | set(old_img.packages)), repos=filter_repos, meta=filter_meta )
title = "Comparing Images"
is_popup = request.GET.get('is_popup', False)
full_path = "%s?" % request.path
for query, values in request.GET.lists():
full_path += "&".join([ '%s=%s' % (query, val) for val in values ])
full_path += "&"
context = {
"title": title,
"opts": self.model._meta,
"app_label": self.model._meta.app_label,
'diff' : diff,
'diffts' : diffts,
'live_diff' : live_diff,
'new_obj' : new_img,
'old_obj' : old_img,
'is_popup' : is_popup,
'trace': trace_reqs,
'issue_ref' : json.dumps(issue_ref),
'packagemetatypes' : list(PackageMetaType.objects.all().prefetch_related("choices")),
"full_path" : full_path,
}
return TemplateResponse(request, 'diff.html', context = context, current_app=self.admin_site.name)
def jump(self, date=None, speed=None):
if not self._enabled:
raise RuntimeError(u'Timewarp is disabled.')
cache.set_many({
u'timewarp.warped_from': self.real_time,
u'timewarp.warped_to': time_orig.mktime(date.timetuple()) if date is not None else self.warped_time,
u'timewarp.speedup': speed if speed is not None else self.speedup,
}, timeout=None)
self._lastupdate = None
def uncache_flag(**kwargs):
flag = kwargs.get('instance')
data = {
keyfmt(FLAG_CACHE_KEY, flag.name): None,
keyfmt(FLAG_USERS_CACHE_KEY, flag.name): None,
keyfmt(FLAG_GROUPS_CACHE_KEY, flag.name): None,
keyfmt(FLAGS_ALL_CACHE_KEY): None
}
cache.set_many(data, 5)
def push_to_remote(self):
"""
Update the remote cache
"""
if self._max_scores_updates:
cache.set_many(
{self._remote_cache_key(key): value for key, value in self._max_scores_updates.items()},
60 * 60 * 24, # 1 day
)
def incr_many(self, keys): values = cache.get_many(keys) for key in keys: if key in values: values[key] += 1 else: values[key] = 0 cache.set_many(values, self.timeout)
def uncache_flag(**kwargs):
flag = kwargs.get('instance')
data = {
FLAG_CACHE_KEY.format(n=hashlib.md5(flag.name).hexdigest()): None,
FLAG_USERS_CACHE_KEY.format(n=hashlib.md5(flag.name).hexdigest()): None,
FLAG_GROUPS_CACHE_KEY.format(n=hashlib.md5(flag.name).hexdigest()): None,
FLAGS_ALL_CACHE_KEY: None
}
cache.set_many(data, 5)
def test_set_many(self):
if not self._run_test():
return
cache.set_many({'hello1':'world1','hello2':'world2'},30)
self.assertEquals(len(cache._logger.log), 0)
self.assertEquals(cache.get("hello1"), "world1")
self.assertEquals(len(cache._logger.log), 0)
cache.close()
self.assertEquals(len(cache._logger.log), 1)
def transform(self, tuples, cache_key=None, dict_key='pk',
empty_value=-1):
"""
data - список pk, tuple или list, в которых pk или нулевой элемент должен быть заменен на объект
Заменяет ID модели на сам объект
tuples - список tuples или значений.
Если tuple, то ID - нулевой
"""
if not tuples or not len(tuples):
return tuples
# Подготовка ключей для кеша
is_list = type(tuples[0]) in [tuple, list]
is_tuple = is_list and type(tuples[0]) is tuple
# Словарь id -> ключ
keys = {}
for t in tuples:
if is_list:
keys[t[0]] = self.key(cache_key, kwargs={dict_key: t[0]})
else:
keys[t] = self.key(cache_key, kwargs={dict_key: t})
# Загрузка всех ключей
values = cache.get_many(keys.values())
# Подготовка ключей для загрузки из БД
objects = {}
if len(values) != len(keys):
# список ключей
not_loaded = set(keys.values()).difference(set(values.keys()))
if not_loaded:
# Перевернуть ключи, чтобы по ключу получить id
keys_un = dict((value, key) for key, value in keys.items())
not_loaded = [keys_un[x] for x in not_loaded]
objects = {o.id: o for o in self.get_query_set().filter(**{dict_key+'__in': not_loaded})}
# Возврат списка объектов
result = []
cache_set = {}
for t in tuples:
id = is_list and t[0] or t
ck = keys[id]
value = values.get(ck, empty_value)
if value is empty_value:
value = objects.get(id, empty_value)
if value is empty_value:
continue
cache_set[ck] = value
if is_list:
if is_tuple:
result.append((value,) + t[1:])
else:
result.append([value] + t[1:])
else:
result.append(value)
if cache_set:
cache.set_many(cache_set)
return result
def test_cluster_admin_counts_cache(self):
""" tests the cache for the admin cluster counts on the status page
these tests will fail if cache is not configured
Verifies:
* existing values are updated
* any of the dict keys can be updated
* keys not in the cache are discarded
"""
ids = ["cluster_admin_0", "cluster_admin_1", "cluster_admin_2"]
data = {
"cluster_admin_0": {"orphaned": 1, "missing": 2, "ready_to_import": 3},
"cluster_admin_1": {"orphaned": 4, "missing": 5, "ready_to_import": 6},
}
cache.set_many(data)
update = {0: 4, 1: 5, 3: 6}
# update orphaned
update_vm_counts("orphaned", update)
cached = cache.get_many(ids)
self.assertTrue("cluster_admin_0" in cached)
self.assertTrue("cluster_admin_1" in cached)
self.assertFalse("cluster_admin_2" in cached)
self.assertEqual(5, cached["cluster_admin_0"]["orphaned"])
self.assertEqual(2, cached["cluster_admin_0"]["missing"])
self.assertEqual(3, cached["cluster_admin_0"]["ready_to_import"])
self.assertEqual(9, cached["cluster_admin_1"]["orphaned"])
self.assertEqual(5, cached["cluster_admin_1"]["missing"])
self.assertEqual(6, cached["cluster_admin_1"]["ready_to_import"])
# update orphaned
update_vm_counts("missing", update)
cached = cache.get_many(ids)
self.assertTrue("cluster_admin_0" in cached)
self.assertTrue("cluster_admin_1" in cached)
self.assertFalse("cluster_admin_2" in cached)
self.assertEqual(5, cached["cluster_admin_0"]["orphaned"])
self.assertEqual(6, cached["cluster_admin_0"]["missing"])
self.assertEqual(3, cached["cluster_admin_0"]["ready_to_import"])
self.assertEqual(9, cached["cluster_admin_1"]["orphaned"])
self.assertEqual(10, cached["cluster_admin_1"]["missing"])
self.assertEqual(6, cached["cluster_admin_1"]["ready_to_import"])
# update ready_to_import
update_vm_counts("ready_to_import", update)
cached = cache.get_many(ids)
self.assertTrue("cluster_admin_0" in cached)
self.assertTrue("cluster_admin_1" in cached)
self.assertFalse("cluster_admin_2" in cached)
self.assertEqual(5, cached["cluster_admin_0"]["orphaned"])
self.assertEqual(6, cached["cluster_admin_0"]["missing"])
self.assertEqual(7, cached["cluster_admin_0"]["ready_to_import"])
self.assertEqual(9, cached["cluster_admin_1"]["orphaned"])
self.assertEqual(10, cached["cluster_admin_1"]["missing"])
self.assertEqual(11, cached["cluster_admin_1"]["ready_to_import"])
def get_users_for_authors(organization_id, authors, user=None):
"""
Returns a dictionary of author_id => user, if a Sentry
user object exists for that email. If there is no matching
Sentry user, a {user, email} dict representation of that
author is returned.
e.g.
{
1: serialized(<User id=1>),
2: {email: '*****@*****.**', name: 'dunno'},
...
}
"""
results = {}
fetched = cache.get_many([
_user_to_author_cache_key(organization_id, author)
for author in authors
])
if fetched:
missed = []
for author in authors:
fetched_user = fetched.get(
_user_to_author_cache_key(organization_id, author))
if fetched_user is None:
missed.append(author)
else:
results[str(author.id)] = fetched_user
else:
missed = authors
if missed:
# Filter users based on the emails provided in the commits
user_emails = list(
UserEmail.objects.filter(
in_iexact("email", [a.email for a in missed])).order_by("id"))
# Filter users belonging to the organization associated with
# the release
users = User.objects.filter(
id__in={ue.user_id
for ue in user_emails},
is_active=True,
sentry_orgmember_set__organization_id=organization_id,
)
users = serialize(list(users), user)
users_by_id = {user["id"]: user for user in users}
# Figure out which email address matches to a user
users_by_email = {}
for email in user_emails:
# force emails to lower case so we can do case insensitive matching
lower_email = email.email.lower()
if lower_email not in users_by_email:
user = users_by_id.get(str(email.user_id), None)
# user can be None if there's a user associated
# with user_email in separate organization
if user:
users_by_email[lower_email] = user
to_cache = {}
for author in missed:
results[str(author.id)] = users_by_email.get(
author.email.lower(), {
"name": author.name,
"email": author.email
})
to_cache[_user_to_author_cache_key(
organization_id, author)] = results[str(author.id)]
cache.set_many(to_cache)
metrics.incr("sentry.release.get_users_for_authors.missed",
amount=len(missed))
metrics.incr("sentry.tasks.process_suspect_commits.total",
amount=len(results))
return results
def set_many(self, values, timeout=None):
raw_values = dict((self._prefix_key(key), value)
for (key, value) in values.items())
cache.set_many(raw_values, timeout)
self._cache_data.update(values)
def set_sessions_active(cls, session_ids):
data = {cls.ACTIVE_CACHE_KEY_PREFIX.format(i): i for i in session_ids}
cache.set_many(data, timeout=5 * 60)
def test_clear(self):
# The cache can be emptied using clear
cache.set_many({'key1': 'spam', 'key2': 'eggs'})
cache.clear()
self.assertIsNone(cache.get('key1'))
self.assertIsNone(cache.get('key2'))
def test_get_many(self, mock_warning, mock_info):
programs = ProgramFactory.create_batch(3)
# Cache details for 2 of 3 programs.
partial_programs = {
PROGRAM_CACHE_KEY_TPL.format(uuid=program['uuid']): program
for program in programs[:2]
}
cache.set_many(partial_programs, None)
# When called before UUIDs are cached, the function should return an
# empty list and log a warning.
self.assertEqual(get_programs(self.site), [])
mock_warning.assert_called_once_with(
'Failed to get program UUIDs from the cache for site {}.'.format(
self.site.domain))
mock_warning.reset_mock()
# Cache UUIDs for all 3 programs.
cache.set(
SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=self.site.domain),
[program['uuid'] for program in programs], None)
actual_programs = get_programs(self.site)
# The 2 cached programs should be returned while info and warning
# messages should be logged for the missing one.
self.assertEqual(
set(program['uuid'] for program in actual_programs),
set(program['uuid'] for program in partial_programs.values()))
mock_info.assert_called_with(
'Failed to get details for 1 programs. Retrying.')
mock_warning.assert_called_with(
'Failed to get details for program {uuid} from the cache.'.format(
uuid=programs[2]['uuid']))
mock_warning.reset_mock()
# We can't use a set comparison here because these values are dictionaries
# and aren't hashable. We've already verified that all programs came out
# of the cache above, so all we need to do here is verify the accuracy of
# the data itself.
for program in actual_programs:
key = PROGRAM_CACHE_KEY_TPL.format(uuid=program['uuid'])
self.assertEqual(program, partial_programs[key])
# Cache details for all 3 programs.
all_programs = {
PROGRAM_CACHE_KEY_TPL.format(uuid=program['uuid']): program
for program in programs
}
cache.set_many(all_programs, None)
actual_programs = get_programs(self.site)
# All 3 programs should be returned.
self.assertEqual(
set(program['uuid'] for program in actual_programs),
set(program['uuid'] for program in all_programs.values()))
self.assertFalse(mock_warning.called)
for program in actual_programs:
key = PROGRAM_CACHE_KEY_TPL.format(uuid=program['uuid'])
self.assertEqual(program, all_programs[key])
def test_get_many(self, mock_warning, mock_info):
pathways = CreditPathwayFactory.create_batch(3)
# Cache details for 2 of 3 programs.
partial_pathways = {
CREDIT_PATHWAY_CACHE_KEY_TPL.format(id=pathway['id']): pathway
for pathway in pathways[:2]
}
cache.set_many(partial_pathways, None)
# When called before pathways are cached, the function should return an
# empty list and log a warning.
self.assertEqual(get_credit_pathways(self.site), [])
mock_warning.assert_called_once_with(
'Failed to get credit pathway ids from the cache.')
mock_warning.reset_mock()
# Cache all 3 pathways
cache.set(
SITE_CREDIT_PATHWAY_IDS_CACHE_KEY_TPL.format(
domain=self.site.domain),
[pathway['id'] for pathway in pathways], None)
actual_pathways = get_credit_pathways(self.site)
# The 2 cached pathways should be returned while info and warning
# messages should be logged for the missing one.
self.assertEqual(
set(pathway['id'] for pathway in actual_pathways),
set(pathway['id'] for pathway in partial_pathways.values()))
mock_info.assert_called_with(
'Failed to get details for 1 pathways. Retrying.')
mock_warning.assert_called_with(
'Failed to get details for credit pathway {id} from the cache.'.
format(id=pathways[2]['id']))
mock_warning.reset_mock()
# We can't use a set comparison here because these values are dictionaries
# and aren't hashable. We've already verified that all pathways came out
# of the cache above, so all we need to do here is verify the accuracy of
# the data itself.
for pathway in actual_pathways:
key = CREDIT_PATHWAY_CACHE_KEY_TPL.format(id=pathway['id'])
self.assertEqual(pathway, partial_pathways[key])
# Cache details for all 3 pathways.
all_pathways = {
CREDIT_PATHWAY_CACHE_KEY_TPL.format(id=pathway['id']): pathway
for pathway in pathways
}
cache.set_many(all_pathways, None)
actual_pathways = get_credit_pathways(self.site)
# All 3 pathways should be returned.
self.assertEqual(
set(pathway['id'] for pathway in actual_pathways),
set(pathway['id'] for pathway in all_pathways.values()))
self.assertFalse(mock_warning.called)
for pathway in actual_pathways:
key = CREDIT_PATHWAY_CACHE_KEY_TPL.format(id=pathway['id'])
self.assertEqual(pathway, all_pathways[key])
def test_set_many(self):
"""Test the set_many cache operation"""
cache.set_many({"a": 1, "b": 2, "c": 3})
res = cache.get_many(["a", "b", "c"])
self.assertEqual(res, {"a": 1, "b": 2, "c": 3})
def diff(self, request, new_imgid, old_imgid):
new_img = Image.objects.select_related(
"container_repo").prefetch_related(
"repo", "container_repo__platform").get(pk=new_imgid)
old_img = Image.objects.select_related(
"container_repo").prefetch_related(
"repo", "container_repo__platform").get(pk=old_imgid)
live_diff = (False, False)
new_repo = new_img.container_repo
old_repo = old_img.container_repo
trace_reqs = _get_trace(old_repo, new_repo)
issue_ref = []
names = []
for repo in new_img.repo.all():
for i in repo.platform.issuetracker_set.all():
if not i.name in names:
issue_ref.append({
'name': i.name,
're': i.re,
'url': i.url
})
names.append(i.name)
filter_repos = set(request.GET.getlist("repo"))
filter_meta = _get_filter_meta(request.GET)
cachekey = "%s%s%s" % ("repodiff", new_repo.id, old_repo.id)
cached = cache.get_many([cachekey, cachekey + 'ts'])
diff = cached.get(cachekey)
diffts = cached.get(cachekey + 'ts')
if diff is None:
diff = _diff_sacks(new_repo, old_repo)
diffts = datetime.datetime.now()
cachelife = (60 *
3) if (live_diff[0] or live_diff[1]) else (60 * 60 *
24)
cache.set_many({
cachekey: diff,
cachekey + 'ts': diffts
}, cachelife)
diff = _sort_filter_diff(
diff,
pkgs=list(set(new_img.packages) | set(old_img.packages)),
repos=filter_repos,
meta=filter_meta)
title = "Comparing Images"
is_popup = request.GET.get('is_popup', False)
full_path = "%s?" % request.path
for query, values in request.GET.lists():
full_path += "&".join(['%s=%s' % (query, val) for val in values])
full_path += "&"
context = {
"title":
title,
"opts":
self.model._meta,
"app_label":
self.model._meta.app_label,
'diff':
diff,
'diffts':
diffts,
'live_diff':
live_diff,
'new_obj':
new_img,
'old_obj':
old_img,
'is_popup':
is_popup,
'trace':
trace_reqs,
'issue_ref':
json.dumps(issue_ref),
'packagemetatypes':
list(PackageMetaType.objects.all().prefetch_related("choices")),
"full_path":
full_path,
}
return TemplateResponse(request,
'diff.html',
context=context,
current_app=self.admin_site.name)
def test_set_many(self):
# Multiple keys can be set using set_many
cache.set_many({'key1': 'spam', 'key2': 'eggs'})
self.assertEqual(cache.get('key1'), 'spam')
self.assertEqual(cache.get('key2'), 'eggs')
def test_delete_with_prefix_with_no_reverse_works(self):
cache.set_many({'K1': 'value', 'K2': 'value2', 'B2': 'Anothervalue'})
assert cache.delete_with_prefix('K') == 2
assert cache.get_many(['K1', 'K2', 'B2']) == {'B2': 'Anothervalue'}
def get_cached_objects(pks, model=None, timeout=CACHE_TIMEOUT, missing=RAISE):
"""
Return a list of objects with given PKs using cache.
Params:
pks - list of Primary Key values to look up or list of content_type_id, pk tuples
model - ContentType instance representing the model's class or the model class itself
timeout - TTL for the items in cache, defaults to CACHE_TIMEOUT
Throws:
model.DoesNotExist is propagated from content_type.get_object_for_this_type
"""
if model is not None:
if not isinstance(model, ContentType):
model = ContentType.objects.get_for_model(model)
pks = [(model, pk) for pk in pks]
else:
pks = [(ContentType.objects.get_for_id(ct_id), pk) for (ct_id, pk) in pks]
keys = [_get_key(KEY_PREFIX, model, pk=pk) for (model, pk) in pks]
cached = cache.get_many(keys)
# keys not in cache
keys_to_set = set(keys) - set(cached.keys())
if keys_to_set:
# build lookup to get model and pks from the key
lookup = dict(zip(keys, pks))
to_get = {}
# group lookups by CT so we can do in_bulk
for k in keys_to_set:
ct, pk = lookup[k]
to_get.setdefault(ct, {})[int(pk)] = k
# take out all the publishables
publishable_ct = ContentType.objects.get_for_model(get_model('core', 'publishable'))
if publishable_ct in to_get:
publishable_keys = to_get.pop(publishable_ct)
models = publishable_ct.model_class()._default_manager.values('content_type_id', 'id').filter(id__in=publishable_keys.keys())
for m in models:
ct = ContentType.objects.get_for_id(m['content_type_id'])
pk = m['id']
# and put them back as their native content_type
to_get.setdefault(ct, {})[pk] = publishable_keys[pk]
to_set = {}
# retrieve all the models from DB
for ct, vals in to_get.items():
models = ct.model_class()._default_manager.in_bulk(vals.keys())
for pk, m in models.items():
k = vals[pk]
cached[k] = to_set[k] = m
if not isinstance(cache, DummyCache):
# write them into cache
cache.set_many(to_set, timeout=timeout)
out = []
for k in keys:
try:
out.append(cached[k])
except KeyError:
if missing == NONE:
out.append(None)
elif missing == SKIP:
pass
elif missing == RAISE:
ct = ContentType.objects.get_for_id(int(k.split(':')[1]))
raise ct.model_class().DoesNotExist(
'%s matching query does not exist.' % ct.model_class()._meta.object_name)
return out
def get_cached(self,
slugs=None,
pks=None,
select_related_media_tag=True,
portal_id=None):
"""
Gets all ideas defined by either `slugs` or `pks`.
`slugs` and `pks` may be a list or tuple of identifiers to use for
request where the elements are of type string / unicode or int,
respectively. You may provide a single string / unicode or int directly
to query only one object.
:returns: An instance or a list of instances of :class:`CosinnusGroup`.
:raises: If a single object is defined a `CosinnusGroup.DoesNotExist`
will be raised in case the requested object does not exist.
"""
if portal_id is None:
portal_id = CosinnusPortal.get_current().id
# Check that at most one of slugs and pks is set
assert not (slugs and pks)
assert not (slugs or pks)
if slugs is not None:
if isinstance(slugs, six.string_types):
# We request a single idea
slugs = [slugs]
# We request multiple ideas by slugs
keys = [self._IDEAS_SLUG_CACHE_KEY % (portal_id, s) for s in slugs]
ideas = cache.get_many(keys)
missing = [key.split('/')[-1] for key in keys if key not in ideas]
if missing:
# we can only find ideas via this function that are in the same portal we run in
query = self.get_queryset().filter(portal__id=portal_id,
is_active=True,
slug__in=missing)
if select_related_media_tag:
query = query.select_related('media_tag')
for idea in query:
ideas[self._IDEAS_SLUG_CACHE_KEY %
(portal_id, idea.slug)] = idea
cache.set_many(ideas, settings.COSINNUS_IDEA_CACHE_TIMEOUT)
# sort by a good sorting function that acknowldges umlauts, etc, case insensitive
idea_list = list(ideas.values())
idea_list = sorted(idea_list, key=sort_key_strcoll_attr('name'))
return idea_list
elif pks is not None:
if isinstance(pks, int):
pks = [pks]
else:
# We request multiple ideas
cached_pks = self.get_pks(portal_id=portal_id)
slugs = [
_f for _f in (cached_pks.get(pk, []) for pk in pks) if _f
]
if slugs:
return self.get_cached(slugs=slugs, portal_id=portal_id)
return [] # We rely on the slug and id maps being up to date
return []
def test_set_many_returns_empty_list_on_success(self):
"""set_many() returns an empty list when all keys are inserted."""
failing_keys = cache.set_many({'key1': 'spam', 'key2': 'eggs'})
self.assertEqual(failing_keys, [])
def handle(self, *args, **options):
top_users = Profile.objects.by_rating()[:3]
top_tags = Tag.objects.hottest()[:3]
cache.set_many({'top_users': top_users, 'top_tags': top_tags})
def count(self, request, ip=True, field=None, period=60):
counters = dict((key, 0) for key in self._keys(request, ip, field))
counters.update(cache.get_many(counters.keys()))
for key in counters:
counters[key] += 1
cache.set_many(counters, timeout=period)
def get_instance(model, instance_or_pk, timeout=None, using=None):
"""
Returns the ``model`` instance with a primary key of ``instance_or_pk``.
If the data is cached it will be returned from there, otherwise the regular
Django ORM is queried for this instance and the data stored in the cache.
If omitted, the timeout value defaults to
``settings.CACHE_TOOLBOX_DEFAULT_TIMEOUT`` instead of 0 (zero).
Example::
>>> get_instance(User, 1) # Cache miss
<User: lamby>
>>> get_instance(User, 1) # Cache hit
<User: lamby>
>>> User.objects.get(pk=1) == get_instance(User, 1)
True
"""
pk = getattr(instance_or_pk, "pk", instance_or_pk)
primary_model = model
descriptors = getattr(primary_model, "_cache_fetch_related", ())
models = [model, *(d.related.field.model for d in descriptors)]
# Note: we're assuming that the relations are primary key foreign keys, and
# so all have the same primary key. This matches the assumption which
# `cache_relation` makes.
keys_to_models = {instance_key(model, instance_or_pk): model for model in models}
data_map = cache.get_many(tuple(keys_to_models.keys()))
instance_map = {}
if data_map.keys() == keys_to_models.keys():
try:
for key, data in data_map.items():
model = keys_to_models[key]
instance_map[key] = deserialise(model, data, pk, using)
except:
# Error when deserialising - remove from the cache; we will
# fallback and return the underlying instance
cache.delete_many(tuple(keys_to_models.keys()))
else:
key = instance_key(primary_model, instance_or_pk)
primary_instance = instance_map[key]
for descriptor in descriptors:
related_instance = instance_map[
instance_key(
descriptor.related.field.model,
instance_or_pk,
)
]
related_cache_name = get_related_cache_name(
get_related_name(descriptor),
)
setattr(primary_instance, related_cache_name, related_instance)
return primary_instance
related_names = [d.related.field.related_query_name() for d in descriptors]
# Use the default manager so we are never filtered by a .get_query_set()
queryset = primary_model._default_manager.using(using)
if related_names:
# NB: select_related without args selects all it can find, which we don't want.
queryset = queryset.select_related(*related_names)
primary_instance = queryset.get(pk=pk)
instances = [
primary_instance,
*(getattr(primary_instance, x, None) for x in related_names),
]
cache_data = {}
for instance in instances:
if instance is None:
continue
key = instance_key(instance._meta.model, instance)
cache_data[key] = serialise(instance)
if timeout is None:
timeout = app_settings.CACHE_TOOLBOX_DEFAULT_TIMEOUT
cache.set_many(cache_data, timeout)
return primary_instance
def _add_entity_to_memcache(model, entity, identifiers):
cache.set_many({x: entity
for x in identifiers},
timeout=CACHE_TIMEOUT_SECONDS)
def post(self, request, *args, **kwargs):
errors = []
try:
request_id = str(uuid4())
file_opts = {}
targetgenes_file, targetgenes_source = get_file(
request, "targetgenes", gene_lists_storage)
filter_tfs_file, filter_tfs_source = get_file(request, "filtertfs")
target_networks, networks_source = get_file(
request, "targetnetworks", networks_storage)
background_genes_file, background_genes_source = get_file(
request, "backgroundgenes")
if background_genes_file:
background_genes = get_background_genes(background_genes_file)
file_opts['target_filter_list'] = background_genes
cache.set(f'{request_id}/background_genes', background_genes)
if targetgenes_file:
user_lists = get_gene_lists(targetgenes_file)
if 'target_filter_list' in file_opts:
user_lists = filter_gene_lists_by_background(
user_lists, file_opts['target_filter_list'])
bad_genes = check_annotations(user_lists[0].index)
if bad_genes and targetgenes_source != 'storage':
errors.append(
f'Genes in Target Genes File not in database: {", ".join(bad_genes)}'
)
if not target_networks:
cache.set(f'{request_id}/target_genes',
user_lists) # cache the user list here
file_opts["user_lists"] = user_lists
if filter_tfs_file:
filter_tfs = get_genes(filter_tfs_file)
file_opts["tf_filter_list"] = filter_tfs
bad_genes = check_annotations(filter_tfs)
if bad_genes and filter_tfs_source != 'storage':
errors.append(
f'Genes in Filter TFs File not in database: {", ".join(bad_genes)}'
)
if target_networks:
network = get_network(
target_networks,
headers=request.POST.get('networkHeaders') == 'true')
bad_genes = check_annotations(network[1]['source'].append(
network[1]['target']))
if bad_genes and networks_source != 'storage':
errors.append(
f'Genes in Network File not in database: {", ".join(bad_genes)}'
)
try:
user_lists = file_opts["user_lists"]
# merge network with current user_lists
network, user_lists = merge_network_lists(
network, user_lists)
except KeyError:
user_lists = network_to_lists(network)
file_opts["user_lists"] = user_lists
try:
tf_filter_list = file_opts["tf_filter_list"]
network, tf_filter_list = merge_network_filter_tfs(
network, tf_filter_list)
except KeyError:
tf_filter_list = network_to_filter_tfs(network)
file_opts["tf_filter_list"] = tf_filter_list
cache.set_many({
f'{request_id}/target_network': network,
f'{request_id}/target_genes': user_lists
})
edges = request.POST.getlist('edges')
query = request.POST['query']
result, metadata, stats, _uid, ids = get_query_result(
query=query,
edges=edges,
size_limit=50_000_000,
uid=request_id,
**file_opts)
columns, merged_cells, result_list = format_data(
result, stats, metadata, ids)
metadata = metadata_to_dict(metadata)
cache.set_many({
f'{request_id}/query':
query.strip() + '\n', # save queries
f'{request_id}/formatted_tabular_output':
(columns, merged_cells, result_list, metadata),
})
res = {
'result': {
'data': result_list,
'mergeCells': merged_cells,
'columns': columns
},
'metadata': metadata,
'request_id': request_id,
'analysis_ids': list(ids.items())
}
if errors:
res['errors'] = errors
return JsonResponse(res, encoder=PandasJSONEncoder)
except (QueryError, BadFile) as e:
return HttpResponseBadRequest(e)
except ValueError as e:
return HttpResponseNotFound(f'Query not available: {e}',
content_type='text/plain')
except MultiValueDictKeyError as e:
return HttpResponseBadRequest(f"Problem with query: {e}")
def search_flights(request):
"""
Search flights based on Airport code ,date,travellers,seating class
"""
GO = goibiboAPI(settings.API_USERNAME, settings.API_PASSWORD)
try:
trip = request.POST.get('trip', request.COOKIES.get('trip'))
startdate = request.POST.get('start', request.COOKIES.get('start'))
date, month, year = startdate.split('/')
departure = year + month + date
source = request.POST.get('iata_code_source',
request.COOKIES.get('s_code')).strip()
destination = request.POST.get('iata_code_destination',
request.COOKIES.get('d_code')).strip()
adult = request.POST.get('f_adults', request.COOKIES.get('adults'))
child = request.POST.get('f_childs',
request.COOKIES.get('childs', '0'))
infant = request.POST.get('f_infants',
request.COOKIES.get('infants', '0'))
s_class = request.POST.get('f_class', request.COOKIES.get('class'))
source_city = request.POST.get('flight_source',
request.COOKIES.get('s_city'))
destination_city = request.POST.get('flight_destination',
request.COOKIES.get('d_city'))
except:
messages.add_message(request, messages.INFO,
'User Entered data Incorrect')
return HttpResponseRedirect(format_redirect_url("/", 'error=101'))
if 'round' in trip:
enddate = request.POST.get('end', request.COOKIES.get('end'))
date, month, year = enddate.split('/')
arrival = year + month + date
# this is return url accessing query and response from api
query, search_flights_response = GO.SearchFlights(
trip, departure, source, destination, adult, child, infant,
s_class, arrival)
# split onwardflight data and return flight data from search_flights_response
onwardflight = search_flights_response['data']['onwardflights']
returnflight = search_flights_response['data']['returnflights']
if onwardflight:
#Flights sorting based on price
sorted_onward_flights = sorted(
onwardflight, key=lambda x: x['fare']['totalfare'])
sorted_return_flights = sorted(
returnflight, key=lambda x: x['fare']['totalfare'])
# Zipping for template convience
zipped = zip(sorted_onward_flights, sorted_return_flights)
# stored flight details in redis cache based on search queries
# Flight row id is unique value
caching_dict = {}
for onwardflights, returnflights in zipped:
if 'rowid' in onwardflights:
rowid = onwardflights['rowid']
caching_dict[rowid] = onwardflights
if 'rowid' in returnflights:
rowid = returnflights['rowid']
caching_dict[rowid] = returnflights
# stored redis cache
cache.set_many(caching_dict)
else:
messages.add_message(
request, messages.INFO,
'No Flights availble this source and destination')
return HttpResponseRedirect(format_redirect_url("/", 'error=102'))
else:
query, search_flights_response = GO.SearchFlights(
trip, departure, source, destination, adult, child, infant,
s_class)
onwardflight = search_flights_response['data']['onwardflights']
if onwardflight:
#Flights sorting based on price
sorted_onward_flights = sorted(
onwardflight, key=lambda x: x['fare']['totalfare'])
# stored flight details in redis cache based on search queries
# Flight row id is unique value
caching_dict = {}
for onwardflights in onwardflight:
if 'rowid' in onwardflights:
rowid = onwardflights['rowid']
caching_dict[rowid] = onwardflights
# stored redis cache
cache.set_many(caching_dict)
else:
messages.add_message(
request, messages.INFO,
'No Flights availble this source and destination')
return HttpResponseRedirect(format_redirect_url("/", 'error=103'))
if 'oneway' in trip:
response = render_to_response('v2/flight/flight_search_list.html', {
'flight': sorted_onward_flights,
'source': source_city,
'destination': destination_city,
'dateofdeparture': startdate,
'trip': trip,
'iata_code_return': destination,
'iata_code_onward': source
},
context_instance=RequestContext(request))
else:
response = render_to_response(
'v2/flight/flight_search_list.html', {
'flight': zipped,
'source': request.POST.get('flight_source', ''),
'destination': request.POST.get('flight_destination', ''),
'dateofarrival': enddate,
'dateofdeparture': startdate,
'trip': trip,
'iata_code_return': destination,
'iata_code_onward': source
},
context_instance=RequestContext(request))
response.set_cookie('end', enddate)
response.set_cookie('trip', trip)
response.set_cookie('start', startdate)
response.set_cookie('s_code', source)
response.set_cookie('d_code', destination)
response.set_cookie('adults', adult)
response.set_cookie('childs', child)
response.set_cookie('infants', infant)
response.set_cookie('class', s_class)
response.set_cookie('s_city', source_city)
response.set_cookie('d_city', destination_city)
return response
def bulk_get_rule_status(
self, rules: Sequence[Rule]) -> Mapping[int, GroupRuleStatus]:
keys = [self._build_rule_status_cache_key(rule.id) for rule in rules]
cache_results: Mapping[str, GroupRuleStatus] = cache.get_many(keys)
missing_rule_ids: Set[int] = set()
rule_statuses: Mapping[int, GroupRuleStatus] = {}
for key, rule in zip(keys, rules):
rule_status = cache_results.get(key)
if not rule_status:
missing_rule_ids.add(rule.id)
else:
rule_statuses[rule.id] = rule_status
if missing_rule_ids:
# If not cached, attempt to fetch status from the database
statuses = GroupRuleStatus.objects.filter(
group=self.group, rule_id__in=missing_rule_ids)
to_cache: Sequence[GroupRuleStatus] = list()
for status in statuses:
rule_statuses[status.rule_id] = status
missing_rule_ids.remove(status.rule_id)
to_cache.append(status)
# We might need to create some statuses if they don't already exist
if missing_rule_ids:
# We use `ignore_conflicts=True` here to avoid race conditions where the statuses
# might be created between when we queried above and attempt to create the rows now.
GroupRuleStatus.objects.bulk_create(
[
GroupRuleStatus(rule_id=rule_id,
group=self.group,
project=self.project)
for rule_id in missing_rule_ids
],
ignore_conflicts=True,
)
# Using `ignore_conflicts=True` prevents the pk from being set on the model
# instances. Re-query the database to fetch the rows, they should all exist at this
# point.
statuses = GroupRuleStatus.objects.filter(
group=self.group, rule_id__in=missing_rule_ids)
for status in statuses:
rule_statuses[status.rule_id] = status
missing_rule_ids.remove(status.rule_id)
to_cache.append(status)
if missing_rule_ids:
# Shouldn't happen, but log just in case
self.logger.error(
"Failed to fetch some GroupRuleStatuses in RuleProcessor",
extra={
"missing_rule_ids": missing_rule_ids,
"group_id": self.group.id
},
)
if to_cache:
cache.set_many({
self._build_rule_status_cache_key(item.rule_id): item
for item in to_cache
})
return rule_statuses
def test_set_many_expiration(self):
# set_many takes a second ``timeout`` parameter
cache.set_many({'key1': 'spam', 'key2': 'eggs'}, 1)
time.sleep(2)
self.assertIsNone(cache.get('key1'))
self.assertIsNone(cache.get('key2'))
def test_set_many_expiration(self):
# set_many takes a second ``timeout`` parameter
cache.set_many({"key1": "spam", "key2": "eggs"}, 1)
time.sleep(2)
self.assertEqual(cache.get("key1"), None)
self.assertEqual(cache.get("key2"), None)
def test_set_many(self):
# Multiple keys can be set using set_many
cache.set_many({"key1": "spam", "key2": "eggs"})
self.assertEqual(cache.get("key1"), "spam")
self.assertEqual(cache.get("key2"), "eggs")
def test_cache_versioning_get_set_many(self):
# set, using default version = 1
cache.set_many({'ford1': 37, 'arthur1': 42})
assert (
cache.get_many(['ford1', 'arthur1']) ==
{'ford1': 37, 'arthur1': 42}
)
assert (
cache.get_many(['ford1', 'arthur1'], version=1) ==
{'ford1': 37, 'arthur1': 42}
)
assert cache.get_many(['ford1', 'arthur1'], version=2) == {}
assert caches['v2'].get_many(['ford1', 'arthur1']) == {}
assert (
caches['v2'].get_many(['ford1', 'arthur1'], version=1) ==
{'ford1': 37, 'arthur1': 42}
)
assert caches['v2'].get_many(['ford1', 'arthur1'], version=2) == {}
# set, default version = 1, but manually override version = 2
cache.set_many({'ford2': 37, 'arthur2': 42}, version=2)
assert cache.get_many(['ford2', 'arthur2']) == {}
assert cache.get_many(['ford2', 'arthur2'], version=1) == {}
assert (
cache.get_many(['ford2', 'arthur2'], version=2) ==
{'ford2': 37, 'arthur2': 42}
)
assert (
caches['v2'].get_many(['ford2', 'arthur2']) ==
{'ford2': 37, 'arthur2': 42}
)
assert caches['v2'].get_many(['ford2', 'arthur2'], version=1) == {}
assert (
caches['v2'].get_many(['ford2', 'arthur2'], version=2) ==
{'ford2': 37, 'arthur2': 42}
)
# v2 set, using default version = 2
caches['v2'].set_many({'ford3': 37, 'arthur3': 42})
assert cache.get_many(['ford3', 'arthur3']) == {}
assert cache.get_many(['ford3', 'arthur3'], version=1) == {}
assert (
cache.get_many(['ford3', 'arthur3'], version=2) ==
{'ford3': 37, 'arthur3': 42}
)
assert (
caches['v2'].get_many(['ford3', 'arthur3']) ==
{'ford3': 37, 'arthur3': 42}
)
assert (
caches['v2'].get_many(['ford3', 'arthur3'], version=1) ==
{}
)
assert (
caches['v2'].get_many(['ford3', 'arthur3'], version=2) ==
{'ford3': 37, 'arthur3': 42}
)
# v2 set, default version = 2, but manually override version = 1
caches['v2'].set_many({'ford4': 37, 'arthur4': 42}, version=1)
assert (
cache.get_many(['ford4', 'arthur4']) ==
{'ford4': 37, 'arthur4': 42}
)
assert (
cache.get_many(['ford4', 'arthur4'], version=1) ==
{'ford4': 37, 'arthur4': 42}
)
assert cache.get_many(['ford4', 'arthur4'], version=2) == {}
assert caches['v2'].get_many(['ford4', 'arthur4']) == {}
assert (
caches['v2'].get_many(['ford4', 'arthur4'], version=1) ==
{'ford4': 37, 'arthur4': 42}
)
assert caches['v2'].get_many(['ford4', 'arthur4'], version=2) == {}
def iterator(self):
cache_key = self.model._generate_cache_key("QUERY:%s" %
self._get_query_hash())
on_cache_query_attr = self.model.value_to_list_on_cache_query()
to_return = None
to_cache = {}
with_aggregates = len(self.query.aggregates) > 0
key_list = self._fetch_from_query_cache(cache_key)
if key_list is None:
if not with_aggregates:
values_list = [on_cache_query_attr]
if len(self.query.extra):
values_list += self.query.extra.keys()
key_list = [v[0] for v in self.values_list(*values_list)]
to_cache[cache_key] = (datetime.datetime.now(), key_list)
else:
to_return = list(super(CachedQuerySet, self).iterator())
to_cache[cache_key] = (datetime.datetime.now(), [
(row.__dict__[on_cache_query_attr],
dict([(k, row.__dict__[k])
for k in self.query.aggregates.keys()]))
for row in to_return
])
elif with_aggregates:
tmp = key_list
key_list = [k[0] for k in tmp]
with_aggregates = [k[1] for k in tmp]
del tmp
if (not to_return) and key_list:
row_keys = [
self.model.infer_cache_key({on_cache_query_attr: attr})
for attr in key_list
]
cached = cache.get_many(row_keys)
to_return = [(ck in cached) and self.obj_from_datadict(cached[ck])
or ToFetch(force_unicode(key_list[i]))
for i, ck in enumerate(row_keys)]
if len(cached) != len(row_keys):
to_fetch = [
unicode(tr) for tr in to_return if isinstance(tr, ToFetch)
]
fetched = dict([
(force_unicode(r.__dict__[on_cache_query_attr]), r)
for r in models.query.QuerySet(self.model).filter(
**{"%s__in" % on_cache_query_attr: to_fetch})
])
to_return = [(isinstance(tr, ToFetch) and fetched[unicode(tr)]
or tr) for tr in to_return]
to_cache.update(
dict([(self.model.infer_cache_key(
{on_cache_query_attr: attr}), r._as_dict())
for attr, r in fetched.items()]))
if with_aggregates:
for i, r in enumerate(to_return):
r.__dict__.update(with_aggregates[i])
if len(to_cache):
cache.set_many(to_cache, 60 * 60)
if to_return:
for row in to_return:
if hasattr(row, 'leaf'):
row = row.leaf
row.reset_original_state()
yield row
# 1）导入缓存功能
from django.core.cache import cache # 导入的cache就是配置的默认缓存
from django.core.cache import caches # caches相当于全部缓存集
from django.views.decorators.cache import never_cache, cache_page, cache_control
def_cache = caches['default']
rds_cache = caches['redis']
assert def_cache == cache
assert rds_cache != cache
print('def_cache', def_cache)
print('rds_cache', rds_cache)
# 2）设置，如果将exp过期时间设置0或负值，就是删除缓存
# cache.set('key', 'value', exp=1000)
cache.set('key', 'value')
print(cache.set_many({'a': 1, 'b': 2, 'c': 3}))
print(cache.get_many(['a', 'b', 'c']))
# 3）获取
cache.get('key')
cache.set('num', 1)
cache.incr('num')
cache.incr('num', 10)
cache.decr('num')
cache.decr('num', 5)
cache.clear()
@cache_page(60 * 15, cache="redis") # 可以选用缓存方式
@cache_page(60 * 15, key_prefix="site1")
@cache_control(max_age=3600)
@never_cache
def handle(self, *args, **options):
if waffle.switch_is_active('populate-multitenant-programs'):
failure = False
logger.info('populate-multitenant-programs switch is ON')
catalog_integration = CatalogIntegration.current()
username = catalog_integration.service_username
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
logger.error(
'Failed to create API client. Service user {username} does not exist.'.format(username=username)
)
raise
programs = {}
for site in Site.objects.all():
site_config = getattr(site, 'configuration', None)
if site_config is None or not site_config.get_value('COURSE_CATALOG_API_URL'):
logger.info('Skipping site {domain}. No configuration.'.format(domain=site.domain))
cache.set(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=site.domain), [], None)
continue
client = create_catalog_api_client(user, site=site)
uuids, program_uuids_failed = self.get_site_program_uuids(client, site)
new_programs, program_details_failed = self.fetch_program_details(client, uuids)
if program_uuids_failed or program_details_failed:
failure = True
programs.update(new_programs)
logger.info('Caching UUIDs for {total} programs for site {site_name}.'.format(
total=len(uuids),
site_name=site.domain,
))
cache.set(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=site.domain), uuids, None)
successful = len(programs)
logger.info('Caching details for {successful} programs.'.format(successful=successful))
cache.set_many(programs, None)
if failure:
# This will fail a Jenkins job running this command, letting site
# operators know that there was a problem.
sys.exit(1)
else:
catalog_integration = CatalogIntegration.current()
username = catalog_integration.service_username
try:
user = User.objects.get(username=username)
client = create_catalog_api_client(user)
except User.DoesNotExist:
logger.error(
'Failed to create API client. Service user {username} does not exist.'.format(username=username)
)
raise
try:
querystring = {
'exclude_utm': 1,
'status': ('active', 'retired'),
'uuids_only': 1,
}
logger.info('Requesting program UUIDs.')
uuids = client.programs.get(**querystring)
except: # pylint: disable=bare-except
logger.error('Failed to retrieve program UUIDs.')
raise
total = len(uuids)
logger.info('Received {total} UUIDs.'.format(total=total))
programs = {}
failure = False
for uuid in uuids:
try:
logger.info('Requesting details for program {uuid}.'.format(uuid=uuid))
program = client.programs(uuid).get(exclude_utm=1)
cache_key = PROGRAM_CACHE_KEY_TPL.format(uuid=uuid)
programs[cache_key] = program
except: # pylint: disable=bare-except
logger.exception('Failed to retrieve details for program {uuid}.'.format(uuid=uuid))
failure = True
continue
successful = len(programs)
logger.info('Caching details for {successful} programs.'.format(successful=successful))
cache.set_many(programs, None)
logger.info('Caching UUIDs for {total} programs.'.format(total=total))
cache.set(PROGRAM_UUIDS_CACHE_KEY, uuids, None)
if failure:
# This will fail a Jenkins job running this command, letting site
# operators know that there was a problem.
sys.exit(1)
def handler(cache_name,
function=None,
page_key="",
force_update_cache=False,
cache_duration=DEFAULT_CACHE_DURATION,
*args,
**kwargs):
"""Handles caching for results and data.
Args:
cache: Name of the cache to use.
function: A function reference that returns some value to be cached. This function must only use **kwargs.
page_key: The page name or page number to use as a key value.
force_update_cache:
cache_duration:
**kwargs: Any parameters that need to be passed into "function".
"""
cached_results = None
# Looks through cache and will perform a search if needed.
try:
log.handler(
cache_name + " - Accessed.",
log.DEBUG,
__logger,
)
# If the function was actually a list, then use set_many and/or get_many
# All items must belong to the same cache
# { page_key: {
# "function": function_value,
# "kwargs": {kwargs_value},
# "args": [args_values],
# },
# ... }
if isinstance(function, dict):
if len(function) == 0:
# Nothing was passed in
return None
else:
# Obtain all the keys from the passed in dictionary
requested_keys = []
for key, value in function.items():
cache_key = generate_cache_key(cache_name, value["args"],
value["kwargs"], key)
log.handler(
cache_name +
" - Multi-execution generated cache key " + cache_key,
log.DEBUG,
__logger,
)
requested_keys.append(cache_key)
# Search cache for all keys
cached_results = cache.get_many(requested_keys)
log.handler(
cache_name + " - Multi-execution detected " +
str(len(cached_results)) + " available keys.",
log.INFO,
__logger,
)
# If nothing was in cache, or cache was expired, run function()
thread_list = []
for cache_key in requested_keys:
if not cached_results.__contains__(cache_key):
key = obtain_key_from_cache_key(cache_key)
thread = ReturnThread(
target=function[key]["function"],
args=function[key]["args"],
kwargs=function[key]["kwargs"],
)
thread.start()
thread_list.append((cache_key, thread))
missing_keys = {}
for key, thread in thread_list:
missing_keys[key] = thread.join()
# Set values in cache for any newly executed functions
if bool(missing_keys):
log.handler(
cache_name + " - Multi-execution detected " +
str(len(missing_keys)) + " missing keys.",
log.INFO,
__logger,
)
cache.set_many(missing_keys, cache_duration)
# Return all results
cached_results.update(missing_keys)
# If results were none, log it.
if cached_results is None:
log.handler(
cache_name +
" - Multi-execution generated no results!",
log.WARNING,
__logger,
)
return cached_results
# Get the cached value
cache_key = generate_cache_key(cache_name, args, kwargs, page_key)
log.handler(
cache_name + " - Generated cache key " + cache_key,
log.DEBUG,
__logger,
)
cached_results = cache.get(cache_key)
# No function was provided, just return bare cache value
if function is None:
log.handler(
cache_name + " - Requested raw cache values.",
log.DEBUG,
__logger,
)
return cached_results
# If the user wants to force update the cache, nothing
# was in cache, or cache was expired, run function()
if cached_results is None or force_update_cache:
function_results = function(*args, **kwargs)
log.handler(
cache_name + " - Function " + function.__name__ +
" has been executed!",
log.INFO,
__logger,
)
cache.set(cache_key, function_results, cache_duration)
return function_results
if cached_results is None:
log.handler(
cache_name + " - No cached results found!",
log.INFO,
__logger,
)
# If a value was in cache and not expired, return that value
return cached_results
except:
# If the function threw an exception, return none.
if isinstance(function, dict):
log.handler(
"Function list failed to execute!",
log.ERROR,
__logger,
)
else:
log.handler(
"Function " + function.__name__ + " failed to execute!",
log.ERROR,
__logger,
)
return None
def diff(self, request, new_repoid, old_repoid):
start = datetime.datetime.now()
is_popup = request.GET.get('is_popup', False)
do_regen = request.GET.get('do_regen', False)
progress_id = request.GET.get('progress_id', None)
if not request.is_ajax() and request.method != 'POST':
progress_id = uuid.uuid4()
context = {
"title": "",
"opts": self.model._meta,
"app_label": self.model._meta.app_label,
"progress_id": progress_id,
"do_regen": do_regen,
}
return TemplateResponse(request,
'diff.html',
context=context,
current_app=self.admin_site.name)
progress_cb = lambda x: None
if progress_id is not None:
progress_cb = lambda x: cache.set(progress_id, x, 60 * 5)
progress_cb("Initializing repositories")
new_repo = Repo.objects.select_related(
"server",
"platform").prefetch_related("projects", "components",
"containers").get(pk=new_repoid)
old_repo = Repo.objects.select_related(
"server",
"platform").prefetch_related("projects", "components",
"containers").get(pk=old_repoid)
live_diff = (new_repo.is_live, old_repo.is_live)
end = datetime.datetime.now() - start
context = {
"title": "",
"opts": self.model._meta,
"app_label": self.model._meta.app_label,
'is_popup': is_popup,
'new_obj': new_repo,
'old_obj': old_repo,
'live_diff': live_diff,
'processing_time': end.total_seconds(),
}
if request.method == 'POST':
progress_cb("Creating request")
if not (live_diff[0] and live_diff[1]):
raise ValidationError("Can only creq on live repos")
submit = request.POST.getlist('submit')
delete = request.POST.getlist('delete')
comment = "Request by %s from repodiff of %s to %s" % (
request.user, new_repo, old_repo)
creq_msg = request.POST.get('creq_msg')
if creq_msg:
comment = "%s\n%s" % (comment, creq_msg)
mesgs, errors = _creq(new_repo, old_repo, submit, delete, comment)
for msg in mesgs:
messages.info(request, msg, extra_tags="safe")
for err in errors:
messages.error(request, err, extra_tags="safe")
progress_cb("Done")
return TemplateResponse(request,
'diff_noprogress.html',
context=context,
current_app=self.admin_site.name)
progress_cb("Generating repository diff")
cachekey = "%s%s%s" % ("repodiff", new_repoid, old_repoid)
cached = cache.get_many([cachekey, cachekey + 'ts'])
diff = cached.get(cachekey)
diffts = cached.get(cachekey + 'ts')
if diff is None or do_regen:
diff = _diff_sacks(new_repo, old_repo, progress_cb)
diffts = datetime.datetime.now()
cachelife = (60 *
3) if (live_diff[0] or live_diff[1]) else (60 * 60 *
24)
cache.set_many({
cachekey: diff,
cachekey + 'ts': diffts
}, cachelife)
filter_repos = set(request.GET.getlist("repo", None))
filter_meta = _get_filter_meta(request.GET)
diff = _sort_filter_diff(diff, repos=filter_repos, meta=filter_meta)
trace_reqs = _get_trace(old_repo, new_repo)
issue_ref = []
names = []
for i in new_repo.platform.issuetracker_set.all():
if not i.name in names:
issue_ref.append({'name': i.name, 're': i.re, 'url': i.url})
names.append(i.name)
full_path = "%s?" % request.path
for query, values in request.GET.lists():
full_path += "&".join(['%s=%s' % (query, val) for val in values])
full_path += "&"
end = datetime.datetime.now() - start
context.update({
'title': "Comparing repositories",
'packagemetatypes': list(PackageMetaType.objects.all()),
'diff': diff,
'diffts': diffts,
'trace': trace_reqs,
'issue_ref': json.dumps(issue_ref),
'full_path': full_path,
'processing_time': end.total_seconds()
})
progress_cb("Done")
return TemplateResponse(request,
"diff_content.html",
context=context,
current_app=self.admin_site.name)
def handle(self, *args, **options):
failure = False
logger.info('populate-multitenant-programs switch is ON')
catalog_integration = CatalogIntegration.current()
username = catalog_integration.service_username
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
logger.error(
'Failed to create API client. Service user {username} does not exist.'.format(username=username)
)
raise
programs = {}
pathways = {}
for site in Site.objects.all():
site_config = getattr(site, 'configuration', None)
if site_config is None or not site_config.get_value('COURSE_CATALOG_API_URL'):
logger.info('Skipping site {domain}. No configuration.'.format(domain=site.domain))
cache.set(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=site.domain), [], None)
cache.set(SITE_PATHWAY_IDS_CACHE_KEY_TPL.format(domain=site.domain), [], None)
continue
client = create_catalog_api_client(user, site=site)
uuids, program_uuids_failed = self.get_site_program_uuids(client, site)
new_programs, program_details_failed = self.fetch_program_details(client, uuids)
new_pathways, pathways_failed = self.get_pathways(client, site)
new_pathways, new_programs, pathway_processing_failed = self.process_pathways(site, new_pathways,
new_programs)
if program_uuids_failed or program_details_failed or pathways_failed or pathway_processing_failed:
failure = True
programs.update(new_programs)
pathways.update(new_pathways)
logger.info('Caching UUIDs for {total} programs for site {site_name}.'.format(
total=len(uuids),
site_name=site.domain,
))
cache.set(SITE_PROGRAM_UUIDS_CACHE_KEY_TPL.format(domain=site.domain), uuids, None)
pathway_ids = new_pathways.keys()
logger.info('Caching ids for {total} pathways for site {site_name}.'.format(
total=len(pathway_ids),
site_name=site.domain,
))
cache.set(SITE_PATHWAY_IDS_CACHE_KEY_TPL.format(domain=site.domain), pathway_ids, None)
successful_programs = len(programs)
logger.info('Caching details for {successful_programs} programs.'.format(
successful_programs=successful_programs))
cache.set_many(programs, None)
successful_pathways = len(pathways)
logger.info('Caching details for {successful_pathways} pathways.'.format(
successful_pathways=successful_pathways))
cache.set_many(pathways, None)
if failure:
# This will fail a Jenkins job running this command, letting site
# operators know that there was a problem.
sys.exit(1)
