def test_sensitive_cache_data_is_encrypted(settings, mocker):
"fields marked as `encrypted` are stored in the cache with encryption"
settings.registry.register('AWX_ENCRYPTED',
field_class=fields.CharField,
category=_('System'),
category_slug='system',
encrypted=True)
def rot13(obj, attribute):
assert obj.pk == 123
return getattr(obj, attribute).encode('rot13')
native_cache = LocMemCache(str(uuid4()), {})
cache = EncryptedCacheProxy(native_cache,
settings.registry,
encrypter=rot13,
decrypter=rot13)
# Insert the setting value into the database; the encryption process will
# use its primary key as part of the encryption key
setting_from_db = mocker.Mock(pk=123, key='AWX_ENCRYPTED', value='SECRET!')
mocks = mocker.Mock(
**{
'order_by.return_value':
mocker.Mock(
**{
'__iter__': lambda self: iter([setting_from_db]),
'first.return_value': setting_from_db
}),
})
with mocker.patch('awx.conf.models.Setting.objects.filter',
return_value=mocks):
cache.set('AWX_ENCRYPTED', 'SECRET!')
assert cache.get('AWX_ENCRYPTED') == 'SECRET!'
assert native_cache.get('AWX_ENCRYPTED') == 'FRPERG!'
class BaseCacheCase(TestCase):
def setUp(self):
self.locmem_cache = LocMemCache('default', {})
self.locmem_cache.clear()
self.patch = patch.object(decorators, 'cache', self.locmem_cache)
self.patch.start()
def tearDown(self):
self.patch.stop()
def test_readonly_sensitive_cache_data_is_encrypted(settings):
"readonly fields marked as `encrypted` are stored in the cache with encryption"
settings.registry.register('AWX_ENCRYPTED', field_class=fields.CharField, category=_('System'), category_slug='system', read_only=True, encrypted=True)
def rot13(obj, attribute):
assert obj.pk is None
return codecs.encode(getattr(obj, attribute), 'rot_13')
native_cache = LocMemCache(str(uuid4()), {})
cache = EncryptedCacheProxy(native_cache, settings.registry, encrypter=rot13, decrypter=rot13)
cache.set('AWX_ENCRYPTED', 'SECRET!')
assert cache.get('AWX_ENCRYPTED') == 'SECRET!'
assert native_cache.get('AWX_ENCRYPTED') == 'FRPERG!'
def test_thread_summary_locmem_cache(self):
cache.cache = LocMemCache('', {}) # Enable local caching
thread = self.q.thread
key = Thread.SUMMARY_CACHE_KEY_TPL % thread.id
self.assertTrue(thread.summary_html_cached())
self.assertIsNotNone(thread.get_cached_summary_html())
###
cache.cache.delete(key) # let's start over
self.assertFalse(thread.summary_html_cached())
self.assertIsNone(thread.get_cached_summary_html())
context = {
'thread': thread,
'question': self.q,
'search_state': DummySearchState(),
}
html = get_template('widgets/question_summary.html').render(context)
filled_html = html.replace('<<<tag1>>>', SearchState.get_empty().add_tag('tag1').full_url())\
.replace('<<<tag2>>>', SearchState.get_empty().add_tag('tag2').full_url())\
.replace('<<<tag3>>>', SearchState.get_empty().add_tag('tag3').full_url())
self.assertEqual(
filled_html,
thread.get_summary_html(search_state=SearchState.get_empty()))
self.assertTrue(thread.summary_html_cached())
self.assertEqual(html, thread.get_cached_summary_html())
###
cache.cache.set(key, 'Test <<<tag1>>>', timeout=100)
self.assertTrue(thread.summary_html_cached())
self.assertEqual('Test <<<tag1>>>', thread.get_cached_summary_html())
self.assertEqual(
'Test %s' % SearchState.get_empty().add_tag('tag1').full_url(),
thread.get_summary_html(search_state=SearchState.get_empty()))
###
cache.cache.set(key, 'TestBBB <<<tag1>>>', timeout=100)
self.assertTrue(thread.summary_html_cached())
self.assertEqual('TestBBB <<<tag1>>>',
thread.get_cached_summary_html())
self.assertEqual(
'TestBBB %s' % SearchState.get_empty().add_tag('tag1').full_url(),
thread.get_summary_html(search_state=SearchState.get_empty()))
###
cache.cache.delete(key)
thread.update_summary_html = lambda dummy: "Monkey-patched <<<tag2>>>"
self.assertFalse(thread.summary_html_cached())
self.assertIsNone(thread.get_cached_summary_html())
self.assertEqual(
'Monkey-patched %s' %
SearchState.get_empty().add_tag('tag2').full_url(),
thread.get_summary_html(search_state=SearchState.get_empty()))
def settings(request):
"""
This fixture initializes a Django settings object that wraps our
`awx.conf.settings.SettingsWrapper` and passes it as an argument into the
test function.
This mimics the work done by `awx.conf.settings.SettingsWrapper.initialize`
on `django.conf.settings`.
"""
cache = LocMemCache(str(uuid4()), {}) # make a new random cache each time
settings = LazySettings()
registry = SettingsRegistry(settings)
defaults = {}
# @pytest.mark.defined_in_file can be used to mark specific setting values
# as "defined in a settings file". This is analogous to manually
# specifying a setting on the filesystem (e.g., in a local_settings.py in
# development, or in /etc/tower/conf.d/<something>.py)
for marker in request.node.own_markers:
if marker.name == 'defined_in_file':
defaults = marker.kwargs
defaults['DEFAULTS_SNAPSHOT'] = {}
settings.configure(**defaults)
settings._wrapped = SettingsWrapper(settings._wrapped, cache, registry)
return settings
def get_request_cache():
"""
Request cache is meant to be cleared before every request/task.
"""
if not hasattr(LOCAL_STORAGE, 'request_cache'):
LOCAL_STORAGE.request_cache = LocMemCache('request_cache@%i' % hash(threading.currentThread()), {})
return LOCAL_STORAGE.request_cache
class InMemoryCache(CacheBase):
def __init__(self, params=None):
self.cache = LocMemCache(name='Hydro', params={})
def get(self, key):
try:
value = self.cache.get(key)
except Exception as err:
value = None
return value
def put(self, key, value, ttl=Configurator.CACHE_IN_MEMORY_KEY_EXPIRE):
# just in case the default was changed during the running
if ttl > Configurator.CACHE_IN_MEMORY_KEY_EXPIRE:
ttl = Configurator.CACHE_IN_MEMORY_KEY_EXPIRE
self.cache.set(key, value, ttl)
class InMemoryCache(CacheBase):
def __init__(self, params=None):
self.cache = LocMemCache(name='Hydro', params={})
def get(self, key):
try:
value = self.cache.get(key)
except Exception, err:
value = None
return value
def setUp(self):
self.create_user()
self.user.set_password('pswd')
self.user.save()
assert self.client.login(username=self.user.username, password='******')
self.create_user(username='******')
self.user2.set_password('pswd')
self.user2.reputation = 10000
self.user2.save()
self.old_cache = cache.cache
cache.cache = LocMemCache('', {}) # Enable local caching
def check_code_token_throttler(rf):
with freeze_time('2020-01-02 13:00:00') as frozen_datetime:
cache = LocMemCache('test_cache', {})
cache.clear()
throttler = get_code_token_throttler(cache)
request = rf.get('/')
assert inspect_cache(cache) == {}
# First two requests should be allowed
assert throttler.allow_request(request, None) is True
assert inspect_cache(cache) == {
':1:drf_jwt_2fa-tc-SHA1(127.0.0.1)': [1577970000.0],
}
frozen_datetime.tick(delta=datetime.timedelta(seconds=1))
throttler = get_code_token_throttler(cache)
assert throttler.allow_request(request, None) is True
assert inspect_cache(cache) == {
':1:drf_jwt_2fa-tc-SHA1(127.0.0.1)': [1577970001.0, 1577970000.0],
}
# Third request should be throttled
throttler = get_code_token_throttler(cache)
frozen_datetime.tick(delta=datetime.timedelta(seconds=1))
assert throttler.allow_request(request, None) is False
assert inspect_cache(cache) == {
':1:drf_jwt_2fa-tc-SHA1(127.0.0.1)': [1577970001.0, 1577970000.0],
}
# After 8s more, total 11s have passed and a new request should
# be allowed
throttler = get_code_token_throttler(cache)
frozen_datetime.tick(delta=datetime.timedelta(seconds=8))
assert throttler.allow_request(request, None) is True
assert inspect_cache(cache) == {
':1:drf_jwt_2fa-tc-SHA1(127.0.0.1)': [1577970010.0, 1577970001.0],
}
class TestCachedTemplate(SimpleTestCase):
"""Test the single cached angular template tag."""
def setUp(self):
"""Patch in a local memory cache."""
self.locmem_cache = LocMemCache('default', {})
self.locmem_cache.clear()
self.patch = mock.patch.object(angular, 'cache', self.locmem_cache)
self.patch.start()
def tearDown(self):
"""Stop the patch."""
self.patch.stop()
@tag('unit')
@mock.patch('web.templatetags.angular.finders.get_finders')
@mock.patch(
'builtins.open',
new_callable=mock.mock_open,
read_data='some_data'
)
def test_not_cached(self, m_file, m_get_finders):
"""Test behaviour of a template that is not cached."""
m_get_finders.return_value = [FakeFinder(['a'])]
key = '_angular_template_a'
self.assertFalse(self.locmem_cache.get(key))
html = CAT('a')
expected = (
'<script type="text/ng-template" id="/static/a">'
'some_data'
'</script>'
)
self.assertEqual(html, expected)
self.assertEqual(self.locmem_cache.get(key), expected)
@tag('unit')
@mock.patch('web.templatetags.angular.finders.get_finders')
@mock.patch(
'builtins.open',
new_callable=mock.mock_open,
read_data='some_data'
)
def test_cached(self, m_file, m_get_finders):
"""Test behavior of a template that is cached."""
m_get_finders.return_value = [FakeFinder(['a'])]
key = '_angular_template_a'
self.locmem_cache.add(key, 'the_data')
html = CAT('a')
expected = 'the_data'
self.assertEqual(html, expected)
self.assertEqual(self.locmem_cache.get(key), expected)
def reg(request):
"""
This fixture initializes an awx settings registry object and passes it as
an argument into the test function.
"""
cache = LocMemCache(str(uuid4()), {}) # make a new random cache each time
settings = LazySettings()
registry = SettingsRegistry(settings)
# @pytest.mark.defined_in_file can be used to mark specific setting values
# as "defined in a settings file". This is analogous to manually
# specifying a setting on the filesystem (e.g., in a local_settings.py in
# development, or in /etc/tower/conf.d/<something>.py)
defaults = request.node.get_marker('defined_in_file')
if defaults:
settings.configure(**defaults.kwargs)
settings._wrapped = SettingsWrapper(settings._wrapped, cache, registry)
return registry
def setUp(self):
self.backend = LocMemCache('', {})
self.cache = TransactionCache(self.backend)
self.cache.timeout = 1000
class TransactionCacheTestCase(base.JohnnyTestCase):
def setUp(self):
self.backend = LocMemCache('', {})
self.cache = TransactionCache(self.backend)
self.cache.timeout = 1000
def test_get(self):
self.backend.set('a', '1')
self.assertEqual(self.cache.get('a'), '1')
self.assertIsNone(self.cache.get('missing'))
def test_local_caching(self):
self.backend.set('a', '1')
self.assertEqual(self.cache.get('a'), '1')
self.backend.clear()
self.assertEqual(self.cache.get('a'), '1')
def test_get_many_local_caching(self):
self.backend.set('a', '1')
self.backend.set('b', '2')
self.assertEqual(
self.cache.get_many(['a', 'b', 'c']),
{'a': '1', 'b': '2'},
)
self.backend.clear()
self.backend.set('c', '3')
self.assertEqual(
self.cache.get_many(['a', 'b', 'c']),
{'a': '1', 'b': '2'},
)
def test_get_many_looks_up_missing(self):
self.cache.get('a')
with patch.object(self.backend, 'get_many') as get_many:
self.cache.get_many(['a', 'b', 'c'])
get_many.assert_called_with(['b', 'c'])
def test_set_get(self):
self.backend.set('a', '1')
self.assertEqual(self.cache.get('a'), '1')
self.cache.set('a', '2')
self.assertEqual(self.cache.get('a'), '2')
self.assertEqual(self.backend.get('a'), '1')
def test_set_many(self):
self.cache.set_many({'a': '1', 'b': '2'})
self.assertEqual(self.cache.get('a'), '1')
self.assertEqual(self.cache.get('b'), '2')
def test_delete_get(self):
self.backend.set('a', '1')
self.assertEqual(self.cache.get('a'), '1')
self.cache.delete('a')
self.assertIsNone(self.cache.get('a'))
self.assertEqual(self.backend.get('a'), '1')
def test_delete_many(self):
self.backend.set_many({'a': '1', 'b': '2'})
self.cache.delete_many(['a', 'b'])
self.assertIsNone(self.cache.get('a'))
self.assertIsNone(self.cache.get('b'))
def test_rollback(self):
self.backend.set('a', '1')
self.assertEqual(self.cache.get('a'), '1')
self.backend.set('a', '2')
self.assertEqual(self.cache.get('a'), '1')
self.cache.rollback()
self.assertEqual(self.cache.get('a'), '2')
def test_commit(self):
self.backend.set('a', '1')
self.backend.set('b', '2')
self.cache.set('a', '3')
self.cache.delete('b')
self.assertEqual(self.backend.get('a'), '1')
self.assertEqual(self.backend.get('b'), '2')
self.cache.commit()
self.assertEqual(self.backend.get('a'), '3')
self.assertIsNone(self.backend.get('b'))
def test_rollback_savepoint(self):
self.cache.set('a', '1')
self.cache.savepoint('sp1')
self.cache.set('a', '2')
self.cache.savepoint('sp2')
self.cache.set('a', '3')
self.assertEqual(self.cache.get('a'), '3')
self.cache.rollback_savepoint('sp2')
self.assertEqual(self.cache.get('a'), '2')
self.cache.rollback_savepoint('sp1')
self.assertEqual(self.cache.get('a'), '1')
self.assertRaises(IndexError, self.cache.rollback_savepoint, 'sp0')
def test_rollback_savepoint_skip(self):
self.cache.set('a', '1')
self.cache.savepoint('sp1')
self.cache.set('a', '2')
self.cache.savepoint('sp2')
self.cache.set('a', '3')
self.assertEqual(self.cache.get('a'), '3')
self.cache.rollback_savepoint('sp1')
self.assertEqual(self.cache.get('a'), '1')
self.assertRaises(IndexError, self.cache.rollback_savepoint, 'sp2')
def test_rollback_savepoint_same_name(self):
self.cache.set('a', '1')
self.cache.savepoint('sp')
self.cache.set('a', '2')
self.cache.savepoint('sp')
self.cache.set('a', '3')
self.assertEqual(self.cache.get('a'), '3')
self.cache.rollback_savepoint('sp')
self.assertEqual(self.cache.get('a'), '2')
self.cache.rollback_savepoint('sp')
self.assertEqual(self.cache.get('a'), '1')
self.assertRaises(IndexError, self.cache.rollback_savepoint, 'sp')
def test_commit_savepoint(self):
self.cache.set('a', '1')
self.cache.savepoint('sp1')
self.cache.set('a', '2')
self.assertEqual(self.cache.get('a'), '2')
self.cache.commit_savepoint('sp1')
self.assertEqual(self.cache.get('a'), '2')
self.assertRaises(IndexError, self.cache.rollback_savepoint, 'sp1')
def test_commit_savepoint_same_name(self):
self.cache.set('a', '1')
self.cache.savepoint('sp')
self.cache.set('a', '2')
self.cache.savepoint('sp')
self.cache.set('a', '3')
self.assertEqual(self.cache.get('a'), '3')
self.cache.commit_savepoint('sp')
self.assertEqual(self.cache.get('a'), '3')
self.cache.rollback_savepoint('sp')
self.assertEqual(self.cache.get('a'), '1')
self.assertRaises(IndexError, self.cache.rollback_savepoint, 'sp')
def get_local_cache():
if not hasattr(LOCAL_STORAGE, 'local_cache'):
LOCAL_STORAGE.local_cache = LocMemCache('local_cache@%i' % hash(threading.currentThread()), {})
return LOCAL_STORAGE.local_cache
def store(self):
c = LocMemCache("test", settings.CACHES["default"])
c.clear()
return OptionsStore(cache=c)
def __init__(self, name, params):
LocMemCache.__init__(self, name, params)
self.default_timeout = float(params.get('TIMEOUT', 0.25))
def __init__(self, name, params):
LocMemCache.__init__(self, name, params)
self.default_timeout = float(params.get("TIMEOUT", 0.01))
from string import ascii_lowercase
from timeit import timeit
from django.core.cache.backends.locmem import LocMemCache
from django_redis.cache import RedisCache
from shared_memory_dict.caches.django import SharedMemoryCache
cache_smc_django = SharedMemoryCache(
'django', params={'OPTIONS': {
'MEMORY_BLOCK_SIZE': 64
}})
cache_django_redis = RedisCache(server='redis://127.0.0.1:6379/1', params={})
cache_django_locmem = LocMemCache('locmem', params={})
def agressive(cache):
for c in ascii_lowercase:
for i in range(5):
cache.set(f'{c}{i}', '1')
for i in range(10, 0, -1):
cache.get(f'{c}{i}')
def fun(cache):
cache.set('fun', 'uhull')
for _ in range(3):
cache.get('fun')
def collect(cache_var_name):
time_of_agressive = timeit(stmt=f'agressive({cache_var_name})',
def setUp(self):
"""Patch in a local memory cache."""
self.locmem_cache = LocMemCache('default', {})
self.locmem_cache.clear()
self.patch = mock.patch.object(angular, 'cache', self.locmem_cache)
self.patch.start()
def setUp(self):
self.locmem_cache = LocMemCache('default', {})
self.locmem_cache.clear()
self.patch = patch.object(decorators, 'cache', self.locmem_cache)
self.patch.start()
__FILENAME__ = compress
import hashlib
import os
import subprocess
from StringIO import StringIO
from django.core.urlresolvers import reverse
from django.core.cache.backends.locmem import LocMemCache
from simplestatic import conf
CACHE = LocMemCache('simplestatic', {})
CHUNK_SIZE = 8192
def uncached_hash_for_paths(paths):
hsh = hashlib.md5()
for path in paths:
full_path = os.path.join(conf.SIMPLESTATIC_DIR, path)
if not os.path.exists(full_path):
# TODO: Log some kind of warning here
continue
with open(full_path, 'r') as f:
while 1:
data = f.read(CHUNK_SIZE)
if not data:
break
def __init__(self, name, params):
LocMemCache.__init__(self, name, params)
self.default_timeout = float(params.get("TIMEOUT", 0.25))
import time
import django.utils.unittest as unittest
from django.test import TestCase
from django.core.cache.backends.locmem import LocMemCache
from django.core.cache.backends.filebased import FileBasedCache
from django.conf import settings
from jcache import JCache
CACHES = {
'default': LocMemCache('unique-snowflake', {'TIMEOUT': 1}),
'secondary': LocMemCache('second-unique-snowflake', {}),
'file-backed': FileBasedCache('/var/tmp/django_jcache_tests', {}),
}
JCACHES = {
# one will never go into stale, because the entries TIMEOUT too fast
'one': JCache(stale=2, expiry=None, cache=CACHES['default']),
# two will pass through stale because we override TIMEOUT by
# having a higher expiry
'two': JCache(stale=2, expiry=3, cache=CACHES['default']),
# secondary doesn't have a TIMEOUT (so it's the default of 300),
# but we override anyway to keep things down
'three': JCache(stale=2, expiry=3, cache=CACHES['secondary']),
# just like three, but uses a backend that can be pickled
'async': JCache(stale=2, expiry=3, cache=CACHES['file-backed']),
# and one without expiry
'async-noexpire': JCache(stale=2, expiry=None,
cache=CACHES['file-backed']),
}
def cache(self):
return LocMemCache(self.keyspace, {})
def __init__(self, params=None):
self.cache = LocMemCache(name='Hydro', params={})
def store(self):
c = LocMemCache('test', {})
c.clear()
return OptionsStore(cache=c)
import os
import pkg_resources
import sqlite3
import sys
import traceback
import uuid
from django.core.cache import cache
from django.core.cache.backends.locmem import LocMemCache
from django.db.backends.postgresql.base import DatabaseWrapper as BaseDatabaseWrapper
from awx.main.utils import memoize
__loc__ = LocMemCache(str(uuid.uuid4()), {})
__all__ = ['DatabaseWrapper']
class RecordedQueryLog(object):
def __init__(self, log, db, dest='/var/log/tower/profile'):
self.log = log
self.db = db
self.dest = dest
try:
self.threshold = cache.get('awx-profile-sql-threshold')
except Exception:
# if we can't reach the cache, just assume profiling's off
self.threshold = None
def append(self, query):
ret = self.log.append(query)
try:
## RED_FLAG: this feels wrong...
#from app_plugins.models import PluginPoint, Plugin
#from app_plugins.models import is_valid_label, construct_template_path
models = apps.get_app_config('app_plugins').models_module
APP_PLUGINS_CACHE_PARAMS = getattr(
settings,
'APP_PLUGINS_CACHE_PARAMS',
{
'cull_frequency': 4,
'max_entries': 3000,
'timeout': 60 * 60 * 24 * 3, # 3 days
})
app_plugin_apps_with_templates = LocMemCache('localhost',
APP_PLUGINS_CACHE_PARAMS)
# at import cache the app names for indexing
app_names = []
for app in settings.INSTALLED_APPS:
name = app.split('.')[-1]
if name not in app_names:
app_names.append(name)
app_names = tuple(app_names)
def callback(func, variables, context, takes_context):
"""
resolve an iterable of Variable objects into a list of args and a dict
of keyword arguments. support full python style keyword argument
processing::
def __init__(self, name, params):
LocMemCache.__init__(self, name, params)
self.default_timeout = float(params.get('TIMEOUT', 0.01))
