def get_installed_models(by_model=False):
cache_key = get_cache_key('models', 'list', by_model)
models = cache.get(cache_key)
if models:
return models
#
# do discovery on models
#
models = {}
for app in INSTALLED_APPS:
module_path = '%s.models' % app
module = get_module(module_path)
if module:
for fixture in dir(module):
if type(getattr(module, fixture)) == ModelBase:
model_fixture = getattr(module, fixture)
model_name = model_fixture._meta.db_table
if by_model:
model_name = fixture
models[model_name] = dict(
name=fixture,
model=getattr(module, fixture)
)
if not by_model:
cache.set(cache_key, models)
return models
def get_page_layout_config(page, default='empty', force_layout=False):
layout_name = page.layout if page else default
if force_layout:
layout_name = default
cache_key = get_cache_key('layout', layout_name)
layout = cache.get(cache_key)
if layout:
layout = get_module(layout)
return layout.config
path = ''
for app in INSTALLED_APPS:
if app.startswith('django'):
continue
path = '%s.templates.layout.%s' % (app, layout_name)
layout = get_module(path)
if layout:
break
if not layout:
return {}
cache.set(cache_key, path)
return layout.config
def latest_revision(cls, article_id: int, language_id: int = 1) -> int:
cache_key = cls.__cache_key_latest_id_of_article__.format(
article_id=article_id, language_id=language_id
)
revision_id = cache.get(cache_key)
if revision_id:
return cls.from_attrs(
revision_id=revision_id,
article_id=article_id,
language_id=language_id,
)
else:
latest_revision = (
cls.query.filter(
and_(
cls.article_id == article_id,
cls.language_id == language_id,
)
)
.order_by(cls.revision_id.desc()) # type: ignore
.limit(1)
.scalar()
)
if not latest_revision:
raise WikiNoRevisions
cache.set(cache_key, latest_revision.revision_id)
return latest_revision
def get_assets_config():
cache_key = get_cache_key('assets')
config = cache.get(cache_key)
if config:
return config
#
# discovery blocks in all apps
#
config = {}
for app in INSTALLED_APPS:
if app.startswith('django'):
continue
module = get_module(app)
if hasattr(module, 'wepo_assets'):
app_assets = module.wepo_assets
for asset_type, assets in app_assets.items():
if not asset_type in config:
config[asset_type] = {}
config[asset_type].update(assets.items())
cache.set(cache_key, config, 86400)
return config
def permissions(self) -> List[str]:
"""
A general function to get the permissions of a user from a permission
model and attributes of their user classes. Locked users are restricted
to the permissions defined for them in the config.
:param key: The cache key to cache the permissions under
:param model: The model to query custom permissions from
:param attr: The attribute of the userclasses that should be queried
"""
from core.permissions.models import SecondaryClass
from core.permissions.models import UserPermission
if self.locked: # Locked accounts have restricted permissions.
return app.config['LOCKED_ACCOUNT_PERMISSIONS']
key = self.__cache_key_permissions__.format(id=self.id)
permissions = cache.get(key)
if not permissions:
permissions = copy(self.user_class_model.permissions)
for class_ in SecondaryClass.from_user(self.id):
permissions += class_.permissions
permissions = set(permissions) # De-dupe
for perm, granted in UserPermission.from_user(self.id).items():
if not granted and perm in permissions:
permissions.remove(perm)
if granted and perm not in permissions:
permissions.add(perm)
cache.set(key, permissions)
return permissions
def get_all_assets():
cache_key = get_cache_key('assets')
assets = cache.get(cache_key)
if assets:
return assets
#
# discovery assets in all apps
#
assets = {'script': [], 'style': [], 'meta': []}
for app in INSTALLED_APPS:
if app.startswith('django'):
continue
module = get_module(app)
if hasattr(module, 'wepo_assets'):
wepo_assets = module.wepo_assets
for asset_type in ['script', 'style', 'meta']:
for asset, asset_conf in wepo_assets[asset_type].__dict__.items():
assets[asset_type].append(dict(name=asset, conf=asset_conf))
cache.set(cache_key, assets, 3600)
return assets
def get_layouts():
cache_key = get_cache_key('layouts')
layouts = cache.get(cache_key)
if layouts:
return layouts
#
# discovery blocks in all apps
#
layouts = []
for app in INSTALLED_APPS:
if app.startswith('django'):
continue
path = '%s/%s/templates/layout' % (PROJECT_ROOT, app)
for root, dir_names, file_names in os.walk(path):
#
# filter out only modules except __init__
#
for filename in fnmatch.filter(file_names, '*.py'):
layout_path = root.replace('%s/%s/templates/layout' % (PROJECT_ROOT, app), '')
layout_module_path = root.replace('%s/' % PROJECT_ROOT, '').replace('/', '.')
if layout_path:
layout_path = layout_path[1:].replace('/', '.')
layout_module = get_module(layout_module_path)
layout_name = layout_module.name if hasattr(layout_module, 'name') else layout_path.replace('.', ' ').title()
layouts.append({'name': layout_name, 'path': layout_path, 'full_path': layout_module_path})
cache.set(cache_key, layouts, 86400)
return layouts
def test_clear_cache_keys(client):
ckey = Notification.__cache_key_notification_count__.format(
user_id=1, type=1
)
cache.set(ckey, 100)
Notification.clear_cache_keys(1, 'subscripple')
assert not cache.has(ckey)
def test_clear_cache_keys_without_type(client):
ckey = Notification.__cache_key_notification_count__.format(
user_id=1, type=1
)
cache.set(ckey, 100)
Notification.clear_cache_keys(1)
assert not cache.has(ckey)
def test_clear_cache_keys_wrong_type(client):
ckey = Notification.__cache_key_notification_count__.format(
user_id=1, type='subscripple'
)
cache.set(ckey, 100)
with pytest.raises(APIException):
Notification.clear_cache_keys(1, 'not_real_type')
def count(
cls,
*,
key: str,
attribute: InstrumentedAttribute,
filter: BinaryExpression = None,
) -> int:
"""
An abstracted function for counting a number of elements matching a query. If the
passed cache key exists, its value will be returned; otherwise, the passed query
will be ran and the resultant count cached and returned.
:param key: The cache key to check
:param attribute: The attribute to count; a model's column
:param filter: The SQLAlchemy filter expression
:return: The number of rows matching the query element
"""
count = cache.get(key)
if not isinstance(count, int):
query = cls._construct_query(
db.session.query(func.count(attribute)), filter)
count = query.scalar()
cache.set(key, count)
return count
def get_form_items():
cache_key = get_cache_key('form_elements')
form_items = cache.get(cache_key)
if form_items:
return form_items
#
# discovery form items in all apps
#
form_items = {}
for app in INSTALLED_APPS:
if app.startswith('django'):
continue
path = '%s.form.elements' % app
module = get_module(path)
if not module:
continue
for element in dir(module):
if not element.startswith('__'):
form_items[element] = path
cache.set(cache_key, form_items, 86400)
return form_items
def test_cache_delete_many(app, client):
cache.set('key_1', 1)
cache.set('key_2', 2)
assert cache.delete_many('key_1', 'key_2', 'key_3')
with app.test_request_context('/test'):
assert 'key_1' in flask.g.cache_keys['delete_many']
assert 'key_2' in flask.g.cache_keys['delete_many']
assert 'key_3' in flask.g.cache_keys['delete_many']
def test_post_edit_history_from_cache(app, authed_client):
cache.set(ForumPostEditHistory.__cache_key_of_post__.format(id=3), [1])
ForumPostEditHistory.from_pk(1) # cache this
history = ForumPostEditHistory.from_post(3)
assert len(history) == 1
assert any(
h.contents == 'Why the fcuk is Gazelle in HPH?' for h in history
)
def test_cache_get_dict_order(app, client):
cache.set('key_4', 4)
cache.set('key_3', 3)
cache.set('key_5', 5)
cache.set('key_1', 1)
cache.set('key_2', 2)
data = cache.get_dict(*('key_2', 'key_1', 'key_3', 'key_4', 'key_5'))
assert list(data.keys()) == ['key_2', 'key_1', 'key_3', 'key_4', 'key_5']
def test_cache_get_dict(app, client):
cache.set('key_1', 1)
cache.set('key_2', 2)
data = cache.get_dict(*('keY_1', 'kEy_2', 'key_3'))
assert data == {'key_1': 1, 'key_2': 2, 'key_3': None}
with app.test_request_context('/test'):
assert 'key_1' in flask.g.cache_keys['get_dict']
assert 'key_2' in flask.g.cache_keys['get_dict']
assert 'key_3' in flask.g.cache_keys['get_dict']
def test_thread_last_viewed_post_cached(app, authed_client):
cache.set(ForumLastViewedPost.__cache_key__.format(thread_id=1, user_id=1),
2)
thread = ForumThread.from_pk(1)
last_post = thread.last_viewed_post
assert last_post.id == 2
assert last_post.thread_id == 3
assert 2 == cache.get(
ForumLastViewedPost.__cache_key__.format(thread_id=1, user_id=1))
def translate(request, phrase, language=None, translation_type=None):
if not language:
language = get_user_language(request)
# Not implemented yet
if translation_type:
cache_key = get_cache_key('translation', language, 'group', translation_type)
translations = cache.get(cache_key)
if not translations:
translations = {}
all_translations = None #Translation.objects.filter(language__code=language, translation_type__name=translation_type)
if all_translations:
for translation in all_translations:
translations[translation.phrase.text] = translation.text
cache.set(cache_key, translations)
if phrase in translations:
return translations[phrase]
return phrase
else:
from core.helper.common_help import get_cache_key_2
phrase_key_name = get_cache_key_2(phrase)
cache_key = get_cache_key('translation', language, phrase_key_name)
translation = cache.get(cache_key)
result = phrase
if not translation:
phrase_obj = Phrase.objects.filter(key_name=phrase_key_name)
if phrase_obj and len(phrase_obj) > 0:
phrase_obj = phrase_obj[0]
translations = Translation.objects.filter(language__code=language, phrase=phrase_obj)
if translations and len(translations) > 0:
translation = translations[0]
stat = cache.set(cache_key, translation)
else:
new_translation = Translation()
new_translation.translation = phrase
new_translation.phrase = phrase_obj
new_translation.language = Language.objects.get(code=language)
new_translation.save()
else:
new_phrase = Phrase()
new_phrase.text = phrase
new_phrase.save()
new_translation = Translation()
new_translation.translation = phrase
new_translation.phrase = new_phrase
new_translation.language = Language.objects.get(code=language)
new_translation.save()
else:
result = translation.translation
return result
def get_pks_of_many(
cls,
key: str = None,
filter: BinaryExpression = None,
order: BinaryExpression = None,
include_dead: bool = False,
expr_override: BinaryExpression = None,
) -> List[Union[int, str]]:
"""
Get a list of object IDs meeting query criteria. Fetching from the
cache with the provided cache key will be attempted first; if the cache
key does not exist then a query will be ran. Calls with ``include_dead=True`` are
saved under a different cache key. ``include_dead`` does not affect the query results
when passing ``expr_override``.
:param key: The cache key to check (and return if present)
:param filter: A SQLAlchemy filter expression to be applied to the query
:param order: A SQLAlchemy order_by expression to be applied to the query
:param include_dead: Whether or not to include dead results in the IDs list
:param expr_override: If passed, this will override filter and order, and be
called verbatim in a ``db.session.execute`` if the cache
key does not exist
:return: A list of IDs
"""
key = f'{key}_incl_dead' if include_dead and key else key
pks = cache.get(key) if key else None
if not pks or not isinstance(pks, list):
if expr_override is not None:
pks = [
x[0] if len(x) == 1 else dict(x)
for x in db.session.execute(expr_override)
]
else:
primary_key = cls.get_primary_key()
if isinstance(primary_key, list):
query = cls._construct_query(
db.session.query(*(getattr(cls, k)
for k in primary_key)),
filter,
order,
)
else:
query = cls._construct_query(
db.session.query(getattr(cls, primary_key)),
filter,
order,
)
if not include_dead and cls.__deletion_attr__:
query = query.filter(
getattr(cls, cls.__deletion_attr__) == 'f')
pks = [
x[0] if len(x) == 1 else x._asdict() for x in query.all()
]
if key:
cache.set(key, pks)
return pks
def get_rules(section: str) -> dict:
if section not in SECTION_NAMES:
raise APIException(f'{section} is not a valid section of the rules.')
rules = cache.get(f'rules_{section}')
if not rules:
filename = os.path.join(os.path.dirname(__file__), f'{section}.json')
with open(filename, 'r') as f:
rules = json.load(f)
cache.set(f'rules_{section}', rules, 0)
return rules
def test_view_all_keys_cached(app, authed_client):
add_permissions(app, ApikeyPermissions.VIEW)
cache_key = APIKey.__cache_key_of_user__.format(user_id=1)
cache.set(cache_key, ['abcdefghij', 'bcdefghijk'], timeout=60)
response = authed_client.get('/api_keys')
data = response.get_json()['response']
assert any('hash' in api_key and api_key['hash'] == CODE_2[:10]
for api_key in data)
assert cache.ttl(cache_key)
def test_category_get_all_cached(app, authed_client):
cache.set(ForumCategory.__cache_key_all__, [1, 3], timeout=60)
categories = ForumCategory.get_all()
assert len(categories) == 2
for category in categories:
if category.name == 'Site' and category.id == 1:
break
else:
raise AssertionError('A real forum not called')
def test_get_from_cache(app, authed_client):
"""Test that cache values are used instead of querying a user."""
add_permissions(app, 'users_view')
user = User.from_pk(1)
data = {}
for attr in user.__table__.columns.keys():
data[attr] = getattr(user, attr, None)
data['username'] = '******'
cache.set(user.cache_key, data)
user = User.from_pk(1)
assert user.username == 'fakeshit'
def test_thread_get_from_forum_cached(app, authed_client):
cache.set(ForumThread.__cache_key_of_forum__.format(id=2), [1, 5],
timeout=60)
ForumThread.from_pk(1)
ForumThread.from_pk(5) # noqa cache these
threads = ForumThread.from_forum(2, page=1, limit=50)
assert len(threads) == 2
for thread in threads:
if thread.topic == 'Donations?' and thread.id == 5:
break
else:
raise AssertionError('A real thread not called')
def test_forum_get_from_category_cached(app, authed_client):
cache.set(Forum.__cache_key_of_category__.format(id=2), ['1', '5'],
timeout=60)
Forum.from_pk(1)
Forum.from_pk(5) # noqa: cache these
forums = Forum.from_category(2)
assert len(forums) == 2
for forum in forums:
if forum.name == 'Yacht Funding' and forum.id == 5:
break
else:
raise AssertionError('A real forum not called')
def test_post_get_from_thread_cached(app, authed_client):
cache.set(
ForumPost.__cache_key_of_thread__.format(id=2), ['1', '6'], timeout=60
)
ForumPost.from_pk(1)
ForumPost.from_pk(6) # noqa cache this
posts = ForumPost.from_thread(2, page=1, limit=50)
assert len(posts) == 2
for post in posts:
if post.contents == 'Delete this' and post.id == 6:
break
else:
raise AssertionError('A real post not called')
def wrapper(*args, **kwargs):
arg = '-'.join(map(str, args))
key = kw + '-' + arg
data = cache.get(key)
if not data:
# print key, 'notin'
data = func(*args, **kwargs)
if timeout:
cache.set(key, data, timeout)
else:
cache.set(key, data)
# else:
# print key, 'bingo'
return data
def test_rate_limit_function_global(app, client, monkeypatch):
"""Test that the per-user rate limit function correctly increments and errors."""
monkeypatch.setattr(
'core.hooks.before.flask.g',
g(
user=User.from_pk(1),
api_key=api_key(hash='abcdefghij'),
cache_keys=defaultdict(set),
),
)
cache.set('rate_limit_user_1', 70, timeout=60)
with pytest.raises(APIException) as e:
for i in range(31):
check_rate_limit()
assert 'User rate limit exceeded.' in e.value.message
def get_model(request, key_name, by_model=False):
cache_key = get_cache_key('model', key_name)
model = cache.get(cache_key)
if model:
return model
#
# get model from installed models
#
models = get_installed_models(by_model=by_model)
if key_name in models:
cache.set(cache_key, models[key_name])
return models[key_name]
return None
def test_clear_cache_keys(client):
for uid in [1, 2]:
for f in ['inbox', 'sentbox', 'deleted']:
cache.set(
PrivateConversation.__cache_key_of_user__.format(user_id=uid,
filter=f),
1,
)
PrivateConversation.clear_cache_keys(1)
for f in ['inbox', 'sentbox', 'deleted']:
assert not cache.has(
PrivateConversation.__cache_key_of_user__.format(user_id=1,
filter=f))
assert cache.has(
PrivateConversation.__cache_key_of_user__.format(user_id=2,
filter=f))
def test_cache_inc_key_already_exists(app, client):
"""Incrementing an already-existing key just increments it."""
assert cache.set('test-inc-key', 3, timeout=15)
value = cache.inc('test-inc-key', 4, timeout=80)
time_left = cache.ttl('test-inc-key')
assert value == 7
assert time_left > 13 and time_left < 16
def get_seo(request, url=None):
url_parts = request.wepo.url_parts
if url:
url, url_parts = get_url_parts(url)
#
# for each part, from longest possible to 0 length
#
if not len(url_parts):
#
# return seo if in cache
#
cache_key = 'seo::/'
seo = cache.get(cache_key)
if seo:
return seo
seo = Seo.objects.filter(url='/')
if seo:
cache.set(cache_key, seo[0], 3600)
return seo[0]
else:
for i in range(0, len(url_parts)):
url = ''
if i == 0:
url = build_url(url_parts)
else:
url = build_url(url_parts, -i)
#
# return seo if in cache
#
cache_key = 'seo::%s' % url
seo = cache.get(cache_key)
if seo:
return seo
#
# return seo if in db
#
seo = Seo.objects.filter(url=url)
if seo:
cache.set(cache_key, seo[0], 3600)
return seo[0]
return False
def test_route():
cache.set('key3', 1)
cache.inc('key1')
cache.get('key2')
cache.ttl('key2')
cache.ttl('key3')
cache.get('key3')
cache.delete('key3')
cache.delete('key4')
assert cache.has('key1')
assert flask.g.cache_keys['inc'] == {'key1'}
assert flask.g.cache_keys['get'] == {'key3'}
assert flask.g.cache_keys['set'] == {'key3'}
assert flask.g.cache_keys['delete'] == {'key3'}
assert flask.g.cache_keys['ttl'] == {'key3'}
assert flask.g.cache_keys['has'] == {'key1'}
return flask.jsonify('complete')
def test_forum_thread_subscriptions_cache_keys_user_ids(app, authed_client):
user_ids = ForumThreadSubscription.user_ids_from_thread(4) # noqa
cache.set(
ForumThreadSubscription.__cache_key_of_user__.format(user_id=1),
[14, 23],
)
cache.set(
ForumThreadSubscription.__cache_key_users__.format(thread_id=4),
[3, 4, 5],
)
assert 3 == len(
cache.get(
ForumThreadSubscription.__cache_key_users__.format(thread_id=4)))
ForumThreadSubscription.clear_cache_keys(user_ids=[1, 2])
assert 3 == len(
cache.get(
ForumThreadSubscription.__cache_key_users__.format(thread_id=4)))
assert not cache.get(
ForumThreadSubscription.__cache_key_of_user__.format(user_id=1))
def test_auth_updates(app, client):
"""Test that the cache key delay for updating an API key works."""
@app.route('/test_api_key')
def test_api_key():
return flask.jsonify('completed')
db.engine.execute(
"UPDATE api_keys SET ip = '127.0.0.1', user_agent = 'pulsar-test-client'"
)
cache.set('api_keys_abcdefghij_updated', 1, timeout=9000)
response = client.get(
'/test_api_key',
environ_base={
'HTTP_USER_AGENT': 'pulsar-test-client',
'REMOTE_ADDR': '127.0.0.1',
},
headers={'Authorization': f'Token abcdefghij{CODE_1}'},
)
check_json_response(response, 'completed')
assert cache.ttl('api_keys_abcdefghij_updated') > 8000
def update_api_key(api_key: APIKey) -> None:
"""
Update the provided session or api key's last seen times,
user agent, and IP fields.
:param session_key: The session or API key to update
"""
cache_key = f'{api_key.cache_key}_updated'
if (
not cache.get(cache_key)
or api_key.ip != flask.request.remote_addr
or api_key.user_agent != flask.request.headers.get('User-Agent')
):
api_key.last_used = datetime.utcnow().replace(tzinfo=pytz.utc)
api_key.user_agent = flask.request.headers.get('User-Agent')
api_key.ip = flask.request.remote_addr
db.session.commit()
cache.delete(api_key.cache_key)
cache.set(
cache_key, 1, timeout=60 * 2
) # 2 minute wait before next update
def get_col_from_many(
cls,
*,
column: InstrumentedAttribute,
key: str = None,
filter: BinaryExpression = None,
order: BinaryExpression = None,
) -> List[Any]:
"""
Get the values of a specific column from every row in the database.
:param column: The desired column
:param key: A cache key to save the resultant list in
:param filter: Filters to apply to the database query
:param order: How to order the values from the database query
:return: A list of values from the column
"""
values = cache.get(key) if key else None
if values is None:
query = cls._construct_query(db.session.query(column), filter,
order)
values = [x[0] for x in query.all()]
cache.set(key, values)
return values
def get_permissions():
cache_key = get_cache_key('permissions')
permissions = cache.get(cache_key)
if permissions:
return permissions
#
# discovery blocks in all apps
#
permissions = {}
for app in INSTALLED_APPS:
if app.startswith('django'):
continue
path = '%s/%s/permissions' % (PROJECT_ROOT, app)
for root, dir_names, file_names in os.walk(path):
for filename in fnmatch.filter(file_names, '*.py'):
permission_module_file_name = filename.replace('.py', '')
permission_module_path = root.replace('%s/' % PROJECT_ROOT, '').replace('/', '.')
module = get_module('%s.%s' % (permission_module_path, permission_module_file_name))
for permission in getmembers(module):
permission_path = ('%s.%s' % (permission_module_path, permission_module_file_name)).replace('.__init__', '')
permission_full_path = '%s.%s' % (permission_module_path, permission_module_file_name)
permission_module = permission_path.split('.')
permission_module = permission_module[-1]
if isfunction(permission[1]) and permission[1].func_globals['__name__'] == permission_full_path:
permission_key_name = '%s.%s' % (permission_module, permission[0])
permissions[permission_key_name] = dict(path=permission_path, name=permission[0], key_name=permission_key_name)
cache.set(cache_key, permissions, 86400)
return permissions
def post_from_attrs(cls, thread_id, user_id):
cache_key = cls.__cache_key__.format(
thread_id=thread_id, user_id=user_id
)
post_id = cache.get(cache_key)
if not post_id:
last_viewed = cls.query.filter(
and_((cls.thread_id == thread_id), (cls.user_id == user_id))
).scalar()
post_id = last_viewed.post_id if last_viewed else None
post = ForumPost.from_pk(post_id, include_dead=True)
if not post:
return None
if post.deleted: # Get the last non-deleted and read post.
post = ForumPost.from_query(
filter=and_(
(ForumPost.thread_id == thread_id),
(ForumPost.id < post.id),
(ForumPost.deleted == 'f'),
),
order=ForumPost.id.desc(),
) # type: ignore
cache.set(cache_key, post.id if post else None)
return post
def test_cache_set_key_bad(app, client, monkeypatch):
"""A bad key should not be added to the DB."""
monkeypatch.setattr('core.cache._client.setex', lambda *a, **k: False)
with app.test_request_context('/test'):
cache.set('blah', 2, timeout=60)
assert 'blah' not in flask.g.cache_keys['set']
