def get_installed_models(by_model=False): cache_key = get_cache_key('models', 'list', by_model) models = cache.get(cache_key) if models: return models # # do discovery on models # models = {} for app in INSTALLED_APPS: module_path = '%s.models' % app module = get_module(module_path) if module: for fixture in dir(module): if type(getattr(module, fixture)) == ModelBase: model_fixture = getattr(module, fixture) model_name = model_fixture._meta.db_table if by_model: model_name = fixture models[model_name] = dict( name=fixture, model=getattr(module, fixture) ) if not by_model: cache.set(cache_key, models) return models
def get_page_layout_config(page, default='empty', force_layout=False): layout_name = page.layout if page else default if force_layout: layout_name = default cache_key = get_cache_key('layout', layout_name) layout = cache.get(cache_key) if layout: layout = get_module(layout) return layout.config path = '' for app in INSTALLED_APPS: if app.startswith('django'): continue path = '%s.templates.layout.%s' % (app, layout_name) layout = get_module(path) if layout: break if not layout: return {} cache.set(cache_key, path) return layout.config
def latest_revision(cls, article_id: int, language_id: int = 1) -> int: cache_key = cls.__cache_key_latest_id_of_article__.format( article_id=article_id, language_id=language_id ) revision_id = cache.get(cache_key) if revision_id: return cls.from_attrs( revision_id=revision_id, article_id=article_id, language_id=language_id, ) else: latest_revision = ( cls.query.filter( and_( cls.article_id == article_id, cls.language_id == language_id, ) ) .order_by(cls.revision_id.desc()) # type: ignore .limit(1) .scalar() ) if not latest_revision: raise WikiNoRevisions cache.set(cache_key, latest_revision.revision_id) return latest_revision
def get_assets_config(): cache_key = get_cache_key('assets') config = cache.get(cache_key) if config: return config # # discovery blocks in all apps # config = {} for app in INSTALLED_APPS: if app.startswith('django'): continue module = get_module(app) if hasattr(module, 'wepo_assets'): app_assets = module.wepo_assets for asset_type, assets in app_assets.items(): if not asset_type in config: config[asset_type] = {} config[asset_type].update(assets.items()) cache.set(cache_key, config, 86400) return config
def permissions(self) -> List[str]: """ A general function to get the permissions of a user from a permission model and attributes of their user classes. Locked users are restricted to the permissions defined for them in the config. :param key: The cache key to cache the permissions under :param model: The model to query custom permissions from :param attr: The attribute of the userclasses that should be queried """ from core.permissions.models import SecondaryClass from core.permissions.models import UserPermission if self.locked: # Locked accounts have restricted permissions. return app.config['LOCKED_ACCOUNT_PERMISSIONS'] key = self.__cache_key_permissions__.format(id=self.id) permissions = cache.get(key) if not permissions: permissions = copy(self.user_class_model.permissions) for class_ in SecondaryClass.from_user(self.id): permissions += class_.permissions permissions = set(permissions) # De-dupe for perm, granted in UserPermission.from_user(self.id).items(): if not granted and perm in permissions: permissions.remove(perm) if granted and perm not in permissions: permissions.add(perm) cache.set(key, permissions) return permissions
def get_all_assets(): cache_key = get_cache_key('assets') assets = cache.get(cache_key) if assets: return assets # # discovery assets in all apps # assets = {'script': [], 'style': [], 'meta': []} for app in INSTALLED_APPS: if app.startswith('django'): continue module = get_module(app) if hasattr(module, 'wepo_assets'): wepo_assets = module.wepo_assets for asset_type in ['script', 'style', 'meta']: for asset, asset_conf in wepo_assets[asset_type].__dict__.items(): assets[asset_type].append(dict(name=asset, conf=asset_conf)) cache.set(cache_key, assets, 3600) return assets
def get_layouts(): cache_key = get_cache_key('layouts') layouts = cache.get(cache_key) if layouts: return layouts # # discovery blocks in all apps # layouts = [] for app in INSTALLED_APPS: if app.startswith('django'): continue path = '%s/%s/templates/layout' % (PROJECT_ROOT, app) for root, dir_names, file_names in os.walk(path): # # filter out only modules except __init__ # for filename in fnmatch.filter(file_names, '*.py'): layout_path = root.replace('%s/%s/templates/layout' % (PROJECT_ROOT, app), '') layout_module_path = root.replace('%s/' % PROJECT_ROOT, '').replace('/', '.') if layout_path: layout_path = layout_path[1:].replace('/', '.') layout_module = get_module(layout_module_path) layout_name = layout_module.name if hasattr(layout_module, 'name') else layout_path.replace('.', ' ').title() layouts.append({'name': layout_name, 'path': layout_path, 'full_path': layout_module_path}) cache.set(cache_key, layouts, 86400) return layouts
def test_clear_cache_keys(client): ckey = Notification.__cache_key_notification_count__.format( user_id=1, type=1 ) cache.set(ckey, 100) Notification.clear_cache_keys(1, 'subscripple') assert not cache.has(ckey)
def test_clear_cache_keys_without_type(client): ckey = Notification.__cache_key_notification_count__.format( user_id=1, type=1 ) cache.set(ckey, 100) Notification.clear_cache_keys(1) assert not cache.has(ckey)
def test_clear_cache_keys_wrong_type(client): ckey = Notification.__cache_key_notification_count__.format( user_id=1, type='subscripple' ) cache.set(ckey, 100) with pytest.raises(APIException): Notification.clear_cache_keys(1, 'not_real_type')
def count( cls, *, key: str, attribute: InstrumentedAttribute, filter: BinaryExpression = None, ) -> int: """ An abstracted function for counting a number of elements matching a query. If the passed cache key exists, its value will be returned; otherwise, the passed query will be ran and the resultant count cached and returned. :param key: The cache key to check :param attribute: The attribute to count; a model's column :param filter: The SQLAlchemy filter expression :return: The number of rows matching the query element """ count = cache.get(key) if not isinstance(count, int): query = cls._construct_query( db.session.query(func.count(attribute)), filter) count = query.scalar() cache.set(key, count) return count
def get_form_items(): cache_key = get_cache_key('form_elements') form_items = cache.get(cache_key) if form_items: return form_items # # discovery form items in all apps # form_items = {} for app in INSTALLED_APPS: if app.startswith('django'): continue path = '%s.form.elements' % app module = get_module(path) if not module: continue for element in dir(module): if not element.startswith('__'): form_items[element] = path cache.set(cache_key, form_items, 86400) return form_items
def test_cache_delete_many(app, client): cache.set('key_1', 1) cache.set('key_2', 2) assert cache.delete_many('key_1', 'key_2', 'key_3') with app.test_request_context('/test'): assert 'key_1' in flask.g.cache_keys['delete_many'] assert 'key_2' in flask.g.cache_keys['delete_many'] assert 'key_3' in flask.g.cache_keys['delete_many']
def test_post_edit_history_from_cache(app, authed_client): cache.set(ForumPostEditHistory.__cache_key_of_post__.format(id=3), [1]) ForumPostEditHistory.from_pk(1) # cache this history = ForumPostEditHistory.from_post(3) assert len(history) == 1 assert any( h.contents == 'Why the fcuk is Gazelle in HPH?' for h in history )
def test_cache_get_dict_order(app, client): cache.set('key_4', 4) cache.set('key_3', 3) cache.set('key_5', 5) cache.set('key_1', 1) cache.set('key_2', 2) data = cache.get_dict(*('key_2', 'key_1', 'key_3', 'key_4', 'key_5')) assert list(data.keys()) == ['key_2', 'key_1', 'key_3', 'key_4', 'key_5']
def test_cache_get_dict(app, client): cache.set('key_1', 1) cache.set('key_2', 2) data = cache.get_dict(*('keY_1', 'kEy_2', 'key_3')) assert data == {'key_1': 1, 'key_2': 2, 'key_3': None} with app.test_request_context('/test'): assert 'key_1' in flask.g.cache_keys['get_dict'] assert 'key_2' in flask.g.cache_keys['get_dict'] assert 'key_3' in flask.g.cache_keys['get_dict']
def test_thread_last_viewed_post_cached(app, authed_client): cache.set(ForumLastViewedPost.__cache_key__.format(thread_id=1, user_id=1), 2) thread = ForumThread.from_pk(1) last_post = thread.last_viewed_post assert last_post.id == 2 assert last_post.thread_id == 3 assert 2 == cache.get( ForumLastViewedPost.__cache_key__.format(thread_id=1, user_id=1))
def translate(request, phrase, language=None, translation_type=None): if not language: language = get_user_language(request) # Not implemented yet if translation_type: cache_key = get_cache_key('translation', language, 'group', translation_type) translations = cache.get(cache_key) if not translations: translations = {} all_translations = None #Translation.objects.filter(language__code=language, translation_type__name=translation_type) if all_translations: for translation in all_translations: translations[translation.phrase.text] = translation.text cache.set(cache_key, translations) if phrase in translations: return translations[phrase] return phrase else: from core.helper.common_help import get_cache_key_2 phrase_key_name = get_cache_key_2(phrase) cache_key = get_cache_key('translation', language, phrase_key_name) translation = cache.get(cache_key) result = phrase if not translation: phrase_obj = Phrase.objects.filter(key_name=phrase_key_name) if phrase_obj and len(phrase_obj) > 0: phrase_obj = phrase_obj[0] translations = Translation.objects.filter(language__code=language, phrase=phrase_obj) if translations and len(translations) > 0: translation = translations[0] stat = cache.set(cache_key, translation) else: new_translation = Translation() new_translation.translation = phrase new_translation.phrase = phrase_obj new_translation.language = Language.objects.get(code=language) new_translation.save() else: new_phrase = Phrase() new_phrase.text = phrase new_phrase.save() new_translation = Translation() new_translation.translation = phrase new_translation.phrase = new_phrase new_translation.language = Language.objects.get(code=language) new_translation.save() else: result = translation.translation return result
def get_pks_of_many( cls, key: str = None, filter: BinaryExpression = None, order: BinaryExpression = None, include_dead: bool = False, expr_override: BinaryExpression = None, ) -> List[Union[int, str]]: """ Get a list of object IDs meeting query criteria. Fetching from the cache with the provided cache key will be attempted first; if the cache key does not exist then a query will be ran. Calls with ``include_dead=True`` are saved under a different cache key. ``include_dead`` does not affect the query results when passing ``expr_override``. :param key: The cache key to check (and return if present) :param filter: A SQLAlchemy filter expression to be applied to the query :param order: A SQLAlchemy order_by expression to be applied to the query :param include_dead: Whether or not to include dead results in the IDs list :param expr_override: If passed, this will override filter and order, and be called verbatim in a ``db.session.execute`` if the cache key does not exist :return: A list of IDs """ key = f'{key}_incl_dead' if include_dead and key else key pks = cache.get(key) if key else None if not pks or not isinstance(pks, list): if expr_override is not None: pks = [ x[0] if len(x) == 1 else dict(x) for x in db.session.execute(expr_override) ] else: primary_key = cls.get_primary_key() if isinstance(primary_key, list): query = cls._construct_query( db.session.query(*(getattr(cls, k) for k in primary_key)), filter, order, ) else: query = cls._construct_query( db.session.query(getattr(cls, primary_key)), filter, order, ) if not include_dead and cls.__deletion_attr__: query = query.filter( getattr(cls, cls.__deletion_attr__) == 'f') pks = [ x[0] if len(x) == 1 else x._asdict() for x in query.all() ] if key: cache.set(key, pks) return pks
def get_rules(section: str) -> dict: if section not in SECTION_NAMES: raise APIException(f'{section} is not a valid section of the rules.') rules = cache.get(f'rules_{section}') if not rules: filename = os.path.join(os.path.dirname(__file__), f'{section}.json') with open(filename, 'r') as f: rules = json.load(f) cache.set(f'rules_{section}', rules, 0) return rules
def test_view_all_keys_cached(app, authed_client): add_permissions(app, ApikeyPermissions.VIEW) cache_key = APIKey.__cache_key_of_user__.format(user_id=1) cache.set(cache_key, ['abcdefghij', 'bcdefghijk'], timeout=60) response = authed_client.get('/api_keys') data = response.get_json()['response'] assert any('hash' in api_key and api_key['hash'] == CODE_2[:10] for api_key in data) assert cache.ttl(cache_key)
def test_category_get_all_cached(app, authed_client): cache.set(ForumCategory.__cache_key_all__, [1, 3], timeout=60) categories = ForumCategory.get_all() assert len(categories) == 2 for category in categories: if category.name == 'Site' and category.id == 1: break else: raise AssertionError('A real forum not called')
def test_get_from_cache(app, authed_client): """Test that cache values are used instead of querying a user.""" add_permissions(app, 'users_view') user = User.from_pk(1) data = {} for attr in user.__table__.columns.keys(): data[attr] = getattr(user, attr, None) data['username'] = '******' cache.set(user.cache_key, data) user = User.from_pk(1) assert user.username == 'fakeshit'
def test_thread_get_from_forum_cached(app, authed_client): cache.set(ForumThread.__cache_key_of_forum__.format(id=2), [1, 5], timeout=60) ForumThread.from_pk(1) ForumThread.from_pk(5) # noqa cache these threads = ForumThread.from_forum(2, page=1, limit=50) assert len(threads) == 2 for thread in threads: if thread.topic == 'Donations?' and thread.id == 5: break else: raise AssertionError('A real thread not called')
def test_forum_get_from_category_cached(app, authed_client): cache.set(Forum.__cache_key_of_category__.format(id=2), ['1', '5'], timeout=60) Forum.from_pk(1) Forum.from_pk(5) # noqa: cache these forums = Forum.from_category(2) assert len(forums) == 2 for forum in forums: if forum.name == 'Yacht Funding' and forum.id == 5: break else: raise AssertionError('A real forum not called')
def test_post_get_from_thread_cached(app, authed_client): cache.set( ForumPost.__cache_key_of_thread__.format(id=2), ['1', '6'], timeout=60 ) ForumPost.from_pk(1) ForumPost.from_pk(6) # noqa cache this posts = ForumPost.from_thread(2, page=1, limit=50) assert len(posts) == 2 for post in posts: if post.contents == 'Delete this' and post.id == 6: break else: raise AssertionError('A real post not called')
def wrapper(*args, **kwargs): arg = '-'.join(map(str, args)) key = kw + '-' + arg data = cache.get(key) if not data: # print key, 'notin' data = func(*args, **kwargs) if timeout: cache.set(key, data, timeout) else: cache.set(key, data) # else: # print key, 'bingo' return data
def test_rate_limit_function_global(app, client, monkeypatch): """Test that the per-user rate limit function correctly increments and errors.""" monkeypatch.setattr( 'core.hooks.before.flask.g', g( user=User.from_pk(1), api_key=api_key(hash='abcdefghij'), cache_keys=defaultdict(set), ), ) cache.set('rate_limit_user_1', 70, timeout=60) with pytest.raises(APIException) as e: for i in range(31): check_rate_limit() assert 'User rate limit exceeded.' in e.value.message
def get_model(request, key_name, by_model=False): cache_key = get_cache_key('model', key_name) model = cache.get(cache_key) if model: return model # # get model from installed models # models = get_installed_models(by_model=by_model) if key_name in models: cache.set(cache_key, models[key_name]) return models[key_name] return None
def test_clear_cache_keys(client): for uid in [1, 2]: for f in ['inbox', 'sentbox', 'deleted']: cache.set( PrivateConversation.__cache_key_of_user__.format(user_id=uid, filter=f), 1, ) PrivateConversation.clear_cache_keys(1) for f in ['inbox', 'sentbox', 'deleted']: assert not cache.has( PrivateConversation.__cache_key_of_user__.format(user_id=1, filter=f)) assert cache.has( PrivateConversation.__cache_key_of_user__.format(user_id=2, filter=f))
def test_cache_inc_key_already_exists(app, client): """Incrementing an already-existing key just increments it.""" assert cache.set('test-inc-key', 3, timeout=15) value = cache.inc('test-inc-key', 4, timeout=80) time_left = cache.ttl('test-inc-key') assert value == 7 assert time_left > 13 and time_left < 16
def get_seo(request, url=None): url_parts = request.wepo.url_parts if url: url, url_parts = get_url_parts(url) # # for each part, from longest possible to 0 length # if not len(url_parts): # # return seo if in cache # cache_key = 'seo::/' seo = cache.get(cache_key) if seo: return seo seo = Seo.objects.filter(url='/') if seo: cache.set(cache_key, seo[0], 3600) return seo[0] else: for i in range(0, len(url_parts)): url = '' if i == 0: url = build_url(url_parts) else: url = build_url(url_parts, -i) # # return seo if in cache # cache_key = 'seo::%s' % url seo = cache.get(cache_key) if seo: return seo # # return seo if in db # seo = Seo.objects.filter(url=url) if seo: cache.set(cache_key, seo[0], 3600) return seo[0] return False
def test_route(): cache.set('key3', 1) cache.inc('key1') cache.get('key2') cache.ttl('key2') cache.ttl('key3') cache.get('key3') cache.delete('key3') cache.delete('key4') assert cache.has('key1') assert flask.g.cache_keys['inc'] == {'key1'} assert flask.g.cache_keys['get'] == {'key3'} assert flask.g.cache_keys['set'] == {'key3'} assert flask.g.cache_keys['delete'] == {'key3'} assert flask.g.cache_keys['ttl'] == {'key3'} assert flask.g.cache_keys['has'] == {'key1'} return flask.jsonify('complete')
def test_forum_thread_subscriptions_cache_keys_user_ids(app, authed_client): user_ids = ForumThreadSubscription.user_ids_from_thread(4) # noqa cache.set( ForumThreadSubscription.__cache_key_of_user__.format(user_id=1), [14, 23], ) cache.set( ForumThreadSubscription.__cache_key_users__.format(thread_id=4), [3, 4, 5], ) assert 3 == len( cache.get( ForumThreadSubscription.__cache_key_users__.format(thread_id=4))) ForumThreadSubscription.clear_cache_keys(user_ids=[1, 2]) assert 3 == len( cache.get( ForumThreadSubscription.__cache_key_users__.format(thread_id=4))) assert not cache.get( ForumThreadSubscription.__cache_key_of_user__.format(user_id=1))
def test_auth_updates(app, client): """Test that the cache key delay for updating an API key works.""" @app.route('/test_api_key') def test_api_key(): return flask.jsonify('completed') db.engine.execute( "UPDATE api_keys SET ip = '127.0.0.1', user_agent = 'pulsar-test-client'" ) cache.set('api_keys_abcdefghij_updated', 1, timeout=9000) response = client.get( '/test_api_key', environ_base={ 'HTTP_USER_AGENT': 'pulsar-test-client', 'REMOTE_ADDR': '127.0.0.1', }, headers={'Authorization': f'Token abcdefghij{CODE_1}'}, ) check_json_response(response, 'completed') assert cache.ttl('api_keys_abcdefghij_updated') > 8000
def update_api_key(api_key: APIKey) -> None: """ Update the provided session or api key's last seen times, user agent, and IP fields. :param session_key: The session or API key to update """ cache_key = f'{api_key.cache_key}_updated' if ( not cache.get(cache_key) or api_key.ip != flask.request.remote_addr or api_key.user_agent != flask.request.headers.get('User-Agent') ): api_key.last_used = datetime.utcnow().replace(tzinfo=pytz.utc) api_key.user_agent = flask.request.headers.get('User-Agent') api_key.ip = flask.request.remote_addr db.session.commit() cache.delete(api_key.cache_key) cache.set( cache_key, 1, timeout=60 * 2 ) # 2 minute wait before next update
def get_col_from_many( cls, *, column: InstrumentedAttribute, key: str = None, filter: BinaryExpression = None, order: BinaryExpression = None, ) -> List[Any]: """ Get the values of a specific column from every row in the database. :param column: The desired column :param key: A cache key to save the resultant list in :param filter: Filters to apply to the database query :param order: How to order the values from the database query :return: A list of values from the column """ values = cache.get(key) if key else None if values is None: query = cls._construct_query(db.session.query(column), filter, order) values = [x[0] for x in query.all()] cache.set(key, values) return values
def get_permissions(): cache_key = get_cache_key('permissions') permissions = cache.get(cache_key) if permissions: return permissions # # discovery blocks in all apps # permissions = {} for app in INSTALLED_APPS: if app.startswith('django'): continue path = '%s/%s/permissions' % (PROJECT_ROOT, app) for root, dir_names, file_names in os.walk(path): for filename in fnmatch.filter(file_names, '*.py'): permission_module_file_name = filename.replace('.py', '') permission_module_path = root.replace('%s/' % PROJECT_ROOT, '').replace('/', '.') module = get_module('%s.%s' % (permission_module_path, permission_module_file_name)) for permission in getmembers(module): permission_path = ('%s.%s' % (permission_module_path, permission_module_file_name)).replace('.__init__', '') permission_full_path = '%s.%s' % (permission_module_path, permission_module_file_name) permission_module = permission_path.split('.') permission_module = permission_module[-1] if isfunction(permission[1]) and permission[1].func_globals['__name__'] == permission_full_path: permission_key_name = '%s.%s' % (permission_module, permission[0]) permissions[permission_key_name] = dict(path=permission_path, name=permission[0], key_name=permission_key_name) cache.set(cache_key, permissions, 86400) return permissions
def post_from_attrs(cls, thread_id, user_id): cache_key = cls.__cache_key__.format( thread_id=thread_id, user_id=user_id ) post_id = cache.get(cache_key) if not post_id: last_viewed = cls.query.filter( and_((cls.thread_id == thread_id), (cls.user_id == user_id)) ).scalar() post_id = last_viewed.post_id if last_viewed else None post = ForumPost.from_pk(post_id, include_dead=True) if not post: return None if post.deleted: # Get the last non-deleted and read post. post = ForumPost.from_query( filter=and_( (ForumPost.thread_id == thread_id), (ForumPost.id < post.id), (ForumPost.deleted == 'f'), ), order=ForumPost.id.desc(), ) # type: ignore cache.set(cache_key, post.id if post else None) return post
def test_cache_set_key_bad(app, client, monkeypatch): """A bad key should not be added to the DB.""" monkeypatch.setattr('core.cache._client.setex', lambda *a, **k: False) with app.test_request_context('/test'): cache.set('blah', 2, timeout=60) assert 'blah' not in flask.g.cache_keys['set']