Exemplo n.º 1
0
def is_verified(verifying_client: VerifyingClient) -> bool:
    verification_model = None
    verification_model_cache = cache.get(
        cache_key('verifications', verifying_client.verification_id))
    if verification_model_cache:
        verification_model = VerificationsModel.from_cache(
            verification_model_cache)

    if not verification_model:
        with session() as s:
            q = s.query(VerificationOrmModel)
            q = q.filter_by(verification_id=verifying_client.verification_id)

            verifications_orm_model = q.one_or_none()
            if verifications_orm_model:
                verification_model = VerificationsModel.from_orm_model(
                    verifications_orm_model)

                cached = verification_model.to_cache()
                timeout = max(1, (verification_model.expires - now()) // 1000)
                cache.set(cache_key('verifications', verification_model.id),
                          cached,
                          timeout=timeout)
            s.commit()

    return verification_model and _is_verifications_valid(
        verifying_client, verification_model)
Exemplo n.º 2
0
def find_by_type(page_type: str) -> 'List[PageModel]':
    _check_page_type(page_type)

    lc = local_cache.get(cache_key('type', page_type))
    if lc:
        return list(map(lambda i: i.copy(), lc))

    pages_by_type_cached = cache.get(cache_key('pages_by_type', page_type))
    if pages_by_type_cached is not None:
        res = list(map(lambda i: PageModel.from_cache(i),
                       pages_by_type_cached))
    else:
        with session() as s:
            q = s.query(PageOrmModel).filter_by(type=page_type)
            q = q.order_by(asc(PageOrmModel.order))
            res = list(map(lambda i: PageModel.from_orm_model(i), q.all()))

            cache.set(cache_key('pages_by_type', page_type),
                      list(map(lambda i: i.to_cache(), res)))

            s.commit()

    local_cache.set(cache_key('type', page_type), res)

    return res
Exemplo n.º 3
0
def find_thread_by_board_thread_refno_with_posts(
        board: BoardModel, thread_refno: int) -> Optional[ThreadModel]:
    thread_cache = cache.get(cache_key('thread', board.name, thread_refno))
    if not thread_cache:
        with session() as s:
            q = s.query(ThreadOrmModel)
            q = q.options(lazyload('posts'))
            q = q.filter(ThreadOrmModel.refno == thread_refno,
                         ThreadOrmModel.board_id == BoardOrmModel.id,
                         BoardOrmModel.name == board.name)
            thread_orm_model = q.one_or_none()

            if not thread_orm_model or not thread_orm_model.posts:
                return None

            # TODO: also load board in q above
            thread = ThreadModel.from_orm_model(thread_orm_model,
                                                include_board=True,
                                                include_posts=True)
            thread_cache = thread.to_cache(include_board=True,
                                           include_posts=True)
            cache.set(cache_key('thread', thread.board.name, thread.refno),
                      thread_cache,
                      timeout=0)
            return thread

    if thread_cache:
        return ThreadModel.from_cache(thread_cache)
    return None
Exemplo n.º 4
0
def _invalidate_thread_cache(s: Session, old_thread: ThreadModel, board: BoardModel):
    """
    Update the memcache version of the specified thread. This will update the thread cache,
    and the thread stub cache.
    """
    key = cache_key('thread', board.name, old_thread.refno)
    stub_key = cache_key('thread_stub', board.name, old_thread.refno)

    # Reuse the parsed html from the old cache.
    old_thread_posts_cache = cache.get(key)
    old_thread_posts = None
    if old_thread_posts_cache:
        old_thread_posts = ThreadModel.from_cache(old_thread_posts_cache).posts

    # Next, query all the new posts
    q = s.query(ThreadOrmModel)
    q = q.filter_by(id=old_thread.id)
    q = q.options(lazyload('posts'))
    res = q.one_or_none()
    if not res:
        cache.delete(key)
        cache.delete(stub_key)
        return

    thread = ThreadModel.from_orm_model(res, include_board=True, include_posts=True,
                                        cached_thread_posts=old_thread_posts)

    thread_cache = thread.to_cache(include_board=True, include_posts=True)
    cache.set(key, thread_cache, timeout=0)

    thread_stub = ThreadStubModel.from_thread(thread, include_snippets=True)
    thread_stub_cache = thread_stub.to_cache()
    cache.set(stub_key, thread_stub_cache, timeout=0)

    return thread, thread_stub
Exemplo n.º 5
0
def create_thread(board: BoardModel, post: PostModel) \
        -> Tuple[PostResultModel, int, int]:
    start_time = now()
    with session() as s:
        board_orm_model = s.query(BoardOrmModel).filter_by(id=board.id).one()

        thread_orm_model = ThreadOrmModel()
        thread_orm_model.last_modified = now()
        thread_orm_model.refno = 0
        thread_orm_model.board_id = board.id

        post_orm_model = post.to_orm_model()

        post_orm_model.thread = thread_orm_model
        post_orm_model.refno = 1
        s.add(thread_orm_model)

        # Atomically update the refno counter
        board_orm_model.refno_counter = BoardOrmModel.refno_counter + 1
        s.commit()

        # Set it to the board after the commit to make sure there aren't any duplicates
        thread_refno = thread_orm_model.refno = board_orm_model.refno_counter

        # Attach file to the post id
        if post.file:
            file_orm_model = post.file.to_orm_model()
            file_orm_model.post_id = post_orm_model.id
            s.add(file_orm_model)

        if post.moderator:
            post_orm_model.moderator_id = post.moderator.id

        # Purge overflowed threads
        threads_refnos_to_purge = _purge_threads(s, board, board.config.pages,
                                                 board.config.per_page)
        s.commit()

        insert_time = now() - start_time
        start_time = now()

        for purging_refno in threads_refnos_to_purge:
            cache.delete(cache_key('thread', board.name, purging_refno))
            cache.delete(cache_key('thread_stub', board.name, purging_refno))

        thread = ThreadModel.from_orm_model(thread_orm_model)
        _invalidate_thread_cache(s, thread, board)
        _invalidate_board_pages_catalog_cache(s, board)

        document_cache.purge_board(board)

        cache_time = now() - start_time

        res = PostResultModel.from_board_name_thread_refno_post_refno(
            board.name, thread_refno, 1)
        return res, insert_time, cache_time
Exemplo n.º 6
0
def _invalidate_board_pages_catalog_cache(s: Session, board: BoardModel):
    """
    Update the memcache version of the specified board.
    This will update the board pages from the already cached thread stubs, and create a new catalog cache.
    """

    q = s.query(ThreadOrmModel)
    q = q.filter(ThreadOrmModel.board_id == board.id)
    threads_orm = q.all()
    thread_models = list(map(lambda j: ThreadModel.from_orm_model(j, ), threads_orm))

    # This builds the board index, stickies first, oldest first, then normal posts, newest first.
    # The pages are split accordingly to the board config,
    # and the catalog is build from only the ops.
    stickies = []
    threads = []
    for thread in thread_models:
        thread_stub_cache = cache.get(cache_key('thread_stub', board.name, thread.refno))
        if not thread_stub_cache:
            thread, thread_stub = _invalidate_thread_cache(s, thread, board)
            # The board and thread selects are done separately and there is thus the
            # possibility that the thread was removed after the board select
            if thread_stub is None:
                continue
        else:
            thread_stub = ThreadStubModel.from_cache(thread_stub_cache)

        stickies.append(thread_stub) if thread_stub.sticky else threads.append(thread_stub)

    stickies = sorted(stickies, key=lambda t: t.last_modified, reverse=False)
    threads = sorted(threads, key=lambda t: t.last_modified, reverse=True)
    all_thread_stubs = stickies + threads

    # The catalog is a CatalogModel with ThreadStubs with only OP's
    catalog = CatalogModel.from_board_thread_stubs(board, all_thread_stubs)
    catalog_cache = catalog.to_cache()
    cache.set(cache_key('board', board.name), catalog_cache, timeout=0)

    # All threads with stubs, divided per page
    # note: there is the possibility that concurrent processes updating this cache
    # mess up the order / create duplicates of the page threads
    # this chance is however very low, and has no ill side effects except for
    # a visual glitch
    board_pages = []
    for i in range(board.config.pages):
        from_index = i * board.config.per_page
        to_index = (i + 1) * board.config.per_page

        board_page = BoardPageModel.from_page_thread_stubs(i, all_thread_stubs[from_index:to_index])
        board_pages.append(board_page)

        board_page_cache = board_page.to_cache()
        cache.set(cache_key('board', board.name, i), board_page_cache, timeout=0)

    return catalog, board_pages
Exemplo n.º 7
0
def create_thread(board: BoardModel, post: PostModel) \
        -> Tuple[PostResultModel, int, int]:
    start_time = now()
    with session() as s:
        board_orm_model = s.query(BoardOrmModel).filter_by(id=board.id).one()

        thread_orm_model = ThreadOrmModel()
        thread_orm_model.last_modified = now()
        thread_orm_model.refno = 0
        thread_orm_model.board_id = board.id

        post_orm_model = post.to_orm_model()

        post_orm_model.thread = thread_orm_model
        post_orm_model.refno = 1
        s.add(thread_orm_model)

        # Atomically update the refno counter
        board_orm_model.refno_counter = BoardOrmModel.refno_counter + 1
        s.commit()

        # Set it to the board after the commit to make sure there aren't any duplicates
        thread_refno = thread_orm_model.refno = board_orm_model.refno_counter

        # Attach file to the post id
        if post.file:
            file_orm_model = post.file.to_orm_model()
            file_orm_model.post_id = post_orm_model.id
            s.add(file_orm_model)

        if post.moderator:
            post_orm_model.moderator_id = post.moderator.id

        # Purge overflowed threads
        threads_refnos_to_purge = _purge_threads(s, board, board.config.pages, board.config.per_page)
        s.commit()

        insert_time = now() - start_time
        start_time = now()

        for purging_refno in threads_refnos_to_purge:
            cache.delete(cache_key('thread', board.name, purging_refno))
            cache.delete(cache_key('thread_stub', board.name, purging_refno))

        thread = ThreadModel.from_orm_model(thread_orm_model)
        _invalidate_thread_cache(s, thread, board)
        _invalidate_board_pages_catalog_cache(s, board)

        document_cache.purge_board(board)

        cache_time = now() - start_time

        res = PostResultModel.from_board_name_thread_refno_post_refno(board.name, thread_refno, 1)
        return res, insert_time, cache_time
Exemplo n.º 8
0
def create(board: BoardModel) -> BoardModel:
    if not validation.check_board_name_validity(board.name):
        raise ArgumentError(MESSAGE_INVALID_NAME)

    with session() as s:
        existing = s.query(BoardOrmModel).filter_by(name=board.name).one_or_none()
        if existing:
            raise ArgumentError(MESSAGE_DUPLICATE_BOARD_NAME)

        orm_board = board.to_orm_model()

        board_config = BoardConfigModel.from_defaults()
        board_config_orm = board_config.to_orm_model()
        s.add(board_config_orm)
        s.flush()

        orm_board.config_id = board_config_orm.id

        s.add(orm_board)
        s.commit()

        board = board.from_orm_model(orm_board)

        cache.set(cache_key('board_and_config', board.name), board.to_cache())

        _set_all_board_names_cache(s)

        s.commit()

        return board
Exemplo n.º 9
0
def get_catalog(board: BoardModel) -> CatalogModel:
    catalog_cache = cache.get(cache_key('board', board.name))
    if not catalog_cache:
        with session() as s:
            catalog, board_pages = _invalidate_board_pages_catalog_cache(s, board)
            return catalog

    return CatalogModel.from_cache(catalog_cache)
Exemplo n.º 10
0
def get_board_page(board: BoardModel, page: int) -> BoardPageModel:
    board_page_cache = cache.get(cache_key('board', board.name, page))
    if not board_page_cache:
        with session() as s:
            catalog, board_pages = _invalidate_board_pages_catalog_cache(s, board)
            return board_pages[page]

    return BoardPageModel.from_cache(board_page_cache)
Exemplo n.º 11
0
def find_by_name(name: str) -> Optional[BoardModel]:
    if not validation.check_board_name_validity(name):
        raise ArgumentError(MESSAGE_INVALID_NAME)

    board_cache = cache.get(cache_key('board_and_config', name))
    if not board_cache:
        with session() as s:
            q = s.query(BoardOrmModel).filter_by(name=name)
            q = q.options(joinedload('config'))
            board_orm_model = q.one_or_none()
            if not board_orm_model:
                return None
            board = BoardModel.from_orm_model(board_orm_model, include_config=True)
            cache.set(cache_key('board_and_config', name), board.to_cache())
            return board

    return BoardModel.from_cache(board_cache)
Exemplo n.º 12
0
def get_board_page(board: BoardModel, page: int) -> BoardPageModel:
    board_page_cache = cache.get(cache_key('board', board.name, page))
    if not board_page_cache:
        with session() as s:
            catalog, board_pages = _invalidate_board_pages_catalog_cache(
                s, board)
            return board_pages[page]

    return BoardPageModel.from_cache(board_page_cache)
Exemplo n.º 13
0
def get_catalog(board: BoardModel) -> CatalogModel:
    catalog_cache = cache.get(cache_key('board', board.name))
    if not catalog_cache:
        with session() as s:
            catalog, board_pages = _invalidate_board_pages_catalog_cache(
                s, board)
            return catalog

    return CatalogModel.from_cache(catalog_cache)
Exemplo n.º 14
0
def delete(page: PageModel):
    with session() as s:
        m = s.query(PageOrmModel).filter_by(id=page.id).one()
        s.delete(m)
        s.flush()

        cache.delete(cache_key('page_by_link_name', page.link_name))
        _cache_pages_by_type(s, page.type)

        s.commit()
Exemplo n.º 15
0
def delete(page: PageModel):
    with session() as s:
        m = s.query(PageOrmModel).filter_by(id=page.id).one()
        s.delete(m)
        s.flush()

        cache.delete(cache_key('page_by_link_name', page.link_name))
        _cache_pages_by_type(s, page.type)

        s.commit()
Exemplo n.º 16
0
def get_site() -> SiteConfigModel:
    local_cached = local_site_config_cache.get('site_config')
    if local_cached:
        return local_cached.copy()

    cached = cache.get(cache_key('config_site'))
    if cached:
        res = SiteConfigModel.from_cache(cached)
    else:
        with session() as s:
            m = s.query(ConfigOrmModel).filter_by(type='site').one_or_none()
            if m:
                res = SiteConfigModel.from_orm_model(m)
            else:
                res = SiteConfigModel.from_defaults()
            s.commit()

            cache.set(cache_key('config_site'), res.to_cache())

    local_site_config_cache.set('site_config', res)
    return res
Exemplo n.º 17
0
def get_site() -> SiteConfigModel:
    local_cached = local_site_config_cache.get('site_config')
    if local_cached:
        return local_cached.copy()

    cached = cache.get(cache_key('config_site'))
    if cached:
        res = SiteConfigModel.from_cache(cached)
    else:
        with session() as s:
            m = s.query(ConfigOrmModel).filter_by(type='site').one_or_none()
            if m:
                res = SiteConfigModel.from_orm_model(m)
            else:
                res = SiteConfigModel.from_defaults()
            s.commit()

            cache.set(cache_key('config_site'), res.to_cache())

    local_site_config_cache.set('site_config', res)
    return res
Exemplo n.º 18
0
def is_verified(verifying_client: VerifyingClient) -> bool:
    verification_model = None
    verification_model_cache = cache.get(cache_key('verifications', verifying_client.verification_id))
    if verification_model_cache:
        verification_model = VerificationsModel.from_cache(verification_model_cache)

    if not verification_model:
        with session() as s:
            q = s.query(VerificationOrmModel)
            q = q.filter_by(verification_id=verifying_client.verification_id)

            verifications_orm_model = q.one_or_none()
            if verifications_orm_model:
                verification_model = VerificationsModel.from_orm_model(verifications_orm_model)

                cached = verification_model.to_cache()
                timeout = max(1, (verification_model.expires - now()) // 1000)
                cache.set(cache_key('verifications', verification_model.id), cached, timeout=timeout)
            s.commit()

    return verification_model and _is_verifications_valid(verifying_client, verification_model)
Exemplo n.º 19
0
def find_by_names(names: List[str]) -> List[BoardModel]:
    """unknown names are ignored!"""

    for name in names:
        if not validation.check_board_name_validity(name):
            raise ArgumentError(MESSAGE_INVALID_NAME)

    boards = []
    with session() as s:
        for name in names:
            board_cache = cache.get(cache_key('board_and_config', name))
            if board_cache:
                boards.append(BoardModel.from_cache(board_cache))
            else:
                board_orm_model = s.query(BoardOrmModel).filter_by(name=name).one_or_none()
                if board_orm_model:
                    board = BoardModel.from_orm_model(board_orm_model, include_config=True)
                    cache.set(cache_key('board_and_config', name), board.to_cache())
                    boards.append(board)

    return boards
Exemplo n.º 20
0
def get_all_board_names() -> List[str]:
    local_cached = local_cache.get('all_board_names')
    if local_cached:
        return local_cached

    all_board_names_cached = cache.get(cache_key('all_board_names'))
    if all_board_names_cached is not None:
        # No need to map a list of strings
        res = all_board_names_cached
    else:
        with session() as s:
            q = s.query(BoardOrmModel).options(load_only('name')).order_by(BoardOrmModel.name)
            # No mapping here either
            res = list(map(lambda i: i.name, q.all()))
            s.commit()

        cache.set(cache_key('all_board_names'), res)

    local_cache.set('all_board_names', res)

    return res
Exemplo n.º 21
0
def find_by_link_name(link_name: str) -> PageModel:
    lc = local_cache.get(cache_key('link_name', link_name))
    if lc:
        return lc.copy()

    page_cached = cache.get(cache_key('page_by_link_name', link_name))
    if page_cached:
        return PageModel.from_cache(page_cached)
    else:
        with session() as s:
            m = s.query(PageOrmModel).filter_by(link_name=link_name).one_or_none()
            res = None
            if m:
                res = PageModel.from_orm_model(m)

                cache.set(cache_key('page_by_link_name', res.link_name), res.to_cache())

    if res:
        local_cache.set(cache_key('link_name', link_name), res)

    return res
Exemplo n.º 22
0
def _invalidate_thread_cache(s: Session, old_thread: ThreadModel,
                             board: BoardModel):
    """
    Update the memcache version of the specified thread. This will update the thread cache,
    and the thread stub cache.
    """
    key = cache_key('thread', board.name, old_thread.refno)
    stub_key = cache_key('thread_stub', board.name, old_thread.refno)

    # Reuse the parsed html from the old cache.
    old_thread_posts_cache = cache.get(key)
    old_thread_posts = None
    if old_thread_posts_cache:
        old_thread_posts = ThreadModel.from_cache(old_thread_posts_cache).posts

    # Next, query all the new posts
    q = s.query(ThreadOrmModel)
    q = q.filter_by(id=old_thread.id)
    q = q.options(lazyload('posts'))
    res = q.one_or_none()
    if not res:
        cache.delete(key)
        cache.delete(stub_key)
        return

    thread = ThreadModel.from_orm_model(res,
                                        include_board=True,
                                        include_posts=True,
                                        cached_thread_posts=old_thread_posts)

    thread_cache = thread.to_cache(include_board=True, include_posts=True)
    cache.set(key, thread_cache, timeout=0)

    thread_stub = ThreadStubModel.from_thread(thread, include_snippets=True)
    thread_stub_cache = thread_stub.to_cache()
    cache.set(stub_key, thread_stub_cache, timeout=0)

    return thread, thread_stub
Exemplo n.º 23
0
def find_by_type(page_type: str) -> 'List[PageModel]':
    _check_page_type(page_type)

    lc = local_cache.get(cache_key('type', page_type))
    if lc:
        return list(map(lambda i: i.copy(), lc))

    pages_by_type_cached = cache.get(cache_key('pages_by_type', page_type))
    if pages_by_type_cached is not None:
        res = list(map(lambda i: PageModel.from_cache(i), pages_by_type_cached))
    else:
        with session() as s:
            q = s.query(PageOrmModel).filter_by(type=page_type)
            q = q.order_by(asc(PageOrmModel.order))
            res = list(map(lambda i: PageModel.from_orm_model(i), q.all()))

            cache.set(cache_key('pages_by_type', page_type), list(map(lambda i: i.to_cache(), res)))

            s.commit()

    local_cache.set(cache_key('type', page_type), res)

    return res
Exemplo n.º 24
0
def delete(board: BoardModel):
    with session() as s:
        b = s.query(BoardOrmModel).filter_by(id=board.id).one()
        s.delete(b)
        s.commit()

        # The pages etc. will fall out of the cache themselves
        # This is the first thing all board related endpoints use, so they get cancelled at the start of the request.
        # If any are still working with caches of this board let them use the left over caches.
        cache.delete(cache_key('board_and_config', board.name))

        _set_all_board_names_cache(s)

        s.commit()
Exemplo n.º 25
0
def find_by_link_name(link_name: str) -> PageModel:
    lc = local_cache.get(cache_key('link_name', link_name))
    if lc:
        return lc.copy()

    page_cached = cache.get(cache_key('page_by_link_name', link_name))
    if page_cached:
        return PageModel.from_cache(page_cached)
    else:
        with session() as s:
            m = s.query(PageOrmModel).filter_by(
                link_name=link_name).one_or_none()
            res = None
            if m:
                res = PageModel.from_orm_model(m)

                cache.set(cache_key('page_by_link_name', res.link_name),
                          res.to_cache())

    if res:
        local_cache.set(cache_key('link_name', link_name), res)

    return res
Exemplo n.º 26
0
def find_thread_by_board_thread_refno_with_posts(board: BoardModel, thread_refno: int) -> Optional[ThreadModel]:
    thread_cache = cache.get(cache_key('thread', board.name, thread_refno))
    if not thread_cache:
        with session() as s:
            q = s.query(ThreadOrmModel)
            q = q.options(lazyload('posts'))
            q = q.filter(ThreadOrmModel.refno == thread_refno,
                         ThreadOrmModel.board_id == BoardOrmModel.id,
                         BoardOrmModel.name == board.name)
            thread_orm_model = q.one_or_none()

            if not thread_orm_model or not thread_orm_model.posts:
                return None

            # TODO: also load board in q above
            thread = ThreadModel.from_orm_model(thread_orm_model, include_board=True, include_posts=True)
            thread_cache = thread.to_cache(include_board=True, include_posts=True)
            cache.set(cache_key('thread', thread.board.name, thread.refno), thread_cache, timeout=0)
            return thread

    if thread_cache:
        return ThreadModel.from_cache(thread_cache)
    return None
Exemplo n.º 27
0
def find_thread_by_board_name_thread_refno(board_name: str, thread_refno: int) -> Optional[ThreadModel]:
    thread_cache = cache.get(cache_key('thread', board_name, thread_refno))
    if not thread_cache:
        with session() as s:
            q = s.query(ThreadOrmModel)
            q = q.filter(ThreadOrmModel.refno == thread_refno,
                         ThreadOrmModel.board_id == BoardOrmModel.id,
                         BoardOrmModel.name == board_name)
            thread_orm_model = q.one_or_none()

            if not thread_orm_model:
                return None

            # TODO: also load board in q above
            thread = ThreadModel.from_orm_model(thread_orm_model, include_board=True)
            return thread

    if thread_cache:
        return ThreadModel.from_cache(thread_cache)
    return None
Exemplo n.º 28
0
def find_thread_by_board_name_thread_refno(
        board_name: str, thread_refno: int) -> Optional[ThreadModel]:
    thread_cache = cache.get(cache_key('thread', board_name, thread_refno))
    if not thread_cache:
        with session() as s:
            q = s.query(ThreadOrmModel)
            q = q.filter(ThreadOrmModel.refno == thread_refno,
                         ThreadOrmModel.board_id == BoardOrmModel.id,
                         BoardOrmModel.name == board_name)
            thread_orm_model = q.one_or_none()

            if not thread_orm_model:
                return None

            # TODO: also load board in q above
            thread = ThreadModel.from_orm_model(thread_orm_model,
                                                include_board=True)
            return thread

    if thread_cache:
        return ThreadModel.from_cache(thread_cache)
    return None
Exemplo n.º 29
0
def _cache_page(s, page: PageModel):
    cache.set(cache_key('page_by_link_name', page.link_name), page.to_cache())
    _cache_pages_by_type(s, page.type)
Exemplo n.º 30
0
def _cache_pages_by_type(s, page_type):
    type_pages_q = s.query(PageOrmModel).filter_by(type=page_type).all()
    type_pages = list(map(lambda i: PageModel.from_orm_model(i).to_cache(), type_pages_q))
    cache.set(cache_key('pages_by_type', page_type), type_pages)
Exemplo n.º 31
0
def update_config(board: BoardModel):
    with session() as s:
        s.merge(board.config.to_orm_model())
        s.commit()
        cache.set(cache_key('board_and_config', board.name), board.to_cache())
Exemplo n.º 32
0
def _cache_pages_by_type(s, page_type):
    type_pages_q = s.query(PageOrmModel).filter_by(type=page_type).all()
    type_pages = list(
        map(lambda i: PageModel.from_orm_model(i).to_cache(), type_pages_q))
    cache.set(cache_key('pages_by_type', page_type), type_pages)
Exemplo n.º 33
0
def _invalidate_board_pages_catalog_cache(s: Session, board: BoardModel):
    """
    Update the memcache version of the specified board.
    This will update the board pages from the already cached thread stubs, and create a new catalog cache.
    """

    q = s.query(ThreadOrmModel)
    q = q.filter(ThreadOrmModel.board_id == board.id)
    threads_orm = q.all()
    thread_models = list(
        map(lambda j: ThreadModel.from_orm_model(j, ), threads_orm))

    # This builds the board index, stickies first, oldest first, then normal posts, newest first.
    # The pages are split accordingly to the board config,
    # and the catalog is build from only the ops.
    stickies = []
    threads = []
    for thread in thread_models:
        thread_stub_cache = cache.get(
            cache_key('thread_stub', board.name, thread.refno))
        if not thread_stub_cache:
            thread, thread_stub = _invalidate_thread_cache(s, thread, board)
            # The board and thread selects are done separately and there is thus the
            # possibility that the thread was removed after the board select
            if thread_stub is None:
                continue
        else:
            thread_stub = ThreadStubModel.from_cache(thread_stub_cache)

        stickies.append(thread_stub) if thread_stub.sticky else threads.append(
            thread_stub)

    stickies = sorted(stickies, key=lambda t: t.last_modified, reverse=False)
    threads = sorted(threads, key=lambda t: t.last_modified, reverse=True)
    all_thread_stubs = stickies + threads

    # The catalog is a CatalogModel with ThreadStubs with only OP's
    catalog = CatalogModel.from_board_thread_stubs(board, all_thread_stubs)
    catalog_cache = catalog.to_cache()
    cache.set(cache_key('board', board.name), catalog_cache, timeout=0)

    # All threads with stubs, divided per page
    # note: there is the possibility that concurrent processes updating this cache
    # mess up the order / create duplicates of the page threads
    # this chance is however very low, and has no ill side effects except for
    # a visual glitch
    board_pages = []
    for i in range(board.config.pages):
        from_index = i * board.config.per_page
        to_index = (i + 1) * board.config.per_page

        board_page = BoardPageModel.from_page_thread_stubs(
            i, all_thread_stubs[from_index:to_index])
        board_pages.append(board_page)

        board_page_cache = board_page.to_cache()
        cache.set(cache_key('board', board.name, i),
                  board_page_cache,
                  timeout=0)

    return catalog, board_pages
Exemplo n.º 34
0
def _set_all_board_names_cache(s):
    all_board_names_q = s.query(BoardOrmModel).options(load_only('name')).order_by(BoardOrmModel.name)
    cache.set(cache_key('all_board_names'), list(map(lambda i: i.name, all_board_names_q.all())))
Exemplo n.º 35
0
def update_site(site_config: SiteConfigModel):
    with session() as s:
        s.merge(site_config.to_orm_model())
        s.commit()
        cache.set(cache_key('config_site'), site_config.to_cache())
Exemplo n.º 36
0
def update_site(site_config: SiteConfigModel):
    with session() as s:
        s.merge(site_config.to_orm_model())
        s.commit()
        cache.set(cache_key('config_site'), site_config.to_cache())
Exemplo n.º 37
0
def _cache_page(s, page: PageModel):
    cache.set(cache_key('page_by_link_name', page.link_name), page.to_cache())
    _cache_pages_by_type(s, page.type)