Пример #1
0
def get_saved_keys(account_id: int, reload: bool = False):
    redis = get_conn()
    key = 'saved_keys:' + str(account_id)
    saved_keys = redis.get(key)
    result = None
    if saved_keys is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            args_dict = dict(account_id=str(account_id))
            query = """
                SELECT id, service, discord_channel_ids, added, dead
                FROM saved_session_keys_with_hashes
                WHERE contributor_id = %(account_id)s
                ORDER BY
                    added DESC
            """
            cursor.execute(query, args_dict)
            result = cursor.fetchall()
            redis.set(key, serialize_dict_list(result), ex=3600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_saved_keys(account_id, reload=reload)
    else:
        result = deserialize_dict_list(saved_keys)
    saved_keys = [Service_Key.from_dict(service_key) for service_key in result]
    return saved_keys
Пример #2
0
def get_saved_key_import_ids(key_id, reload=False):
    redis = get_conn()
    key = 'saved_key_import_ids:' + str(key_id)
    saved_key_import_ids = redis.get(key)
    if saved_key_import_ids is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            # TODO: select columns
            query = """
                SELECT *
                FROM saved_session_key_import_ids
                WHERE key_id = %s
            """
            cursor.execute(query, (int(key_id), ))
            saved_key_import_ids = cursor.fetchall()
            redis.set(key, serialize_dict_list(saved_key_import_ids), ex=3600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_saved_key_import_ids(key_id, reload=reload)
    else:
        saved_key_import_ids = deserialize_dict_list(saved_key_import_ids)

    return saved_key_import_ids
Пример #3
0
def count_banned_artists(reload: bool = False) -> int:
    redis = get_conn()
    redis_key = construct_banned_artists_count_key()
    artist_count = redis.get(redis_key)
    result = None

    if artist_count and not reload:
        result = int(artist_count)
        return result

    lock = KemonoRedisLock(redis, redis_key, expire=60, auto_renewal=True)

    if not lock.acquire(blocking=False):
        time.sleep(0.1)

        return count_banned_artists(reload=reload)

    cursor = get_cursor()
    query = """
        SELECT COUNT(*) as artist_count
        FROM dnp
    """
    cursor.execute(query)
    result = cursor.fetchone()
    artist_count: int = result['artist_count']
    redis.set(redis_key, str(artist_count), ex=600)
    lock.release()

    return artist_count
Пример #4
0
def revoke_saved_keys(key_ids: List[int], account_id: int):
    cursor = get_cursor()
    query_args = dict(key_ids=key_ids, account_id=account_id)
    query1 = """
        DELETE
        FROM saved_session_key_import_ids skid
        USING saved_session_keys_with_hashes sk
        WHERE
            skid.key_id = sk.id
            AND sk.id = ANY (%(key_ids)s)
            AND sk.contributor_id = %(account_id)s
    """
    cursor.execute(query1, query_args)
    query2 = """
        DELETE
        FROM saved_session_keys_with_hashes
        WHERE
            id = ANY (%(key_ids)s)
            AND contributor_id = %(account_id)s
    """
    cursor.execute(query2, query_args)
    redis = get_conn()
    key = 'saved_keys:' + str(account_id)
    redis.delete(key)
    return True
Пример #5
0
def get_account_notifications(account_id: int,
                              reload: bool = False) -> List[Notification]:
    redis = get_conn()
    key = f"notifications_for_account:{account_id}"
    notifications = redis.get(key)
    result = None

    if notifications is None or reload:
        args_dict = dict(account_id=account_id)

        cursor = get_cursor()
        query = """
            SELECT id, account_id, type, created_at, is_seen, extra_info
            FROM notifications
            WHERE account_id = %(account_id)s
            ORDER BY
                created_at DESC
        """
        cursor.execute(query, args_dict)
        result = cursor.fetchall()
        redis.set(key, serialize_dict_list(result), ex=60)
    else:
        result = deserialize_dict_list(notifications)
    # TODO: fix this mess
    notifications = [
        Notification.from_dict(notification) for notification in result
    ]
    return notifications
Пример #6
0
def get_all_dms_by_query_count(text_query: str, reload: bool = False) -> int:
    redis = get_conn()
    key = 'all_dms_by_query_count:' + base64.b64encode(
        text_query.encode('utf-8')).decode('utf-8')
    count = redis.get(key)

    if count and not reload:
        return int(count)

    lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)

    if not lock.acquire(blocking=False):
        time.sleep(0.1)
        return get_all_dms_by_query_count(text_query, reload=reload)

    cursor = get_cursor()
    query_args = dict(text_query=text_query)
    query = """
        SELECT COUNT(*)
        FROM dms
        WHERE
            to_tsvector(\'english\', content) @@ websearch_to_tsquery(%(text_query)s)
    """
    cursor.execute(query, query_args)
    count = int(cursor.fetchone()['count'])
    redis.set(key, str(count), ex=600)
    lock.release()

    return count
Пример #7
0
def get_all_dms_count(reload: bool = False) -> int:
    redis = get_conn()
    key = 'all_dms_count'
    count = redis.get(key)

    if count and not reload:
        return int(count)

    lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)

    if not lock.acquire(blocking=False):
        time.sleep(0.1)
        return get_all_dms_count(reload=reload)

    cursor = get_cursor()
    query = """
        SELECT COUNT(*)
        FROM dms
    """
    cursor.execute(query)
    count = int(cursor.fetchone()['count'])
    redis.set(key, str(count), ex=600)
    lock.release()

    return count
Пример #8
0
def get_count_of_artists_faved(reload=False):
    redis = get_conn()
    key = 'artists_faved'
    count = redis.get(key)
    if count is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = """
                SELECT count(distinct(l.id, l.service))
                FROM lookup l
                INNER JOIN account_artist_favorite aaf
                    ON l.id = aaf.artist_id AND l.service = aaf.service
                WHERE aaf.service != 'discord-channel'
            """
            cursor.execute(query)
            count = cursor.fetchone()['count']
            redis.set(key, count, ex=3600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_count_of_artists_faved(reload=reload)
    else:
        count = int(count)
    return count
Пример #9
0
def get_top_artists_by_faves(offset, count, reload=False):
    redis = get_conn()
    key = 'top_artists:' + str(offset) + ':' + str(count)
    artists = redis.get(key)
    if artists is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = """
                SELECT l.*, count(*)
                FROM lookup l
                INNER JOIN account_artist_favorite aaf
                    ON l.id = aaf.artist_id AND l.service = aaf.service
                WHERE aaf.service != 'discord-channel'
                GROUP BY (l.id, l.service)
                ORDER BY count(*) DESC
                OFFSET %s
                LIMIT %s
            """
            cursor.execute(query, (offset, count,))
            artists = cursor.fetchall()
            redis.set(key, serialize_artists(artists), ex=3600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_top_artists_by_faves(offset, count, reload=reload)
    else:
        artists = deserialize_artists(artists)
    return artists
Пример #10
0
def get_artist_posts(artist_id,
                     service,
                     offset,
                     limit,
                     sort='id',
                     reload=False):
    redis = get_conn()
    key = 'artist_posts_offset:' + service + ':' + str(artist_id) + ':' + str(
        offset)
    posts = redis.get(key)
    if posts is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = 'SELECT * FROM posts WHERE \"user\" = %s AND service = %s ORDER BY ' + sort + ' OFFSET %s LIMIT %s'
            cursor.execute(query, (
                artist_id,
                service,
                offset,
                limit,
            ))
            posts = cursor.fetchall()
            redis.set(key, serialize_posts(posts), ex=600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_artist_posts(artist_id,
                                    service,
                                    offset,
                                    limit,
                                    sort=sort,
                                    reload=reload)
    else:
        posts = deserialize_posts(posts)
    return posts
Пример #11
0
def load_account(account_id: str = None,
                 reload: bool = False) -> Optional[TDAccount]:
    """
    TODO: Make it return an instance of `Account`.
    """
    if account_id is None and 'account_id' in session:
        return load_account(session['account_id'], reload)
    elif account_id is None and 'account_id' not in session:
        return None

    redis = get_conn()
    key = 'account:' + str(account_id)
    account = redis.get(key)
    if account is None or reload:
        cursor = get_cursor()
        query = """
            SELECT id, username, created_at, role
            FROM account
            WHERE id = %s
        """
        cursor.execute(query, (account_id, ))
        account = cursor.fetchone()
        redis.set(key, serialize_account(account))
    else:
        account = deserialize_account(account)

    return account
Пример #12
0
def count_user_dms(service: str, user_id: str, reload: bool = False) -> int:
    redis = get_conn()
    key = f"dms_count:{service}:{user_id}"
    count = redis.get(key)

    if count and not reload:
        return int(count)

    lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)

    if not lock.acquire(blocking=False):
        time.sleep(0.1)
        return count_user_dms(service, user_id, reload=reload)

    cursor = get_cursor()
    query_args = dict(service=service, user_id=user_id)
    query = """
        SELECT COUNT(*)
        FROM dms
        WHERE
            service = %(service)s
            AND "user" = %(user_id)s

    """
    cursor.execute(query, query_args)
    result = cursor.fetchall()
    count = result[0]['count']
    redis.set(key, str(count), ex=600)
    lock.release()

    return count
Пример #13
0
def get_artists(pagination_db: TDPaginationDB,
                params: TDArtistsParams = default_params,
                reload: bool = False) -> List[TDArtist]:
    """
    Get all artist information.
    @TODO return dataclass
    """
    redis = get_conn()
    service = params["service"]
    sort_by = params["sort_by"]
    # encoded_name = encode_text_query(params["name"])
    redis_key = construct_artists_key(
        *("service", service) if service else "",
        *("sort_by", sort_by),
        # *("name", encoded_name) if name else "",
        str(pagination_db["pagination_init"]["current_page"]))

    artists = redis.get(redis_key)

    if artists is not None and not reload:
        return deserialize_dict_list(artists)

    lock = KemonoRedisLock(redis, redis_key, expire=60, auto_renewal=True)

    if not lock.acquire(blocking=False):
        time.sleep(0.1)
        return get_artists(pagination_db, params, reload=reload)

    cursor = get_cursor()
    arg_dict = dict(
        offset=pagination_db["offset"],
        limit=pagination_db["sql_limit"],
        service=service,
        # name=name
    )
    # name_query = f"AND to_tsvector('english', name) @@ websearch_to_tsquery(%(name)s)" if name else ""
    sort_query = sort_queries[sort_by]

    query = f"""
        SELECT id, indexed, name, service, updated
        FROM lookup
        WHERE
            service != 'discord-channel'
            {
                "AND service = %(service)s"
                if service
                else ""
            }
        ORDER BY
            {sort_query}
        OFFSET %(offset)s
        LIMIT %(limit)s
    """
    cursor.execute(query, arg_dict)
    artists: List[TDArtist] = cursor.fetchall()
    redis.set(redis_key, serialize_dict_list(artists), ex=600)
    lock.release()

    return artists
Пример #14
0
def get_importer_logs(import_id: str):
    redis = get_conn()
    key = f'importer_logs:{import_id}'
    llen = redis.llen(key)
    messages = []
    if llen > 0:
        messages = redis.lrange(key, 0, llen)
        redis.expire(key, 60 * 60 * 48)

    return json.dumps(list(map(lambda msg: msg.decode('utf-8'),
                               messages))), 200
Пример #15
0
def importer_submit():
    if not session.get('account_id') and request.form.get("save_dms"):
        return 'You must be logged in to import direct messages.', 401

    if not request.form.get("session_key"):
        return "Session key missing.", 401

    key = request.form.get("session_key").strip()
    result = validate_import_key(key, request.form.get("service"))

    if not result.is_valid:
        return ("\n".join(result.errors), 422)

    formatted_key = result.modified_result if result.modified_result else key

    try:
        redis = get_conn()

        for _import in scan_keys('imports:*'):
            _import = _import.decode('utf8')
            existing_import = redis.get(_import)
            existing_import_data = json.loads(existing_import)
            if existing_import_data['key'] == formatted_key:
                props = SuccessProps(
                    message=
                    'This key is already being used for an import. Redirecting to logs...',
                    currentPage='import',
                    redirect=
                    f"/importer/status/{_import.split(':')[1]}{ '?dms=1' if request.form.get('save_dms') else '' }"
                )

                return make_response(
                    render_template('success.html', props=props), 200)

        import_id = get_import_id(formatted_key)
        data = dict(key=formatted_key,
                    service=request.form.get("service"),
                    channel_ids=request.form.get("channel_ids"),
                    auto_import=request.form.get("auto_import"),
                    save_session_key=request.form.get("save_session_key"),
                    save_dms=request.form.get("save_dms"),
                    contributor_id=session.get("account_id"))
        redis.set(f'imports:{import_id}', json.dumps(data))

        props = SuccessProps(
            currentPage='import',
            redirect=
            f'/importer/status/{import_id}{"?dms=1" if request.form.get("save_dms") else "" }'
        )

        return make_response(render_template('success.html', props=props), 200)
    except Exception:
        current_app.logger.exception('Error connecting to archiver')
        return 'Error while pushing import request. Is Redis running?', 500
Пример #16
0
def get_previous_post_id(post_id, artist_id, service, reload=False):
    redis = get_conn()
    key = 'previous_post:' + service + ':' + str(artist_id) + ':' + str(
        post_id)
    prev_post = redis.get(key)
    if prev_post is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = """
                SELECT id
                FROM posts
                WHERE
                    posts.user = %s
                    AND service = %s
                    AND published > (
                        SELECT published
                        FROM posts
                        WHERE
                            id = %s
                            AND "user" = %s
                            AND service = %s
                        LIMIT 1
                    )
                ORDER BY published ASC
                LIMIT 1
            """
            cursor.execute(query, (
                artist_id,
                service,
                post_id,
                artist_id,
                service,
            ))
            prev_post = cursor.fetchone()
            if prev_post is None:
                prev_post = ""
            else:
                prev_post = prev_post['id']
            redis.set(key, str(prev_post), ex=600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_previous_post_id(post_id,
                                        artist_id,
                                        service,
                                        reload=reload)
    else:
        prev_post = prev_post.decode('utf-8')

    if prev_post == "":
        return None
    else:
        return prev_post
Пример #17
0
def is_post_favorited(account_id, service, artist_id, post_id, reload=False):
    redis = get_conn()
    key = 'post_favorited:' + str(account_id) + ':' + str(service) + str(
        artist_id) + ':' + str(post_id)
    value = redis.get(key)
    if value is None or reload:
        cursor = get_cursor()
        query = "select 1 from account_post_favorite where account_id = %s and service = %s and artist_id = %s and post_id = %s"
        cursor.execute(query, (account_id, service, artist_id, post_id))
        value = cursor.fetchone() is not None
        redis.set(key, str(value))
    else:
        value = value.decode('utf-8') == 'True'

    return value
Пример #18
0
def count_artists(
        service: str = None,
        # name: str = None,
        reload: bool = False) -> int:
    redis = get_conn()
    # encoded_name = encode_text_query(name)
    redis_key = construct_artists_count_key(
        *("service", service) if service else "",
        # *("name", encoded_name) if name else ""
    )
    artist_count = redis.get(redis_key)
    result = None

    if artist_count and not reload:
        result = int(artist_count)
        return result

    lock = KemonoRedisLock(redis, redis_key, expire=60, auto_renewal=True)

    if not lock.acquire(blocking=False):
        time.sleep(0.1)
        return count_artists(
            service,
            # name,
            reload=reload)

    cursor = get_cursor()
    query_args = dict(service=service,
                      # name=name
                      )

    # name_query = f"AND to_tsvector('english', name) @@ websearch_to_tsquery(%(name)s)" if name else ""

    query = f"""
        SELECT COUNT(*) as artist_count
        FROM lookup
        WHERE
            service != 'discord-channel'
            {"AND service = %(service)s" if service else ""}
    """
    cursor.execute(query, query_args)
    result = cursor.fetchone()
    artist_count: int = result['artist_count']
    redis.set(redis_key, str(artist_count), ex=600)
    lock.release()

    return artist_count
Пример #19
0
def get_banned_artists(pagination_db: TDPaginationDB, reload: bool = False):
    redis = get_conn()
    redis_key = construct_banned_artists_key(
        str(pagination_db["pagination_init"]["current_page"]))
    banned_artists = redis.get(redis_key)
    result = None

    if banned_artists is not None and not reload:
        result = deserialize_dict_list(banned_artists)
        return result

    lock = KemonoRedisLock(redis, redis_key, expire=60, auto_renewal=True)

    if not lock.acquire(blocking=False):
        time.sleep(0.1)

        return get_banned_artists(
            pagination_db,
            reload=reload,
        )

    cursor = get_cursor()
    query_args = dict(offset=pagination_db["offset"],
                      limit=pagination_db["sql_limit"])
    query = """
        SELECT
            artist.id,
            artist.indexed,
            artist.name,
            artist.service,
            artist.updated
        FROM
            dnp as banned,
            lookup as artist
        WHERE
            banned.id = artist.id
            AND banned.service = artist.service
        OFFSET %(offset)s
        LIMIT %(limit)s
    """
    cursor.execute(query, query_args)
    result: List[TDArtist] = cursor.fetchall()
    redis.set(redis_key, serialize_dict_list(banned_artists), ex=600)
    lock.release()

    return result
Пример #20
0
def get_artist_post_count(service, artist_id, reload=False):
    redis = get_conn()
    key = 'artist_post_count:' + service + ':' + str(artist_id)
    count = redis.get(key)
    if count is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = 'SELECT count(*) as count FROM posts WHERE \"user\" = %s AND service = %s'
            cursor.execute(query, (artist_id, service,))
            count = cursor.fetchone()['count']
            redis.set(key, str(count), ex=600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_artist_post_count(service, artist_id, reload=reload)
    else:
        count = int(count)
    return count
Пример #21
0
def get_random_artist_keys(count, reload=False):
    redis = get_conn()
    key = 'random_artist_keys:' + str(count)
    artist_keys = redis.get(key)
    if artist_keys is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = "SELECT id, service FROM lookup WHERE service != 'discord-channel' ORDER BY random() LIMIT %s"
            cursor.execute(query, (count,))
            artist_keys = cursor.fetchall()
            redis.set(key, ujson.dumps(artist_keys), ex=600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_random_artist_keys(count, reload=reload)
    else:
        artist_keys = ujson.loads(artist_keys)
    return artist_keys
Пример #22
0
def get_all_posts(offset: int, reload=False):
    redis = get_conn()
    key = 'all_posts:' + str(offset)
    all_posts = redis.get(key)
    if all_posts is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = 'SELECT * FROM posts ORDER BY added desc OFFSET %s LIMIT 25'
            cursor.execute(query, (offset, ))
            all_posts = cursor.fetchall()
            redis.set(key, serialize_dict_list(all_posts), ex=600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_all_posts(offset, reload=reload)
    else:
        all_posts = deserialize_dict_list(all_posts)
    return all_posts
Пример #23
0
def get_artist(service: str, artist_id: str, reload: bool = False) -> dict:
    redis = get_conn()
    key = 'artist:' + service + ':' + str(artist_id)
    artist = redis.get(key)
    if artist is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = 'SELECT * FROM lookup WHERE id = %s AND service = %s'
            cursor.execute(query, (artist_id, service,))
            artist = cursor.fetchone()
            redis.set(key, serialize_artist(artist), ex=600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_artist(service, artist_id, reload=reload)
    else:
        artist = deserialize_artist(artist)
    return artist
Пример #24
0
def get_all_posts_by_artist(artist_id, service, reload=False):
    redis = get_conn()
    key = 'posts_by_artist:' + service + ':' + str(artist_id)
    posts = redis.get(key)
    if posts is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = 'SELECT * FROM posts WHERE \"user\" = %s AND service = %s'
            cursor.execute(query, (artist_id, service))
            posts = cursor.fetchall()
            redis.set(key, serialize_posts(posts), ex=600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_all_posts_by_artist(artist_id, service, reload=reload)
    else:
        posts = deserialize_posts(posts)
    return posts
Пример #25
0
def get_artists_by_service(service, reload=False):
    redis = get_conn()
    key = 'artists_by_service:' + service
    artists = redis.get(key)
    if artists is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = "SELECT * FROM lookup WHERE service = %s"
            cursor.execute(query, (service,))
            artists = cursor.fetchall()
            redis.set(key, serialize_artists(artists), ex=600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_artists_by_service(service, reload=reload)
    else:
        artists = deserialize_artists(artists)
    return artists
Пример #26
0
def get_post_comments(post_id, service, reload=False):
    redis = get_conn()
    key = 'comments:' + service + ':' + str(post_id)
    comments = redis.get(key)
    if comments is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = 'SELECT * FROM comments WHERE post_id = %s AND service = %s'
            cursor.execute(query, (post_id, service))
            comments = cursor.fetchall()
            redis.set(key, serialize_comments(comments), ex=600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_post_comments(post_id, service, reload=reload)
    else:
        comments = deserialize_comments(comments)
    return comments
Пример #27
0
def get_all_post_keys(reload=False):
    redis = get_conn()
    key = 'all_post_keys'
    post_keys = redis.get(key)
    if post_keys is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = "SELECT id, \"user\", service FROM posts"
            cursor.execute(query)
            post_keys = cursor.fetchall()
            redis.set(key, ujson.dumps(post_keys), ex=600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_all_post_keys(reload=reload)
    else:
        post_keys = ujson.loads(post_keys)
    return post_keys
Пример #28
0
def get_random_posts_keys(count, reload=False):
    redis = get_conn()
    key = 'random_post_keys:' + str(count)
    post_keys = redis.get(key)
    if post_keys is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = "SELECT id, \"user\", service FROM posts WHERE file != '{}' AND attachments != '{}' ORDER BY random() LIMIT %s"
            cursor.execute(query, (count, ))
            post_keys = cursor.fetchall()
            redis.set(key, ujson.dumps(post_keys), ex=600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_random_posts_keys(count, reload=reload)
    else:
        post_keys = ujson.loads(post_keys)
    return post_keys
Пример #29
0
def count_all_posts(reload=False):
    redis = get_conn()
    key = 'global_post_count'
    count = redis.get(key)
    if count is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = 'SELECT COUNT(*) FROM posts'
            cursor.execute(query)
            count = cursor.fetchone()
            redis.set(key, str(count['count']), ex=600)
            count = int(count['count'])
            lock.release()
        else:
            time.sleep(0.1)
            return count_all_posts(reload=reload)
    else:
        count = int(count)
    return count
Пример #30
0
def get_all_dms_by_query(text_query: str,
                         offset: int,
                         limit: int,
                         reload: bool = False) -> List[Approved_DM]:
    transformed_query = base64.b64encode(
        text_query.encode('utf-8')).decode('utf-8')
    redis = get_conn()
    key = f'all_dms_by_query:{transformed_query}:{str(offset)}'
    dms = redis.get(key)
    result = None

    if dms and not reload:
        result = deserialize_dms(dms)
        return [Approved_DM.from_dict(dm) for dm in result] if result else []

    lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
    if not lock.acquire(blocking=False):
        time.sleep(0.1)
        return get_all_dms_by_query(text_query, offset, limit, reload=reload)

    cursor = get_cursor()
    query_args = dict(text_query=text_query, offset=offset, limit=limit)
    query = """
        SELECT
            id, "user", service, content, embed, file, added, published
        FROM dms
        WHERE
            to_tsvector(\'english\', content) @@ websearch_to_tsquery(%(text_query)s)
        ORDER BY
            added DESC
        OFFSET %(offset)s
        LIMIT %(limit)s
    """
    cursor.execute(query, query_args)
    result = cursor.fetchall()
    redis.set(key, serialize_dms(result), ex=600)
    lock.release()

    dms = [Approved_DM.from_dict(dm) for dm in result] if result else []

    return dms