Beispiel #1
0
def get_saved_keys(account_id: int, reload: bool = False):
    redis = get_conn()
    key = 'saved_keys:' + str(account_id)
    saved_keys = redis.get(key)
    result = None
    if saved_keys is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            args_dict = dict(account_id=str(account_id))
            query = """
                SELECT id, service, discord_channel_ids, added, dead
                FROM saved_session_keys_with_hashes
                WHERE contributor_id = %(account_id)s
                ORDER BY
                    added DESC
            """
            cursor.execute(query, args_dict)
            result = cursor.fetchall()
            redis.set(key, serialize_dict_list(result), ex=3600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_saved_keys(account_id, reload=reload)
    else:
        result = deserialize_dict_list(saved_keys)
    saved_keys = [Service_Key.from_dict(service_key) for service_key in result]
    return saved_keys
Beispiel #2
0
def count_banned_artists(reload: bool = False) -> int:
    redis = get_conn()
    redis_key = construct_banned_artists_count_key()
    artist_count = redis.get(redis_key)
    result = None

    if artist_count and not reload:
        result = int(artist_count)
        return result

    lock = KemonoRedisLock(redis, redis_key, expire=60, auto_renewal=True)

    if not lock.acquire(blocking=False):
        time.sleep(0.1)

        return count_banned_artists(reload=reload)

    cursor = get_cursor()
    query = """
        SELECT COUNT(*) as artist_count
        FROM dnp
    """
    cursor.execute(query)
    result = cursor.fetchone()
    artist_count: int = result['artist_count']
    redis.set(redis_key, str(artist_count), ex=600)
    lock.release()

    return artist_count
Beispiel #3
0
def get_count_of_artists_faved(reload=False):
    redis = get_conn()
    key = 'artists_faved'
    count = redis.get(key)
    if count is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = """
                SELECT count(distinct(l.id, l.service))
                FROM lookup l
                INNER JOIN account_artist_favorite aaf
                    ON l.id = aaf.artist_id AND l.service = aaf.service
                WHERE aaf.service != 'discord-channel'
            """
            cursor.execute(query)
            count = cursor.fetchone()['count']
            redis.set(key, count, ex=3600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_count_of_artists_faved(reload=reload)
    else:
        count = int(count)
    return count
Beispiel #4
0
def get_all_dms_count(reload: bool = False) -> int:
    redis = get_conn()
    key = 'all_dms_count'
    count = redis.get(key)

    if count and not reload:
        return int(count)

    lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)

    if not lock.acquire(blocking=False):
        time.sleep(0.1)
        return get_all_dms_count(reload=reload)

    cursor = get_cursor()
    query = """
        SELECT COUNT(*)
        FROM dms
    """
    cursor.execute(query)
    count = int(cursor.fetchone()['count'])
    redis.set(key, str(count), ex=600)
    lock.release()

    return count
Beispiel #5
0
def get_top_artists_by_faves(offset, count, reload=False):
    redis = get_conn()
    key = 'top_artists:' + str(offset) + ':' + str(count)
    artists = redis.get(key)
    if artists is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = """
                SELECT l.*, count(*)
                FROM lookup l
                INNER JOIN account_artist_favorite aaf
                    ON l.id = aaf.artist_id AND l.service = aaf.service
                WHERE aaf.service != 'discord-channel'
                GROUP BY (l.id, l.service)
                ORDER BY count(*) DESC
                OFFSET %s
                LIMIT %s
            """
            cursor.execute(query, (offset, count,))
            artists = cursor.fetchall()
            redis.set(key, serialize_artists(artists), ex=3600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_top_artists_by_faves(offset, count, reload=reload)
    else:
        artists = deserialize_artists(artists)
    return artists
Beispiel #6
0
def get_all_dms_by_query_count(text_query: str, reload: bool = False) -> int:
    redis = get_conn()
    key = 'all_dms_by_query_count:' + base64.b64encode(
        text_query.encode('utf-8')).decode('utf-8')
    count = redis.get(key)

    if count and not reload:
        return int(count)

    lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)

    if not lock.acquire(blocking=False):
        time.sleep(0.1)
        return get_all_dms_by_query_count(text_query, reload=reload)

    cursor = get_cursor()
    query_args = dict(text_query=text_query)
    query = """
        SELECT COUNT(*)
        FROM dms
        WHERE
            to_tsvector(\'english\', content) @@ websearch_to_tsquery(%(text_query)s)
    """
    cursor.execute(query, query_args)
    count = int(cursor.fetchone()['count'])
    redis.set(key, str(count), ex=600)
    lock.release()

    return count
Beispiel #7
0
def count_user_dms(service: str, user_id: str, reload: bool = False) -> int:
    redis = get_conn()
    key = f"dms_count:{service}:{user_id}"
    count = redis.get(key)

    if count and not reload:
        return int(count)

    lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)

    if not lock.acquire(blocking=False):
        time.sleep(0.1)
        return count_user_dms(service, user_id, reload=reload)

    cursor = get_cursor()
    query_args = dict(service=service, user_id=user_id)
    query = """
        SELECT COUNT(*)
        FROM dms
        WHERE
            service = %(service)s
            AND "user" = %(user_id)s

    """
    cursor.execute(query, query_args)
    result = cursor.fetchall()
    count = result[0]['count']
    redis.set(key, str(count), ex=600)
    lock.release()

    return count
Beispiel #8
0
def get_artist_posts(artist_id,
                     service,
                     offset,
                     limit,
                     sort='id',
                     reload=False):
    redis = get_conn()
    key = 'artist_posts_offset:' + service + ':' + str(artist_id) + ':' + str(
        offset)
    posts = redis.get(key)
    if posts is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = 'SELECT * FROM posts WHERE \"user\" = %s AND service = %s ORDER BY ' + sort + ' OFFSET %s LIMIT %s'
            cursor.execute(query, (
                artist_id,
                service,
                offset,
                limit,
            ))
            posts = cursor.fetchall()
            redis.set(key, serialize_posts(posts), ex=600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_artist_posts(artist_id,
                                    service,
                                    offset,
                                    limit,
                                    sort=sort,
                                    reload=reload)
    else:
        posts = deserialize_posts(posts)
    return posts
Beispiel #9
0
def get_saved_key_import_ids(key_id, reload=False):
    redis = get_conn()
    key = 'saved_key_import_ids:' + str(key_id)
    saved_key_import_ids = redis.get(key)
    if saved_key_import_ids is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            # TODO: select columns
            query = """
                SELECT *
                FROM saved_session_key_import_ids
                WHERE key_id = %s
            """
            cursor.execute(query, (int(key_id), ))
            saved_key_import_ids = cursor.fetchall()
            redis.set(key, serialize_dict_list(saved_key_import_ids), ex=3600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_saved_key_import_ids(key_id, reload=reload)
    else:
        saved_key_import_ids = deserialize_dict_list(saved_key_import_ids)

    return saved_key_import_ids
Beispiel #10
0
def get_artists(pagination_db: TDPaginationDB,
                params: TDArtistsParams = default_params,
                reload: bool = False) -> List[TDArtist]:
    """
    Get all artist information.
    @TODO return dataclass
    """
    redis = get_conn()
    service = params["service"]
    sort_by = params["sort_by"]
    # encoded_name = encode_text_query(params["name"])
    redis_key = construct_artists_key(
        *("service", service) if service else "",
        *("sort_by", sort_by),
        # *("name", encoded_name) if name else "",
        str(pagination_db["pagination_init"]["current_page"]))

    artists = redis.get(redis_key)

    if artists is not None and not reload:
        return deserialize_dict_list(artists)

    lock = KemonoRedisLock(redis, redis_key, expire=60, auto_renewal=True)

    if not lock.acquire(blocking=False):
        time.sleep(0.1)
        return get_artists(pagination_db, params, reload=reload)

    cursor = get_cursor()
    arg_dict = dict(
        offset=pagination_db["offset"],
        limit=pagination_db["sql_limit"],
        service=service,
        # name=name
    )
    # name_query = f"AND to_tsvector('english', name) @@ websearch_to_tsquery(%(name)s)" if name else ""
    sort_query = sort_queries[sort_by]

    query = f"""
        SELECT id, indexed, name, service, updated
        FROM lookup
        WHERE
            service != 'discord-channel'
            {
                "AND service = %(service)s"
                if service
                else ""
            }
        ORDER BY
            {sort_query}
        OFFSET %(offset)s
        LIMIT %(limit)s
    """
    cursor.execute(query, arg_dict)
    artists: List[TDArtist] = cursor.fetchall()
    redis.set(redis_key, serialize_dict_list(artists), ex=600)
    lock.release()

    return artists
Beispiel #11
0
def get_previous_post_id(post_id, artist_id, service, reload=False):
    redis = get_conn()
    key = 'previous_post:' + service + ':' + str(artist_id) + ':' + str(
        post_id)
    prev_post = redis.get(key)
    if prev_post is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = """
                SELECT id
                FROM posts
                WHERE
                    posts.user = %s
                    AND service = %s
                    AND published > (
                        SELECT published
                        FROM posts
                        WHERE
                            id = %s
                            AND "user" = %s
                            AND service = %s
                        LIMIT 1
                    )
                ORDER BY published ASC
                LIMIT 1
            """
            cursor.execute(query, (
                artist_id,
                service,
                post_id,
                artist_id,
                service,
            ))
            prev_post = cursor.fetchone()
            if prev_post is None:
                prev_post = ""
            else:
                prev_post = prev_post['id']
            redis.set(key, str(prev_post), ex=600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_previous_post_id(post_id,
                                        artist_id,
                                        service,
                                        reload=reload)
    else:
        prev_post = prev_post.decode('utf-8')

    if prev_post == "":
        return None
    else:
        return prev_post
Beispiel #12
0
def count_artists(
        service: str = None,
        # name: str = None,
        reload: bool = False) -> int:
    redis = get_conn()
    # encoded_name = encode_text_query(name)
    redis_key = construct_artists_count_key(
        *("service", service) if service else "",
        # *("name", encoded_name) if name else ""
    )
    artist_count = redis.get(redis_key)
    result = None

    if artist_count and not reload:
        result = int(artist_count)
        return result

    lock = KemonoRedisLock(redis, redis_key, expire=60, auto_renewal=True)

    if not lock.acquire(blocking=False):
        time.sleep(0.1)
        return count_artists(
            service,
            # name,
            reload=reload)

    cursor = get_cursor()
    query_args = dict(service=service,
                      # name=name
                      )

    # name_query = f"AND to_tsvector('english', name) @@ websearch_to_tsquery(%(name)s)" if name else ""

    query = f"""
        SELECT COUNT(*) as artist_count
        FROM lookup
        WHERE
            service != 'discord-channel'
            {"AND service = %(service)s" if service else ""}
    """
    cursor.execute(query, query_args)
    result = cursor.fetchone()
    artist_count: int = result['artist_count']
    redis.set(redis_key, str(artist_count), ex=600)
    lock.release()

    return artist_count
Beispiel #13
0
def get_banned_artists(pagination_db: TDPaginationDB, reload: bool = False):
    redis = get_conn()
    redis_key = construct_banned_artists_key(
        str(pagination_db["pagination_init"]["current_page"]))
    banned_artists = redis.get(redis_key)
    result = None

    if banned_artists is not None and not reload:
        result = deserialize_dict_list(banned_artists)
        return result

    lock = KemonoRedisLock(redis, redis_key, expire=60, auto_renewal=True)

    if not lock.acquire(blocking=False):
        time.sleep(0.1)

        return get_banned_artists(
            pagination_db,
            reload=reload,
        )

    cursor = get_cursor()
    query_args = dict(offset=pagination_db["offset"],
                      limit=pagination_db["sql_limit"])
    query = """
        SELECT
            artist.id,
            artist.indexed,
            artist.name,
            artist.service,
            artist.updated
        FROM
            dnp as banned,
            lookup as artist
        WHERE
            banned.id = artist.id
            AND banned.service = artist.service
        OFFSET %(offset)s
        LIMIT %(limit)s
    """
    cursor.execute(query, query_args)
    result: List[TDArtist] = cursor.fetchall()
    redis.set(redis_key, serialize_dict_list(banned_artists), ex=600)
    lock.release()

    return result
Beispiel #14
0
def get_artist_post_count(service, artist_id, reload=False):
    redis = get_conn()
    key = 'artist_post_count:' + service + ':' + str(artist_id)
    count = redis.get(key)
    if count is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = 'SELECT count(*) as count FROM posts WHERE \"user\" = %s AND service = %s'
            cursor.execute(query, (artist_id, service,))
            count = cursor.fetchone()['count']
            redis.set(key, str(count), ex=600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_artist_post_count(service, artist_id, reload=reload)
    else:
        count = int(count)
    return count
Beispiel #15
0
def get_all_posts_by_artist(artist_id, service, reload=False):
    redis = get_conn()
    key = 'posts_by_artist:' + service + ':' + str(artist_id)
    posts = redis.get(key)
    if posts is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = 'SELECT * FROM posts WHERE \"user\" = %s AND service = %s'
            cursor.execute(query, (artist_id, service))
            posts = cursor.fetchall()
            redis.set(key, serialize_posts(posts), ex=600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_all_posts_by_artist(artist_id, service, reload=reload)
    else:
        posts = deserialize_posts(posts)
    return posts
Beispiel #16
0
def get_post_comments(post_id, service, reload=False):
    redis = get_conn()
    key = 'comments:' + service + ':' + str(post_id)
    comments = redis.get(key)
    if comments is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = 'SELECT * FROM comments WHERE post_id = %s AND service = %s'
            cursor.execute(query, (post_id, service))
            comments = cursor.fetchall()
            redis.set(key, serialize_comments(comments), ex=600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_post_comments(post_id, service, reload=reload)
    else:
        comments = deserialize_comments(comments)
    return comments
Beispiel #17
0
def get_artists_by_service(service, reload=False):
    redis = get_conn()
    key = 'artists_by_service:' + service
    artists = redis.get(key)
    if artists is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = "SELECT * FROM lookup WHERE service = %s"
            cursor.execute(query, (service,))
            artists = cursor.fetchall()
            redis.set(key, serialize_artists(artists), ex=600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_artists_by_service(service, reload=reload)
    else:
        artists = deserialize_artists(artists)
    return artists
Beispiel #18
0
def get_all_post_keys(reload=False):
    redis = get_conn()
    key = 'all_post_keys'
    post_keys = redis.get(key)
    if post_keys is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = "SELECT id, \"user\", service FROM posts"
            cursor.execute(query)
            post_keys = cursor.fetchall()
            redis.set(key, ujson.dumps(post_keys), ex=600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_all_post_keys(reload=reload)
    else:
        post_keys = ujson.loads(post_keys)
    return post_keys
Beispiel #19
0
def get_random_posts_keys(count, reload=False):
    redis = get_conn()
    key = 'random_post_keys:' + str(count)
    post_keys = redis.get(key)
    if post_keys is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = "SELECT id, \"user\", service FROM posts WHERE file != '{}' AND attachments != '{}' ORDER BY random() LIMIT %s"
            cursor.execute(query, (count, ))
            post_keys = cursor.fetchall()
            redis.set(key, ujson.dumps(post_keys), ex=600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_random_posts_keys(count, reload=reload)
    else:
        post_keys = ujson.loads(post_keys)
    return post_keys
Beispiel #20
0
def get_random_artist_keys(count, reload=False):
    redis = get_conn()
    key = 'random_artist_keys:' + str(count)
    artist_keys = redis.get(key)
    if artist_keys is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = "SELECT id, service FROM lookup WHERE service != 'discord-channel' ORDER BY random() LIMIT %s"
            cursor.execute(query, (count,))
            artist_keys = cursor.fetchall()
            redis.set(key, ujson.dumps(artist_keys), ex=600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_random_artist_keys(count, reload=reload)
    else:
        artist_keys = ujson.loads(artist_keys)
    return artist_keys
Beispiel #21
0
def get_artist(service: str, artist_id: str, reload: bool = False) -> dict:
    redis = get_conn()
    key = 'artist:' + service + ':' + str(artist_id)
    artist = redis.get(key)
    if artist is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = 'SELECT * FROM lookup WHERE id = %s AND service = %s'
            cursor.execute(query, (artist_id, service,))
            artist = cursor.fetchone()
            redis.set(key, serialize_artist(artist), ex=600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_artist(service, artist_id, reload=reload)
    else:
        artist = deserialize_artist(artist)
    return artist
Beispiel #22
0
def get_all_posts(offset: int, reload=False):
    redis = get_conn()
    key = 'all_posts:' + str(offset)
    all_posts = redis.get(key)
    if all_posts is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = 'SELECT * FROM posts ORDER BY added desc OFFSET %s LIMIT 25'
            cursor.execute(query, (offset, ))
            all_posts = cursor.fetchall()
            redis.set(key, serialize_dict_list(all_posts), ex=600)
            lock.release()
        else:
            time.sleep(0.1)
            return get_all_posts(offset, reload=reload)
    else:
        all_posts = deserialize_dict_list(all_posts)
    return all_posts
Beispiel #23
0
def count_all_posts(reload=False):
    redis = get_conn()
    key = 'global_post_count'
    count = redis.get(key)
    if count is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = 'SELECT COUNT(*) FROM posts'
            cursor.execute(query)
            count = cursor.fetchone()
            redis.set(key, str(count['count']), ex=600)
            count = int(count['count'])
            lock.release()
        else:
            time.sleep(0.1)
            return count_all_posts(reload=reload)
    else:
        count = int(count)
    return count
Beispiel #24
0
def get_all_dms_by_query(text_query: str,
                         offset: int,
                         limit: int,
                         reload: bool = False) -> List[Approved_DM]:
    transformed_query = base64.b64encode(
        text_query.encode('utf-8')).decode('utf-8')
    redis = get_conn()
    key = f'all_dms_by_query:{transformed_query}:{str(offset)}'
    dms = redis.get(key)
    result = None

    if dms and not reload:
        result = deserialize_dms(dms)
        return [Approved_DM.from_dict(dm) for dm in result] if result else []

    lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
    if not lock.acquire(blocking=False):
        time.sleep(0.1)
        return get_all_dms_by_query(text_query, offset, limit, reload=reload)

    cursor = get_cursor()
    query_args = dict(text_query=text_query, offset=offset, limit=limit)
    query = """
        SELECT
            id, "user", service, content, embed, file, added, published
        FROM dms
        WHERE
            to_tsvector(\'english\', content) @@ websearch_to_tsquery(%(text_query)s)
        ORDER BY
            added DESC
        OFFSET %(offset)s
        LIMIT %(limit)s
    """
    cursor.execute(query, query_args)
    result = cursor.fetchall()
    redis.set(key, serialize_dms(result), ex=600)
    lock.release()

    dms = [Approved_DM.from_dict(dm) for dm in result] if result else []

    return dms
Beispiel #25
0
def get_unapproved_dms(import_id: str,
                       account_id: int,
                       reload: bool = False) -> List[Unapproved_DM]:
    """
    TODO: fix `account_id` type
    """
    redis = get_conn()
    key = f'unapproved_dms:{import_id}:{str(account_id)}'
    dms = redis.get(key)
    result = None

    if dms and not reload:
        result = deserialize_dms(dms)
        return [Unapproved_DM.from_dict(dm) for dm in result if result]

    lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)

    if not lock.acquire(blocking=False):
        time.sleep(0.1)
        return get_unapproved_dms(import_id, account_id, reload=reload)

    cursor = get_cursor()
    args_dict = dict(import_id=import_id, account_id=str(account_id))
    query = """
        SELECT
            id, import_id, contributor_id, "user", service, content, embed, added, published, file
        FROM unapproved_dms
        WHERE
            import_id = %(import_id)s
            AND contributor_id = %(account_id)s
    """
    cursor.execute(query, args_dict)
    result = cursor.fetchall()
    redis.set(key, serialize_dms(result), ex=1)
    lock.release()

    dms = [Unapproved_DM.from_dict(dm) for dm in result] if result else []

    return dms
Beispiel #26
0
def count_all_posts_for_query(q: str, reload=False):
    if q.strip() == '':
        return count_all_posts()
    redis = get_conn()
    key = 'global_post_count_for_query:' + q
    count = redis.get(key)
    if count is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = "SELECT COUNT(*) FROM posts WHERE to_tsvector('english', content || ' ' || title) @@ websearch_to_tsquery(%s)"
            cursor.execute(query, (q, ))
            count = cursor.fetchone()
            redis.set(key, str(count['count']), ex=600)
            count = int(count['count'])
            lock.release()
        else:
            time.sleep(0.1)
            return count_all_posts_for_query(q, reload=reload)
    else:
        count = int(count)
    return count
Beispiel #27
0
def get_artists_by_update_time(offset, reload=False):
    redis = get_conn()
    key = 'artists_by_update_time:' + str(offset)
    artists = redis.get(key)
    if artists is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = "SELECT * FROM lookup WHERE service != 'discord-channel' ORDER BY updated desc "
            params = ()
            query += "OFFSET %s "
            params += (offset,)
            query += "LIMIT 25"
            cursor.execute(query, (params,))
            artists = cursor.fetchall()
            redis.set(key, serialize_artists(artists), ex=600)
            lock.release()
        else:
            time.sleep(0.1)
            get_artists_by_update_time(offset, reload=reload)
    else:
        artists = deserialize_artists(artists)
    return artists
Beispiel #28
0
def get_all_dms(offset: int,
                limit: int,
                reload: bool = False) -> List[Approved_DM]:
    redis = get_conn()
    key = f'all_dms:{str(offset)}'
    dms = redis.get(key)
    result = None

    if dms and not reload:
        result = deserialize_dms(dms)
        return [Approved_DM.from_dict(dm) for dm in result] if result else []

    lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)

    if not lock.acquire(blocking=False):
        time.sleep(0.1)
        return get_all_dms(offset, limit, reload=reload)

    cursor = get_cursor()
    query_args = dict(offset=offset, limit=limit)
    query = """
        SELECT
            id, "user", service, content, embed, file, added, published
        FROM dms
        ORDER BY
            added DESC
        OFFSET %(offset)s
        LIMIT %(limit)s
    """
    cursor.execute(query, query_args)
    result = cursor.fetchall()
    redis.set(key, serialize_dms(result), ex=600)
    lock.release()

    dms = [Approved_DM.from_dict(dm) for dm in result] if result else []

    return dms
Beispiel #29
0
def is_post_flagged(service, artist_id, post_id, reload=False):
    redis = get_conn()
    key = 'is_post_flagged:' + service + ':' + str(artist_id) + ':' + str(
        post_id)
    flagged = redis.get(key)
    if flagged is None or reload:
        lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)
        if lock.acquire(blocking=False):
            cursor = get_cursor()
            query = "SELECT * FROM booru_flags WHERE id = %s AND \"user\" = %s AND service = %s"
            cursor.execute(query, (
                post_id,
                artist_id,
                service,
            ))
            flagged = cursor.fetchone() is not None
            redis.set(key, str(flagged), ex=600)
            lock.release()
        else:
            time.sleep(0.1)
            return is_post_flagged(service, artist_id, post_id, reload=reload)
    else:
        flagged = flagged.decode('utf-8') == 'True'
    return flagged
Beispiel #30
0
def get_artist_dms(service: str,
                   artist_id: int,
                   reload: bool = False) -> List[Approved_DM]:
    redis = get_conn()
    key = f'dms:{service}:{str(artist_id)}'
    dms = redis.get(key)
    result = None

    if dms and not reload:
        result = deserialize_dms(dms)
        return [Approved_DM.from_dict(dm) for dm in result] if result else []

    lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True)

    if not lock.acquire(blocking=False):
        time.sleep(0.1)
        return get_artist_dms(service, artist_id, reload=reload)

    cursor = get_cursor()
    query_args = dict(service=service, artist_id=artist_id)
    query = """
        SELECT
            id, "user", service, content, embed, file, added, published
        FROM dms
        WHERE
            service = %(service)s
            AND "user" = %(artist_id)s
    """
    cursor.execute(query, query_args)
    result = cursor.fetchall()
    redis.set(key, serialize_dms(result), ex=600)
    lock.release()

    dms = [Approved_DM.from_dict(dm) for dm in result] if result else []

    return dms