def get_saved_keys(account_id: int, reload: bool = False): redis = get_conn() key = 'saved_keys:' + str(account_id) saved_keys = redis.get(key) result = None if saved_keys is None or reload: lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True) if lock.acquire(blocking=False): cursor = get_cursor() args_dict = dict(account_id=str(account_id)) query = """ SELECT id, service, discord_channel_ids, added, dead FROM saved_session_keys_with_hashes WHERE contributor_id = %(account_id)s ORDER BY added DESC """ cursor.execute(query, args_dict) result = cursor.fetchall() redis.set(key, serialize_dict_list(result), ex=3600) lock.release() else: time.sleep(0.1) return get_saved_keys(account_id, reload=reload) else: result = deserialize_dict_list(saved_keys) saved_keys = [Service_Key.from_dict(service_key) for service_key in result] return saved_keys
def count_banned_artists(reload: bool = False) -> int: redis = get_conn() redis_key = construct_banned_artists_count_key() artist_count = redis.get(redis_key) result = None if artist_count and not reload: result = int(artist_count) return result lock = KemonoRedisLock(redis, redis_key, expire=60, auto_renewal=True) if not lock.acquire(blocking=False): time.sleep(0.1) return count_banned_artists(reload=reload) cursor = get_cursor() query = """ SELECT COUNT(*) as artist_count FROM dnp """ cursor.execute(query) result = cursor.fetchone() artist_count: int = result['artist_count'] redis.set(redis_key, str(artist_count), ex=600) lock.release() return artist_count
def get_count_of_artists_faved(reload=False): redis = get_conn() key = 'artists_faved' count = redis.get(key) if count is None or reload: lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True) if lock.acquire(blocking=False): cursor = get_cursor() query = """ SELECT count(distinct(l.id, l.service)) FROM lookup l INNER JOIN account_artist_favorite aaf ON l.id = aaf.artist_id AND l.service = aaf.service WHERE aaf.service != 'discord-channel' """ cursor.execute(query) count = cursor.fetchone()['count'] redis.set(key, count, ex=3600) lock.release() else: time.sleep(0.1) return get_count_of_artists_faved(reload=reload) else: count = int(count) return count
def get_all_dms_count(reload: bool = False) -> int: redis = get_conn() key = 'all_dms_count' count = redis.get(key) if count and not reload: return int(count) lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True) if not lock.acquire(blocking=False): time.sleep(0.1) return get_all_dms_count(reload=reload) cursor = get_cursor() query = """ SELECT COUNT(*) FROM dms """ cursor.execute(query) count = int(cursor.fetchone()['count']) redis.set(key, str(count), ex=600) lock.release() return count
def get_top_artists_by_faves(offset, count, reload=False): redis = get_conn() key = 'top_artists:' + str(offset) + ':' + str(count) artists = redis.get(key) if artists is None or reload: lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True) if lock.acquire(blocking=False): cursor = get_cursor() query = """ SELECT l.*, count(*) FROM lookup l INNER JOIN account_artist_favorite aaf ON l.id = aaf.artist_id AND l.service = aaf.service WHERE aaf.service != 'discord-channel' GROUP BY (l.id, l.service) ORDER BY count(*) DESC OFFSET %s LIMIT %s """ cursor.execute(query, (offset, count,)) artists = cursor.fetchall() redis.set(key, serialize_artists(artists), ex=3600) lock.release() else: time.sleep(0.1) return get_top_artists_by_faves(offset, count, reload=reload) else: artists = deserialize_artists(artists) return artists
def get_all_dms_by_query_count(text_query: str, reload: bool = False) -> int: redis = get_conn() key = 'all_dms_by_query_count:' + base64.b64encode( text_query.encode('utf-8')).decode('utf-8') count = redis.get(key) if count and not reload: return int(count) lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True) if not lock.acquire(blocking=False): time.sleep(0.1) return get_all_dms_by_query_count(text_query, reload=reload) cursor = get_cursor() query_args = dict(text_query=text_query) query = """ SELECT COUNT(*) FROM dms WHERE to_tsvector(\'english\', content) @@ websearch_to_tsquery(%(text_query)s) """ cursor.execute(query, query_args) count = int(cursor.fetchone()['count']) redis.set(key, str(count), ex=600) lock.release() return count
def count_user_dms(service: str, user_id: str, reload: bool = False) -> int: redis = get_conn() key = f"dms_count:{service}:{user_id}" count = redis.get(key) if count and not reload: return int(count) lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True) if not lock.acquire(blocking=False): time.sleep(0.1) return count_user_dms(service, user_id, reload=reload) cursor = get_cursor() query_args = dict(service=service, user_id=user_id) query = """ SELECT COUNT(*) FROM dms WHERE service = %(service)s AND "user" = %(user_id)s """ cursor.execute(query, query_args) result = cursor.fetchall() count = result[0]['count'] redis.set(key, str(count), ex=600) lock.release() return count
def get_artist_posts(artist_id, service, offset, limit, sort='id', reload=False): redis = get_conn() key = 'artist_posts_offset:' + service + ':' + str(artist_id) + ':' + str( offset) posts = redis.get(key) if posts is None or reload: lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True) if lock.acquire(blocking=False): cursor = get_cursor() query = 'SELECT * FROM posts WHERE \"user\" = %s AND service = %s ORDER BY ' + sort + ' OFFSET %s LIMIT %s' cursor.execute(query, ( artist_id, service, offset, limit, )) posts = cursor.fetchall() redis.set(key, serialize_posts(posts), ex=600) lock.release() else: time.sleep(0.1) return get_artist_posts(artist_id, service, offset, limit, sort=sort, reload=reload) else: posts = deserialize_posts(posts) return posts
def get_saved_key_import_ids(key_id, reload=False): redis = get_conn() key = 'saved_key_import_ids:' + str(key_id) saved_key_import_ids = redis.get(key) if saved_key_import_ids is None or reload: lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True) if lock.acquire(blocking=False): cursor = get_cursor() # TODO: select columns query = """ SELECT * FROM saved_session_key_import_ids WHERE key_id = %s """ cursor.execute(query, (int(key_id), )) saved_key_import_ids = cursor.fetchall() redis.set(key, serialize_dict_list(saved_key_import_ids), ex=3600) lock.release() else: time.sleep(0.1) return get_saved_key_import_ids(key_id, reload=reload) else: saved_key_import_ids = deserialize_dict_list(saved_key_import_ids) return saved_key_import_ids
def get_artists(pagination_db: TDPaginationDB, params: TDArtistsParams = default_params, reload: bool = False) -> List[TDArtist]: """ Get all artist information. @TODO return dataclass """ redis = get_conn() service = params["service"] sort_by = params["sort_by"] # encoded_name = encode_text_query(params["name"]) redis_key = construct_artists_key( *("service", service) if service else "", *("sort_by", sort_by), # *("name", encoded_name) if name else "", str(pagination_db["pagination_init"]["current_page"])) artists = redis.get(redis_key) if artists is not None and not reload: return deserialize_dict_list(artists) lock = KemonoRedisLock(redis, redis_key, expire=60, auto_renewal=True) if not lock.acquire(blocking=False): time.sleep(0.1) return get_artists(pagination_db, params, reload=reload) cursor = get_cursor() arg_dict = dict( offset=pagination_db["offset"], limit=pagination_db["sql_limit"], service=service, # name=name ) # name_query = f"AND to_tsvector('english', name) @@ websearch_to_tsquery(%(name)s)" if name else "" sort_query = sort_queries[sort_by] query = f""" SELECT id, indexed, name, service, updated FROM lookup WHERE service != 'discord-channel' { "AND service = %(service)s" if service else "" } ORDER BY {sort_query} OFFSET %(offset)s LIMIT %(limit)s """ cursor.execute(query, arg_dict) artists: List[TDArtist] = cursor.fetchall() redis.set(redis_key, serialize_dict_list(artists), ex=600) lock.release() return artists
def get_previous_post_id(post_id, artist_id, service, reload=False): redis = get_conn() key = 'previous_post:' + service + ':' + str(artist_id) + ':' + str( post_id) prev_post = redis.get(key) if prev_post is None or reload: lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True) if lock.acquire(blocking=False): cursor = get_cursor() query = """ SELECT id FROM posts WHERE posts.user = %s AND service = %s AND published > ( SELECT published FROM posts WHERE id = %s AND "user" = %s AND service = %s LIMIT 1 ) ORDER BY published ASC LIMIT 1 """ cursor.execute(query, ( artist_id, service, post_id, artist_id, service, )) prev_post = cursor.fetchone() if prev_post is None: prev_post = "" else: prev_post = prev_post['id'] redis.set(key, str(prev_post), ex=600) lock.release() else: time.sleep(0.1) return get_previous_post_id(post_id, artist_id, service, reload=reload) else: prev_post = prev_post.decode('utf-8') if prev_post == "": return None else: return prev_post
def count_artists( service: str = None, # name: str = None, reload: bool = False) -> int: redis = get_conn() # encoded_name = encode_text_query(name) redis_key = construct_artists_count_key( *("service", service) if service else "", # *("name", encoded_name) if name else "" ) artist_count = redis.get(redis_key) result = None if artist_count and not reload: result = int(artist_count) return result lock = KemonoRedisLock(redis, redis_key, expire=60, auto_renewal=True) if not lock.acquire(blocking=False): time.sleep(0.1) return count_artists( service, # name, reload=reload) cursor = get_cursor() query_args = dict(service=service, # name=name ) # name_query = f"AND to_tsvector('english', name) @@ websearch_to_tsquery(%(name)s)" if name else "" query = f""" SELECT COUNT(*) as artist_count FROM lookup WHERE service != 'discord-channel' {"AND service = %(service)s" if service else ""} """ cursor.execute(query, query_args) result = cursor.fetchone() artist_count: int = result['artist_count'] redis.set(redis_key, str(artist_count), ex=600) lock.release() return artist_count
def get_banned_artists(pagination_db: TDPaginationDB, reload: bool = False): redis = get_conn() redis_key = construct_banned_artists_key( str(pagination_db["pagination_init"]["current_page"])) banned_artists = redis.get(redis_key) result = None if banned_artists is not None and not reload: result = deserialize_dict_list(banned_artists) return result lock = KemonoRedisLock(redis, redis_key, expire=60, auto_renewal=True) if not lock.acquire(blocking=False): time.sleep(0.1) return get_banned_artists( pagination_db, reload=reload, ) cursor = get_cursor() query_args = dict(offset=pagination_db["offset"], limit=pagination_db["sql_limit"]) query = """ SELECT artist.id, artist.indexed, artist.name, artist.service, artist.updated FROM dnp as banned, lookup as artist WHERE banned.id = artist.id AND banned.service = artist.service OFFSET %(offset)s LIMIT %(limit)s """ cursor.execute(query, query_args) result: List[TDArtist] = cursor.fetchall() redis.set(redis_key, serialize_dict_list(banned_artists), ex=600) lock.release() return result
def get_artist_post_count(service, artist_id, reload=False): redis = get_conn() key = 'artist_post_count:' + service + ':' + str(artist_id) count = redis.get(key) if count is None or reload: lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True) if lock.acquire(blocking=False): cursor = get_cursor() query = 'SELECT count(*) as count FROM posts WHERE \"user\" = %s AND service = %s' cursor.execute(query, (artist_id, service,)) count = cursor.fetchone()['count'] redis.set(key, str(count), ex=600) lock.release() else: time.sleep(0.1) return get_artist_post_count(service, artist_id, reload=reload) else: count = int(count) return count
def get_all_posts_by_artist(artist_id, service, reload=False): redis = get_conn() key = 'posts_by_artist:' + service + ':' + str(artist_id) posts = redis.get(key) if posts is None or reload: lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True) if lock.acquire(blocking=False): cursor = get_cursor() query = 'SELECT * FROM posts WHERE \"user\" = %s AND service = %s' cursor.execute(query, (artist_id, service)) posts = cursor.fetchall() redis.set(key, serialize_posts(posts), ex=600) lock.release() else: time.sleep(0.1) return get_all_posts_by_artist(artist_id, service, reload=reload) else: posts = deserialize_posts(posts) return posts
def get_post_comments(post_id, service, reload=False): redis = get_conn() key = 'comments:' + service + ':' + str(post_id) comments = redis.get(key) if comments is None or reload: lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True) if lock.acquire(blocking=False): cursor = get_cursor() query = 'SELECT * FROM comments WHERE post_id = %s AND service = %s' cursor.execute(query, (post_id, service)) comments = cursor.fetchall() redis.set(key, serialize_comments(comments), ex=600) lock.release() else: time.sleep(0.1) return get_post_comments(post_id, service, reload=reload) else: comments = deserialize_comments(comments) return comments
def get_artists_by_service(service, reload=False): redis = get_conn() key = 'artists_by_service:' + service artists = redis.get(key) if artists is None or reload: lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True) if lock.acquire(blocking=False): cursor = get_cursor() query = "SELECT * FROM lookup WHERE service = %s" cursor.execute(query, (service,)) artists = cursor.fetchall() redis.set(key, serialize_artists(artists), ex=600) lock.release() else: time.sleep(0.1) return get_artists_by_service(service, reload=reload) else: artists = deserialize_artists(artists) return artists
def get_all_post_keys(reload=False): redis = get_conn() key = 'all_post_keys' post_keys = redis.get(key) if post_keys is None or reload: lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True) if lock.acquire(blocking=False): cursor = get_cursor() query = "SELECT id, \"user\", service FROM posts" cursor.execute(query) post_keys = cursor.fetchall() redis.set(key, ujson.dumps(post_keys), ex=600) lock.release() else: time.sleep(0.1) return get_all_post_keys(reload=reload) else: post_keys = ujson.loads(post_keys) return post_keys
def get_random_posts_keys(count, reload=False): redis = get_conn() key = 'random_post_keys:' + str(count) post_keys = redis.get(key) if post_keys is None or reload: lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True) if lock.acquire(blocking=False): cursor = get_cursor() query = "SELECT id, \"user\", service FROM posts WHERE file != '{}' AND attachments != '{}' ORDER BY random() LIMIT %s" cursor.execute(query, (count, )) post_keys = cursor.fetchall() redis.set(key, ujson.dumps(post_keys), ex=600) lock.release() else: time.sleep(0.1) return get_random_posts_keys(count, reload=reload) else: post_keys = ujson.loads(post_keys) return post_keys
def get_random_artist_keys(count, reload=False): redis = get_conn() key = 'random_artist_keys:' + str(count) artist_keys = redis.get(key) if artist_keys is None or reload: lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True) if lock.acquire(blocking=False): cursor = get_cursor() query = "SELECT id, service FROM lookup WHERE service != 'discord-channel' ORDER BY random() LIMIT %s" cursor.execute(query, (count,)) artist_keys = cursor.fetchall() redis.set(key, ujson.dumps(artist_keys), ex=600) lock.release() else: time.sleep(0.1) return get_random_artist_keys(count, reload=reload) else: artist_keys = ujson.loads(artist_keys) return artist_keys
def get_artist(service: str, artist_id: str, reload: bool = False) -> dict: redis = get_conn() key = 'artist:' + service + ':' + str(artist_id) artist = redis.get(key) if artist is None or reload: lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True) if lock.acquire(blocking=False): cursor = get_cursor() query = 'SELECT * FROM lookup WHERE id = %s AND service = %s' cursor.execute(query, (artist_id, service,)) artist = cursor.fetchone() redis.set(key, serialize_artist(artist), ex=600) lock.release() else: time.sleep(0.1) return get_artist(service, artist_id, reload=reload) else: artist = deserialize_artist(artist) return artist
def get_all_posts(offset: int, reload=False): redis = get_conn() key = 'all_posts:' + str(offset) all_posts = redis.get(key) if all_posts is None or reload: lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True) if lock.acquire(blocking=False): cursor = get_cursor() query = 'SELECT * FROM posts ORDER BY added desc OFFSET %s LIMIT 25' cursor.execute(query, (offset, )) all_posts = cursor.fetchall() redis.set(key, serialize_dict_list(all_posts), ex=600) lock.release() else: time.sleep(0.1) return get_all_posts(offset, reload=reload) else: all_posts = deserialize_dict_list(all_posts) return all_posts
def count_all_posts(reload=False): redis = get_conn() key = 'global_post_count' count = redis.get(key) if count is None or reload: lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True) if lock.acquire(blocking=False): cursor = get_cursor() query = 'SELECT COUNT(*) FROM posts' cursor.execute(query) count = cursor.fetchone() redis.set(key, str(count['count']), ex=600) count = int(count['count']) lock.release() else: time.sleep(0.1) return count_all_posts(reload=reload) else: count = int(count) return count
def get_all_dms_by_query(text_query: str, offset: int, limit: int, reload: bool = False) -> List[Approved_DM]: transformed_query = base64.b64encode( text_query.encode('utf-8')).decode('utf-8') redis = get_conn() key = f'all_dms_by_query:{transformed_query}:{str(offset)}' dms = redis.get(key) result = None if dms and not reload: result = deserialize_dms(dms) return [Approved_DM.from_dict(dm) for dm in result] if result else [] lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True) if not lock.acquire(blocking=False): time.sleep(0.1) return get_all_dms_by_query(text_query, offset, limit, reload=reload) cursor = get_cursor() query_args = dict(text_query=text_query, offset=offset, limit=limit) query = """ SELECT id, "user", service, content, embed, file, added, published FROM dms WHERE to_tsvector(\'english\', content) @@ websearch_to_tsquery(%(text_query)s) ORDER BY added DESC OFFSET %(offset)s LIMIT %(limit)s """ cursor.execute(query, query_args) result = cursor.fetchall() redis.set(key, serialize_dms(result), ex=600) lock.release() dms = [Approved_DM.from_dict(dm) for dm in result] if result else [] return dms
def get_unapproved_dms(import_id: str, account_id: int, reload: bool = False) -> List[Unapproved_DM]: """ TODO: fix `account_id` type """ redis = get_conn() key = f'unapproved_dms:{import_id}:{str(account_id)}' dms = redis.get(key) result = None if dms and not reload: result = deserialize_dms(dms) return [Unapproved_DM.from_dict(dm) for dm in result if result] lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True) if not lock.acquire(blocking=False): time.sleep(0.1) return get_unapproved_dms(import_id, account_id, reload=reload) cursor = get_cursor() args_dict = dict(import_id=import_id, account_id=str(account_id)) query = """ SELECT id, import_id, contributor_id, "user", service, content, embed, added, published, file FROM unapproved_dms WHERE import_id = %(import_id)s AND contributor_id = %(account_id)s """ cursor.execute(query, args_dict) result = cursor.fetchall() redis.set(key, serialize_dms(result), ex=1) lock.release() dms = [Unapproved_DM.from_dict(dm) for dm in result] if result else [] return dms
def count_all_posts_for_query(q: str, reload=False): if q.strip() == '': return count_all_posts() redis = get_conn() key = 'global_post_count_for_query:' + q count = redis.get(key) if count is None or reload: lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True) if lock.acquire(blocking=False): cursor = get_cursor() query = "SELECT COUNT(*) FROM posts WHERE to_tsvector('english', content || ' ' || title) @@ websearch_to_tsquery(%s)" cursor.execute(query, (q, )) count = cursor.fetchone() redis.set(key, str(count['count']), ex=600) count = int(count['count']) lock.release() else: time.sleep(0.1) return count_all_posts_for_query(q, reload=reload) else: count = int(count) return count
def get_artists_by_update_time(offset, reload=False): redis = get_conn() key = 'artists_by_update_time:' + str(offset) artists = redis.get(key) if artists is None or reload: lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True) if lock.acquire(blocking=False): cursor = get_cursor() query = "SELECT * FROM lookup WHERE service != 'discord-channel' ORDER BY updated desc " params = () query += "OFFSET %s " params += (offset,) query += "LIMIT 25" cursor.execute(query, (params,)) artists = cursor.fetchall() redis.set(key, serialize_artists(artists), ex=600) lock.release() else: time.sleep(0.1) get_artists_by_update_time(offset, reload=reload) else: artists = deserialize_artists(artists) return artists
def get_all_dms(offset: int, limit: int, reload: bool = False) -> List[Approved_DM]: redis = get_conn() key = f'all_dms:{str(offset)}' dms = redis.get(key) result = None if dms and not reload: result = deserialize_dms(dms) return [Approved_DM.from_dict(dm) for dm in result] if result else [] lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True) if not lock.acquire(blocking=False): time.sleep(0.1) return get_all_dms(offset, limit, reload=reload) cursor = get_cursor() query_args = dict(offset=offset, limit=limit) query = """ SELECT id, "user", service, content, embed, file, added, published FROM dms ORDER BY added DESC OFFSET %(offset)s LIMIT %(limit)s """ cursor.execute(query, query_args) result = cursor.fetchall() redis.set(key, serialize_dms(result), ex=600) lock.release() dms = [Approved_DM.from_dict(dm) for dm in result] if result else [] return dms
def is_post_flagged(service, artist_id, post_id, reload=False): redis = get_conn() key = 'is_post_flagged:' + service + ':' + str(artist_id) + ':' + str( post_id) flagged = redis.get(key) if flagged is None or reload: lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True) if lock.acquire(blocking=False): cursor = get_cursor() query = "SELECT * FROM booru_flags WHERE id = %s AND \"user\" = %s AND service = %s" cursor.execute(query, ( post_id, artist_id, service, )) flagged = cursor.fetchone() is not None redis.set(key, str(flagged), ex=600) lock.release() else: time.sleep(0.1) return is_post_flagged(service, artist_id, post_id, reload=reload) else: flagged = flagged.decode('utf-8') == 'True' return flagged
def get_artist_dms(service: str, artist_id: int, reload: bool = False) -> List[Approved_DM]: redis = get_conn() key = f'dms:{service}:{str(artist_id)}' dms = redis.get(key) result = None if dms and not reload: result = deserialize_dms(dms) return [Approved_DM.from_dict(dm) for dm in result] if result else [] lock = KemonoRedisLock(redis, key, expire=60, auto_renewal=True) if not lock.acquire(blocking=False): time.sleep(0.1) return get_artist_dms(service, artist_id, reload=reload) cursor = get_cursor() query_args = dict(service=service, artist_id=artist_id) query = """ SELECT id, "user", service, content, embed, file, added, published FROM dms WHERE service = %(service)s AND "user" = %(artist_id)s """ cursor.execute(query, query_args) result = cursor.fetchall() redis.set(key, serialize_dms(result), ex=600) lock.release() dms = [Approved_DM.from_dict(dm) for dm in result] if result else [] return dms