def get_forum_posts(anime: Anime) -> int:
"""
Requests forum posts from Jikan, then sums up all the episode discussion
thread replies.
Where N is the weekly interval forum posts are scored:
if this week is not a multiple of N, return 0
Otherwise:
Gathers all the posts in the forum discussion threads of the past N weeks,
Currently uses regex to match for Episode Discussion. This is brittle and
will stop working once we get over 15 forum threads on an anime.
ASAP we should fix Jikan's API so that it can filter for episode discussion threads
in its search. Alternatively, we could remove the 15 search result limit but that
seems harder to do.
Also TODO: add functionality to subtract a certain number of forum posts,
i.e. posts made before season started
"""
week = config.getint("weekly info", "current-week")
n_week = config.getint("scoring info", "forum-posts-every-n-weeks")
# don't score forum posts if this isn't the right week
if week % n_week != 0:
return 0
jikan = jikanpy.Jikan()
forum_threads: List[Dict[str,
Any]] = jikan.anime(anime.id,
extension="forum")["topics"]
or_alias = ""
if anime.alias:
or_alias = f"|{anime.alias}"
# here we build the regex OR statement based on all the week numbers
# since the last time we checked forum posts
episode_nums_str = "|".join(
[str(num) for num in range(max(1, week - n_week + 1), week + 1)])
episode_discussions = [
thread for thread in forum_threads if re.fullmatch(
f"({anime.name}{or_alias}) Episode ({episode_nums_str}) Discussion",
thread["title"],
)
]
if not episode_discussions:
print(f"""
WARNING: did not find as many episode discussion threads for {anime.name}.
Double check that this is expected and manually update if necessary.
(Found {len(episode_discussions)} discussions in the {n_week} weeks
before week {week})
""")
return sum([disc["replies"] for disc in episode_discussions])
def main(username, force, rate_limit, blur_factor, blur_radius):
"""
Main function module for ScriptTaste.
Arguments:
username:
MyAnimeList username to query for list of episodes watched of anime.
force:
Forces data to be repulled from MyAnimeList even if a cached version exists.
rate_limit: float
Rate limit to sleep between MAL queries.
blur_factor:
Number of times to apply a blurring operation to diffuse wasted space.
blur_radius:
Radius of neighbourhood for use as Gaussian blurring parameter.
"""
jikan = jikanpy.Jikan()
try:
user_animes = src.url_utils.fetch_user_animes(jikan, username, force)
except jikanpy.exceptions.APIException as err:
if err.args[0].split(" ", 1)[0] == "400":
print(RED + "Username not found, was it typed correctly?" + END)
exit(0)
raise err
try:
src.url_utils.fetch_images(user_animes, force, rate_limit)
src.url_utils.fetch_anime_metadata(jikan, user_animes, force,
rate_limit)
except jikanpy.exceptions.APIException as err:
code = err.args[0].split(" ", 1)[0]
if code == "429":
print(RED + "Rate limit exceeded!" +
"\nNo worries, data up to this point has been cached." +
"\nSimply wait a bit and try again." + END)
exit(0)
elif code == "400":
print(
RED +
"Failure to connect with API, this behaviour is unexpected." +
"\nUser may need to debug on their end. If Resolved please" +
"\nsubmit a PR to help others in the future." + END)
raise err
src.collage.draw(user_animes, username, blur_factor, blur_radius)
def get_anime_stats_from_jikan(anime: Anime) -> AnimeStats:
"""
Makes requests to Jikan given an anime id
and returns a Dict containing all the information needed to
populate an AnimeWeeklyStat object
"""
jikan = jikanpy.Jikan()
general_anime_info = jikan.anime(anime.id)
jikan_anime_stats = jikan.anime(anime.id, extension="stats")
return AnimeStats(
watching=jikan_anime_stats["watching"],
completed=jikan_anime_stats["completed"],
dropped=jikan_anime_stats["dropped"],
score=general_anime_info.get("score", 0),
favorites=general_anime_info["favorites"],
forum_posts=get_forum_posts(anime),
)
def get_total_forum_posts(self, anime: Anime) -> int:
"""
Requests episode discussion threads from Jikan, then sums up all the thread replies.
Where N is the weekly interval forum posts are scored:
if this week is not a multiple of N, return 0
Otherwise:
Gathers all the posts in the forum discussion threads of the past N weeks,
Also TODO: add functionality to subtract a certain number of forum posts,
i.e. posts made before season started
"""
jikan = jikanpy.Jikan()
episode_discussions: List[Dict[str, Any]] = jikan.anime(
anime.mal_id, extension="forum/episodes")["topics"]
if not episode_discussions:
print(f"""
WARNING: did not find any episode discussion threads for {anime}.
Double check that this is expected and manually update if necessary.
""")
return sum([disc["replies"] for disc in episode_discussions])
def team_ages() -> None:
"""
Potentially can help spot alt accounts
"""
jikan = jikanpy.Jikan()
season_of_year = config.get("season info", "season")
year = config.getint("season info", "year")
with session_scope() as session:
teams = Season.get_season_from_database(season_of_year, year,
session).teams
for team in cast(Iterable[Team], teams):
if not team.mal_join_date:
print(f"Getting join date for {team.name}")
assert team.name
try:
team.mal_join_date = parse(jikan.user(team.name)["joined"])
session.commit()
# time.sleep(config.get('jikanpy', 'request-interval'))
except Exception as e:
print(
f"Ran into issues figuring out join date with {team.name}: {str(e)}"
)
continue
import time
import jikanpy
import requests
import backoff # type: ignore[import]
from typing import Dict, Any, Iterator, List
from . import fibo_long
j = jikanpy.Jikan("http://localhost:8000/v3/")
@backoff.on_exception(
fibo_long, # fibonacci sequence backoff
(jikanpy.exceptions.JikanException, jikanpy.exceptions.APIException),
max_tries=3,
on_backoff=lambda x: print("backing off"),
)
def get_page(username: str, page: int) -> Dict[str, Any]:
# this does block the main thread for 10 seconds at a time, but
# thats probably okay, not a ton of requests get sent to this
# bot that arent background tasks
# the above iterator in main.py that iterates over the results
# of download_users_list does an await asyncio.sleep to allow
# other tasks to run if they exist
time.sleep(10)
return j.user(username, "animelist", argument="all", page=page)
def download_users_list(username: str) -> Iterator[List[Dict[str, Any]]]:
#!/usr/bin/python3.7
import jikanpy
import re
import time
import configparser
jikan = jikanpy.Jikan()
def check(name: str) -> None:
"""Check if the profile exists based on the name"""
print(name)
try:
user = jikan.user(name)
assert name.lower() == user["username"].lower()
except jikanpy.exceptions.APIException:
print(f"username doesn't exist anymore: {name}")
def main() -> None:
config = configparser.ConfigParser()
config.read("config.ini")
f_name = "lists/team_headcount.txt"
names = re.findall(r"\[b\](.+?)\[/b\]\n", open(f_name).read())
for name in names:
check(name)
An example of how to transparently cache jikanpy requests using the
cachecontrol/requests modules
To install:
pip install --user cachecontrol[filecache] jikanpy
"""
import time
import requests
import jikanpy
from cachecontrol import CacheControl
from cachecontrol.heuristics import ExpiresAfter
from cachecontrol.caches.file_cache import FileCache
# define heuristic, how long requests should stay in cache
# you can modify this to fit whatever you want
# it accepts the same kwargs as datetime.timedelta:
# https://docs.python.org/3/library/datetime.html#datetime.timedelta
expires = ExpiresAfter(days=1)
# create session and mount file cache
session = CacheControl(requests.Session(), heuristic=expires, cache=FileCache("cache_dir"))
# use session for jikanpy
j = jikanpy.Jikan(session=session)
# the second request here is cached
print(j.anime(1)["title"])
print(j.anime(1)["title"])
import requests
import jikanpy
from functools import partial
import urllib3
urllib3.disable_warnings(urllib3.exceptions.SubjectAltNameWarning)
PROXY_IP = os.environ["PROXY_IP"]
PROXY_PORT = os.environ["PROXY_PORT"]
PROXY_TOKEN = os.environ["PROXY_TOKEN"]
PROXY_FULL = "https://{}:{}".format(PROXY_IP, PROXY_PORT)
print("Requesting to", PROXY_FULL)
session = requests.Session()
# re-assign session.get with partial that validates HTTPS
session.get = partial(session.get, verify='cert.pem')
# update default headers to include authentication
session.headers.update({"Proxy-Token": PROXY_TOKEN})
jikan_instance = jikanpy.Jikan(selected_base=PROXY_FULL, session=session)
resp = session.get(PROXY_FULL, headers={"Proxy-To": "https://httpbin.org"})
resp.raise_for_status()
print(resp)
print(jikan_instance.anime(1)["title"])
def __init__(self, wait, retry_max):
self.wait = wait
self.jikan = jikanpy.Jikan()
self.retry_max = retry_max
self.last_scrape = time.time() - (self.wait * 0.5)
