def observe():
try:
with rabbitpy.Connection('amqp://{username}:{password}@{host}:{port}/{vhost}'.format(
username=settings.get('RABBITMQ_USERNAME'),
password=settings.get('RABBITMQ_PASSWORD'),
host=settings.get('RABBITMQ_HOST'),
port=settings.get('RABBITMQ_PORT'),
vhost=settings.get('RABBITMQ_VHOST')
)) as conn:
with conn.channel() as channel:
channel.enable_publisher_confirms()
Ayumi.set_rabbitpy_channel(channel)
Ayumi.info("Now connected AMQP provider.", color=Ayumi.GREEN)
event_handler = IzumiHandler(channel)
observer = Observer()
observer.schedule(event_handler, settings.get('KOTEN_WATCH_PATH', DEFAULT_WATCH_PATH), recursive=True)
observer.start()
Ayumi.info("Now observing: {}".format(settings.get('KOTEN_WATCH_PATH', DEFAULT_WATCH_PATH)), color=Ayumi.BLUE)
try:
while True:
time.sleep(settings.get('KOTEN_SLEEP_INTERVAL', DEFAULT_SLEEP_INTERVAL))
except:
Ayumi.warning("Detected SIGKILL or error, returning...", color=Ayumi.YELLOW)
observer.stop()
observer.join()
except rabbitpy.exceptions.AMQPConnectionForced:
Ayumi.rabbitpy_channel = None
Ayumi.critical("Operator manually closed RabbitMQ connection, shutting down.", color=Ayumi.RED)
# Use return for now because in some cases, calling exit() may invoke the retry() header.
return
def _add_list_entries(list_name, list_json):
"""
Helper method to add all list entries into a medialist (watching/paused/ptw)
Params:
list_name: the name that the list appears to be from Anilist ("Watching")
list_json: anilist's raw api response (json format) {'data':'MediaListCollection'}
Returns: A list with populated Anilist Media entries (a list of dicts)
"""
try:
entries = list()
media_lists = list_json['data']['MediaListCollection']['lists']
for media_list in media_lists:
if list_name.lower() == media_list['name'].lower():
for entry in media_list['entries']:
entries.append(entry['media'])
return entries
except:
Ayumi.warning("Kishi was unable to process list entries for {}".format(
list_name))
raise Exception()
def _get_main_studio_info(studios):
"""
Goes through the studio edges and returns the main (studio name, siteurl)
Params:
studios - The studios body from the Anilist GraphQL json response
Returns: A tuple (studio name: str, site url: str), or None if not found
"""
try:
edges = studios['edges']
for edge in edges:
Ayumi.debug("Checking edge {}".format(edge['node']['name']))
if edge['isMain']:
Ayumi.debug("Found main studio edge, returning tuple")
node = edge['node']
return (node['name'], node['siteUrl'])
# If a main studio isn't found, return None
Ayumi.debug("Didn't find any main studio edge, returning None")
return (None, None)
except Exception as e:
Ayumi.warning(
"Didn't find any main studio edge due to error, returning None")
Ayumi.warning(e)
return (None, None)
def find_closest_title(title):
"""
Finds the closest title from Anilist for Airing and Not_Yet_Released shows
"""
now = datetime.datetime.now()
date_next_month = int((now + datetime.timedelta(weeks=4)).strftime("%Y%m%d"))
date_last_month = int((now - datetime.timedelta(weeks=4)).strftime("%Y%m%d"))
shows = list()
heap = list()
shows.extend(_get_shows("RELEASING"))
shows.extend(_get_shows("NOT_YET_RELEASED", start_date=date_next_month))
shows.extend(_get_shows("FINISHED", end_date=date_last_month))
for show in shows:
ratio = _similarity(title, show)
Ayumi.debug('Matched "{}" against "{}" for a ratio of {}'.format(title, show, ratio))
heapq.heappush(heap, (ratio, show))
top_5 = heapq.nlargest(5, heap)
Ayumi.info("Displaying (up to) top 5 matches of {} results:".format(len(heap)), color=Ayumi.LBLUE)
for top in top_5:
Ayumi.info("{}: {}".format(top[1], top[0]), color=Ayumi.LBLUE)
if top_5:
Ayumi.info('Returning top match: {}'.format(top_5[0][1]), color=Ayumi.LGREEN)
return top_5[0][1]
else:
Ayumi.warning("No shows were fetched by Naomi, returning None", color=Ayumi.LYELLOW)
return None
def _clean_episode_name(unclean_name):
"""
Clean the episode name for new usage.
Parameter unclean_name should only be the file name, no paths.
"""
info = anitopy.parse(unclean_name)
new_name = info['anime_title']
if 'anime_season' in info:
Ayumi.debug('Found anime_season "{}"'.format(info['anime_season']))
new_name = new_name + " S" + str(info['anime_season'])
if 'episode_number' in info:
Ayumi.debug('Found episode_number "{}"'.format(info['episode_number']))
new_name = new_name + " - " + str(info['episode_number'])
if 'video_resolution' in info:
Ayumi.debug('Found video_resolution "{}"'.format(info['video_resolution']))
new_name = new_name + " [{}]".format(info['video_resolution'])
if 'other' in info and 'uncensored' in info['other'].lower():
Ayumi.debug('Detected this episode is uncensored, adding "(Uncensored)" to the title.')
_, ext = os.path.splitext(unclean_name)
new_name += ext
Ayumi.debug('Complete new file name: {}'.format(new_name))
return new_name
def _load_amqp_message_body(message) -> Dict:
try:
feeditem = loads(message.body.decode('utf-8'))
return feeditem
except JSONDecodeError:
Ayumi.warning(
"Received an AMQP message that is invalid JSON, will not process...", color=Ayumi.RED)
return None
def _get_kitsu_id(title):
"""Gets Kitsu's ID for a show." Returns -1 if not found."""
try:
kitsu_id = _kitsu_basic_search(title)['data'][0]['id']
return int(kitsu_id)
except:
Ayumi.debug("Error occured when fetching Kitsu ID, returning -1.")
return -1
def _parse_shows_map(show_title) -> Tuple[str, str]:
if "->" in show_title:
original_title, override_title = show_title.split(" -> ")
Ayumi.debug(
"Found '->' in title, mapping (unstripped) {} to {}".format(original_title, override_title))
return (_strip_title(original_title), override_title)
else:
return (_strip_title(show_title), show_title)
def _load_history() -> List[str]:
try:
with open("data.json", "r") as data:
history = json.load(data)
Ayumi.debug("Loaded history: {}".format(history))
except:
history = []
Ayumi.debug("No history loaded - using empty list")
return history
def _clean_title(title):
"""
Removes potentially problematic characters.
"""
try:
clean_str = title.replace('"', '')
Ayumi.debug("Cleaned {} to {}.".format(title, clean_str))
return clean_str
except:
Ayumi.debug(
"Cleaner wasn't provided a valid title ({}), returning None.".
format(title))
return None
def search(show):
airing = _single_search(show, "RELEASING")
if airing:
Ayumi.info(
"Found show {} in RELEASING, returning Hisha object.".format(show),
color=Ayumi.LCYAN)
return _create_hisha_object(airing, show)
finished = _page_search(show, "FINISHED")
if finished:
Ayumi.info(
"Found show {} in FINISHED, returning Hisha object.".format(show),
color=Ayumi.LCYAN)
return _create_hisha_object(finished, show)
not_yet_released = _single_search(show, "NOT_YET_RELEASED")
if not_yet_released:
Ayumi.info(
"Found show {} in NOT_YET_RELEASED, returning Hisha object.".
format(show),
color=Ayumi.LCYAN)
return _create_hisha_object(not_yet_released, show)
# None of them found a show, so create a dummy Hisha object.
Ayumi.info(
"Creating dummy Hisha object for {} with default values.".format(show),
color=Ayumi.LCYAN)
return _create_hisha_object(None, show)
def _fetch_retry(user, listurl, listname, times=5):
"""
Jikan.moe is susceptible to randomly failing. This method allows us to try multiple times before really "failing"
Params: See fetch_list()
Returns: See fetch_list() if successful, or raises an Exception() otherwise
"""
for i in range(times):
try:
Ayumi.debug("Attempt #{} to contact Jikan.moe for {}".format(
i + 1, listname))
anime = _fetch_list(user, listurl, listname)
Ayumi.debug("Attempt #{} succeeded".format(i + 1))
return anime
except:
# Sleep 5 seconds, and then try again
Ayumi.debug(
"Attempt #{} failed, sleeping 5 seconds and trying again...".
format(i + 1))
sleep(5)
# If this point is reached, then there has been too many errors. Raise an exception
Ayumi.debug("Akari was unable to contact Jikan.moe")
raise Exception()
def _check_exists(config: str, source: str, job: Job) -> bool:
"""
Checks if the jobs exists under the source.
Note: Does not validate under which path, just validates that jobs exists somewhere in:
source/(...probably job.show)/job.episode
"""
try:
response = _run([
"rclone", "--config={}".format(config), "lsjson", "-R",
"{}/{}/".format(source, job.show)
])
episode_list = loads(response.stdout.decode('utf-8'))
for episode in episode_list:
Ayumi.debug("Checking {} against episode {}".format(
job.episode, episode['Name']))
if episode['Name'] == job.episode:
Ayumi.info("Found episode {} in {}".format(
job.episode, source))
return True
Ayumi.info("Didn't find episode {} in {}".format(job.episode, source))
return False
except:
# Typically hit if the source doesn't exist.
Ayumi.warning(
"Error occured while checking source {} - does it exist?".format(
source))
return False
def _run(command: Iterable[str]) -> str:
# Subprocess passes signals properly to the ffmpeg child, so we can just run it as is.
try:
response = subprocess.run(command, capture_output=True, timeout=3600)
except subprocess.TimeoutExpired:
Ayumi.warning("Command expired by timeout, killing...")
raise ShikyouResponseException()
if response.returncode != 0:
Ayumi.warning(
"Command caused return code of {}, returning None.".format(
response.returncode))
raise ShikyouResponseException()
return response
def _query_request(query, search, status) -> dict:
"""
Makes requests to Anlist, returns response in JSON.
Query: One of the Queries objects.
Search: Name of show to search for.
Status: Status of the show to search for.
"""
try:
Ayumi.debug(
"Making request to {}, searching for {} under status {}".format(
ANILIST_API_URL, search, status))
ani_res = requests.post(ANILIST_API_URL,
json={
'query': query,
'variables': {
'search': search,
'status': status
}
},
timeout=10)
if ani_res.status_code != 200:
Ayumi.warning(
"Anilist returned unaccepted HTTP code {} upon request.".
format(ani_res.status_code),
color=Ayumi.LRED)
raise Exception()
# Get request response as JSON object.
try:
ani_json = ani_res.json()
return ani_json['data']
except:
Ayumi.warning("Anilist returned a non-JSON response.",
color=Ayumi.LRED)
raise Exception()
except requests.exceptions.Timeout:
Ayumi.warning("Request to Anilist timed out.", color=Ayumi.LRED)
raise Exception()
except requests.exceptions.ConnectionError:
Ayumi.warning("Unable to contact Anilist, maybe it's down?",
color=Ayumi.LRED)
raise Exception()
def is_user_watching_names(user, show_name):
"""
Determines whether or not an Anilist user is watching a show
Checks by show name
Params:
user: username to look up
show_name: name of the show to look up. this should already be the anilist name.
Returns: a boolean - True if watching, False if not
"""
try:
watching, paused, ptw = _kishi_list(user)
for show in watching:
for title in show['title'].values():
if _check_equality(title, show_name):
Ayumi.debug("Matched {} to {} in {}".format(
title, show_name, "watching"))
return True
for show in paused:
for title in show['title'].values():
if _check_equality(title, show_name):
Ayumi.debug("Matched {} to {} in {}".format(
title, show_name, "paused"))
return True
for show in ptw:
for title in show['title'].values():
if _check_equality(title, show_name):
Ayumi.debug("Matched {} to {} in {}".format(
title, show_name, "planning"))
return True
Ayumi.debug("Didn't find a match for {}".format(show_name))
return False
except:
# If any errors are encountered, return True (default assumption)
Ayumi.warning(
"An error was encountered while contacting Anilist. Defaulting to TRUE"
)
return True
def _generate_new_filename(dl_file):
info = anitopy.parse(dl_file)
new_dl_file = info['anime_title']
if 'anime_season' in info:
Ayumi.debug('Found anime_season "{}"'.format(info['anime_season']))
new_dl_file = new_dl_file + " S" + str(info['anime_season'])
if 'episode_number' in info:
Ayumi.debug('Found episode_number "{}"'.format(info['episode_number']))
new_dl_file = new_dl_file + " - " + str(info['episode_number'])
if 'video_resolution' in info:
Ayumi.debug('Found video_resolution "{}"'.format(
info['video_resolution']))
new_dl_file = new_dl_file + " [{}]".format(info['video_resolution'])
if 'other' in info and 'uncensored' in info['other'].lower():
Ayumi.debug(
'Detected this episode is uncensored, adding "(Uncensored)" to the title.')
new_dl_file += " (Uncensored)"
_, ext = os.path.splitext(dl_file)
new_dl_file += ext
Ayumi.debug('returning new_dl_file: {}'.format(new_dl_file))
return new_dl_file
def upload(job: Job, destinations: List[str], upload_file: str, config: str,
flags: str) -> None:
"""
Upload the completed new hardsub file into the rclone destinations
Returns a boolean based on success
job: Job to do! This is the job of the HARDSUB file
destinations: list of rlcone destinations (e.g., EncoderConf.uploading_destinations)
upload_file: Path to the file to be uploaded
rclone_config: Path to the rclone config file
flags: rclone flag
This method will upload the file and include its show name:
e.g., 'temp.mp4' --> destination/show/episode.mp4
"""
for dest in destinations:
rclone_dest = "{}/{}/{}".format(_clean(dest), job.show, job.episode)
command = [
"rclone", "--config={}".format(config), "copyto", upload_file,
rclone_dest
]
command.extend(flags.split())
Ayumi.debug("Now running command: {}".format(" ".join(command)))
Ayumi.info("Now uploading file to {}".format(dest))
try:
_run(command)
except ShikyouResponseException:
Ayumi.error(
"Rclone command returned a bad return code, contact the developer.",
color=Ayumi.LRED)
raise ShikyouResponseException()
except ShikyouTimeoutException:
Ayumi.error("Rclone command timed out, are your sources okay?",
color=Ayumi.LRED)
raise ShikyouTimeoutException()
Ayumi.info("Completed uploading files.", color=Ayumi.LGREEN)
def on_created(self, event):
Ayumi.info('Detected new creation event under path "{}", running preflight checks.'.format(event.src_path), color=Ayumi.LCYAN)
# Do not process new directory events
if event.is_directory:
Ayumi.info('Detected new event directory under path "{}", ignoring.'.format(event.src_path), color=Ayumi.LYELLOW)
return
if not os.path.exists(event.src_path):
Ayumi.info("Event detected as false positive, will not proceed.", color=Ayumi.LYELLOW)
return
Ayumi.info("Preflight check good, starting pipeline.", color=Ayumi.LGREEN)
on_new_file(event.src_path, self.channel)
def _kishi_list(user):
"""
Helper method to get all of a user's anime list.
Params:
user: String, username of Anilist user
Returns: A tuple of three lists.
The first list is all the Watching
The second list is all the PTW
The third list is all the Paused
Throws an exception if anything goes wrong. This should be caught by any method using this.
"""
watching = list()
paused = list()
ptw = list()
# Anilist API is much nicer to play with.
try:
# Make the request to Anilist, and pass in the userName as the user query
anilist_res = requests.post(_ANILIST_API_URL,
json={
'query': _USER_ANIME_QUERY,
'variables': {
'userName': user
}
})
if anilist_res.status_code != 200:
Ayumi.error(
"Anilist returned a bad status code when attempting to get {}'s lists"
.format(user))
raise Exception()
try:
anilist_res_json = anilist_res.json()
except:
Ayumi.error(
"Anilist returned a response that was not parseable into JSON")
raise Exception()
watching = _add_list_entries("Watching", anilist_res_json)
paused = _add_list_entries("Paused", anilist_res_json)
ptw = _add_list_entries("Planning", anilist_res_json)
except:
Ayumi.critical("Kishi was unable to properly contact Anilist")
raise Exception()
return (watching, paused, ptw)
def on_new_file(src_path, channel):
show_name = None
episode_name = None
new_file = src_path.replace(os.path.commonpath([settings.get('KOTEN_WATCH_PATH', DEFAULT_WATCH_PATH), src_path]) + "/", "")
if m := util._show_manually_specified(new_file):
Ayumi.info("Detected show name and episode name in event, using Mode 1.")
show_name = m.group(1)
episode_name = util._clean_episode_name(m.group(2))
Ayumi.info("New show name: {}".format(show_name), color=Ayumi.LYELLOW)
Ayumi.info("New episode name: {}".format(episode_name), color=Ayumi.LYELLOW)
def _load_shows_map() -> Dict[str, str]:
shows_map = settings.get(
'ACQUISITION_DOWNLOAD_BITTORRENT_SHOWS_MAP', [])
Ayumi.debug("Fetched shows map from config: {}".format(shows_map))
show_name_map = dict()
Ayumi.debug("Creating a stripped title to override title mapping...")
for show in shows_map:
stripped_title, override_title = _parse_shows_map(show)
show_name_map[stripped_title] = override_title
Ayumi.debug("Mapped {} to key {}.".format(
override_title, stripped_title))
return show_name_map
def _kitsu_basic_search(title):
"""Quick Kitsu implementation"""
title_lower = title.lower()
request_url = requests.utils.requote_uri(KITSU_API_URL + title_lower)
Ayumi.debug("Created Kitsu request URL: {}".format(request_url))
try:
kitsu_res = requests.get(request_url, timeout=10)
try:
kitsu_json = kitsu_res.json()
return kitsu_json
except:
Ayumi.warning("Kitsu did not return a valid JSON response.",
color=Ayumi.RED)
raise Exception()
except requests.exceptions.Timeout:
Ayumi.warning("Kitsu request timed out.", color=Ayumi.LRED)
raise Exception()
except requests.exceptions.ConnectionError:
Ayumi.warning("Unable to contact Kitsu, maybe it's down?",
color=Ayumi.LRED)
raise Exception()
def is_user_watching_id(user, malID, times=5):
"""
Is a user watching this show or not?
Params:
user: username to lookup
malID: malID to match against
"""
Ayumi.debug("Now finding if \"{}\" is in {}'s list".format(malID, user))
anime_list = akari_list(user, times)
for show in anime_list:
if str(show['mal_id']) == str(malID):
Ayumi.info("\"{}\" was found in {}'s list".format(malID, user),
color=Ayumi.LGREEN)
return True
Ayumi.info("\"{}\" was not found in {}'s list".format(malID, user),
color=Ayumi.LYELLOW)
return False
def is_user_watching_names(user, show_name, times=5):
"""
Is a user watching this show or not?
Params:
user: username to lookup
show_name: show name to match against
Returns True if the show was found in the list, false if not
"""
Ayumi.debug("Now finding if \"{}\" is in {}'s list".format(
show_name, user))
anime_list = akari_list(user, times)
for show in anime_list:
if show['title'] == show_name:
Ayumi.info("\"{}\" was found in {}'s list".format(show_name, user),
color=Ayumi.LGREEN)
return True
Ayumi.info("\"{}\" was not found in {}'s list".format(show_name, user),
color=Ayumi.LYELLOW)
return False
def consume():
try:
with rabbitpy.Connection(
'amqp://{username}:{password}@{host}:{port}/{vhost}'.format(
username=settings.get('RABBITMQ_USERNAME'),
password=settings.get('RABBITMQ_PASSWORD'),
host=settings.get('RABBITMQ_HOST'),
port=settings.get('RABBITMQ_PORT'),
vhost=settings.get('RABBITMQ_VHOST'))) as conn:
with conn.channel() as channel:
Ayumi.set_rabbitpy_channel(channel)
queue = rabbitpy.Queue(
channel,
settings.get('NOTIFICATIONS_DISCORD_WEBHOOK_QUEUE',
'nonexistent'))
queue.declare(passive=True)
Ayumi.info("Now listening for messages from AMQP provider.",
color=Ayumi.YELLOW)
for message in queue.consume(prefetch=1):
try:
job = json.loads(message.body.decode('utf-8'))
except json.JSONDecodeError:
Ayumi.warning(
"Received a job that is invalid json, not processing.",
color=Ayumi.LRED)
message.reject()
continue
Ayumi.info("Received a new job: {}".format(
json.dumps(job)),
color=Ayumi.CYAN)
if metsuke.validate(job):
Ayumi.debug("Loaded show: {}".format(job['show']))
Ayumi.debug("Loaded episode: {}".format(
job['episode']))
Ayumi.debug("Loaded filesize: {}".format(
job['filesize']))
Ayumi.debug("Loaded sub type: {}".format(job['sub']))
embed = _generate_embed(metsuke.generate(job),
hisha.search(job['show']))
Ayumi.info(
"Beginning sending embeds to webhook endpoints.",
color=Ayumi.CYAN)
for endpoint in settings.get(
'NOTIFICATIONS_DISCORD_WEBHOOK_ENDPOINTS'
).to_list():
try:
requests.post(endpoint, json=embed, timeout=5)
Ayumi.debug(
"Sent embed to {}".format(endpoint))
except:
Ayumi.warning(
"Failed to send embed to {}".format(
endpoint),
color=Ayumi.RED)
else:
Ayumi.warning(
"Received a job that Metsuke was not able to validate.",
color=Ayumi.LRED)
Ayumi.warning(json.dumps(job), color=Ayumi.LRED)
Ayumi.info(
"Completed processing this message for {}".format(
job['episode']),
color=Ayumi.LGREEN)
message.ack()
except rabbitpy.exceptions.AMQPConnectionForced:
Ayumi.rabbitpy_channel = None
Ayumi.critical(
"Operator manually closed RabbitMQ connection, shutting down.",
color=Ayumi.RED)
# Use return for now because in some cases, calling exit() may invoke the retry() header.
return
def sig_handler(sig, frame):
Ayumi.warning("SIG command {} detected, exiting...".format(sig),
color=Ayumi.LRED)
sys.exit()
'name':
'Links',
'value':
"[MyAnimeList]({}) | [Anilist]({}) | [Kitsu]({})".format(
str.strip(_MAL_ANI_BASE + str(hisha.id_mal)),
str.strip(_ANI_ANI_BASE + str(hisha.id)),
str.strip(_KIT_ANI_BASE + str(hisha.id_kitsu)))
})
webhook['embeds'].append(embed)
return webhook
# Retry on all instances except
@retry(delay=60, backoff=1.5, max_delay=3600, logger=Ayumi.get_logger())
def consume():
try:
with rabbitpy.Connection(
'amqp://{username}:{password}@{host}:{port}/{vhost}'.format(
username=settings.get('RABBITMQ_USERNAME'),
password=settings.get('RABBITMQ_PASSWORD'),
host=settings.get('RABBITMQ_HOST'),
port=settings.get('RABBITMQ_PORT'),
vhost=settings.get('RABBITMQ_VHOST'))) as conn:
with conn.channel() as channel:
Ayumi.set_rabbitpy_channel(channel)
queue = rabbitpy.Queue(
channel,
def _fetch_list(user, listurl, listname):
"""
Helper method to get a user's anime list.
Doesn't handle errors, just raises them. Needs to be called by a wrapper.
Params:
User: Username of MAL user
listurl: URL to use to fetch info
listname: Watching, Plan to Watch (for logging)
"""
anime = list()
try:
jikan_res = requests.get(listurl.format(user, ""))
# Verify status code
if jikan_res.status_code != 200:
Ayumi.debug(
"jikan mode returned a bad status code on attempt to get {}'s {} list."
.format(user, listname))
raise Exception()
# Make sure an anime actually exists undeer this name
try:
jikan_res_json = jikan_res.json()
except:
jikan_res_json = dict() # Clean handling
if 'anime' not in jikan_res_json:
Ayumi.debug(
"Jikan.moe did not return an anime list on attempt to get {}'s {} list."
.format(user, listname))
raise Exception()
# Add all anime in the first page.
for entry in jikan_res_json['anime']:
anime.append(entry)
Ayumi.debug("Added {} show {} to processing list.".format(
listname, entry['title']))
page = 2
while (len(jikan_res_json['anime']) == 300):
jikan_res = requests.get(listurl.format(user, str(page)))
if jikan_res.status_code != 200 or 'anime' not in jikan_res:
Ayumi.debug(
"Jikan returned a bad status code when attempting to get {}'s page {} {} list."
.format(user, str(page), listname))
raise Exception()
try:
jikan_res_json = jikan_res.json()
except:
jikan_res_json = dict()
if 'anime' not in jikan_res_json:
Ayumi.debug(
"Jikan.moe did not return an anime list on attempt to get {}'s page {} {} list."
.format(user, str(page), listname))
raise Exception()
for entry in jikan_res_json['anime']:
anime.append(entry)
Ayumi.debug(
"Added {} page {} show {} to processing list.".format(
listname, str(page), entry['title']))
page += 1
Ayumi.debug("Returning list with {} entires.".format(str(len(anime))))
return anime
except:
# raise some kind of exception - somehow Jikan couldn't be reached
Ayumi.debug(
"Akari encountered an unknown error when attempting to fetch {}'s {} list"
.format(user, listname))
raise Exception()
def rss(last_guid=None):
try:
with rabbitpy.Connection(
'amqp://{username}:{password}@{host}:{port}/{vhost}'.format(
username=settings.get_fresh('RABBITMQ_USERNAME'),
password=settings.get_fresh('RABBITMQ_PASSWORD'),
host=settings.get_fresh('RABBITMQ_HOST'),
port=settings.get_fresh('RABBITMQ_PORT'),
vhost=settings.get_fresh('RABBITMQ_VHOST'))) as conn:
with conn.channel() as channel:
Ayumi.set_rabbitpy_channel(channel)
channel.enable_publisher_confirms()
while True:
Ayumi.info("Now starting feed fetch.", color=Ayumi.LCYAN)
feed = feedparser.parse(
settings.get('ACQUISITION_RSS_FEED_URL', None))
accepted_shows = _load_accepted_shows()
Ayumi.debug(
"Loaded accepted shows map: {}".format(accepted_shows))
history = _load_history()
new_history = list()
for entry in feed.entries:
# Fetch data first
title, link, guid = entry.title, entry.link, entry.guid
Ayumi.debug(
'Encountered RSS item with title "{}", and guid "{}"'
.format(title, guid))
# If feed item with last GUID encountered, do not process any further
if guid == last_guid:
Ayumi.debug(
"Encountered RSS item with last_guid {} matching argument, breaking and writing history."
.format(last_guid),
color=Ayumi.YELLOW)
break
# Check the title data
# Use the parsed title to match user provided titles.
parsed_title = anitopy.parse(title)['anime_title']
if _strip_title(parsed_title) not in accepted_shows:
Ayumi.info(
'Feed item with title "{}" (show title: "{}") is not in accepted shows, skipping.'
.format(title, parsed_title))
else:
if guid in history:
# This item has been previously processed, skip it.
Ayumi.info(
'Feed item with title "{}" (show title: "{}") has already been processed, skipping.'
.format(title, parsed_title),
color=Ayumi.GREEN)
else:
# A new feeditem! Let us process it.
Ayumi.info(
'Feed item with title "{}" (show title: "{}") is in accepted shows, processing.'
.format(title, parsed_title),
color=Ayumi.YELLOW)
message = rabbitpy.Message(
channel,
json.dumps({
"title":
title,
"link":
link,
"guid":
guid,
"show_title":
accepted_shows[_strip_title(
parsed_title)]
}))
acquisition_rss_exchange_name = settings.get(
'ACQUISITION_RSS_EXCHANGE')
while not message.publish(
acquisition_rss_exchange_name,
mandatory=True):
Ayumi.warning(
'Failed to publish feed item with title "{}" to exchange "{}", retrying in 60s...'
.format(title,
acquisition_rss_exchange_name),
color=Ayumi.RED)
sleep(60)
Ayumi.info(
'Published feed item with title "{}" to exchange "{}".'
.format(
title,
acquisition_rss_exchange_name,
),
color=Ayumi.LGREEN)
# Keep all items processed in the new history - it will be auto deleted by the expiry of the RSS
Ayumi.debug(
'Appending item "{}" with title "{}" (show title: "{}") to new_history for write.'
.format(guid, title, parsed_title),
color=Ayumi.YELLOW)
new_history.append(guid)
_write_history(new_history)
# Sleep till the next iteration
sleep_duration = settings.get(
'ACQUISITION_RSS_SLEEP_INTERVAL',
_DEFAULT_SLEEP_INTERVAL)
Ayumi.info(
"Now sleeping {} seconds.".format(sleep_duration),
color=Ayumi.LCYAN)
sleep(sleep_duration)
except rabbitpy.exceptions.AMQPConnectionForced:
Ayumi.rabbitpy_channel = None
Ayumi.critical(
"Operator manually closed RabbitMQ connection, shutting down.",
color=Ayumi.RED)
# Use return for now because in some cases, calling exit() may invoke the retry() header.
return
