def _clean_episode_name(unclean_name): """ Clean the episode name for new usage. Parameter unclean_name should only be the file name, no paths. """ info = anitopy.parse(unclean_name) new_name = info['anime_title'] if 'anime_season' in info: Ayumi.debug('Found anime_season "{}"'.format(info['anime_season'])) new_name = new_name + " S" + str(info['anime_season']) if 'episode_number' in info: Ayumi.debug('Found episode_number "{}"'.format(info['episode_number'])) new_name = new_name + " - " + str(info['episode_number']) if 'video_resolution' in info: Ayumi.debug('Found video_resolution "{}"'.format(info['video_resolution'])) new_name = new_name + " [{}]".format(info['video_resolution']) if 'other' in info and 'uncensored' in info['other'].lower(): Ayumi.debug('Detected this episode is uncensored, adding "(Uncensored)" to the title.') _, ext = os.path.splitext(unclean_name) new_name += ext Ayumi.debug('Complete new file name: {}'.format(new_name)) return new_name
def _check_exists(config: str, source: str, job: Job) -> bool: """ Checks if the jobs exists under the source. Note: Does not validate under which path, just validates that jobs exists somewhere in: source/(...probably job.show)/job.episode """ try: response = _run([ "rclone", "--config={}".format(config), "lsjson", "-R", "{}/{}/".format(source, job.show) ]) episode_list = loads(response.stdout.decode('utf-8')) for episode in episode_list: Ayumi.debug("Checking {} against episode {}".format( job.episode, episode['Name'])) if episode['Name'] == job.episode: Ayumi.info("Found episode {} in {}".format( job.episode, source)) return True Ayumi.info("Didn't find episode {} in {}".format(job.episode, source)) return False except: # Typically hit if the source doesn't exist. Ayumi.warning( "Error occured while checking source {} - does it exist?".format( source)) return False
def find_closest_title(title): """ Finds the closest title from Anilist for Airing and Not_Yet_Released shows """ now = datetime.datetime.now() date_next_month = int((now + datetime.timedelta(weeks=4)).strftime("%Y%m%d")) date_last_month = int((now - datetime.timedelta(weeks=4)).strftime("%Y%m%d")) shows = list() heap = list() shows.extend(_get_shows("RELEASING")) shows.extend(_get_shows("NOT_YET_RELEASED", start_date=date_next_month)) shows.extend(_get_shows("FINISHED", end_date=date_last_month)) for show in shows: ratio = _similarity(title, show) Ayumi.debug('Matched "{}" against "{}" for a ratio of {}'.format(title, show, ratio)) heapq.heappush(heap, (ratio, show)) top_5 = heapq.nlargest(5, heap) Ayumi.info("Displaying (up to) top 5 matches of {} results:".format(len(heap)), color=Ayumi.LBLUE) for top in top_5: Ayumi.info("{}: {}".format(top[1], top[0]), color=Ayumi.LBLUE) if top_5: Ayumi.info('Returning top match: {}'.format(top_5[0][1]), color=Ayumi.LGREEN) return top_5[0][1] else: Ayumi.warning("No shows were fetched by Naomi, returning None", color=Ayumi.LYELLOW) return None
def _get_kitsu_id(title): """Gets Kitsu's ID for a show." Returns -1 if not found.""" try: kitsu_id = _kitsu_basic_search(title)['data'][0]['id'] return int(kitsu_id) except: Ayumi.debug("Error occured when fetching Kitsu ID, returning -1.") return -1
def _parse_shows_map(show_title) -> Tuple[str, str]: if "->" in show_title: original_title, override_title = show_title.split(" -> ") Ayumi.debug( "Found '->' in title, mapping (unstripped) {} to {}".format(original_title, override_title)) return (_strip_title(original_title), override_title) else: return (_strip_title(show_title), show_title)
def _load_history() -> List[str]: try: with open("data.json", "r") as data: history = json.load(data) Ayumi.debug("Loaded history: {}".format(history)) except: history = [] Ayumi.debug("No history loaded - using empty list") return history
def download(job: Job, sources: List[str], tempfolder: str, config: str, flags: str) -> str: """ Download the provided episode from sources Returns the path of the downloaded file job: Job to do! sources: list of rclone sources (EncoderConf.downloading_sources) tmppath: Path of the temporary folder rclone_config: Path to the rclone config file flags: rclone flags """ for source in sources: Ayumi.debug("Checking for existence from source: {}".format(source)) if _check_exists(config, source, job): Ayumi.info( "Now downloading episode from source: {}".format(source)) src_file = "{}/{}/{}".format(_clean(source), job.show, job.episode) Ayumi.debug("Sourcing from rclone path: {}".format(src_file)) dest_file = "{}/{}".format(_clean(tempfolder), "temp") Ayumi.debug("Downloading to destination: {}".format(dest_file)) command = [ "rclone", "--config={}".format(config), "copyto", src_file, dest_file ] command.extend(flags.split()) Ayumi.debug("Now running command: {}".format(" ".join(command))) Ayumi.info("Now starting download.", color=Ayumi.LCYAN) try: _run(command) except ShikyouResponseException: Ayumi.error( "Rclone command returned a bad return code, contact the developer.", color=Ayumi.LRED) raise ShikyouResponseException() except ShikyouTimeoutException: Ayumi.error("Rclone command timed out, are your sources okay?", color=Ayumi.LRED) raise ShikyouTimeoutException() Ayumi.info("Completed downloading files.", color=Ayumi.LGREEN) return dest_file else: Ayumi.debug( "Requested episode doesn't exist under source {}".format( source)) Ayumi.warning("No download sources contained the file.") return None
def _clean_title(title): """ Removes potentially problematic characters. """ try: clean_str = title.replace('"', '') Ayumi.debug("Cleaned {} to {}.".format(title, clean_str)) return clean_str except: Ayumi.debug( "Cleaner wasn't provided a valid title ({}), returning None.". format(title)) return None
def _fetch_retry(user, listurl, listname, times=5): """ Jikan.moe is susceptible to randomly failing. This method allows us to try multiple times before really "failing" Params: See fetch_list() Returns: See fetch_list() if successful, or raises an Exception() otherwise """ for i in range(times): try: Ayumi.debug("Attempt #{} to contact Jikan.moe for {}".format( i + 1, listname)) anime = _fetch_list(user, listurl, listname) Ayumi.debug("Attempt #{} succeeded".format(i + 1)) return anime except: # Sleep 5 seconds, and then try again Ayumi.debug( "Attempt #{} failed, sleeping 5 seconds and trying again...". format(i + 1)) sleep(5) # If this point is reached, then there has been too many errors. Raise an exception Ayumi.debug("Akari was unable to contact Jikan.moe") raise Exception()
def _query_request(query, search, status) -> dict: """ Makes requests to Anlist, returns response in JSON. Query: One of the Queries objects. Search: Name of show to search for. Status: Status of the show to search for. """ try: Ayumi.debug( "Making request to {}, searching for {} under status {}".format( ANILIST_API_URL, search, status)) ani_res = requests.post(ANILIST_API_URL, json={ 'query': query, 'variables': { 'search': search, 'status': status } }, timeout=10) if ani_res.status_code != 200: Ayumi.warning( "Anilist returned unaccepted HTTP code {} upon request.". format(ani_res.status_code), color=Ayumi.LRED) raise Exception() # Get request response as JSON object. try: ani_json = ani_res.json() return ani_json['data'] except: Ayumi.warning("Anilist returned a non-JSON response.", color=Ayumi.LRED) raise Exception() except requests.exceptions.Timeout: Ayumi.warning("Request to Anilist timed out.", color=Ayumi.LRED) raise Exception() except requests.exceptions.ConnectionError: Ayumi.warning("Unable to contact Anilist, maybe it's down?", color=Ayumi.LRED) raise Exception()
def is_user_watching_id(user, malID, times=5): """ Is a user watching this show or not? Params: user: username to lookup malID: malID to match against """ Ayumi.debug("Now finding if \"{}\" is in {}'s list".format(malID, user)) anime_list = akari_list(user, times) for show in anime_list: if str(show['mal_id']) == str(malID): Ayumi.info("\"{}\" was found in {}'s list".format(malID, user), color=Ayumi.LGREEN) return True Ayumi.info("\"{}\" was not found in {}'s list".format(malID, user), color=Ayumi.LYELLOW) return False
def is_user_watching_id(user, show_id): """ Determines whether or not an Anilist user is watching a show Checks by show ID Params: user: username to look up id: id of the show to look up Returns: a boolean - True if watching, False if not """ try: show_id = int(show_id) # Get the int equivalent value of the ID except: # Why would you not pass an integer in? Ayumi.critical( "Kishi ID search requires an input that can be converted to an int. Returning FALSE" ) return False try: watching, paused, ptw = _kishi_list(user) for show in watching: if show_id == show['id']: Ayumi.debug("Found show ID {} in {}".format( show_id, "watching")) return True for show in paused: if show_id == show['id']: Ayumi.debug("Found show ID {} in {}".format(show_id, "paused")) return True for show in ptw: if show_id == show['id']: Ayumi.debug("Found show ID {} in {}".format( show_id, "planning")) return True Ayumi.debug("Didn't find a match for {}".format(show_id)) return False except: # If any errors are encountered, return True (default assumption) Ayumi.warning( "An error was encountered while contacting Anilist. Defaulting to TRUE" ) return True
def _generate_new_filename(dl_file): info = anitopy.parse(dl_file) new_dl_file = info['anime_title'] if 'anime_season' in info: Ayumi.debug('Found anime_season "{}"'.format(info['anime_season'])) new_dl_file = new_dl_file + " S" + str(info['anime_season']) if 'episode_number' in info: Ayumi.debug('Found episode_number "{}"'.format(info['episode_number'])) new_dl_file = new_dl_file + " - " + str(info['episode_number']) if 'video_resolution' in info: Ayumi.debug('Found video_resolution "{}"'.format( info['video_resolution'])) new_dl_file = new_dl_file + " [{}]".format(info['video_resolution']) if 'other' in info and 'uncensored' in info['other'].lower(): Ayumi.debug( 'Detected this episode is uncensored, adding "(Uncensored)" to the title.') new_dl_file += " (Uncensored)" _, ext = os.path.splitext(dl_file) new_dl_file += ext Ayumi.debug('returning new_dl_file: {}'.format(new_dl_file)) return new_dl_file
def _get_main_studio_info(studios): """ Goes through the studio edges and returns the main (studio name, siteurl) Params: studios - The studios body from the Anilist GraphQL json response Returns: A tuple (studio name: str, site url: str), or None if not found """ try: edges = studios['edges'] for edge in edges: Ayumi.debug("Checking edge {}".format(edge['node']['name'])) if edge['isMain']: Ayumi.debug("Found main studio edge, returning tuple") node = edge['node'] return (node['name'], node['siteUrl']) # If a main studio isn't found, return None Ayumi.debug("Didn't find any main studio edge, returning None") return (None, None) except Exception as e: Ayumi.warning( "Didn't find any main studio edge due to error, returning None") Ayumi.warning(e) return (None, None)
def is_user_watching_names(user, show_name, times=5): """ Is a user watching this show or not? Params: user: username to lookup show_name: show name to match against Returns True if the show was found in the list, false if not """ Ayumi.debug("Now finding if \"{}\" is in {}'s list".format( show_name, user)) anime_list = akari_list(user, times) for show in anime_list: if show['title'] == show_name: Ayumi.info("\"{}\" was found in {}'s list".format(show_name, user), color=Ayumi.LGREEN) return True Ayumi.info("\"{}\" was not found in {}'s list".format(show_name, user), color=Ayumi.LYELLOW) return False
def upload(job: Job, destinations: List[str], upload_file: str, config: str, flags: str) -> None: """ Upload the completed new hardsub file into the rclone destinations Returns a boolean based on success job: Job to do! This is the job of the HARDSUB file destinations: list of rlcone destinations (e.g., EncoderConf.uploading_destinations) upload_file: Path to the file to be uploaded rclone_config: Path to the rclone config file flags: rclone flag This method will upload the file and include its show name: e.g., 'temp.mp4' --> destination/show/episode.mp4 """ for dest in destinations: rclone_dest = "{}/{}/{}".format(_clean(dest), job.show, job.episode) command = [ "rclone", "--config={}".format(config), "copyto", upload_file, rclone_dest ] command.extend(flags.split()) Ayumi.debug("Now running command: {}".format(" ".join(command))) Ayumi.info("Now uploading file to {}".format(dest)) try: _run(command) except ShikyouResponseException: Ayumi.error( "Rclone command returned a bad return code, contact the developer.", color=Ayumi.LRED) raise ShikyouResponseException() except ShikyouTimeoutException: Ayumi.error("Rclone command timed out, are your sources okay?", color=Ayumi.LRED) raise ShikyouTimeoutException() Ayumi.info("Completed uploading files.", color=Ayumi.LGREEN)
def _kitsu_basic_search(title): """Quick Kitsu implementation""" title_lower = title.lower() request_url = requests.utils.requote_uri(KITSU_API_URL + title_lower) Ayumi.debug("Created Kitsu request URL: {}".format(request_url)) try: kitsu_res = requests.get(request_url, timeout=10) try: kitsu_json = kitsu_res.json() return kitsu_json except: Ayumi.warning("Kitsu did not return a valid JSON response.", color=Ayumi.RED) raise Exception() except requests.exceptions.Timeout: Ayumi.warning("Kitsu request timed out.", color=Ayumi.LRED) raise Exception() except requests.exceptions.ConnectionError: Ayumi.warning("Unable to contact Kitsu, maybe it's down?", color=Ayumi.LRED) raise Exception()
def is_user_watching_names(user, show_name): """ Determines whether or not an Anilist user is watching a show Checks by show name Params: user: username to look up show_name: name of the show to look up. this should already be the anilist name. Returns: a boolean - True if watching, False if not """ try: watching, paused, ptw = _kishi_list(user) for show in watching: for title in show['title'].values(): if _check_equality(title, show_name): Ayumi.debug("Matched {} to {} in {}".format( title, show_name, "watching")) return True for show in paused: for title in show['title'].values(): if _check_equality(title, show_name): Ayumi.debug("Matched {} to {} in {}".format( title, show_name, "paused")) return True for show in ptw: for title in show['title'].values(): if _check_equality(title, show_name): Ayumi.debug("Matched {} to {} in {}".format( title, show_name, "planning")) return True Ayumi.debug("Didn't find a match for {}".format(show_name)) return False except: # If any errors are encountered, return True (default assumption) Ayumi.warning( "An error was encountered while contacting Anilist. Defaulting to TRUE" ) return True
def _load_shows_map() -> Dict[str, str]: shows_map = settings.get( 'ACQUISITION_DOWNLOAD_BITTORRENT_SHOWS_MAP', []) Ayumi.debug("Fetched shows map from config: {}".format(shows_map)) show_name_map = dict() Ayumi.debug("Creating a stripped title to override title mapping...") for show in shows_map: stripped_title, override_title = _parse_shows_map(show) show_name_map[stripped_title] = override_title Ayumi.debug("Mapped {} to key {}.".format( override_title, stripped_title)) return show_name_map
def rss(last_guid=None): try: with rabbitpy.Connection( 'amqp://{username}:{password}@{host}:{port}/{vhost}'.format( username=settings.get_fresh('RABBITMQ_USERNAME'), password=settings.get_fresh('RABBITMQ_PASSWORD'), host=settings.get_fresh('RABBITMQ_HOST'), port=settings.get_fresh('RABBITMQ_PORT'), vhost=settings.get_fresh('RABBITMQ_VHOST'))) as conn: with conn.channel() as channel: Ayumi.set_rabbitpy_channel(channel) channel.enable_publisher_confirms() while True: Ayumi.info("Now starting feed fetch.", color=Ayumi.LCYAN) feed = feedparser.parse( settings.get('ACQUISITION_RSS_FEED_URL', None)) accepted_shows = _load_accepted_shows() Ayumi.debug( "Loaded accepted shows map: {}".format(accepted_shows)) history = _load_history() new_history = list() for entry in feed.entries: # Fetch data first title, link, guid = entry.title, entry.link, entry.guid Ayumi.debug( 'Encountered RSS item with title "{}", and guid "{}"' .format(title, guid)) # If feed item with last GUID encountered, do not process any further if guid == last_guid: Ayumi.debug( "Encountered RSS item with last_guid {} matching argument, breaking and writing history." .format(last_guid), color=Ayumi.YELLOW) break # Check the title data # Use the parsed title to match user provided titles. parsed_title = anitopy.parse(title)['anime_title'] if _strip_title(parsed_title) not in accepted_shows: Ayumi.info( 'Feed item with title "{}" (show title: "{}") is not in accepted shows, skipping.' .format(title, parsed_title)) else: if guid in history: # This item has been previously processed, skip it. Ayumi.info( 'Feed item with title "{}" (show title: "{}") has already been processed, skipping.' .format(title, parsed_title), color=Ayumi.GREEN) else: # A new feeditem! Let us process it. Ayumi.info( 'Feed item with title "{}" (show title: "{}") is in accepted shows, processing.' .format(title, parsed_title), color=Ayumi.YELLOW) message = rabbitpy.Message( channel, json.dumps({ "title": title, "link": link, "guid": guid, "show_title": accepted_shows[_strip_title( parsed_title)] })) acquisition_rss_exchange_name = settings.get( 'ACQUISITION_RSS_EXCHANGE') while not message.publish( acquisition_rss_exchange_name, mandatory=True): Ayumi.warning( 'Failed to publish feed item with title "{}" to exchange "{}", retrying in 60s...' .format(title, acquisition_rss_exchange_name), color=Ayumi.RED) sleep(60) Ayumi.info( 'Published feed item with title "{}" to exchange "{}".' .format( title, acquisition_rss_exchange_name, ), color=Ayumi.LGREEN) # Keep all items processed in the new history - it will be auto deleted by the expiry of the RSS Ayumi.debug( 'Appending item "{}" with title "{}" (show title: "{}") to new_history for write.' .format(guid, title, parsed_title), color=Ayumi.YELLOW) new_history.append(guid) _write_history(new_history) # Sleep till the next iteration sleep_duration = settings.get( 'ACQUISITION_RSS_SLEEP_INTERVAL', _DEFAULT_SLEEP_INTERVAL) Ayumi.info( "Now sleeping {} seconds.".format(sleep_duration), color=Ayumi.LCYAN) sleep(sleep_duration) except rabbitpy.exceptions.AMQPConnectionForced: Ayumi.rabbitpy_channel = None Ayumi.critical( "Operator manually closed RabbitMQ connection, shutting down.", color=Ayumi.RED) # Use return for now because in some cases, calling exit() may invoke the retry() header. return
def _write_history(new_history) -> None: with open('data.json', 'w') as data: json.dump(new_history, data, indent=4) Ayumi.debug( "Wrote new_history to data.json. Contents: {}".format(new_history), color=Ayumi.YELLOW)
def _page_search(search, status): """ Searches for a show using the page query. Params: search - show to search for status - status to filter under. Returns: Show data if it's found, or None. """ try: info = _query_request(PAGE_QUERY, search, status)['Page']['media'] except: Ayumi.debug("No data provided for {} in {}, returning None.".format( search, status)) return None for show in info: # Check if any of the titles match Ayumi.debug("Checking for matches in media titles...") for title in show['title'].values(): if _check_equality(search, title): Ayumi.debug("Matched {} to {}, returning.".format( search, title)) return show else: Ayumi.debug("Did not match {} to {}.".format(search, title)) # Check if any of the synonyms match Ayumi.debug("Checking for matches in media synonyms...") for title in show['synonyms']: if _check_equality(search, title): Ayumi.debug("Matched {} to {}, returning.".format( search, title)) return show else: Ayumi.debug("Did not match {} to {}.".format(search, title)) # No matches, return None Ayumi.debug("Didn't find any matches for {} in {}, returning None.".format( search, status)) return None
def bittorrent(): try: with rabbitpy.Connection('amqp://{username}:{password}@{host}:{port}/{vhost}'.format( username=settings.get('RABBITMQ_USERNAME'), password=settings.get('RABBITMQ_PASSWORD'), host=settings.get('RABBITMQ_HOST'), port=settings.get('RABBITMQ_PORT'), vhost=settings.get('RABBITMQ_VHOST') )) as conn: with conn.channel() as channel: Ayumi.set_rabbitpy_channel(channel) channel.enable_publisher_confirms() queue_name = settings.get('ACQUISITION_BITTORRENT_QUEUE') Ayumi.debug("Connecting to queue: {}".format(queue_name)) queue = rabbitpy.Queue(channel, queue_name) queue.declare(passive=True) Ayumi.info('Now listening for messages on queue: {}...'.format( queue_name), color=Ayumi.LYELLOW) for message in queue.consume(prefetch=1): Ayumi.info( "Received new message, starting...", color=Ayumi.CYAN) feeditem_preprocess = _load_amqp_message_body(message) Ayumi.debug('Loaded message raw: "{}"'.format( feeditem_preprocess)) if not feeditem_preprocess or not metsuke.validate_feeditem(feeditem_preprocess): Ayumi.error('Invalid message received, rejecting. Output: "{}"'.format( feeditem_preprocess), color=Ayumi.RED) message.reject() continue # Load initial data feeditem: metsuke.FeedItem = metsuke.generate_feeditem( feeditem_preprocess) shows_map = _load_shows_map() overload_title = feeditem.show_title Ayumi.info( 'Setting overload title: "{}"'.format(overload_title)) # If there is a central override, use it instead. if _strip_title(anitopy.parse(feeditem.title)['anime_title']) in shows_map: central_overload_title = shows_map[_strip_title( feeditem.title)] Ayumi.info('Overwriting overload title with central overload title: "{}"'.format( central_overload_title)) overload_title = central_overload_title with tempfile.TemporaryDirectory() as temp_dir: Ayumi.debug( 'Created temporary directory under path: "{}"'.format(temp_dir)) # Download the episode try: res = subprocess.run( [ "aria2c", "--seed-time=0", "--rpc-save-upload-metadata=false", "--bt-save-metadata=false", "--dir={}".format(temp_dir), feeditem.link ] ) if res.returncode != 0: Ayumi.warning( "Aria2 did not return a 0 exit code, assuming download errored and nacking.", color=Ayumi.RED) message.nack() continue except subprocess.TimeoutExpired: Ayumi.warning( "Download via webtorrent timed out - nacking.", color=Ayumi.RED) message.nack() continue if res.returncode != 0: Ayumi.warning( "Webtorrent did not have a return code of 0, nacking.", color=Ayumi.RED) message.nack() continue # Rename it potential_files = [f for f in os.listdir( temp_dir) if f.endswith(".mkv")] Ayumi.debug( "Loaded potential files: {}".format(potential_files)) if len(potential_files) != 1: Ayumi.warning( "Found more than one .mkv file, rejecting this job.", color=Ayumi.RED) message.reject() continue dl_file = potential_files[0] Ayumi.info('Found file: "{}"'.format(dl_file)) dl_file_path = os.path.abspath( '{}/{}'.format(_clean_title(temp_dir), potential_files[0])) Ayumi.debug( 'dl_file_path: "{}"'.format(dl_file_path)) # Remove unneeded files # TODO: THIS IS A HOTFIX, CHANGE LOGIC IN B2 bad_files = [f for f in os.listdir( temp_dir) if not f.endswith(".mkv")] Ayumi.debug("Found bad files: {}".format(bad_files)) for bf in bad_files: try: Ayumi.debug("Removing bad file: {}".format(bf)) os.remove( '{}/{}'.format(_clean_title(temp_dir), bf)) except: Ayumi.debug("Removing bad tree: {}".format(bf)) shutil.rmtree( '{}/{}'.format(_clean_title(temp_dir), bf)) # Move the file to proper layout with updated name dl_file_new_name = _generate_new_filename(dl_file) Ayumi.info('Generated new episode name: "{}"'.format( dl_file_new_name)) dl_file_new_dir = "{}/{}".format( temp_dir, overload_title) Ayumi.debug( 'dl_file_new_dir: "{}"'.format(dl_file_new_dir)) dl_file_new_path = "{}/{}".format( dl_file_new_dir, dl_file_new_name) Ayumi.debug( 'dl_file_new_path: "{}"'.format( dl_file_new_path)) Ayumi.debug('Moving "{}" to "{}"'.format( dl_file_path, dl_file_new_path)) os.mkdir(dl_file_new_dir) shutil.move(dl_file_path, dl_file_new_path) # Upload the file to rclone destination with tempfile.NamedTemporaryFile(suffix=".conf", mode="w+b") as rconf: rconf.write(str.encode( settings.get("RCLONE_CONFIG_FILE"))) rconf.flush() Ayumi.debug( 'Created temporary rclone file under path: "{}"'.format(rconf.name)) rclone_dest = _clean_title(settings.get( "ACQUISITION_BITTORRENT_RCLONE_DEST")) rclone_flags = settings.get("RCLONE_FLAGS", "") command = [ "rclone", "--config={}".format(rconf.name), "copy", temp_dir, rclone_dest] command.extend(rclone_flags.split()) Ayumi.debug( 'Rclone command to be run: "{}"'.format(command)) try: Ayumi.info( 'Now uploading new blob to: "{}"'.format(rclone_dest)) rclone_res = subprocess.run( command, timeout=3600) if rclone_res.returncode != 0: Ayumi.warning('Rclone returned non-zero code of {}, nacking.'.format( rclone_res.returncode), color=Ayumi.LRED) message.nack() except subprocess.TimeoutExpired: Ayumi.warning( 'Rclone upload timed out, nacking.', color=Ayumi.LRED) message.nack() continue # Fetch information on the file to create a job new_message = rabbitpy.Message(channel, dumps( { "show": overload_title, "episode": dl_file_new_name, "filesize": int(os.path.getsize(dl_file_new_path)), "sub": "SOFTSUB" } )) acquisition_bittorrent_exchange_name = settings.get( 'ACQUISITION_BITTORRENT_EXCHANGE') Ayumi.info('Sending to exchange: "{}"'.format( acquisition_bittorrent_exchange_name), color=Ayumi.CYAN) while not new_message.publish(acquisition_bittorrent_exchange_name, mandatory=True): Ayumi.warning( "Failed to publish feed item, trying again in 60 seconds") sleep(60) Ayumi.info("Published feed item with title: " + overload_title, color=Ayumi.LGREEN) message.ack() except rabbitpy.exceptions.AMQPConnectionForced: Ayumi.warning( "Operator manually closed RabbitMQ connection, shutting down.", color=Ayumi.LYELLOW) return
def _get_shows(status: str, **kwargs): """ Fetch all the airing, recently finished, and soon to be airing shows Run string siimlarity and return the string most likely to be the same show. """ shows = list() page = 1 has_next_page = True while has_next_page: Ayumi.info("Now requesting shows from page {} of status {}...".format(page, status), color=Ayumi.CYAN) variables = { 'page': page, 'status': status } k_year = kwargs.get('year') if k_year: Ayumi.debug("Set seasonYear argument to {}".format(k_year), color=Ayumi.CYAN) variables['seasonYear'] = int(k_year) k_sd = kwargs.get('start_date') if k_sd: Ayumi.debug("Set startDate_lesser argument to {}".format(k_sd), color=Ayumi.CYAN) variables['startDate'] = int(k_sd) k_ed = kwargs.get('end_date') if k_ed: Ayumi.debug("Set endDate_greater argument to {}".format(k_ed), color=Ayumi.CYAN) variables['endDate'] = int(k_ed) try: ani_res = requests.post( ANILIST_API_URL, json={ 'query': QUERY, 'variables': variables }) except requests.exceptions.ConnectionError: Ayumi.warning("Unable to contact Anilist, the site or your connection may be down.", color=Ayumi.LRED) return shows if ani_res.status_code != 200: Ayumi.warning("Anilist returned unaccepted HTTP code {} upon request.".format(ani_res.status_code), color=Ayumi.LRED) return shows try: ani_json = ani_res.json()['data']['Page'] has_next_page = ani_json['pageInfo']['hasNextPage'] page += 1 for media in ani_json['media']: for media_title in media['title'].values(): if media_title: Ayumi.debug("Adding show {} to show list".format(media_title)) shows.append(media_title) except: Ayumi.warning("Unable to parse JSON response from Anilist.", color=Ayumi.LRED) return shows return shows
def consume(): try: with rabbitpy.Connection('amqp://{username}:{password}@{host}:{port}/{vhost}'.format( username=settings.get('RABBITMQ_USERNAME'), password=settings.get('RABBITMQ_PASSWORD'), host=settings.get('RABBITMQ_HOST'), port=settings.get('RABBITMQ_PORT'), vhost=settings.get('RABBITMQ_VHOST') )) as conn: with conn.channel() as channel: Ayumi.set_rabbitpy_channel(channel) queue = rabbitpy.Queue(channel, settings.get('DISTRIBUTORS_RCLONE_QUEUE')) queue.declare(passive=True) Ayumi.info("Now listening for messages from AMQP provider.", color=Ayumi.YELLOW) for message in queue.consume(prefetch=1): try: job = json.loads(message.body.decode('utf-8')) except json.JSONDecodeError: Ayumi.warning("Received a job that is invalid json, not processing.", color=Ayumi.LRED) message.reject() continue Ayumi.info("Received a new job: {}".format(json.dumps(job)), color=Ayumi.CYAN) if metsuke.validate(job): Ayumi.debug("Loaded show: {}".format(job['show'])) Ayumi.debug("Loaded episode: {}".format(job['episode'])) Ayumi.debug("Loaded filesize: {}".format(job['filesize'])) Ayumi.debug("Loaded sub type: {}".format(job['sub'])) metsuke_job = metsuke.Job( job['show'], job['episode'], job['filesize'], job['sub']) with tempfile.NamedTemporaryFile(suffix=".conf", mode="w+b") as rconf, tempfile.TemporaryDirectory() as tempdir: Ayumi.debug("Opening context managed rclone config file under path: {}.".format(rconf.name)) Ayumi.debug("Opening context managed rclone temporary directory under path: {}".format(tempdir)) rconf.write(str.encode(settings.get("RCLONE_CONFIG_FILE"))) rconf.flush() # YOU MUST FLUSH THE FILE SO RCLONE CAN READ IT! Ayumi.debug("Configurations written to temporary file. Size is {} bytes.".format(rconf.tell())) dl_sources = None up_dests = None if job['sub'].lower() == "softsub": dl_sources = settings.get("DISTRIBUTORS_RCLONE_SOFTSUB_DOWNLOAD") up_dests = settings.get("DISTRIBUTORS_RCLONE_SOFTSUB_UPLOAD") elif job['sub'].lower() == "hardsub": dl_sources = settings.get("DISTRIBUTORS_RCLONE_HARDSUB_DOWNLOAD") up_dests = settings.get("DISTRIBUTORS_RCLONE_HARDSUB_UPLOAD") Ayumi.debug("Fresh fetched download sources as: {}".format(" ".join(dl_sources))) Ayumi.debug("Fresh fetched upload sources as: {}".format(" ".join(up_dests))) try: temp_ep = shikyou.download(metsuke_job, dl_sources, tempdir, rconf.name, settings.get("RCLONE_FLAGS", "")) if temp_ep: shikyou.upload(metsuke_job, up_dests, temp_ep, rconf.name, settings.get("RCLONE_FLAGS", "")) else: Ayumi.warning("Unable to find requested job in any sources, nacking...", color=Ayumi.RED) message.nack() continue except shikyou.ShikyouResponseException: Ayumi.critical("Rclone threw an unexpected response code, rejecting.", color=Ayumi.RED) message.reject() continue except shikyou.ShikyouTimeoutException: Ayumi.warning("Rclone timed out whilhe executing, nacking.", color=Ayumi.RED) message.nack() continue Ayumi.debug("Closed context managed rclone config file.") Ayumi.debug("Closed context managed temporary directory.") else: Ayumi.warning("Received a job that Metsuke was not able to validate.", color=Ayumi.LRED) Ayumi.warning(json.dumps(job), color=Ayumi.LRED) Ayumi.info("Completed processing this message for {}".format(job['episode']), color=Ayumi.LGREEN) message.ack() except rabbitpy.exceptions.AMQPConnectionForced: Ayumi.rabbitpy_channel = None Ayumi.critical("Operator manually closed RabbitMQ connection, shutting down.", color=Ayumi.RED) # Use return for now because in some cases, calling exit() may invoke the retry() header. return
def _fetch_list(user, listurl, listname): """ Helper method to get a user's anime list. Doesn't handle errors, just raises them. Needs to be called by a wrapper. Params: User: Username of MAL user listurl: URL to use to fetch info listname: Watching, Plan to Watch (for logging) """ anime = list() try: jikan_res = requests.get(listurl.format(user, "")) # Verify status code if jikan_res.status_code != 200: Ayumi.debug( "jikan mode returned a bad status code on attempt to get {}'s {} list." .format(user, listname)) raise Exception() # Make sure an anime actually exists undeer this name try: jikan_res_json = jikan_res.json() except: jikan_res_json = dict() # Clean handling if 'anime' not in jikan_res_json: Ayumi.debug( "Jikan.moe did not return an anime list on attempt to get {}'s {} list." .format(user, listname)) raise Exception() # Add all anime in the first page. for entry in jikan_res_json['anime']: anime.append(entry) Ayumi.debug("Added {} show {} to processing list.".format( listname, entry['title'])) page = 2 while (len(jikan_res_json['anime']) == 300): jikan_res = requests.get(listurl.format(user, str(page))) if jikan_res.status_code != 200 or 'anime' not in jikan_res: Ayumi.debug( "Jikan returned a bad status code when attempting to get {}'s page {} {} list." .format(user, str(page), listname)) raise Exception() try: jikan_res_json = jikan_res.json() except: jikan_res_json = dict() if 'anime' not in jikan_res_json: Ayumi.debug( "Jikan.moe did not return an anime list on attempt to get {}'s page {} {} list." .format(user, str(page), listname)) raise Exception() for entry in jikan_res_json['anime']: anime.append(entry) Ayumi.debug( "Added {} page {} show {} to processing list.".format( listname, str(page), entry['title'])) page += 1 Ayumi.debug("Returning list with {} entires.".format(str(len(anime)))) return anime except: # raise some kind of exception - somehow Jikan couldn't be reached Ayumi.debug( "Akari encountered an unknown error when attempting to fetch {}'s {} list" .format(user, listname)) raise Exception()
def _check_equality(name1, name2) -> bool: Ayumi.debug("Regex comparing: {} | {}".format(name1, name2)) try: # Anilist sometimes has weird leading/trailing spaces re_str1 = re.sub(r'[^\w]', '', name1) Ayumi.debug("Name 1 without puncutation: {}".format(re_str1)) re_str2 = re.sub(r'[^\w]', '', name2) Ayumi.debug("Name 2 without puncutation: {}".format(re_str2)) if re_str1 == re_str2: Ayumi.debug("Both show names are matching, returning True.") return True else: Ayumi.debug("Show names do not match, returning False.") return False except: Ayumi.debug( "Error occured while matching show names, returning False.") return False
def consume(): try: with rabbitpy.Connection( 'amqp://{username}:{password}@{host}:{port}/{vhost}'.format( username=settings.get('RABBITMQ_USERNAME'), password=settings.get('RABBITMQ_PASSWORD'), host=settings.get('RABBITMQ_HOST'), port=settings.get('RABBITMQ_PORT'), vhost=settings.get('RABBITMQ_VHOST'))) as conn: with conn.channel() as channel: Ayumi.set_rabbitpy_channel(channel) queue = rabbitpy.Queue( channel, settings.get('NOTIFICATIONS_DISCORD_WEBHOOK_QUEUE', 'nonexistent')) queue.declare(passive=True) Ayumi.info("Now listening for messages from AMQP provider.", color=Ayumi.YELLOW) for message in queue.consume(prefetch=1): try: job = json.loads(message.body.decode('utf-8')) except json.JSONDecodeError: Ayumi.warning( "Received a job that is invalid json, not processing.", color=Ayumi.LRED) message.reject() continue Ayumi.info("Received a new job: {}".format( json.dumps(job)), color=Ayumi.CYAN) if metsuke.validate(job): Ayumi.debug("Loaded show: {}".format(job['show'])) Ayumi.debug("Loaded episode: {}".format( job['episode'])) Ayumi.debug("Loaded filesize: {}".format( job['filesize'])) Ayumi.debug("Loaded sub type: {}".format(job['sub'])) embed = _generate_embed(metsuke.generate(job), hisha.search(job['show'])) Ayumi.info( "Beginning sending embeds to webhook endpoints.", color=Ayumi.CYAN) for endpoint in settings.get( 'NOTIFICATIONS_DISCORD_WEBHOOK_ENDPOINTS' ).to_list(): try: requests.post(endpoint, json=embed, timeout=5) Ayumi.debug( "Sent embed to {}".format(endpoint)) except: Ayumi.warning( "Failed to send embed to {}".format( endpoint), color=Ayumi.RED) else: Ayumi.warning( "Received a job that Metsuke was not able to validate.", color=Ayumi.LRED) Ayumi.warning(json.dumps(job), color=Ayumi.LRED) Ayumi.info( "Completed processing this message for {}".format( job['episode']), color=Ayumi.LGREEN) message.ack() except rabbitpy.exceptions.AMQPConnectionForced: Ayumi.rabbitpy_channel = None Ayumi.critical( "Operator manually closed RabbitMQ connection, shutting down.", color=Ayumi.RED) # Use return for now because in some cases, calling exit() may invoke the retry() header. return
@retry(delay=60, tries=3, exceptions=(shikyou.ShikyouResponseException, shikyou.ShikyouTimeoutException), backoff=1.5, max_delay=3600, logger=Ayumi.get_logger()) def on_new_file(src_path, channel): show_name = None episode_name = None new_file = src_path.replace(os.path.commonpath([settings.get('KOTEN_WATCH_PATH', DEFAULT_WATCH_PATH), src_path]) + "/", "") if m := util._show_manually_specified(new_file): Ayumi.info("Detected show name and episode name in event, using Mode 1.") show_name = m.group(1) episode_name = util._clean_episode_name(m.group(2)) Ayumi.info("New show name: {}".format(show_name), color=Ayumi.LYELLOW) Ayumi.info("New episode name: {}".format(episode_name), color=Ayumi.LYELLOW) else: Ayumi.debug("Non-conformant episode provided, using Naomi to find show name.") episode_name = util._clean_episode_name(pathlib.PurePath(src_path).name) show_name = naomi.find_closest_title(anitopy.parse(new_file)['anime_title']) Ayumi.info("New show name: {}".format(show_name), color=Ayumi.LYELLOW) Ayumi.info("New episode name: {}".format(episode_name), color=Ayumi.LYELLOW) # There is an event where Anilist is down, and Naomi could return None. # In this case, use the assumed-parsed show as the title if not show_name: show_name = anitopy.parse(new_file)['anime_title'] job = { "show": show_name, "episode": episode_name, "filesize": os.path.getsize(src_path), "sub": "softsub"