async def search(app: AppContext, query: str) -> List[SearchResult]: """ Searches for a query on nyaa.si. Parameters ---------- query: str The search query. """ url = NyaaSearcher._get_query_url(query) loop = asyncio.get_running_loop() feed = await loop.run_in_executor(None, feedparser.parse, url) found = [] for item in feed["entries"]: try: anitopy.parse(item["title"]) except Exception as e: logger.warn( f"Anitopy - Could not Parse '{item['title']}', skipping") continue found.append(SearchResult.from_dict(app, item)) return found
def parse_from_content(content: "content_type", *, name_processor: Callable[[content_type], str] = lambda x: x, stream_url_processor: Callable[[content_type], str] = lambda x: x, overrides: Dict = {}, episode_parsed: bool = False): anitopy_result = anitopy.parse(name_processor(content)) returnee = {"stream_url": stream_url_processor(content)} video_res = anitopy_result.get("video_resolution") or "" if not episode_parsed: returnee.update( {"episode": int(anitopy_result.get("episode_number", 0) or 0)}) if isinstance(video_res, str): stripped = video_res.strip("p") if stripped.isdigit(): returnee.update({"quality": int(stripped)}) returnee.update(overrides) return returnee
def get_episode_number(self, file_name: str) -> int: """ Using the `file_name` argument, you must parse the file name in order to get the episode of the release. Failure to do so will result in incorrect matching. Returning `None` will stop the matching. """ parsed = anitopy.parse(file_name) if "anime_type" in parsed.keys(): return extra_info = parsed.get("release_information", "") if "batch" in extra_info.lower(): return try: episode = parsed["episode_number"] except KeyError: return try: return int(episode) except (ValueError, TypeError): return
def _load_show(self, args, episode): """ Gets the show name from the ArgumentHandler, or finds it if a new episode was downloaded without including the name of the show. Params: conf: A ConfigHandler object that is already populated args: An ArgumentHandler object that is already populated Returns: The name of the show of the episode as a string """ # Args may already have determined the show from the passed-in args. If so, just return that. show = args.show if show: self._logger.info(self._prints.SHOW_LOADED_ARG.format(show)) return show # Unique to Acquisition2: Because Izumi is no longer supported, just use the provided show name. data = anitopy.parse(episode) if 'anime_title' in data: self._logger.info( self._prints.SHOW_LOADED_ARG.format(data['anime_title'])) return data['anime_title'] # If the show wasn't included, this means the episode was included on its own. # What's nice about this? The Hisha library takes care of it for us. show = hisha2.hisha(self._episode) self._logger.info(self._prints.SHOW_LOADED_HISHA.format(show)) # There should be a case where Hisha doesn't find it # But hisha3 will need to be developed first return show
def _clean_episode_name(unclean_name): """ Clean the episode name for new usage. Parameter unclean_name should only be the file name, no paths. """ info = anitopy.parse(unclean_name) new_name = info['anime_title'] if 'anime_season' in info: Ayumi.debug('Found anime_season "{}"'.format(info['anime_season'])) new_name = new_name + " S" + str(info['anime_season']) if 'episode_number' in info: Ayumi.debug('Found episode_number "{}"'.format(info['episode_number'])) new_name = new_name + " - " + str(info['episode_number']) if 'video_resolution' in info: Ayumi.debug('Found video_resolution "{}"'.format(info['video_resolution'])) new_name = new_name + " [{}]".format(info['video_resolution']) if 'other' in info and 'uncensored' in info['other'].lower(): Ayumi.debug('Detected this episode is uncensored, adding "(Uncensored)" to the title.') _, ext = os.path.splitext(unclean_name) new_name += ext Ayumi.debug('Complete new file name: {}'.format(new_name)) return new_name
def test_fails(self): failed = 0 working_tests = [] for index, entry in enumerate(failing_table): filename = entry[0] options = self.parse_options(entry[1]) try: print('Index %d "%s"' % (index, filename)) except: print(('Index %d "%s"' % (index, filename)).encode("utf-8")) elements = anitopy.parse(filename, options=options) expected = dict(entry[2]) if 'id' in expected.keys(): del expected['id'] try: self.assertEqual(expected, elements) working_tests.append(index) except AssertionError as err: failed += 1 print(err) print( '----------------------------------------------------------------------' ) # noqa E501 print('\nFailed %d of %d failing cases tests' % (failed, len(failing_table))) if working_tests: print( 'There are {} working tests from the failing cases: {}'.format( len(working_tests), working_tests))
async def get_episodes(self) -> List[int]: """ Returns a list of episodes that are contained within the torrent. Returns ------- List[int]: List of episodes. """ files = await self._app.dl_client.get_file_structure(self.torrent_link) episodes = [] for file in files: try: parsed = anitopy.parse(file) except Exception as e: logger.warn(f"anitopy - Could not Parse '{file}', skipping") continue if "anime_type" in parsed.keys(): continue try: episodes.append(int(parsed["episode_number"])) except (KeyError, ValueError, TypeError): pass return episodes
def hisha(episode, title="userPreferred"): # returns the show name data = anitopy.parse(episode) names = _names(data['anime_title']) res = _hisha(names) return res.json()['data']['Media']['title'][title]
def check_torrent(title, feed_id): info = anitopy.parse(title) check_title = False check_resolution = False if len(feeds[feed_id]['resolutions']) == 0: feeds[feed_id]['resolutions'] = [config['resolution']] try: if info['video_resolution'] in feeds[feed_id]['resolutions']: check_resolution = True except KeyError: check_resolution = False if len(feeds[feed_id]['anime-list']) == 0: check_title = True else: for anime in feeds[feed_id]['anime-list']: if anime['title'] == info['anime_title']: check_title = True try: if anime['resolution'] == info['video_resolution']: check_resolution = True else: check_resolution = False except KeyError: print('No resolution was set for the anime: %s' % anime['title']) return check_title and check_resolution
def hisha(episode, title="userPreferred"): """ The default Hisha module, for standard airing cases. This will attempt to find the show in airing, and then not_yet_aired, and then premiered. It also tries multiple versions of the search: 1. Last word removed (i.e., some kind of Season indicator) 2. Replace the first " - " to ": " and remove the other " - " Forces os-level application exit if _hisha fails to find a show. Params: episode: The full string of the episode to find title: A string for which language the Anilist title should return Returns: The show of the episode provided """ data = anitopy.parse(episode) names = _names(data['anime_title']) # Get the show name with the helper. try: res = _hisha(names) except: # An exception will be thrown if nothing was found. We exit if that's the case. print("No show was found when searching. The system will now exit.") raise Exception() # Backwards-compatability: Hisha will return the title return res.json()['data']['Media']['title'][title]
def get_show_name(self, file_name: str) -> str: """ Using the `file_name` argument, you must parse the file name in order to get the name of the show. Failure to do so will result in incorrect matching. """ parsed = anitopy.parse(file_name) return parsed["anime_title"]
def parse(): feeds = open("../rsc/feeds.txt").readlines() l = list() rlist = list() for f in feeds: l.append(feedparser.parse(f)) for rss in l: for item in rss.entries: rlist.append(anitopy.parse(item.title)) return rlist
def test_table(self): for index, entry in enumerate(table): filename = entry[0] options = self.parse_options(entry[1]) elements = anitopy.parse(filename, options=options) expected = dict(entry[2]) if 'id' in expected.keys(): del expected['id'] self.assertEqual(expected, elements, 'on entry number %d' % index)
def get_title_from_name(name): if not is_empty(name) and isinstance(name, str): try: info = dict_2_default_dict(anitopy.parse(name)) extracted_name = info["anime_title"] if extracted_name: return extracted_name ptn_info = dict_2_default_dict(PTN.parse(name)) if ptn_info["title"]: return ptn_info["title"] return None except TypeError: return None return None
def hisha2(episode): """ Hisha2, unlike hisha, simply returns all the json it gets. """ data = anitopy.parse(episode) names = _names(data['anime_title']) try: res = _hisha(names) except: # An exception will be thrown if nothing was found. We exit if that's the case. print("No show was found when searching. The system will now exit.") raise Exception() return res.json()
def _rename_mass(filename): """ Usage for mass renaming files within a folder. Strip out fansub names from the filename Params: - filename: The original, dirty filename Does not change extension, returns new filename """ p = anitopy.parse(filename) new_file = p['anime_title'] # Season if 'anime_season' in p: new_file += " S" + p['anime_season'] # Episode number, NCED if 'episode_number' in p: new_file += " - " + p['episode_number'] # Release version if 'release_version' in p: new_file += "v" + p['release_version'] # If uncensored, mark it if 'other' in p and 'uncensored' in p['other'].lower(): new_file += " (Uncensored)" # Video res might not be included if 'video_resolution' in p: if "1920x1080" in p['video_resolution']: p['video_resolution'] = "1080p" p['video_resolution'] = p['video_resolution'].replace("P", "p") if 'p' not in p['video_resolution'].lower(): p['video_resolution'] += "p" new_file += " [" + p['video_resolution'] + "]" new_file += "." + p['file_extension'] return new_file
def __parse_name(self, name: str) -> dict: anitopy_options = dict({ "parse_episode_number": True, "parse_episode_title": False, "parse_file_extension": False, "parse_release_group": False, }) parse_result = anitopy.parse(name, options=anitopy_options) if (parse_result["anime_title"] is not None) and (parse_result["episode_number"] is not None): parse_result["anime_title"] = pathlib.Path( parse_result["anime_title"]).name return dict({ "title": parse_result["anime_title"], "ep_num": parse_result["episode_number"], }) else: return None
def get_files_to_mux(input_files_from_options_file): #files may not be in same directory files_to_mux = {} for input_file_path in input_files_from_options_file: #the extension of the input file input_file_extension = pathlib.Path(input_file_path).suffix files_added = 1 for entry in os.scandir(r'{0}'.format( os.path.dirname(input_file_path))): #used when anitopy can't get episode number from file if not entry.is_dir(): #they should be the same extension as the original input_file_path in input_files_from_options_file if pathlib.Path(entry.path).suffix == input_file_extension: parsed_file = anitopy.parse(entry.name) #find out where to insert the file insertion_index = files_added if 'episode_number' in parsed_file: insertion_index = parsed_file['episode_number'] #TODO test anitopy names that return a list for episode number #ask user to manually input episode number? input can be for every episode or it can be for a rule to select the first, second, ... n element in the episdoe number list if isinstance(insertion_index, list): print( "Anitopy detected this file's episode number as a list. This may effect the accuracy of the mux." ) insertion_index = insertion_index[0] if insertion_index in files_to_mux: files_to_mux[insertion_index].append(entry.path) else: files_to_mux[insertion_index] = [entry.path] files_added += 1 #confirm that each mux_file_group is the same length. If not then one input directory has more files than the other confirm_files_are_mapped_properly = {} for mux_file_group in files_to_mux.values(): confirm_files_are_mapped_properly[len(mux_file_group)] = 0 if len(confirm_files_are_mapped_properly) > 1: print("Files could not be mapped properly!") print("One of your directories has more files than the other") return None return files_to_mux
def ignore_logic(self, item: dict) -> bool: """ Optional This is the first function a parser runs. If this returns False, the item will instantly be ignored by Tsundoku. Any other return value will continue operation as normal. """ parsed = anitopy.parse(item["title"]) title = parsed.get("anime_title") episode = parsed.get("episode_number") resolution = parsed.get("video_resolution") if title is None or episode is None or resolution is None: return False elif resolution != "1080p": return False
def _generate_new_filename(dl_file): info = anitopy.parse(dl_file) new_dl_file = info['anime_title'] if 'anime_season' in info: Ayumi.debug('Found anime_season "{}"'.format(info['anime_season'])) new_dl_file = new_dl_file + " S" + str(info['anime_season']) if 'episode_number' in info: Ayumi.debug('Found episode_number "{}"'.format(info['episode_number'])) new_dl_file = new_dl_file + " - " + str(info['episode_number']) if 'video_resolution' in info: Ayumi.debug('Found video_resolution "{}"'.format( info['video_resolution'])) new_dl_file = new_dl_file + " [{}]".format(info['video_resolution']) if 'other' in info and 'uncensored' in info['other'].lower(): Ayumi.debug( 'Detected this episode is uncensored, adding "(Uncensored)" to the title.') new_dl_file += " (Uncensored)" _, ext = os.path.splitext(dl_file) new_dl_file += ext Ayumi.debug('returning new_dl_file: {}'.format(new_dl_file)) return new_dl_file
def resolve_file(self, root: Path, episode: int) -> Optional[Path]: """ Searches a directory tree for a specific episode file. If the passed path is a file, return it. Parameters ---------- path: Path The directory. episode: int The episode to search for. Returns ------- Optional[Path] The found Path. It is a file. """ if root.is_file(): return root root.resolve() for subpath in root.rglob("*"): try: parsed = anitopy.parse(subpath.name) except Exception: logger.debug( f"Anitopy - Could not parse `{subpath.name}`, skipping") continue try: found = int(parsed["episode_number"]) except (KeyError, ValueError, TypeError): continue if found == episode: return subpath return None
def parse_episode_from_name(name): episode, season = None, None if name and not is_empty(name) and isinstance(name, str): try: ptn_info = dict_2_default_dict(PTN.parse(name)) episode = ptn_info["episode"] season = ptn_info["season"] except: pass if not episode or not season: try: info = dict_2_default_dict(anitopy.parse(name)) episode = info["episode_number"] if not episode else episode season = info["anime_season"] if not season else season except: print("anitopy error") pass try: return {"episode": parse_episode(episode) if episode else None, "season": get_number_from_string(season) if season else None} except ValueError or AttributeError: return {"episode": None, 'season': None} return {"episode": None, 'season': None}
def _generate_new_episode(self, episode): """ Takes the old filename and generates a new, clean name for it. Params: episode - the name of the old episode file Return: A "cleaned" filename """ # Parse the filename a = anitopy.parse(episode) # Generate the new episode name new_episode = a['anime_title'] if 'anime_season' in a: new_episode = new_episode + " S" + str(a['anime_season']) if 'episode_number' in a: new_episode = new_episode + " - " + a['episode_number'] # Mark video resolution if 'video_resolution' in a: new_episode = new_episode + " [" + a['video_resolution'] + "]" # Mark if uncensored if 'other' in a and 'uncensored' in a['other'].lower(): new_episode += " (Uncensored)" # Add the extension _, ext = os.path.splitext(episode) new_episode += ext self._logger.warning(self._prints.EPISODE_NEW_NAME.format(new_episode)) return new_episode
def database_builder(): list_database = [] list_origin_database = [] list_torrents_dir = [ f for f in os.listdir('torrents') if os.path.isfile(os.path.join('torrents', f)) ] origin_database_loaded = False database_loaded = False full_loaded = False if os.path.isfile('anime_database.json'): anime_database_file = open('anime_database.json', encoding='utf-8') anime_database_obj = json.load(anime_database_file) else: color.color_print(Fore.CYAN, '[INFO]', 'MISSING OFFLINE DATABASE. DOWNLOADING...') color.color_print( Fore.CYAN, '[INFO]', 'THANK YOU FOR MANAMI PROJECT TO PROVIDE ME A OFFLINE DATABASE') urllib.request.urlretrieve( 'https://raw.githubusercontent.com/manami-project/anime-offline-database/master/anime-offline-database.json', 'anime_database.json') color.color_print(Fore.CYAN, '[INFO]', 'DOWNLOAD COMPLETED\n') if os.path.isfile('output\\database_original.json'): origin_database_json_file = open('output\\database_original.json', encoding='utf-8') try: origin_database_obj = json.load(origin_database_json_file) origin_database_loaded = True except: origin_database_loaded = False database_json_file = open('output\\database.json', encoding='utf-8') try: database_obj = json.load(database_json_file) database_loaded = True except: database_loaded = False if origin_database_loaded and database_loaded: full_loaded = True list_database = database_obj list_origin_database = origin_database_obj added_new_state = False color.color_print(Fore.LIGHTMAGENTA_EX, '[COPYRIGHT]', 'MANAMI PROJECT: ANIME OFFLINE DATABASE') color.color_print(Fore.LIGHTMAGENTA_EX, '[COPYRIGHT]', 'IGORCMOURA: ANITOPY') color.color_print(Fore.LIGHTMAGENTA_EX, '[COPYRIGHT]', 'JCUL: BENCODE\n') color.color_print(Fore.YELLOW, '[PROCESSING]', 'PARSE TORRENTS\n') for i in list_torrents_dir: torrent_filename = i torrent_full_path = 'torrents\\' + i with open(torrent_full_path, 'rb') as fh: torrent_data = fh.read() if not search_database(list_database, i) or not full_loaded: torrent = bencode.decode(torrent_data) torrent_announces = [] torrent_files = [] torrent_creation_date = '' torrent_hash = '' torrent_magnet = '' torrent_total_length = 0 for im in torrent: torrent_creation_date = ( datetime.utcfromtimestamp(int(im[b'creation date'])) - timedelta(hours=9)).strftime('%Y-%m-%d %H:%M:%S') torrent_temp_announce = [] for imfw in im[b'announce-list']: torrent_temp_announce.append(imfw[0].decode("utf-8")) torrent_hash = str( hashlib.sha1(bencoding.bencode( im[b'info'])).hexdigest()) magnet_temp = 'magnet:?xt=urn:btih:{}'.format(torrent_hash) torrent_magnet = magnet_temp torrent_announces = torrent_temp_announce if b'files' in im[b'info']: for imf in im[b'info'][b'files']: torrent_files.append({ 'name': imf[b'path'][0].decode("utf-8"), 'size': format_size_units(imf[b'length']) }) torrent_total_length += imf[b'length'] else: torrent_total_length = im[b'info'][b'length'] torrent_files.append({ 'name': im[b'info'][b'name'].decode("utf-8"), 'size': format_size_units(im[b'info'][b'length']) }) torrent_size = format_size_units(torrent_total_length) info_id = random_string_digits(10) result_anitopy = anitopy.parse(torrent_filename) anime_db_result = search_anime(anime_database_obj, result_anitopy['anime_title']) json_data_for_add = {} json_data_for_add['id'] = info_id json_data_for_add['file_name'] = torrent_filename json_data_for_add['title'] = result_anitopy['anime_title'] if 'episode_number' in result_anitopy: json_data_for_add['episode'] = result_anitopy['episode_number'] else: json_data_for_add['episode'] = None json_data_for_add['hash'] = torrent_hash json_data_for_add['size'] = torrent_size if 'video_resolution' in result_anitopy: json_data_for_add['resolution'] = result_anitopy[ 'video_resolution'] else: json_data_for_add['resolution'] = None if 'video_term' in result_anitopy: json_data_for_add['video_codec'] = result_anitopy['video_term'] else: json_data_for_add['video_codec'] = None if 'audio_term' in result_anitopy: json_data_for_add['audio_codec'] = result_anitopy['audio_term'] else: json_data_for_add['audio_codec'] = None if 'release_group' in result_anitopy: json_data_for_add['release_group'] = result_anitopy[ 'release_group'] else: json_data_for_add['release_group'] = None json_data_for_add['created_date'] = torrent_creation_date json_data_for_add['magnet_url'] = torrent_magnet json_data_for_add[ 'torrent_url'] = 'https://anime.cryental.dev/download/' + info_id + '.torrent' json_data_for_add['extra'] = { 'announces': torrent_announces, 'files': torrent_files } if not anime_db_result: json_data_for_add['metadata'] = None else: info_type = None info_episodes = None info_picture = None info_thumbnail = None info_status = None if 'type' in anime_db_result: info_type = anime_db_result['type'] if 'episodes' in anime_db_result: info_episodes = anime_db_result['episodes'] if 'picture' in anime_db_result: info_picture = anime_db_result['picture'] if 'thumbnail' in anime_db_result: info_thumbnail = anime_db_result['thumbnail'] if 'status' in anime_db_result: info_status = anime_db_result['status'] json_data_for_add['metadata'] = { 'type': info_type, 'episodes': info_episodes, 'picture': info_picture, 'thumbnail': info_thumbnail, 'status': info_status } list_database.append(json_data_for_add) if not search_database(list_origin_database, i) or not full_loaded: json_original_data_for_add = {} json_original_data_for_add['id'] = info_id json_original_data_for_add['file_name'] = torrent_filename json_original_data_for_add['hash'] = torrent_hash with open(torrent_full_path, "rb") as f: encodedZip = base64.b64encode(f.read()) json_original_data_for_add['raw_data'] = encodedZip.decode( ) json_original_data_for_add[ 'created_date'] = torrent_creation_date list_origin_database.append(json_original_data_for_add) added_new_state = True color.color_print(Fore.YELLOW, '[PROCESSED] ', i) else: print(Fore.LIGHTRED_EX + '[SKIPPED] ' + Style.RESET_ALL + i) if added_new_state or not full_loaded: color.color_print(Fore.YELLOW, '[PROCESSING]', 'SORTING LIST') list_database.sort(key=sortSecond, reverse=True) color.color_print(Fore.YELLOW, '[PROCESSING]', 'SORTING ORIGINAL LIST') list_origin_database.sort(key=sortSecond, reverse=True) color.color_print(Fore.YELLOW, '[PROCESSING]', 'DISK ACCESSING') with open('output\\database.json', 'w') as outfile: color.color_print(Fore.YELLOW, '[PROCESSING]', 'WRITING LIST') json.dump(list_database, outfile) color.color_print(Fore.YELLOW, '[PROCESSING]', 'DISK ACCESSING') with open('output\\database_original.json', 'w') as outfile: color.color_print(Fore.YELLOW, '[PROCESSING]', 'WRITING LIST ORIGINAL') json.dump(list_origin_database, outfile) color.color_print(Fore.YELLOW, '[PROCESSING]', 'WRITING UPDATED DATE') today = datetime.now() new_days = open('output\\updated_on.txt', 'w') new_days.write(today.strftime("%Y-%m-%d %H:%M:%S")) new_days.close() color.color_print(Fore.YELLOW, '[PROCESSING]', 'WRITING HASH FILES') database_md5 = str(md5('output\\database.json')) origin_database_md5 = str(md5('output\\database_original.json')) updated_md5 = str(md5('output\\updated_on.txt')) with open('output\\database.json.md5', 'w') as outfile: json.dump(database_md5, outfile) with open('output\\database_original.json.md5', 'w') as outfile: json.dump(origin_database_md5, outfile) with open('output\\updated_on.txt.md5', 'w') as outfile: json.dump(updated_md5, outfile) color.color_print(Fore.YELLOW, '[DONE]', 'COMPLETED\n')
show_name = None episode_name = None new_file = src_path.replace(os.path.commonpath([settings.get('KOTEN_WATCH_PATH', DEFAULT_WATCH_PATH), src_path]) + "/", "") if m := util._show_manually_specified(new_file): Ayumi.info("Detected show name and episode name in event, using Mode 1.") show_name = m.group(1) episode_name = util._clean_episode_name(m.group(2)) Ayumi.info("New show name: {}".format(show_name), color=Ayumi.LYELLOW) Ayumi.info("New episode name: {}".format(episode_name), color=Ayumi.LYELLOW) else: Ayumi.debug("Non-conformant episode provided, using Naomi to find show name.") episode_name = util._clean_episode_name(pathlib.PurePath(src_path).name) show_name = naomi.find_closest_title(anitopy.parse(new_file)['anime_title']) Ayumi.info("New show name: {}".format(show_name), color=Ayumi.LYELLOW) Ayumi.info("New episode name: {}".format(episode_name), color=Ayumi.LYELLOW) # There is an event where Anilist is down, and Naomi could return None. # In this case, use the assumed-parsed show as the title if not show_name: show_name = anitopy.parse(new_file)['anime_title'] job = { "show": show_name, "episode": episode_name, "filesize": os.path.getsize(src_path), "sub": "softsub" }
def rss(last_guid=None): try: with rabbitpy.Connection( 'amqp://{username}:{password}@{host}:{port}/{vhost}'.format( username=settings.get_fresh('RABBITMQ_USERNAME'), password=settings.get_fresh('RABBITMQ_PASSWORD'), host=settings.get_fresh('RABBITMQ_HOST'), port=settings.get_fresh('RABBITMQ_PORT'), vhost=settings.get_fresh('RABBITMQ_VHOST'))) as conn: with conn.channel() as channel: Ayumi.set_rabbitpy_channel(channel) channel.enable_publisher_confirms() while True: Ayumi.info("Now starting feed fetch.", color=Ayumi.LCYAN) feed = feedparser.parse( settings.get('ACQUISITION_RSS_FEED_URL', None)) accepted_shows = _load_accepted_shows() Ayumi.debug( "Loaded accepted shows map: {}".format(accepted_shows)) history = _load_history() new_history = list() for entry in feed.entries: # Fetch data first title, link, guid = entry.title, entry.link, entry.guid Ayumi.debug( 'Encountered RSS item with title "{}", and guid "{}"' .format(title, guid)) # If feed item with last GUID encountered, do not process any further if guid == last_guid: Ayumi.debug( "Encountered RSS item with last_guid {} matching argument, breaking and writing history." .format(last_guid), color=Ayumi.YELLOW) break # Check the title data # Use the parsed title to match user provided titles. parsed_title = anitopy.parse(title)['anime_title'] if _strip_title(parsed_title) not in accepted_shows: Ayumi.info( 'Feed item with title "{}" (show title: "{}") is not in accepted shows, skipping.' .format(title, parsed_title)) else: if guid in history: # This item has been previously processed, skip it. Ayumi.info( 'Feed item with title "{}" (show title: "{}") has already been processed, skipping.' .format(title, parsed_title), color=Ayumi.GREEN) else: # A new feeditem! Let us process it. Ayumi.info( 'Feed item with title "{}" (show title: "{}") is in accepted shows, processing.' .format(title, parsed_title), color=Ayumi.YELLOW) message = rabbitpy.Message( channel, json.dumps({ "title": title, "link": link, "guid": guid, "show_title": accepted_shows[_strip_title( parsed_title)] })) acquisition_rss_exchange_name = settings.get( 'ACQUISITION_RSS_EXCHANGE') while not message.publish( acquisition_rss_exchange_name, mandatory=True): Ayumi.warning( 'Failed to publish feed item with title "{}" to exchange "{}", retrying in 60s...' .format(title, acquisition_rss_exchange_name), color=Ayumi.RED) sleep(60) Ayumi.info( 'Published feed item with title "{}" to exchange "{}".' .format( title, acquisition_rss_exchange_name, ), color=Ayumi.LGREEN) # Keep all items processed in the new history - it will be auto deleted by the expiry of the RSS Ayumi.debug( 'Appending item "{}" with title "{}" (show title: "{}") to new_history for write.' .format(guid, title, parsed_title), color=Ayumi.YELLOW) new_history.append(guid) _write_history(new_history) # Sleep till the next iteration sleep_duration = settings.get( 'ACQUISITION_RSS_SLEEP_INTERVAL', _DEFAULT_SLEEP_INTERVAL) Ayumi.info( "Now sleeping {} seconds.".format(sleep_duration), color=Ayumi.LCYAN) sleep(sleep_duration) except rabbitpy.exceptions.AMQPConnectionForced: Ayumi.rabbitpy_channel = None Ayumi.critical( "Operator manually closed RabbitMQ connection, shutting down.", color=Ayumi.RED) # Use return for now because in some cases, calling exit() may invoke the retry() header. return
def bittorrent(): try: with rabbitpy.Connection('amqp://{username}:{password}@{host}:{port}/{vhost}'.format( username=settings.get('RABBITMQ_USERNAME'), password=settings.get('RABBITMQ_PASSWORD'), host=settings.get('RABBITMQ_HOST'), port=settings.get('RABBITMQ_PORT'), vhost=settings.get('RABBITMQ_VHOST') )) as conn: with conn.channel() as channel: Ayumi.set_rabbitpy_channel(channel) channel.enable_publisher_confirms() queue_name = settings.get('ACQUISITION_BITTORRENT_QUEUE') Ayumi.debug("Connecting to queue: {}".format(queue_name)) queue = rabbitpy.Queue(channel, queue_name) queue.declare(passive=True) Ayumi.info('Now listening for messages on queue: {}...'.format( queue_name), color=Ayumi.LYELLOW) for message in queue.consume(prefetch=1): Ayumi.info( "Received new message, starting...", color=Ayumi.CYAN) feeditem_preprocess = _load_amqp_message_body(message) Ayumi.debug('Loaded message raw: "{}"'.format( feeditem_preprocess)) if not feeditem_preprocess or not metsuke.validate_feeditem(feeditem_preprocess): Ayumi.error('Invalid message received, rejecting. Output: "{}"'.format( feeditem_preprocess), color=Ayumi.RED) message.reject() continue # Load initial data feeditem: metsuke.FeedItem = metsuke.generate_feeditem( feeditem_preprocess) shows_map = _load_shows_map() overload_title = feeditem.show_title Ayumi.info( 'Setting overload title: "{}"'.format(overload_title)) # If there is a central override, use it instead. if _strip_title(anitopy.parse(feeditem.title)['anime_title']) in shows_map: central_overload_title = shows_map[_strip_title( feeditem.title)] Ayumi.info('Overwriting overload title with central overload title: "{}"'.format( central_overload_title)) overload_title = central_overload_title with tempfile.TemporaryDirectory() as temp_dir: Ayumi.debug( 'Created temporary directory under path: "{}"'.format(temp_dir)) # Download the episode try: res = subprocess.run( [ "aria2c", "--seed-time=0", "--rpc-save-upload-metadata=false", "--bt-save-metadata=false", "--dir={}".format(temp_dir), feeditem.link ] ) if res.returncode != 0: Ayumi.warning( "Aria2 did not return a 0 exit code, assuming download errored and nacking.", color=Ayumi.RED) message.nack() continue except subprocess.TimeoutExpired: Ayumi.warning( "Download via webtorrent timed out - nacking.", color=Ayumi.RED) message.nack() continue if res.returncode != 0: Ayumi.warning( "Webtorrent did not have a return code of 0, nacking.", color=Ayumi.RED) message.nack() continue # Rename it potential_files = [f for f in os.listdir( temp_dir) if f.endswith(".mkv")] Ayumi.debug( "Loaded potential files: {}".format(potential_files)) if len(potential_files) != 1: Ayumi.warning( "Found more than one .mkv file, rejecting this job.", color=Ayumi.RED) message.reject() continue dl_file = potential_files[0] Ayumi.info('Found file: "{}"'.format(dl_file)) dl_file_path = os.path.abspath( '{}/{}'.format(_clean_title(temp_dir), potential_files[0])) Ayumi.debug( 'dl_file_path: "{}"'.format(dl_file_path)) # Remove unneeded files # TODO: THIS IS A HOTFIX, CHANGE LOGIC IN B2 bad_files = [f for f in os.listdir( temp_dir) if not f.endswith(".mkv")] Ayumi.debug("Found bad files: {}".format(bad_files)) for bf in bad_files: try: Ayumi.debug("Removing bad file: {}".format(bf)) os.remove( '{}/{}'.format(_clean_title(temp_dir), bf)) except: Ayumi.debug("Removing bad tree: {}".format(bf)) shutil.rmtree( '{}/{}'.format(_clean_title(temp_dir), bf)) # Move the file to proper layout with updated name dl_file_new_name = _generate_new_filename(dl_file) Ayumi.info('Generated new episode name: "{}"'.format( dl_file_new_name)) dl_file_new_dir = "{}/{}".format( temp_dir, overload_title) Ayumi.debug( 'dl_file_new_dir: "{}"'.format(dl_file_new_dir)) dl_file_new_path = "{}/{}".format( dl_file_new_dir, dl_file_new_name) Ayumi.debug( 'dl_file_new_path: "{}"'.format( dl_file_new_path)) Ayumi.debug('Moving "{}" to "{}"'.format( dl_file_path, dl_file_new_path)) os.mkdir(dl_file_new_dir) shutil.move(dl_file_path, dl_file_new_path) # Upload the file to rclone destination with tempfile.NamedTemporaryFile(suffix=".conf", mode="w+b") as rconf: rconf.write(str.encode( settings.get("RCLONE_CONFIG_FILE"))) rconf.flush() Ayumi.debug( 'Created temporary rclone file under path: "{}"'.format(rconf.name)) rclone_dest = _clean_title(settings.get( "ACQUISITION_BITTORRENT_RCLONE_DEST")) rclone_flags = settings.get("RCLONE_FLAGS", "") command = [ "rclone", "--config={}".format(rconf.name), "copy", temp_dir, rclone_dest] command.extend(rclone_flags.split()) Ayumi.debug( 'Rclone command to be run: "{}"'.format(command)) try: Ayumi.info( 'Now uploading new blob to: "{}"'.format(rclone_dest)) rclone_res = subprocess.run( command, timeout=3600) if rclone_res.returncode != 0: Ayumi.warning('Rclone returned non-zero code of {}, nacking.'.format( rclone_res.returncode), color=Ayumi.LRED) message.nack() except subprocess.TimeoutExpired: Ayumi.warning( 'Rclone upload timed out, nacking.', color=Ayumi.LRED) message.nack() continue # Fetch information on the file to create a job new_message = rabbitpy.Message(channel, dumps( { "show": overload_title, "episode": dl_file_new_name, "filesize": int(os.path.getsize(dl_file_new_path)), "sub": "SOFTSUB" } )) acquisition_bittorrent_exchange_name = settings.get( 'ACQUISITION_BITTORRENT_EXCHANGE') Ayumi.info('Sending to exchange: "{}"'.format( acquisition_bittorrent_exchange_name), color=Ayumi.CYAN) while not new_message.publish(acquisition_bittorrent_exchange_name, mandatory=True): Ayumi.warning( "Failed to publish feed item, trying again in 60 seconds") sleep(60) Ayumi.info("Published feed item with title: " + overload_title, color=Ayumi.LGREEN) message.ack() except rabbitpy.exceptions.AMQPConnectionForced: Ayumi.warning( "Operator manually closed RabbitMQ connection, shutting down.", color=Ayumi.LYELLOW) return
def search(self, query, ep, is_movie=False): url = f'{self.url}{query}{self.opts}' res = requests.get(url) if not res.status_code == requests.codes.ok: return False soup = BeautifulSoup(res.content, features='lxml') torrents = soup.findAll('tr', {'class': 'success'}) search_results = [] for torrent in torrents: info = torrent.select('td', {'class': 'text-center'}) img = info[0].select('img') img_desc = img[0].get('alt', 'null') if not 'Anime - English-translated' in img_desc: print(img_desc) continue title_info = info[1].select('a') if len(title_info) > 1: title = title_info[1].get('title', 'null') else: title = title_info[0].get('title', 'null') # title can then be run through the parser from nyaasi to extract relevant info magnet = '' torrent_links = info[2].select('a') for torrent_link in torrent_links: link = torrent_link.get('href', 'null') if link.startswith('magnet'): magnet = link file_size = info[3].text date_upload = info[4].text seeders = info[5].text # don't bother if 0 seeders if int(seeders) < 1: continue # returns # 'anime_season' # 'anime_title': # 'audio_term': # 'episode_number # 'file_name': # 'release_group': # 'source': # 'video_resolution': # 'video_term': title_info = anitopy.parse(title) # if no episode_number, assume it's batch if not is_movie: episode_number = title_info.get('episode_number', None) if not episode_number: episode_number = 'Batch' if not isinstance(episode_number, list): if not episode_number.isdigit(): episode_number = 'Batch' elif not episode_number == ep: # wrong episode, check next torrent continue info_obj = { 'title': title, 'magnet': magnet, 'size': file_size, 'uploaded on': date_upload, 'seeders': seeders, 'resolution': title_info.get('video_resolution', 'undefined'), 'episode_number': episode_number, 'season': title_info.get('anime_season', '1') } else: info_obj = { 'title': title, 'magnet': magnet, 'seeders': seeders, 'resolution': title_info.get('video_resolution', 'undefined') } search_results.append(info_obj) return search_results