class AnimeIndex(): """ Search on Anime Index using HTML. """ def __init__(self, network): self.log = LoggerManager().get_logger("AnimeIndex") self.network = network def search(self, text="", dic=None, category=5, active=1): """ Returned dictionary struct: dict[RESULT_NUMBER]["title"] = str dict[RESULT_NUMBER]["link"] = str dict[RESULT_NUMBER]["date"] = datetime dict[RESULT_NUMBER]["downloads"] = int :type text: str :param text: search_terms :type dic: dict :param dic: where the data will be stored. If None, a new dict is created. :type category: int :param category: site specific filter to search only for anime. :type active: int :param active: site specific filter to search only for active downloads. """ if dic is None: dic = {} url = self.__get_url(text, category, active) html = self.network.get_data(url) return self.__parse_html(html, dic, text) @staticmethod def __get_url(text, category, active): url = "" if text != "": positive_words = [ word for word in text.split(" ") if not word.startswith("-") ] positive_text = "+".join(positive_words) url = "tracker.anime-index.org/index.php?page=torrents&search=%s&category=%s&active=%s" % ( positive_text, str(category), str(active)) return url def __parse_html(self, html, dic, text): parser = HTMLparser() parser.dict = dic parser.cont = len(dic) parser.search_terms = text.strip().split(" ") try: parser.feed(html) parser.close() except CleanExit: return parser.dict except Exception as error: self.log.print_traceback(error, self.log.error) return parser.dict
class AnimeIndex(): """ Search on Anime Index using HTML. """ def __init__(self,network): self.log = LoggerManager().get_logger("AnimeIndex") self.network = network def search(self,text="",dic=None,category=5,active=1): """ Returned dictionary struct: dict[RESULT_NUMBER]["title"] = str dict[RESULT_NUMBER]["link"] = str dict[RESULT_NUMBER]["date"] = datetime dict[RESULT_NUMBER]["downloads"] = int :type text: str :param text: search_terms :type dic: dict :param dic: where the data will be stored. If None, a new dict is created. :type category: int :param category: site specific filter to search only for anime. :type active: int :param active: site specific filter to search only for active downloads. """ if dic is None: dic = {} url = self.__get_url(text,category,active) html = self.network.get_data(url) return self.__parse_html(html, dic, text) @staticmethod def __get_url(text,category,active): url = "" if text!="": positive_words = [word for word in text.split(" ") if not word.startswith("-")] positive_text = "+".join(positive_words) url = "tracker.anime-index.org/index.php?page=torrents&search=%s&category=%s&active=%s" % (positive_text,str(category),str(active)) return url def __parse_html(self,html, dic, text): parser = HTMLparser() parser.dict = dic parser.cont = len(dic) parser.search_terms = text.strip().split(" ") try: parser.feed(html) parser.close() except CleanExit: return parser.dict except Exception as error: self.log.print_traceback(error,self.log.error) return parser.dict
class Tokyotosho(): """ Search on Tokyotosho using HTML. """ def __init__(self, network): self.log = LoggerManager().get_logger("Tokyotosho") self.network = network def search(self, text="", dic=None, category=1): """ Returned dictionary struct: dict[RESULT_NUMBER]["title"] = str dict[RESULT_NUMBER]["link"] = str dict[RESULT_NUMBER]["date"] = datetime :type text: str :param text: search_terms :type dic: dict :param dic: where the data will be stored. If None, a new dict is created. :type category: str :param category: site specific filter to search only for anime. """ if dic is None: dic = {} url = self.__get_url(text, category) html = self.network.get_data(url) html = html.replace("<span class=\"s\"> </span>", "") return self.__parse_html(html, dic, text) @staticmethod def __get_url(text, category): url = "" if text != "": text = text.strip().replace(" ", "+") url = "www.tokyotosho.info/search.php?terms=%s&type=%s" % ( text, str(category)) return url def __parse_html(self, html, dic, text): parser = HTMLparser() parser.dict = dic parser.cont = len(dic) parser.search_terms = text.strip().split(" ") try: parser.feed(html) parser.close() except CleanExit: return parser.dict except Exception as error: self.log.print_traceback(error, self.log.error) return parser.dict
class Tokyotosho(): """ Search on Tokyotosho using HTML. """ def __init__(self,network): self.log = LoggerManager().get_logger("Tokyotosho") self.network = network def search(self,text="",dic=None,category=1): """ Returned dictionary struct: dict[RESULT_NUMBER]["title"] = str dict[RESULT_NUMBER]["link"] = str dict[RESULT_NUMBER]["date"] = datetime :type text: str :param text: search_terms :type dic: dict :param dic: where the data will be stored. If None, a new dict is created. :type category: str :param category: site specific filter to search only for anime. """ if dic is None: dic = {} url = self.__get_url(text,category) html = self.network.get_data(url) html = html.replace("<span class=\"s\"> </span>","") return self.__parse_html(html, dic, text) @staticmethod def __get_url(text,category): url = "" if text!="": text = text.strip().replace(" ","+") url = "www.tokyotosho.info/search.php?terms=%s&type=%s" % (text,str(category)) return url def __parse_html(self,html,dic,text): parser = HTMLparser() parser.dict = dic parser.cont = len(dic) parser.search_terms = text.strip().split(" ") try: parser.feed(html) parser.close() except CleanExit: return parser.dict except Exception as error: self.log.print_traceback(error,self.log.error) return parser.dict
class AnimeTosho(): """ Search on Anime Tosho using HTML. """ def __init__(self, network): self.log = LoggerManager().get_logger("AnimeTosho") self.network = network def search(self, text="", dic=None): """ Returned dictionary struct: dict[RESULT_NUMBER]["title"] = str dict[RESULT_NUMBER]["link"] = str dict[RESULT_NUMBER]["date"] = datetime :type text: str :param text: search_terms :type dic: dict :param dic: where the data will be stored. If None, a new dict is created. """ if dic is None: dic = {} url = self.__get_url(text) html = self.network.get_data(url) return self.__parse_html(html, dic, text) @staticmethod def __get_url(text): url = "" if text != "": positive_words = [ word for word in text.split(" ") if not word.startswith("-") ] positive_text = "+".join(positive_words) url = "https://animetosho.org/search?q=%s" % positive_text return url def __parse_html(self, html, dic, text): parser = HTMLparser() parser.dict = dic parser.cont = len(dic) parser.search_terms = text.strip().split(" ") try: parser.feed(html) parser.close() except CleanExit: return parser.dict except Exception as error: self.log.print_traceback(error, self.log.error) return parser.dict
def execute_once_search_tokyotosho(self, anime): """ Executes search on this site. :type anime: db.Anime :param anime: Contains search terms. """ self.log = LoggerManager().get_logger("Downloader-Once-TT") self.network = Network() tokyotosho = Tokyotosho(self.network) self.stopping_thread = False self.runningSearch.emit() dict_tokyotosho = None if anime.check_tokyotosho and not self.stopping_thread: dict_tokyotosho = tokyotosho.search(anime.search_terms) self.searchResult.emit(dict_tokyotosho) self.finish.emit()
def execute_once_search_anime_index(self, anime): """ Executes search on this site. :type anime: db.Anime :param anime: Contains search terms. """ self.log = LoggerManager().get_logger("Downloader-Once-AI") self.network = Network() anime_index = AnimeIndex(self.network) self.stopping_thread = False self.runningSearch.emit() dict_anime_index = None if anime.check_anime_index and not self.stopping_thread: dict_anime_index = anime_index.search(anime.search_terms) self.searchResult.emit(dict_anime_index) self.finish.emit()
def execute_once_search_nyaa(self, anime): """ Executes search on this site. :type anime: db.Anime :param anime: Contains search terms. """ self.log = LoggerManager().get_logger("Downloader-Once-NY") self.network = Network() if DBManager().get_config().prefer_rss: nyaa = NyaaRSS(self.network) else: nyaa = Nyaa(self.network) self.stopping_thread = False self.runningSearch.emit() dict_nyaa = None if anime.check_nyaa and not self.stopping_thread: dict_nyaa = nyaa.search(anime.search_terms) self.searchResult.emit(dict_nyaa) self.finish.emit()
def execute_once(self): """ Initializes variables necessary to search for new episodes. """ self.log = LoggerManager().get_logger("Downloader") self.log.debug("#####################") self.dbManager = DBManager() self.animeList = self.dbManager.get_anime_list() self.config = self.dbManager.get_config() self.network = Network() self.anime_index = AnimeIndex(self.network) self.anime_tosho = AnimeTosho(self.network) self.anirena = Anirena(self.network) if self.config.prefer_rss: self.nyaa = NyaaRSS(self.network) else: self.nyaa = Nyaa(self.network) self.tokyotosho = Tokyotosho(self.network) self.stopping_thread = False self.timer = QtCore.QTimer() self.running.emit() self.log.debug("****************************") number_of_downloaded_episodes = self.__search_new_episodes() msg = "No" if number_of_downloaded_episodes == 0 else str( number_of_downloaded_episodes) if not self.stopping_thread: self.log.info("%s episodes downloaded, sleeping for %s seconds" % (msg, self.config.sleep_time)) self.restart.emit() else: self.log.info("%s episodes downloaded, stopping downloader" % msg) self.finish.emit()
def execute_once_search_anime_index(self,anime): """ Executes search on this site. :type anime: db.Anime :param anime: Contains search terms. """ self.log = LoggerManager().get_logger("Downloader-Once-AI") self.network = Network() anime_index = AnimeIndex(self.network) self.stopping_thread = False self.runningSearch.emit() dict_anime_index = None if anime.check_anime_index and not self.stopping_thread: dict_anime_index = anime_index.search(anime.search_terms) self.searchResult.emit(dict_anime_index) self.finish.emit()
def execute_once_search_tokyotosho(self,anime): """ Executes search on this site. :type anime: db.Anime :param anime: Contains search terms. """ self.log = LoggerManager().get_logger("Downloader-Once-TT") self.network = Network() tokyotosho = Tokyotosho(self.network) self.stopping_thread = False self.runningSearch.emit() dict_tokyotosho = None if anime.check_tokyotosho and not self.stopping_thread: dict_tokyotosho = tokyotosho.search(anime.search_terms) self.searchResult.emit(dict_tokyotosho) self.finish.emit()
def execute_once_search_nyaa(self,anime): """ Executes search on this site. :type anime: db.Anime :param anime: Contains search terms. """ self.log = LoggerManager().get_logger("Downloader-Once-NY") self.network = Network() if DBManager().get_config().prefer_rss: nyaa = NyaaRSS(self.network) else: nyaa = Nyaa(self.network) self.stopping_thread = False self.runningSearch.emit() dict_nyaa = None if anime.check_nyaa and not self.stopping_thread: dict_nyaa = nyaa.search(anime.search_terms) self.searchResult.emit(dict_nyaa) self.finish.emit()
def execute_once(self): """ Initializes variables necessary to search for new episodes. """ self.log = LoggerManager().get_logger("Downloader") self.log.debug("#####################") self.dbManager = DBManager() self.animeList = self.dbManager.get_anime_list() self.config = self.dbManager.get_config() self.network = Network() self.anime_index = AnimeIndex(self.network) self.anime_tosho = AnimeTosho(self.network) self.anirena = Anirena(self.network) if self.config.prefer_rss: self.nyaa = NyaaRSS(self.network) else: self.nyaa = Nyaa(self.network) self.tokyotosho = Tokyotosho(self.network) self.stopping_thread = False self.timer = QtCore.QTimer() self.running.emit() self.log.debug("****************************") number_of_downloaded_episodes = self.__search_new_episodes() msg = "No" if number_of_downloaded_episodes==0 else str(number_of_downloaded_episodes) if not self.stopping_thread: self.log.info("%s episodes downloaded, sleeping for %s seconds" % (msg, self.config.sleep_time)) self.restart.emit() else: self.log.info("%s episodes downloaded, stopping downloader" % msg) self.finish.emit()
class Nyaa(): """ Search on Nyaa using HTML. """ def __init__(self, network): self.log = LoggerManager().get_logger("Nyaa") self.network = network def search(self, text="", dic=None, category="1_37"): #"1_37" = English translated anime """ Returned dictionary struct: dict[RESULT_NUMBER]["title"] = str dict[RESULT_NUMBER]["link"] = str dict[RESULT_NUMBER]["date"] = datetime dict[RESULT_NUMBER]["downloads"] = int :type text: str :param text: search_terms :type dic: dict :param dic: where the data will be stored. If None, a new dict is created. :type category: str :param category: site specific filter to search only for anime. """ if dic is None: dic = {} search_terms = text.strip().split(" ") url = self.__get_url(text, category) html = self.network.get_data(url) if "viewtorrentname" in html: return self.__parse_html_1_result(html, dic, search_terms) else: return self.__parse_html(html, dic, search_terms) @staticmethod def __get_url(text, category): url = "" if text != "": text = text.strip().replace(" ", "+") url = "https://www.nyaa.se/?page=search&term=%s&cats=%s" % ( text, category) return url def __parse_html_1_result(self, html, dic, search_terms): parser = HTMLparser1Result() parser.dict = dic parser.cont = len(dic) parser.search_terms = search_terms parser.debug = self try: parser.feed(html) parser.close() except CleanExit: return parser.dict except Exception as error: self.log.print_traceback(error, self.log.error) return parser.dict def __parse_html(self, html, dic, search_terms): parser = HTMLparser() parser.dict = dic parser.cont = len(dic) parser.search_terms = search_terms try: parser.feed(html) parser.close() except CleanExit: self.log.info("HTML successfully parsed (%i results)" % (len(parser.dict))) return parser.dict except Exception as error: self.log.print_traceback(error, self.log.error) else: self.log.info("HTML successfully parsed (%i results)" % (len(parser.dict))) #return dic return parser.dict
class Main(QtCore.QObject): """ Main class, instantiated when the application starts. Creates main window/system tray icon, and stops the user from opnening more than one instance of the application. It also starts/stops the downloader when the user requests or during auto-start on Windows startup. """ def __init__(self): QtCore.QObject.__init__(self) # Make sure the required folders/files exist if not os.path.isdir(constant.DATA_PATH): os.makedirs(constant.DATA_PATH) self.log = LoggerManager().get_logger("MAIN") try: if not os.path.isfile(constant.DB_PATH): shutil.copyfile("dbTemplate.db", constant.DB_PATH) except (shutil.Error, IOError) as error: self.log.print_traceback(error, self.log.critical) sys.exit(1) try: if not os.path.isdir(constant.DEFAULT_TORRENTS_PATH): os.makedirs(constant.DEFAULT_TORRENTS_PATH) except Exception as error: self.log.print_traceback(error, self.log.critical) sys.exit(1) self.app = QtSingleApplication(constant.GUID, sys.argv) self.log.info("---STARTING APPLICATION---") if self.app.isRunning(): self.log.warning( "---The launch of another instance of this application will be cancelled---" ) self.app.sendMessage() sys.exit(0) self.app.messageReceived.connect(self.another_instance_opened) self.app.setQuitOnLastWindowClosed(False) self.window = None self.tray_icon = None self.thread = None self.downloader = None self.timer = None self.downloader_is_running = False self.downloader_is_restarting = False self.downloader_is_stopping = False try: self.window = WindowMain(self) self.tray_icon = SystemTrayIcon(self, self.window) self.tray_icon.show() show_gui = "-nogui" not in sys.argv if show_gui: if self.downloader_is_running: self.window.downloader_started() else: self.window.downloader_stopped() self.window.show() elif not self.window.is_visible(): self.log.info("STARTING DOWNLOADER") self.start_downloader() self.app.exec_() except Exception as unforeseenError: self.log.critical("UNFORESEEN ERROR") self.log.print_traceback(unforeseenError, self.log.critical) if self.tray_icon is not None: self.show_tray_message("Unforeseen error occurred...") exit() def quit(self): """ Finishes the application gracefully - at least tries to, teehee (^_^;) """ if self.tray_icon is not None: self.tray_icon.hide() self.tray_icon.deleteLater() if self.timer is not None: self.timer.stop() if self.thread is not None and self.thread.isRunning(): self.stop_downloader() #self.app.closeAllWindows() self.app.quit() def another_instance_opened(self, _): """ Called when the user tries to open another instance of the application. Instead of allowing it, will open the current one to avoid any errors. :type _: QtCore.QString :param _: message received, see class QtSingleApplication below. """ self.window.show() def start_downloader(self): """ Starts the downloader in a thread. """ # Don't know how to reproduce, but in some really rare cases the downloader might start without the user requesting it. # These logs try to collect information that might help pinpoint what causes that. # Actually, it's been so long since the last time this error was observed that I don't know if it still happens # or if whatever caused it was fixed... self.log.debug("stack ([1][3]):") i = 0 for item in inspect.stack(): self.log.debug("[" + str(i) + "]= " + str(item)) i += 1 self.log.debug("downloader_is_running: " + str(self.downloader_is_running)) self.log.debug("downloader_is_restarting: " + str(self.downloader_is_restarting)) self.log.debug("downloader_is_stopping: " + str(self.downloader_is_stopping)) if not self.downloader_is_stopping: if self.downloader_is_restarting: self.log.info("RESTARTING DOWNLOADER THREAD") self.downloader_is_restarting = False else: self.log.info("STARTING DOWNLOADER THREAD") self.window.downloader_starting() self.thread = QtCore.QThread(self) self.downloader = Downloader() self.downloader.moveToThread(self.thread) self.downloader.running.connect(self.downloader_started) self.downloader.finish.connect(self.thread.quit) self.downloader.restart.connect(self.restart_downloader) self.downloader.showMessage.connect(self.show_tray_message) self.downloader.update_ui.connect(self.update_ui) # noinspection PyUnresolvedReferences self.thread.started.connect( self.downloader.execute_once ) # PyCharm doesn't recognize started.connect()... # noinspection PyUnresolvedReferences self.thread.finished.connect( self.downloader_stopped ) # PyCharm doesn't recognize finished.connect()... self.thread.start() else: self.downloader_is_stopping = False self.downloader_is_restarting = False def stop_downloader(self): """ Stops the downloader (¬_¬) """ self.log.info("TERMINATING DOWNLOADER THREAD") self.window.downloader_stopping() self.downloader_is_stopping = True self.downloader_is_restarting = False if self.thread.isRunning(): self.downloader.stop_thread() thread_stopped_gracefully = self.thread.wait(300) if self.thread.isRunning(): thread_stopped_gracefully = self.thread.quit() self.log.info("THREAD STOPPED CORRECTLY: %s" % thread_stopped_gracefully) if not thread_stopped_gracefully: self.thread.terminate() else: self.downloader_stopped() try: self.timer.stop() except AttributeError: pass # Happens when the downloader is interrupted before being able to fully execute at least once. def restart_downloader(self): """ Finishes the current downloader thread and starts a timer. When the timer times out a new downloader thread is created. """ self.downloader_is_restarting = True self.thread.quit() self.log.info("THREAD FINISHED CORRECTLY: %s" % self.thread.wait(300)) self.timer = QtCore.QTimer() # noinspection PyUnresolvedReferences self.timer.timeout.connect( self.start_downloader ) # PyCharm doesn't recognize timeout.connect()... self.timer.setSingleShot(True) self.timer.start(db.DBManager().get_config().sleep_time * 1000) @QtCore.pyqtSlot() def downloader_started(self): """ Downloader thread started correctly; notifies the user. """ self.downloader_is_running = True self.window.downloader_started() @QtCore.pyqtSlot() def downloader_stopped(self): """ Downloader thread stopped correctly; notifies the user. """ if not self.downloader_is_restarting: self.downloader_is_running = False self.downloader_is_stopping = False self.downloader_is_restarting = False self.window.downloader_stopped() @QtCore.pyqtSlot(str) def show_tray_message(self, message): """ Uses the system tray icon to notify the user about something. :type message: str :param message: Message to be shown to the user. """ # TODO: Would it be better if this were moved to manager.system_tray_icon? self.tray_icon.showMessage(constant.TRAY_MESSAGE_TITLE, message, QtGui.QSystemTrayIcon.Information, 5000) @QtCore.pyqtSlot(str) def update_ui(self, message): """ Updates the anime table in the main window. Also, shows a message to the user using the system tray icon. :type message: str :param message: Message to be shown to the user. """ if self.window is not None: self.window.update_anime_table() self.show_tray_message(message)
class Downloader(QtCore.QObject): """ Processes the search requests from the [Add anime window]. Also responsible for actually searching for new episodes automatically according to the frequency chosen by the user. """ running = QtCore.pyqtSignal() finish = QtCore.pyqtSignal() restart = QtCore.pyqtSignal() showMessage = QtCore.pyqtSignal(str) update_ui = QtCore.pyqtSignal(str) runningSearch = QtCore.pyqtSignal() searchResult = QtCore.pyqtSignal(object) #Might be dict or None def __init__(self, parent=None): super(Downloader, self).__init__(parent) self.log = None self.dbManager = None self.animeList = None self.config = None self.network = None self.anime_index = None self.anime_tosho = None self.anirena = None self.nyaa = None self.tokyotosho = None self.stopping_thread = None self.timer = None @QtCore.pyqtSlot() def execute_once(self): """ Initializes variables necessary to search for new episodes. """ self.log = LoggerManager().get_logger("Downloader") self.log.debug("#####################") self.dbManager = DBManager() self.animeList = self.dbManager.get_anime_list() self.config = self.dbManager.get_config() self.network = Network() self.anime_index = AnimeIndex(self.network) self.anime_tosho = AnimeTosho(self.network) self.anirena = Anirena(self.network) if self.config.prefer_rss: self.nyaa = NyaaRSS(self.network) else: self.nyaa = Nyaa(self.network) self.tokyotosho = Tokyotosho(self.network) self.stopping_thread = False self.timer = QtCore.QTimer() self.running.emit() self.log.debug("****************************") number_of_downloaded_episodes = self.__search_new_episodes() msg = "No" if number_of_downloaded_episodes==0 else str(number_of_downloaded_episodes) if not self.stopping_thread: self.log.info("%s episodes downloaded, sleeping for %s seconds" % (msg, self.config.sleep_time)) self.restart.emit() else: self.log.info("%s episodes downloaded, stopping downloader" % msg) self.finish.emit() def __search_new_episodes(self): """ Searches for new episodes for all enabled anime. """ downloaded_episodes = 0 current_anime = 0 for anime in self.animeList: if anime.enabled and not self.stopping_thread: current_anime += 1 self.log.info("(%d/%d) searching for episode %s of '%s'" % (current_anime, self.dbManager.number_of_anime_enabled, anime.episode, escape_unicode(anime.name))) dict_anime_index,dict_anime_tosho,dict_anirena,dict_nyaa,dict_tokyotosho = self.search(anime) anime_dictionary = {} if dict_anime_index is not None: anime_dictionary = dict(anime_dictionary.items()+dict_anime_index.items()) if dict_anime_tosho is not None: anime_dictionary = dict(anime_dictionary.items()+dict_anime_tosho.items()) if dict_anirena is not None: anime_dictionary = dict(anime_dictionary.items()+dict_anirena.items()) if dict_nyaa is not None: anime_dictionary = dict(anime_dictionary.items()+dict_nyaa.items()) if dict_tokyotosho is not None: anime_dictionary = dict(anime_dictionary.items()+dict_tokyotosho.items()) title = "ERROR(No value)" if not self.stopping_thread: try: for key in anime_dictionary.keys(): self.log.debug("'%s' is a match, checking episode number" % (escape_unicode(anime_dictionary[key]["title"]))) title = anime_dictionary[key]["title"] if (".mkv" or ".mp4") in title: self.log.debug("title = "+title) regex_episode_and_version_number = re.compile("\\b[\s_~\-+.]?(%02d)[\s_~\-+.]*(\d*)[\s_]*v?(\d)?[\s\w.]*(\[|\(|.mkv|.mp4)" % anime.episode) else: regex_episode_and_version_number = re.compile("\\b[\s_~\-+.]?(%02d)[\s_~\-+.]*(\d*)[\s_]*v?(\d)?" % anime.episode) result = regex_episode_and_version_number.findall(title) self.log.debug("regex result = "+str(result)) #self.log.debug("REGEX result = '%s' (len(result)>0: %s)" % (result, len(result)>0)) if len(result)>0: self.log.info("A torrent has been found") try: last_episode_number = int(result[0][1]) self.log.info("It's a double episode! The last episode was number %d" % last_episode_number) except (TypeError,IndexError,ValueError): last_episode_number = int(result[0][0]) self.log.info("It's a normal episode") try: last_version_number = int(result[0][2]) self.log.info("It's version %d" % last_version_number) except (TypeError,IndexError,ValueError): last_version_number = 1 if not self.stopping_thread: result = False try: result = self.network.download_torrent(anime_dictionary[key]["link"], "%s %d" % (anime.name,anime.episode), constant.DEFAULT_TORRENTS_PATH, anime.download_folder) except Exception as error: self.showMessage.emit("Error: %s (%s - %s)" % (type(error).__name__,anime.name,anime.episode)) if result: downloaded_episodes+=1 #self.log.debug("Updating EpisodeNumber") anime.update_episode(last_episode_number+1) #self.log.debug("Updating LastVersionNumber") anime.update_version(last_version_number) #self.log.debug("Updating LastFileDownloaded") anime.update_last_file_downloaded(anime_dictionary[key]["title"]) self.log.debug("Notifying user") self.update_ui.emit("%s - %s" % (anime.name, anime.episode-1)) break except Exception as error: self.log.error("ERROR while analysing '%s'" % escape_unicode(title)) self.log.print_traceback(error,self.log.error) if self.stopping_thread: break return downloaded_episodes def search(self,anime): """ Searches for new episodes of a given anime. :type anime: db.Anime :param anime: Anime to search for new episode. :rtype: dict or None,dict or None,dict or None,dict or None,dict or None :return: results for each site: Anime Index, Anime Tosho, Anirena, Nyaa and Tokyotosho """ text = "%s %02d" % (anime.search_terms,anime.episode) dict_anime_index = None dict_anime_tosho = None dict_anirena = None dict_nyaa = None dict_tokyotosho = None if anime.check_anime_index and not self.stopping_thread: dict_anime_index = self.anime_index.search(text) if anime.check_anime_tosho and not self.stopping_thread: dict_anime_tosho = self.anime_tosho.search(text) if anime.check_anirena and not self.stopping_thread: dict_anirena = self.anirena.search(text) if anime.check_nyaa and not self.stopping_thread: dict_nyaa = self.nyaa.search(text) if anime.check_tokyotosho and not self.stopping_thread: dict_tokyotosho = self.tokyotosho.search(text) return dict_anime_index,dict_anime_tosho,dict_anirena,dict_nyaa,dict_tokyotosho def execute_once_search_anime_index(self,anime): """ Executes search on this site. :type anime: db.Anime :param anime: Contains search terms. """ self.log = LoggerManager().get_logger("Downloader-Once-AI") self.network = Network() anime_index = AnimeIndex(self.network) self.stopping_thread = False self.runningSearch.emit() dict_anime_index = None if anime.check_anime_index and not self.stopping_thread: dict_anime_index = anime_index.search(anime.search_terms) self.searchResult.emit(dict_anime_index) self.finish.emit() def execute_once_search_anime_tosho(self,anime): """ Executes search on this site. :type anime: db.Anime :param anime: Contains search terms. """ self.log = LoggerManager().get_logger("Downloader-Once-AT") self.network = Network() anime_tosho = AnimeTosho(self.network) self.stopping_thread = False self.runningSearch.emit() dict_anime_tosho = None if anime.check_anime_tosho and not self.stopping_thread: dict_anime_tosho = anime_tosho.search(anime.search_terms) self.searchResult.emit(dict_anime_tosho) self.finish.emit() def execute_once_search_anirena(self,anime): """ Executes search on this site. :type anime: db.Anime :param anime: Contains search terms. """ self.log = LoggerManager().get_logger("Downloader-Once-AR") self.network = Network() anirena = Anirena(self.network) self.stopping_thread = False self.runningSearch.emit() dict_anirena = None if anime.check_anirena and not self.stopping_thread: dict_anirena = anirena.search(anime.search_terms) self.searchResult.emit(dict_anirena) self.finish.emit() def execute_once_search_nyaa(self,anime): """ Executes search on this site. :type anime: db.Anime :param anime: Contains search terms. """ self.log = LoggerManager().get_logger("Downloader-Once-NY") self.network = Network() if DBManager().get_config().prefer_rss: nyaa = NyaaRSS(self.network) else: nyaa = Nyaa(self.network) self.stopping_thread = False self.runningSearch.emit() dict_nyaa = None if anime.check_nyaa and not self.stopping_thread: dict_nyaa = nyaa.search(anime.search_terms) self.searchResult.emit(dict_nyaa) self.finish.emit() def execute_once_search_tokyotosho(self,anime): """ Executes search on this site. :type anime: db.Anime :param anime: Contains search terms. """ self.log = LoggerManager().get_logger("Downloader-Once-TT") self.network = Network() tokyotosho = Tokyotosho(self.network) self.stopping_thread = False self.runningSearch.emit() dict_tokyotosho = None if anime.check_tokyotosho and not self.stopping_thread: dict_tokyotosho = tokyotosho.search(anime.search_terms) self.searchResult.emit(dict_tokyotosho) self.finish.emit() def stop_thread(self): """ Stops requests being executed to allow thread to finish gracefully. """ self.log.info("STOPPING DOWNLOADER THREAD") self.stopping_thread = True self.network.stop_thread()
def __init__(self, network): self.log = LoggerManager().get_logger("Nyaa") self.network = network
def __init__(self,network): self.log = LoggerManager().get_logger("AnimeIndex") self.network = network
class Downloader(QtCore.QObject): """ Processes the search requests from the [Add anime window]. Also responsible for actually searching for new episodes automatically according to the frequency chosen by the user. """ running = QtCore.pyqtSignal() finish = QtCore.pyqtSignal() restart = QtCore.pyqtSignal() showMessage = QtCore.pyqtSignal(str) update_ui = QtCore.pyqtSignal(str) runningSearch = QtCore.pyqtSignal() searchResult = QtCore.pyqtSignal(object) #Might be dict or None def __init__(self, parent=None): super(Downloader, self).__init__(parent) self.log = None self.dbManager = None self.animeList = None self.config = None self.network = None self.anime_index = None self.anime_tosho = None self.anirena = None self.nyaa = None self.tokyotosho = None self.stopping_thread = None self.timer = None @QtCore.pyqtSlot() def execute_once(self): """ Initializes variables necessary to search for new episodes. """ self.log = LoggerManager().get_logger("Downloader") self.log.debug("#####################") self.dbManager = DBManager() self.animeList = self.dbManager.get_anime_list() self.config = self.dbManager.get_config() self.network = Network() self.anime_index = AnimeIndex(self.network) self.anime_tosho = AnimeTosho(self.network) self.anirena = Anirena(self.network) if self.config.prefer_rss: self.nyaa = NyaaRSS(self.network) else: self.nyaa = Nyaa(self.network) self.tokyotosho = Tokyotosho(self.network) self.stopping_thread = False self.timer = QtCore.QTimer() self.running.emit() self.log.debug("****************************") number_of_downloaded_episodes = self.__search_new_episodes() msg = "No" if number_of_downloaded_episodes == 0 else str( number_of_downloaded_episodes) if not self.stopping_thread: self.log.info("%s episodes downloaded, sleeping for %s seconds" % (msg, self.config.sleep_time)) self.restart.emit() else: self.log.info("%s episodes downloaded, stopping downloader" % msg) self.finish.emit() def __search_new_episodes(self): """ Searches for new episodes for all enabled anime. """ downloaded_episodes = 0 current_anime = 0 for anime in self.animeList: if anime.enabled and not self.stopping_thread: current_anime += 1 self.log.info( "(%d/%d) searching for episode %s of '%s'" % (current_anime, self.dbManager.number_of_anime_enabled, anime.episode, escape_unicode(anime.name))) dict_anime_index, dict_anime_tosho, dict_anirena, dict_nyaa, dict_tokyotosho = self.search( anime) anime_dictionary = {} if dict_anime_index is not None: anime_dictionary = dict(anime_dictionary.items() + dict_anime_index.items()) if dict_anime_tosho is not None: anime_dictionary = dict(anime_dictionary.items() + dict_anime_tosho.items()) if dict_anirena is not None: anime_dictionary = dict(anime_dictionary.items() + dict_anirena.items()) if dict_nyaa is not None: anime_dictionary = dict(anime_dictionary.items() + dict_nyaa.items()) if dict_tokyotosho is not None: anime_dictionary = dict(anime_dictionary.items() + dict_tokyotosho.items()) title = "ERROR(No value)" if not self.stopping_thread: try: for key in anime_dictionary.keys(): self.log.debug( "'%s' is a match, checking episode number" % (escape_unicode( anime_dictionary[key]["title"]))) title = anime_dictionary[key]["title"] if (".mkv" or ".mp4") in title: self.log.debug("title = " + title) regex_episode_and_version_number = re.compile( "\\b[\s_~\-+.]?(%02d)[\s_~\-+.]*(\d*)[\s_]*v?(\d)?[\s\w.]*(\[|\(|.mkv|.mp4)" % anime.episode) else: regex_episode_and_version_number = re.compile( "\\b[\s_~\-+.]?(%02d)[\s_~\-+.]*(\d*)[\s_]*v?(\d)?" % anime.episode) result = regex_episode_and_version_number.findall( title) self.log.debug("regex result = " + str(result)) #self.log.debug("REGEX result = '%s' (len(result)>0: %s)" % (result, len(result)>0)) if len(result) > 0: self.log.info("A torrent has been found") try: last_episode_number = int(result[0][1]) self.log.info( "It's a double episode! The last episode was number %d" % last_episode_number) except (TypeError, IndexError, ValueError): last_episode_number = int(result[0][0]) self.log.info("It's a normal episode") try: last_version_number = int(result[0][2]) self.log.info("It's version %d" % last_version_number) except (TypeError, IndexError, ValueError): last_version_number = 1 if not self.stopping_thread: result = False try: result = self.network.download_torrent( anime_dictionary[key]["link"], "%s %d" % (anime.name, anime.episode), constant.DEFAULT_TORRENTS_PATH, anime.download_folder) except Exception as error: self.showMessage.emit( "Error: %s (%s - %s)" % (type(error).__name__, anime.name, anime.episode)) if result: downloaded_episodes += 1 #self.log.debug("Updating EpisodeNumber") anime.update_episode( last_episode_number + 1) #self.log.debug("Updating LastVersionNumber") anime.update_version( last_version_number) #self.log.debug("Updating LastFileDownloaded") anime.update_last_file_downloaded( anime_dictionary[key]["title"]) self.log.debug("Notifying user") self.update_ui.emit( "%s - %s" % (anime.name, anime.episode - 1)) break except Exception as error: self.log.error("ERROR while analysing '%s'" % escape_unicode(title)) self.log.print_traceback(error, self.log.error) if self.stopping_thread: break return downloaded_episodes def search(self, anime): """ Searches for new episodes of a given anime. :type anime: db.Anime :param anime: Anime to search for new episode. :rtype: dict or None,dict or None,dict or None,dict or None,dict or None :return: results for each site: Anime Index, Anime Tosho, Anirena, Nyaa and Tokyotosho """ text = "%s %02d" % (anime.search_terms, anime.episode) dict_anime_index = None dict_anime_tosho = None dict_anirena = None dict_nyaa = None dict_tokyotosho = None if anime.check_anime_index and not self.stopping_thread: dict_anime_index = self.anime_index.search(text) if anime.check_anime_tosho and not self.stopping_thread: dict_anime_tosho = self.anime_tosho.search(text) if anime.check_anirena and not self.stopping_thread: dict_anirena = self.anirena.search(text) if anime.check_nyaa and not self.stopping_thread: dict_nyaa = self.nyaa.search(text) if anime.check_tokyotosho and not self.stopping_thread: dict_tokyotosho = self.tokyotosho.search(text) return dict_anime_index, dict_anime_tosho, dict_anirena, dict_nyaa, dict_tokyotosho def execute_once_search_anime_index(self, anime): """ Executes search on this site. :type anime: db.Anime :param anime: Contains search terms. """ self.log = LoggerManager().get_logger("Downloader-Once-AI") self.network = Network() anime_index = AnimeIndex(self.network) self.stopping_thread = False self.runningSearch.emit() dict_anime_index = None if anime.check_anime_index and not self.stopping_thread: dict_anime_index = anime_index.search(anime.search_terms) self.searchResult.emit(dict_anime_index) self.finish.emit() def execute_once_search_anime_tosho(self, anime): """ Executes search on this site. :type anime: db.Anime :param anime: Contains search terms. """ self.log = LoggerManager().get_logger("Downloader-Once-AT") self.network = Network() anime_tosho = AnimeTosho(self.network) self.stopping_thread = False self.runningSearch.emit() dict_anime_tosho = None if anime.check_anime_tosho and not self.stopping_thread: dict_anime_tosho = anime_tosho.search(anime.search_terms) self.searchResult.emit(dict_anime_tosho) self.finish.emit() def execute_once_search_anirena(self, anime): """ Executes search on this site. :type anime: db.Anime :param anime: Contains search terms. """ self.log = LoggerManager().get_logger("Downloader-Once-AR") self.network = Network() anirena = Anirena(self.network) self.stopping_thread = False self.runningSearch.emit() dict_anirena = None if anime.check_anirena and not self.stopping_thread: dict_anirena = anirena.search(anime.search_terms) self.searchResult.emit(dict_anirena) self.finish.emit() def execute_once_search_nyaa(self, anime): """ Executes search on this site. :type anime: db.Anime :param anime: Contains search terms. """ self.log = LoggerManager().get_logger("Downloader-Once-NY") self.network = Network() if DBManager().get_config().prefer_rss: nyaa = NyaaRSS(self.network) else: nyaa = Nyaa(self.network) self.stopping_thread = False self.runningSearch.emit() dict_nyaa = None if anime.check_nyaa and not self.stopping_thread: dict_nyaa = nyaa.search(anime.search_terms) self.searchResult.emit(dict_nyaa) self.finish.emit() def execute_once_search_tokyotosho(self, anime): """ Executes search on this site. :type anime: db.Anime :param anime: Contains search terms. """ self.log = LoggerManager().get_logger("Downloader-Once-TT") self.network = Network() tokyotosho = Tokyotosho(self.network) self.stopping_thread = False self.runningSearch.emit() dict_tokyotosho = None if anime.check_tokyotosho and not self.stopping_thread: dict_tokyotosho = tokyotosho.search(anime.search_terms) self.searchResult.emit(dict_tokyotosho) self.finish.emit() def stop_thread(self): """ Stops requests being executed to allow thread to finish gracefully. """ self.log.info("STOPPING DOWNLOADER THREAD") self.stopping_thread = True self.network.stop_thread()
def __init__(self): self.log = LoggerManager().get_logger("Network") self.stopping_thread = False
class Nyaa(): """ Search on Nyaa using HTML. """ def __init__(self, network): self.log = LoggerManager().get_logger("Nyaa") self.network = network def search(self, text="", dic=None, category="1_37"): #"1_37" = English translated anime """ Returned dictionary struct: dict[RESULT_NUMBER]["title"] = str dict[RESULT_NUMBER]["link"] = str dict[RESULT_NUMBER]["date"] = datetime dict[RESULT_NUMBER]["downloads"] = int :type text: str :param text: search_terms :type dic: dict :param dic: where the data will be stored. If None, a new dict is created. :type category: str :param category: site specific filter to search only for anime. """ if dic is None: dic = {} search_terms = text.strip().split(" ") url = self.__get_url(text, category) html = self.network.get_data(url) if "viewtorrentname" in html: return self.__parse_html_1_result(html, dic, search_terms) else: return self.__parse_html(html, dic, search_terms) @staticmethod def __get_url(text, category): url = "" if text!="": text = text.strip().replace(" ","+") url = "https://www.nyaa.se/?page=search&term=%s&cats=%s" % (text,category) return url def __parse_html_1_result(self, html, dic, search_terms): parser = HTMLparser1Result() parser.dict = dic parser.cont = len(dic) parser.search_terms = search_terms parser.debug = self try: parser.feed(html) parser.close() except CleanExit: return parser.dict except Exception as error: self.log.print_traceback(error,self.log.error) return parser.dict def __parse_html(self,html, dic, search_terms): parser = HTMLparser() parser.dict = dic parser.cont = len(dic) parser.search_terms = search_terms try: parser.feed(html) parser.close() except CleanExit: self.log.info("HTML successfully parsed (%i results)" % (len(parser.dict))) return parser.dict except Exception as error: self.log.print_traceback(error,self.log.error) else: self.log.info("HTML successfully parsed (%i results)" % (len(parser.dict))) #return dic return parser.dict
class Network(): """ Controls data requests. """ def __init__(self): self.log = LoggerManager().get_logger("Network") self.stopping_thread = False def stop_thread(self): """ Data requests can't be stopped immediatelly, so this method warns the Network instance to stop as soon as possible. """ self.stopping_thread = True def get_data(self, url, is_binary=False): """ Requests data, be it the html/rss of a site or a torrent file. :type url: str :param url: Link to the data to be downloaded. :type is_binary: bool :param is_binary: If the data is binary (torrent files) or not. :rtype: str :return: the data requested """ if url == "": self.log.error("No link has been received (aka empty URL)!") return None resp = "" if url.find("://") == -1: url = "http://" + url self.log.debug("link: %s" % url) timeout = True tries = 0 while timeout and not self.stopping_thread: try: if url.startswith("https"): response = requests.get(url, timeout=10, verify=constant.CACERT_PATH) else: response = requests.get(url, timeout=10) timeout = False if is_binary: resp = response.content else: resp = response.text #resp = HTMLParser().unescape(response.text) except (ReadTimeout, ConnectTimeout): tries += 1 if tries == 3: self.log.warning( "Retried 3 times... Will try again later.") return "" self.log.debug("Retrying (%i)" % tries) except Exception as error: self.log.print_traceback(error, self.log.error) return resp def download_torrent(self, url, file_name, torrent_path, anime_folder): """ Requests download of the torrent file, saves it, and opens it with torrent application. :type url: str :param url: Link to the torrent file. :type file_name: str :param file_name: The name with which the torrent file will be saved. :type torrent_path: str :param torrent_path: The folder where the torrent file will be saved. :type anime_folder: str :param anime_folder: The folder where the torrent application should save the episode (uTorrent only). :rtype: bool :return: if the torrent was sent to the torrent application or not. """ if os.path.isdir(torrent_path): try: data = self.get_data(url, is_binary=True) if self.__is_torrent(data): self.__save_torrent(data, file_name, torrent_path, anime_folder) return True return False except Exception as error: raise error else: self.log.error( "The .torrent was NOT downloaded (does the specified folder (%s) exists?)" % torrent_path) return False @staticmethod def __is_torrent(file_data): """ Makes sure the file downloaded is indeed a ".torrent". :param file_data: The file. :rtype: bool :return: If it's a torrent file or not. """ # seems like torrent files start with this. So yeah, it's POG, but for now it's working ^^ return str(file_data).find("d8:announce") == 0 def __save_torrent(self, data, file_name, torrent_path, anime_folder=""): """ Saves the torrent file and opens it with the torrent application selected. :param data: The file data. :type file_name: str or unicode :param file_name: The name with which the torrent file will be saved. :type torrent_path: str :param torrent_path: The folder where the torrent file will be saved. :type anime_folder: str :param anime_folder: The folder where the torrent application should save the episode (uTorrent only). """ if os.path.isdir(torrent_path): title = strings.remove_special_chars(file_name) torrent_file_path = "%s\\%s.torrent" % (torrent_path, title) with open(torrent_file_path, "wb") as torrent: torrent.write(data) self.log.info(".torrent saved") try: application_fullpath = torrent_application.fullpath() self.log.debug("Opening '%s' with '%s'" % (torrent_file_path, application_fullpath)) if torrent_application.is_utorrent() and os.path.isdir( anime_folder): self.log.debug("Using uTorrent, save in folder '%s'" % strings.escape_unicode(anime_folder)) params = ['/DIRECTORY', anime_folder, torrent_file_path] else: params = [torrent_file_path] # http://stackoverflow.com/questions/1910275/unicode-filenames-on-windows-with-python-subprocess-popen # TLDR: Python 2.X's subprocess.Popen doesn't work well with unicode QtCore.QProcess().startDetached(application_fullpath, params) self.log.info(".torrent opened") except Exception as error: self.log.error("Error opening torrent application") self.log.print_traceback(error, self.log.error) raise error else: self.log.error( "The .torrent was NOT saved. Apparently the specified folder (%s) does NOT exist." % torrent_path)
def __init__(self, network): self.log = LoggerManager().get_logger("AnimeTosho") self.network = network
def __init__(self): QtCore.QObject.__init__(self) # Make sure the required folders/files exist if not os.path.isdir(constant.DATA_PATH): os.makedirs(constant.DATA_PATH) self.log = LoggerManager().get_logger("MAIN") try: if not os.path.isfile(constant.DB_PATH): shutil.copyfile("dbTemplate.db", constant.DB_PATH) except (shutil.Error, IOError) as error: self.log.print_traceback(error, self.log.critical) sys.exit(1) try: if not os.path.isdir(constant.DEFAULT_TORRENTS_PATH): os.makedirs(constant.DEFAULT_TORRENTS_PATH) except Exception as error: self.log.print_traceback(error, self.log.critical) sys.exit(1) self.app = QtSingleApplication(constant.GUID, sys.argv) self.log.info("---STARTING APPLICATION---") if self.app.isRunning(): self.log.warning( "---The launch of another instance of this application will be cancelled---" ) self.app.sendMessage() sys.exit(0) self.app.messageReceived.connect(self.another_instance_opened) self.app.setQuitOnLastWindowClosed(False) self.window = None self.tray_icon = None self.thread = None self.downloader = None self.timer = None self.downloader_is_running = False self.downloader_is_restarting = False self.downloader_is_stopping = False try: self.window = WindowMain(self) self.tray_icon = SystemTrayIcon(self, self.window) self.tray_icon.show() show_gui = "-nogui" not in sys.argv if show_gui: if self.downloader_is_running: self.window.downloader_started() else: self.window.downloader_stopped() self.window.show() elif not self.window.is_visible(): self.log.info("STARTING DOWNLOADER") self.start_downloader() self.app.exec_() except Exception as unforeseenError: self.log.critical("UNFORESEEN ERROR") self.log.print_traceback(unforeseenError, self.log.critical) if self.tray_icon is not None: self.show_tray_message("Unforeseen error occurred...") exit()
def __init__(self,network): self.log = LoggerManager().get_logger("Tokyotosho") self.network = network