class AnimeIndex():
"""
Search on Anime Index using HTML.
"""
def __init__(self, network):
self.log = LoggerManager().get_logger("AnimeIndex")
self.network = network
def search(self, text="", dic=None, category=5, active=1):
"""
Returned dictionary struct:
dict[RESULT_NUMBER]["title"] = str
dict[RESULT_NUMBER]["link"] = str
dict[RESULT_NUMBER]["date"] = datetime
dict[RESULT_NUMBER]["downloads"] = int
:type text: str
:param text: search_terms
:type dic: dict
:param dic: where the data will be stored. If None, a new dict is created.
:type category: int
:param category: site specific filter to search only for anime.
:type active: int
:param active: site specific filter to search only for active downloads.
"""
if dic is None:
dic = {}
url = self.__get_url(text, category, active)
html = self.network.get_data(url)
return self.__parse_html(html, dic, text)
@staticmethod
def __get_url(text, category, active):
url = ""
if text != "":
positive_words = [
word for word in text.split(" ") if not word.startswith("-")
]
positive_text = "+".join(positive_words)
url = "tracker.anime-index.org/index.php?page=torrents&search=%s&category=%s&active=%s" % (
positive_text, str(category), str(active))
return url
def __parse_html(self, html, dic, text):
parser = HTMLparser()
parser.dict = dic
parser.cont = len(dic)
parser.search_terms = text.strip().split(" ")
try:
parser.feed(html)
parser.close()
except CleanExit:
return parser.dict
except Exception as error:
self.log.print_traceback(error, self.log.error)
return parser.dict
class AnimeIndex():
"""
Search on Anime Index using HTML.
"""
def __init__(self,network):
self.log = LoggerManager().get_logger("AnimeIndex")
self.network = network
def search(self,text="",dic=None,category=5,active=1):
"""
Returned dictionary struct:
dict[RESULT_NUMBER]["title"] = str
dict[RESULT_NUMBER]["link"] = str
dict[RESULT_NUMBER]["date"] = datetime
dict[RESULT_NUMBER]["downloads"] = int
:type text: str
:param text: search_terms
:type dic: dict
:param dic: where the data will be stored. If None, a new dict is created.
:type category: int
:param category: site specific filter to search only for anime.
:type active: int
:param active: site specific filter to search only for active downloads.
"""
if dic is None:
dic = {}
url = self.__get_url(text,category,active)
html = self.network.get_data(url)
return self.__parse_html(html, dic, text)
@staticmethod
def __get_url(text,category,active):
url = ""
if text!="":
positive_words = [word for word in text.split(" ") if not word.startswith("-")]
positive_text = "+".join(positive_words)
url = "tracker.anime-index.org/index.php?page=torrents&search=%s&category=%s&active=%s" % (positive_text,str(category),str(active))
return url
def __parse_html(self,html, dic, text):
parser = HTMLparser()
parser.dict = dic
parser.cont = len(dic)
parser.search_terms = text.strip().split(" ")
try:
parser.feed(html)
parser.close()
except CleanExit:
return parser.dict
except Exception as error:
self.log.print_traceback(error,self.log.error)
return parser.dict
class Tokyotosho():
"""
Search on Tokyotosho using HTML.
"""
def __init__(self, network):
self.log = LoggerManager().get_logger("Tokyotosho")
self.network = network
def search(self, text="", dic=None, category=1):
"""
Returned dictionary struct:
dict[RESULT_NUMBER]["title"] = str
dict[RESULT_NUMBER]["link"] = str
dict[RESULT_NUMBER]["date"] = datetime
:type text: str
:param text: search_terms
:type dic: dict
:param dic: where the data will be stored. If None, a new dict is created.
:type category: str
:param category: site specific filter to search only for anime.
"""
if dic is None:
dic = {}
url = self.__get_url(text, category)
html = self.network.get_data(url)
html = html.replace("<span class=\"s\"> </span>", "")
return self.__parse_html(html, dic, text)
@staticmethod
def __get_url(text, category):
url = ""
if text != "":
text = text.strip().replace(" ", "+")
url = "www.tokyotosho.info/search.php?terms=%s&type=%s" % (
text, str(category))
return url
def __parse_html(self, html, dic, text):
parser = HTMLparser()
parser.dict = dic
parser.cont = len(dic)
parser.search_terms = text.strip().split(" ")
try:
parser.feed(html)
parser.close()
except CleanExit:
return parser.dict
except Exception as error:
self.log.print_traceback(error, self.log.error)
return parser.dict
class Tokyotosho():
"""
Search on Tokyotosho using HTML.
"""
def __init__(self,network):
self.log = LoggerManager().get_logger("Tokyotosho")
self.network = network
def search(self,text="",dic=None,category=1):
"""
Returned dictionary struct:
dict[RESULT_NUMBER]["title"] = str
dict[RESULT_NUMBER]["link"] = str
dict[RESULT_NUMBER]["date"] = datetime
:type text: str
:param text: search_terms
:type dic: dict
:param dic: where the data will be stored. If None, a new dict is created.
:type category: str
:param category: site specific filter to search only for anime.
"""
if dic is None:
dic = {}
url = self.__get_url(text,category)
html = self.network.get_data(url)
html = html.replace("<span class=\"s\"> </span>","")
return self.__parse_html(html, dic, text)
@staticmethod
def __get_url(text,category):
url = ""
if text!="":
text = text.strip().replace(" ","+")
url = "www.tokyotosho.info/search.php?terms=%s&type=%s" % (text,str(category))
return url
def __parse_html(self,html,dic,text):
parser = HTMLparser()
parser.dict = dic
parser.cont = len(dic)
parser.search_terms = text.strip().split(" ")
try:
parser.feed(html)
parser.close()
except CleanExit:
return parser.dict
except Exception as error:
self.log.print_traceback(error,self.log.error)
return parser.dict
class AnimeTosho():
"""
Search on Anime Tosho using HTML.
"""
def __init__(self, network):
self.log = LoggerManager().get_logger("AnimeTosho")
self.network = network
def search(self, text="", dic=None):
"""
Returned dictionary struct:
dict[RESULT_NUMBER]["title"] = str
dict[RESULT_NUMBER]["link"] = str
dict[RESULT_NUMBER]["date"] = datetime
:type text: str
:param text: search_terms
:type dic: dict
:param dic: where the data will be stored. If None, a new dict is created.
"""
if dic is None:
dic = {}
url = self.__get_url(text)
html = self.network.get_data(url)
return self.__parse_html(html, dic, text)
@staticmethod
def __get_url(text):
url = ""
if text != "":
positive_words = [
word for word in text.split(" ") if not word.startswith("-")
]
positive_text = "+".join(positive_words)
url = "https://animetosho.org/search?q=%s" % positive_text
return url
def __parse_html(self, html, dic, text):
parser = HTMLparser()
parser.dict = dic
parser.cont = len(dic)
parser.search_terms = text.strip().split(" ")
try:
parser.feed(html)
parser.close()
except CleanExit:
return parser.dict
except Exception as error:
self.log.print_traceback(error, self.log.error)
return parser.dict
class Downloader(QtCore.QObject):
"""
Processes the search requests from the [Add anime window].
Also responsible for actually searching for new episodes automatically according to the frequency chosen by the user.
"""
running = QtCore.pyqtSignal()
finish = QtCore.pyqtSignal()
restart = QtCore.pyqtSignal()
showMessage = QtCore.pyqtSignal(str)
update_ui = QtCore.pyqtSignal(str)
runningSearch = QtCore.pyqtSignal()
searchResult = QtCore.pyqtSignal(object) #Might be dict or None
def __init__(self, parent=None):
super(Downloader, self).__init__(parent)
self.log = None
self.dbManager = None
self.animeList = None
self.config = None
self.network = None
self.anime_index = None
self.anime_tosho = None
self.anirena = None
self.nyaa = None
self.tokyotosho = None
self.stopping_thread = None
self.timer = None
@QtCore.pyqtSlot()
def execute_once(self):
"""
Initializes variables necessary to search for new episodes.
"""
self.log = LoggerManager().get_logger("Downloader")
self.log.debug("#####################")
self.dbManager = DBManager()
self.animeList = self.dbManager.get_anime_list()
self.config = self.dbManager.get_config()
self.network = Network()
self.anime_index = AnimeIndex(self.network)
self.anime_tosho = AnimeTosho(self.network)
self.anirena = Anirena(self.network)
if self.config.prefer_rss:
self.nyaa = NyaaRSS(self.network)
else:
self.nyaa = Nyaa(self.network)
self.tokyotosho = Tokyotosho(self.network)
self.stopping_thread = False
self.timer = QtCore.QTimer()
self.running.emit()
self.log.debug("****************************")
number_of_downloaded_episodes = self.__search_new_episodes()
msg = "No" if number_of_downloaded_episodes==0 else str(number_of_downloaded_episodes)
if not self.stopping_thread:
self.log.info("%s episodes downloaded, sleeping for %s seconds" % (msg, self.config.sleep_time))
self.restart.emit()
else:
self.log.info("%s episodes downloaded, stopping downloader" % msg)
self.finish.emit()
def __search_new_episodes(self):
"""
Searches for new episodes for all enabled anime.
"""
downloaded_episodes = 0
current_anime = 0
for anime in self.animeList:
if anime.enabled and not self.stopping_thread:
current_anime += 1
self.log.info("(%d/%d) searching for episode %s of '%s'" %
(current_anime, self.dbManager.number_of_anime_enabled, anime.episode, escape_unicode(anime.name)))
dict_anime_index,dict_anime_tosho,dict_anirena,dict_nyaa,dict_tokyotosho = self.search(anime)
anime_dictionary = {}
if dict_anime_index is not None:
anime_dictionary = dict(anime_dictionary.items()+dict_anime_index.items())
if dict_anime_tosho is not None:
anime_dictionary = dict(anime_dictionary.items()+dict_anime_tosho.items())
if dict_anirena is not None:
anime_dictionary = dict(anime_dictionary.items()+dict_anirena.items())
if dict_nyaa is not None:
anime_dictionary = dict(anime_dictionary.items()+dict_nyaa.items())
if dict_tokyotosho is not None:
anime_dictionary = dict(anime_dictionary.items()+dict_tokyotosho.items())
title = "ERROR(No value)"
if not self.stopping_thread:
try:
for key in anime_dictionary.keys():
self.log.debug("'%s' is a match, checking episode number" % (escape_unicode(anime_dictionary[key]["title"])))
title = anime_dictionary[key]["title"]
if (".mkv" or ".mp4") in title:
self.log.debug("title = "+title)
regex_episode_and_version_number = re.compile("\\b[\s_~\-+.]?(%02d)[\s_~\-+.]*(\d*)[\s_]*v?(\d)?[\s\w.]*(\[|\(|.mkv|.mp4)" % anime.episode)
else:
regex_episode_and_version_number = re.compile("\\b[\s_~\-+.]?(%02d)[\s_~\-+.]*(\d*)[\s_]*v?(\d)?" % anime.episode)
result = regex_episode_and_version_number.findall(title)
self.log.debug("regex result = "+str(result))
#self.log.debug("REGEX result = '%s' (len(result)>0: %s)" % (result, len(result)>0))
if len(result)>0:
self.log.info("A torrent has been found")
try:
last_episode_number = int(result[0][1])
self.log.info("It's a double episode! The last episode was number %d" % last_episode_number)
except (TypeError,IndexError,ValueError):
last_episode_number = int(result[0][0])
self.log.info("It's a normal episode")
try:
last_version_number = int(result[0][2])
self.log.info("It's version %d" % last_version_number)
except (TypeError,IndexError,ValueError):
last_version_number = 1
if not self.stopping_thread:
result = False
try:
result = self.network.download_torrent(anime_dictionary[key]["link"],
"%s %d" % (anime.name,anime.episode),
constant.DEFAULT_TORRENTS_PATH,
anime.download_folder)
except Exception as error:
self.showMessage.emit("Error: %s (%s - %s)" % (type(error).__name__,anime.name,anime.episode))
if result:
downloaded_episodes+=1
#self.log.debug("Updating EpisodeNumber")
anime.update_episode(last_episode_number+1)
#self.log.debug("Updating LastVersionNumber")
anime.update_version(last_version_number)
#self.log.debug("Updating LastFileDownloaded")
anime.update_last_file_downloaded(anime_dictionary[key]["title"])
self.log.debug("Notifying user")
self.update_ui.emit("%s - %s" % (anime.name, anime.episode-1))
break
except Exception as error:
self.log.error("ERROR while analysing '%s'" % escape_unicode(title))
self.log.print_traceback(error,self.log.error)
if self.stopping_thread:
break
return downloaded_episodes
def search(self,anime):
"""
Searches for new episodes of a given anime.
:type anime: db.Anime
:param anime: Anime to search for new episode.
:rtype: dict or None,dict or None,dict or None,dict or None,dict or None
:return: results for each site: Anime Index, Anime Tosho, Anirena, Nyaa and Tokyotosho
"""
text = "%s %02d" % (anime.search_terms,anime.episode)
dict_anime_index = None
dict_anime_tosho = None
dict_anirena = None
dict_nyaa = None
dict_tokyotosho = None
if anime.check_anime_index and not self.stopping_thread:
dict_anime_index = self.anime_index.search(text)
if anime.check_anime_tosho and not self.stopping_thread:
dict_anime_tosho = self.anime_tosho.search(text)
if anime.check_anirena and not self.stopping_thread:
dict_anirena = self.anirena.search(text)
if anime.check_nyaa and not self.stopping_thread:
dict_nyaa = self.nyaa.search(text)
if anime.check_tokyotosho and not self.stopping_thread:
dict_tokyotosho = self.tokyotosho.search(text)
return dict_anime_index,dict_anime_tosho,dict_anirena,dict_nyaa,dict_tokyotosho
def execute_once_search_anime_index(self,anime):
"""
Executes search on this site.
:type anime: db.Anime
:param anime: Contains search terms.
"""
self.log = LoggerManager().get_logger("Downloader-Once-AI")
self.network = Network()
anime_index = AnimeIndex(self.network)
self.stopping_thread = False
self.runningSearch.emit()
dict_anime_index = None
if anime.check_anime_index and not self.stopping_thread:
dict_anime_index = anime_index.search(anime.search_terms)
self.searchResult.emit(dict_anime_index)
self.finish.emit()
def execute_once_search_anime_tosho(self,anime):
"""
Executes search on this site.
:type anime: db.Anime
:param anime: Contains search terms.
"""
self.log = LoggerManager().get_logger("Downloader-Once-AT")
self.network = Network()
anime_tosho = AnimeTosho(self.network)
self.stopping_thread = False
self.runningSearch.emit()
dict_anime_tosho = None
if anime.check_anime_tosho and not self.stopping_thread:
dict_anime_tosho = anime_tosho.search(anime.search_terms)
self.searchResult.emit(dict_anime_tosho)
self.finish.emit()
def execute_once_search_anirena(self,anime):
"""
Executes search on this site.
:type anime: db.Anime
:param anime: Contains search terms.
"""
self.log = LoggerManager().get_logger("Downloader-Once-AR")
self.network = Network()
anirena = Anirena(self.network)
self.stopping_thread = False
self.runningSearch.emit()
dict_anirena = None
if anime.check_anirena and not self.stopping_thread:
dict_anirena = anirena.search(anime.search_terms)
self.searchResult.emit(dict_anirena)
self.finish.emit()
def execute_once_search_nyaa(self,anime):
"""
Executes search on this site.
:type anime: db.Anime
:param anime: Contains search terms.
"""
self.log = LoggerManager().get_logger("Downloader-Once-NY")
self.network = Network()
if DBManager().get_config().prefer_rss:
nyaa = NyaaRSS(self.network)
else:
nyaa = Nyaa(self.network)
self.stopping_thread = False
self.runningSearch.emit()
dict_nyaa = None
if anime.check_nyaa and not self.stopping_thread:
dict_nyaa = nyaa.search(anime.search_terms)
self.searchResult.emit(dict_nyaa)
self.finish.emit()
def execute_once_search_tokyotosho(self,anime):
"""
Executes search on this site.
:type anime: db.Anime
:param anime: Contains search terms.
"""
self.log = LoggerManager().get_logger("Downloader-Once-TT")
self.network = Network()
tokyotosho = Tokyotosho(self.network)
self.stopping_thread = False
self.runningSearch.emit()
dict_tokyotosho = None
if anime.check_tokyotosho and not self.stopping_thread:
dict_tokyotosho = tokyotosho.search(anime.search_terms)
self.searchResult.emit(dict_tokyotosho)
self.finish.emit()
def stop_thread(self):
"""
Stops requests being executed to allow thread to finish gracefully.
"""
self.log.info("STOPPING DOWNLOADER THREAD")
self.stopping_thread = True
self.network.stop_thread()
class Network():
"""
Controls data requests.
"""
def __init__(self):
self.log = LoggerManager().get_logger("Network")
self.stopping_thread = False
def stop_thread(self):
"""
Data requests can't be stopped immediatelly, so this method warns the Network instance to stop as soon as possible.
"""
self.stopping_thread = True
def get_data(self, url, is_binary=False):
"""
Requests data, be it the html/rss of a site or a torrent file.
:type url: str
:param url: Link to the data to be downloaded.
:type is_binary: bool
:param is_binary: If the data is binary (torrent files) or not.
:rtype: str
:return: the data requested
"""
if url == "":
self.log.error("No link has been received (aka empty URL)!")
return None
resp = ""
if url.find("://") == -1:
url = "http://" + url
self.log.debug("link: %s" % url)
timeout = True
tries = 0
while timeout and not self.stopping_thread:
try:
if url.startswith("https"):
response = requests.get(url,
timeout=10,
verify=constant.CACERT_PATH)
else:
response = requests.get(url, timeout=10)
timeout = False
if is_binary:
resp = response.content
else:
resp = response.text
#resp = HTMLParser().unescape(response.text)
except (ReadTimeout, ConnectTimeout):
tries += 1
if tries == 3:
self.log.warning(
"Retried 3 times... Will try again later.")
return ""
self.log.debug("Retrying (%i)" % tries)
except Exception as error:
self.log.print_traceback(error, self.log.error)
return resp
def download_torrent(self, url, file_name, torrent_path, anime_folder):
"""
Requests download of the torrent file, saves it, and opens it with torrent application.
:type url: str
:param url: Link to the torrent file.
:type file_name: str
:param file_name: The name with which the torrent file will be saved.
:type torrent_path: str
:param torrent_path: The folder where the torrent file will be saved.
:type anime_folder: str
:param anime_folder: The folder where the torrent application should save the episode (uTorrent only).
:rtype: bool
:return: if the torrent was sent to the torrent application or not.
"""
if os.path.isdir(torrent_path):
try:
data = self.get_data(url, is_binary=True)
if self.__is_torrent(data):
self.__save_torrent(data, file_name, torrent_path,
anime_folder)
return True
return False
except Exception as error:
raise error
else:
self.log.error(
"The .torrent was NOT downloaded (does the specified folder (%s) exists?)"
% torrent_path)
return False
@staticmethod
def __is_torrent(file_data):
"""
Makes sure the file downloaded is indeed a ".torrent".
:param file_data: The file.
:rtype: bool
:return: If it's a torrent file or not.
"""
# seems like torrent files start with this. So yeah, it's POG, but for now it's working ^^
return str(file_data).find("d8:announce") == 0
def __save_torrent(self, data, file_name, torrent_path, anime_folder=""):
"""
Saves the torrent file and opens it with the torrent application selected.
:param data: The file data.
:type file_name: str or unicode
:param file_name: The name with which the torrent file will be saved.
:type torrent_path: str
:param torrent_path: The folder where the torrent file will be saved.
:type anime_folder: str
:param anime_folder: The folder where the torrent application should save the episode (uTorrent only).
"""
if os.path.isdir(torrent_path):
title = strings.remove_special_chars(file_name)
torrent_file_path = "%s\\%s.torrent" % (torrent_path, title)
with open(torrent_file_path, "wb") as torrent:
torrent.write(data)
self.log.info(".torrent saved")
try:
application_fullpath = torrent_application.fullpath()
self.log.debug("Opening '%s' with '%s'" %
(torrent_file_path, application_fullpath))
if torrent_application.is_utorrent() and os.path.isdir(
anime_folder):
self.log.debug("Using uTorrent, save in folder '%s'" %
strings.escape_unicode(anime_folder))
params = ['/DIRECTORY', anime_folder, torrent_file_path]
else:
params = [torrent_file_path]
# http://stackoverflow.com/questions/1910275/unicode-filenames-on-windows-with-python-subprocess-popen
# TLDR: Python 2.X's subprocess.Popen doesn't work well with unicode
QtCore.QProcess().startDetached(application_fullpath, params)
self.log.info(".torrent opened")
except Exception as error:
self.log.error("Error opening torrent application")
self.log.print_traceback(error, self.log.error)
raise error
else:
self.log.error(
"The .torrent was NOT saved. Apparently the specified folder (%s) does NOT exist."
% torrent_path)
class Downloader(QtCore.QObject):
"""
Processes the search requests from the [Add anime window].
Also responsible for actually searching for new episodes automatically according to the frequency chosen by the user.
"""
running = QtCore.pyqtSignal()
finish = QtCore.pyqtSignal()
restart = QtCore.pyqtSignal()
showMessage = QtCore.pyqtSignal(str)
update_ui = QtCore.pyqtSignal(str)
runningSearch = QtCore.pyqtSignal()
searchResult = QtCore.pyqtSignal(object) #Might be dict or None
def __init__(self, parent=None):
super(Downloader, self).__init__(parent)
self.log = None
self.dbManager = None
self.animeList = None
self.config = None
self.network = None
self.anime_index = None
self.anime_tosho = None
self.anirena = None
self.nyaa = None
self.tokyotosho = None
self.stopping_thread = None
self.timer = None
@QtCore.pyqtSlot()
def execute_once(self):
"""
Initializes variables necessary to search for new episodes.
"""
self.log = LoggerManager().get_logger("Downloader")
self.log.debug("#####################")
self.dbManager = DBManager()
self.animeList = self.dbManager.get_anime_list()
self.config = self.dbManager.get_config()
self.network = Network()
self.anime_index = AnimeIndex(self.network)
self.anime_tosho = AnimeTosho(self.network)
self.anirena = Anirena(self.network)
if self.config.prefer_rss:
self.nyaa = NyaaRSS(self.network)
else:
self.nyaa = Nyaa(self.network)
self.tokyotosho = Tokyotosho(self.network)
self.stopping_thread = False
self.timer = QtCore.QTimer()
self.running.emit()
self.log.debug("****************************")
number_of_downloaded_episodes = self.__search_new_episodes()
msg = "No" if number_of_downloaded_episodes == 0 else str(
number_of_downloaded_episodes)
if not self.stopping_thread:
self.log.info("%s episodes downloaded, sleeping for %s seconds" %
(msg, self.config.sleep_time))
self.restart.emit()
else:
self.log.info("%s episodes downloaded, stopping downloader" % msg)
self.finish.emit()
def __search_new_episodes(self):
"""
Searches for new episodes for all enabled anime.
"""
downloaded_episodes = 0
current_anime = 0
for anime in self.animeList:
if anime.enabled and not self.stopping_thread:
current_anime += 1
self.log.info(
"(%d/%d) searching for episode %s of '%s'" %
(current_anime, self.dbManager.number_of_anime_enabled,
anime.episode, escape_unicode(anime.name)))
dict_anime_index, dict_anime_tosho, dict_anirena, dict_nyaa, dict_tokyotosho = self.search(
anime)
anime_dictionary = {}
if dict_anime_index is not None:
anime_dictionary = dict(anime_dictionary.items() +
dict_anime_index.items())
if dict_anime_tosho is not None:
anime_dictionary = dict(anime_dictionary.items() +
dict_anime_tosho.items())
if dict_anirena is not None:
anime_dictionary = dict(anime_dictionary.items() +
dict_anirena.items())
if dict_nyaa is not None:
anime_dictionary = dict(anime_dictionary.items() +
dict_nyaa.items())
if dict_tokyotosho is not None:
anime_dictionary = dict(anime_dictionary.items() +
dict_tokyotosho.items())
title = "ERROR(No value)"
if not self.stopping_thread:
try:
for key in anime_dictionary.keys():
self.log.debug(
"'%s' is a match, checking episode number" %
(escape_unicode(
anime_dictionary[key]["title"])))
title = anime_dictionary[key]["title"]
if (".mkv" or ".mp4") in title:
self.log.debug("title = " + title)
regex_episode_and_version_number = re.compile(
"\\b[\s_~\-+.]?(%02d)[\s_~\-+.]*(\d*)[\s_]*v?(\d)?[\s\w.]*(\[|\(|.mkv|.mp4)"
% anime.episode)
else:
regex_episode_and_version_number = re.compile(
"\\b[\s_~\-+.]?(%02d)[\s_~\-+.]*(\d*)[\s_]*v?(\d)?"
% anime.episode)
result = regex_episode_and_version_number.findall(
title)
self.log.debug("regex result = " + str(result))
#self.log.debug("REGEX result = '%s' (len(result)>0: %s)" % (result, len(result)>0))
if len(result) > 0:
self.log.info("A torrent has been found")
try:
last_episode_number = int(result[0][1])
self.log.info(
"It's a double episode! The last episode was number %d"
% last_episode_number)
except (TypeError, IndexError, ValueError):
last_episode_number = int(result[0][0])
self.log.info("It's a normal episode")
try:
last_version_number = int(result[0][2])
self.log.info("It's version %d" %
last_version_number)
except (TypeError, IndexError, ValueError):
last_version_number = 1
if not self.stopping_thread:
result = False
try:
result = self.network.download_torrent(
anime_dictionary[key]["link"],
"%s %d" %
(anime.name, anime.episode),
constant.DEFAULT_TORRENTS_PATH,
anime.download_folder)
except Exception as error:
self.showMessage.emit(
"Error: %s (%s - %s)" %
(type(error).__name__, anime.name,
anime.episode))
if result:
downloaded_episodes += 1
#self.log.debug("Updating EpisodeNumber")
anime.update_episode(
last_episode_number + 1)
#self.log.debug("Updating LastVersionNumber")
anime.update_version(
last_version_number)
#self.log.debug("Updating LastFileDownloaded")
anime.update_last_file_downloaded(
anime_dictionary[key]["title"])
self.log.debug("Notifying user")
self.update_ui.emit(
"%s - %s" %
(anime.name, anime.episode - 1))
break
except Exception as error:
self.log.error("ERROR while analysing '%s'" %
escape_unicode(title))
self.log.print_traceback(error, self.log.error)
if self.stopping_thread:
break
return downloaded_episodes
def search(self, anime):
"""
Searches for new episodes of a given anime.
:type anime: db.Anime
:param anime: Anime to search for new episode.
:rtype: dict or None,dict or None,dict or None,dict or None,dict or None
:return: results for each site: Anime Index, Anime Tosho, Anirena, Nyaa and Tokyotosho
"""
text = "%s %02d" % (anime.search_terms, anime.episode)
dict_anime_index = None
dict_anime_tosho = None
dict_anirena = None
dict_nyaa = None
dict_tokyotosho = None
if anime.check_anime_index and not self.stopping_thread:
dict_anime_index = self.anime_index.search(text)
if anime.check_anime_tosho and not self.stopping_thread:
dict_anime_tosho = self.anime_tosho.search(text)
if anime.check_anirena and not self.stopping_thread:
dict_anirena = self.anirena.search(text)
if anime.check_nyaa and not self.stopping_thread:
dict_nyaa = self.nyaa.search(text)
if anime.check_tokyotosho and not self.stopping_thread:
dict_tokyotosho = self.tokyotosho.search(text)
return dict_anime_index, dict_anime_tosho, dict_anirena, dict_nyaa, dict_tokyotosho
def execute_once_search_anime_index(self, anime):
"""
Executes search on this site.
:type anime: db.Anime
:param anime: Contains search terms.
"""
self.log = LoggerManager().get_logger("Downloader-Once-AI")
self.network = Network()
anime_index = AnimeIndex(self.network)
self.stopping_thread = False
self.runningSearch.emit()
dict_anime_index = None
if anime.check_anime_index and not self.stopping_thread:
dict_anime_index = anime_index.search(anime.search_terms)
self.searchResult.emit(dict_anime_index)
self.finish.emit()
def execute_once_search_anime_tosho(self, anime):
"""
Executes search on this site.
:type anime: db.Anime
:param anime: Contains search terms.
"""
self.log = LoggerManager().get_logger("Downloader-Once-AT")
self.network = Network()
anime_tosho = AnimeTosho(self.network)
self.stopping_thread = False
self.runningSearch.emit()
dict_anime_tosho = None
if anime.check_anime_tosho and not self.stopping_thread:
dict_anime_tosho = anime_tosho.search(anime.search_terms)
self.searchResult.emit(dict_anime_tosho)
self.finish.emit()
def execute_once_search_anirena(self, anime):
"""
Executes search on this site.
:type anime: db.Anime
:param anime: Contains search terms.
"""
self.log = LoggerManager().get_logger("Downloader-Once-AR")
self.network = Network()
anirena = Anirena(self.network)
self.stopping_thread = False
self.runningSearch.emit()
dict_anirena = None
if anime.check_anirena and not self.stopping_thread:
dict_anirena = anirena.search(anime.search_terms)
self.searchResult.emit(dict_anirena)
self.finish.emit()
def execute_once_search_nyaa(self, anime):
"""
Executes search on this site.
:type anime: db.Anime
:param anime: Contains search terms.
"""
self.log = LoggerManager().get_logger("Downloader-Once-NY")
self.network = Network()
if DBManager().get_config().prefer_rss:
nyaa = NyaaRSS(self.network)
else:
nyaa = Nyaa(self.network)
self.stopping_thread = False
self.runningSearch.emit()
dict_nyaa = None
if anime.check_nyaa and not self.stopping_thread:
dict_nyaa = nyaa.search(anime.search_terms)
self.searchResult.emit(dict_nyaa)
self.finish.emit()
def execute_once_search_tokyotosho(self, anime):
"""
Executes search on this site.
:type anime: db.Anime
:param anime: Contains search terms.
"""
self.log = LoggerManager().get_logger("Downloader-Once-TT")
self.network = Network()
tokyotosho = Tokyotosho(self.network)
self.stopping_thread = False
self.runningSearch.emit()
dict_tokyotosho = None
if anime.check_tokyotosho and not self.stopping_thread:
dict_tokyotosho = tokyotosho.search(anime.search_terms)
self.searchResult.emit(dict_tokyotosho)
self.finish.emit()
def stop_thread(self):
"""
Stops requests being executed to allow thread to finish gracefully.
"""
self.log.info("STOPPING DOWNLOADER THREAD")
self.stopping_thread = True
self.network.stop_thread()
class Nyaa():
"""
Search on Nyaa using HTML.
"""
def __init__(self, network):
self.log = LoggerManager().get_logger("Nyaa")
self.network = network
def search(self,
text="",
dic=None,
category="1_37"): #"1_37" = English translated anime
"""
Returned dictionary struct:
dict[RESULT_NUMBER]["title"] = str
dict[RESULT_NUMBER]["link"] = str
dict[RESULT_NUMBER]["date"] = datetime
dict[RESULT_NUMBER]["downloads"] = int
:type text: str
:param text: search_terms
:type dic: dict
:param dic: where the data will be stored. If None, a new dict is created.
:type category: str
:param category: site specific filter to search only for anime.
"""
if dic is None:
dic = {}
search_terms = text.strip().split(" ")
url = self.__get_url(text, category)
html = self.network.get_data(url)
if "viewtorrentname" in html:
return self.__parse_html_1_result(html, dic, search_terms)
else:
return self.__parse_html(html, dic, search_terms)
@staticmethod
def __get_url(text, category):
url = ""
if text != "":
text = text.strip().replace(" ", "+")
url = "https://www.nyaa.se/?page=search&term=%s&cats=%s" % (
text, category)
return url
def __parse_html_1_result(self, html, dic, search_terms):
parser = HTMLparser1Result()
parser.dict = dic
parser.cont = len(dic)
parser.search_terms = search_terms
parser.debug = self
try:
parser.feed(html)
parser.close()
except CleanExit:
return parser.dict
except Exception as error:
self.log.print_traceback(error, self.log.error)
return parser.dict
def __parse_html(self, html, dic, search_terms):
parser = HTMLparser()
parser.dict = dic
parser.cont = len(dic)
parser.search_terms = search_terms
try:
parser.feed(html)
parser.close()
except CleanExit:
self.log.info("HTML successfully parsed (%i results)" %
(len(parser.dict)))
return parser.dict
except Exception as error:
self.log.print_traceback(error, self.log.error)
else:
self.log.info("HTML successfully parsed (%i results)" %
(len(parser.dict)))
#return dic
return parser.dict
class Main(QtCore.QObject):
"""
Main class, instantiated when the application starts.
Creates main window/system tray icon, and stops the user from opnening more than one instance of the application.
It also starts/stops the downloader when the user requests or during auto-start on Windows startup.
"""
def __init__(self):
QtCore.QObject.__init__(self)
# Make sure the required folders/files exist
if not os.path.isdir(constant.DATA_PATH):
os.makedirs(constant.DATA_PATH)
self.log = LoggerManager().get_logger("MAIN")
try:
if not os.path.isfile(constant.DB_PATH):
shutil.copyfile("dbTemplate.db", constant.DB_PATH)
except (shutil.Error, IOError) as error:
self.log.print_traceback(error, self.log.critical)
sys.exit(1)
try:
if not os.path.isdir(constant.DEFAULT_TORRENTS_PATH):
os.makedirs(constant.DEFAULT_TORRENTS_PATH)
except Exception as error:
self.log.print_traceback(error, self.log.critical)
sys.exit(1)
self.app = QtSingleApplication(constant.GUID, sys.argv)
self.log.info("---STARTING APPLICATION---")
if self.app.isRunning():
self.log.warning(
"---The launch of another instance of this application will be cancelled---"
)
self.app.sendMessage()
sys.exit(0)
self.app.messageReceived.connect(self.another_instance_opened)
self.app.setQuitOnLastWindowClosed(False)
self.window = None
self.tray_icon = None
self.thread = None
self.downloader = None
self.timer = None
self.downloader_is_running = False
self.downloader_is_restarting = False
self.downloader_is_stopping = False
try:
self.window = WindowMain(self)
self.tray_icon = SystemTrayIcon(self, self.window)
self.tray_icon.show()
show_gui = "-nogui" not in sys.argv
if show_gui:
if self.downloader_is_running:
self.window.downloader_started()
else:
self.window.downloader_stopped()
self.window.show()
elif not self.window.is_visible():
self.log.info("STARTING DOWNLOADER")
self.start_downloader()
self.app.exec_()
except Exception as unforeseenError:
self.log.critical("UNFORESEEN ERROR")
self.log.print_traceback(unforeseenError, self.log.critical)
if self.tray_icon is not None:
self.show_tray_message("Unforeseen error occurred...")
exit()
def quit(self):
"""
Finishes the application gracefully - at least tries to, teehee (^_^;)
"""
if self.tray_icon is not None:
self.tray_icon.hide()
self.tray_icon.deleteLater()
if self.timer is not None:
self.timer.stop()
if self.thread is not None and self.thread.isRunning():
self.stop_downloader()
#self.app.closeAllWindows()
self.app.quit()
def another_instance_opened(self, _):
"""
Called when the user tries to open another instance of the application.
Instead of allowing it, will open the current one to avoid any errors.
:type _: QtCore.QString
:param _: message received, see class QtSingleApplication below.
"""
self.window.show()
def start_downloader(self):
"""
Starts the downloader in a thread.
"""
# Don't know how to reproduce, but in some really rare cases the downloader might start without the user requesting it.
# These logs try to collect information that might help pinpoint what causes that.
# Actually, it's been so long since the last time this error was observed that I don't know if it still happens
# or if whatever caused it was fixed...
self.log.debug("stack ([1][3]):")
i = 0
for item in inspect.stack():
self.log.debug("[" + str(i) + "]= " + str(item))
i += 1
self.log.debug("downloader_is_running: " +
str(self.downloader_is_running))
self.log.debug("downloader_is_restarting: " +
str(self.downloader_is_restarting))
self.log.debug("downloader_is_stopping: " +
str(self.downloader_is_stopping))
if not self.downloader_is_stopping:
if self.downloader_is_restarting:
self.log.info("RESTARTING DOWNLOADER THREAD")
self.downloader_is_restarting = False
else:
self.log.info("STARTING DOWNLOADER THREAD")
self.window.downloader_starting()
self.thread = QtCore.QThread(self)
self.downloader = Downloader()
self.downloader.moveToThread(self.thread)
self.downloader.running.connect(self.downloader_started)
self.downloader.finish.connect(self.thread.quit)
self.downloader.restart.connect(self.restart_downloader)
self.downloader.showMessage.connect(self.show_tray_message)
self.downloader.update_ui.connect(self.update_ui)
# noinspection PyUnresolvedReferences
self.thread.started.connect(
self.downloader.execute_once
) # PyCharm doesn't recognize started.connect()...
# noinspection PyUnresolvedReferences
self.thread.finished.connect(
self.downloader_stopped
) # PyCharm doesn't recognize finished.connect()...
self.thread.start()
else:
self.downloader_is_stopping = False
self.downloader_is_restarting = False
def stop_downloader(self):
"""
Stops the downloader (¬_¬)
"""
self.log.info("TERMINATING DOWNLOADER THREAD")
self.window.downloader_stopping()
self.downloader_is_stopping = True
self.downloader_is_restarting = False
if self.thread.isRunning():
self.downloader.stop_thread()
thread_stopped_gracefully = self.thread.wait(300)
if self.thread.isRunning():
thread_stopped_gracefully = self.thread.quit()
self.log.info("THREAD STOPPED CORRECTLY: %s" %
thread_stopped_gracefully)
if not thread_stopped_gracefully:
self.thread.terminate()
else:
self.downloader_stopped()
try:
self.timer.stop()
except AttributeError:
pass # Happens when the downloader is interrupted before being able to fully execute at least once.
def restart_downloader(self):
"""
Finishes the current downloader thread and starts a timer.
When the timer times out a new downloader thread is created.
"""
self.downloader_is_restarting = True
self.thread.quit()
self.log.info("THREAD FINISHED CORRECTLY: %s" % self.thread.wait(300))
self.timer = QtCore.QTimer()
# noinspection PyUnresolvedReferences
self.timer.timeout.connect(
self.start_downloader
) # PyCharm doesn't recognize timeout.connect()...
self.timer.setSingleShot(True)
self.timer.start(db.DBManager().get_config().sleep_time * 1000)
@QtCore.pyqtSlot()
def downloader_started(self):
"""
Downloader thread started correctly; notifies the user.
"""
self.downloader_is_running = True
self.window.downloader_started()
@QtCore.pyqtSlot()
def downloader_stopped(self):
"""
Downloader thread stopped correctly; notifies the user.
"""
if not self.downloader_is_restarting:
self.downloader_is_running = False
self.downloader_is_stopping = False
self.downloader_is_restarting = False
self.window.downloader_stopped()
@QtCore.pyqtSlot(str)
def show_tray_message(self, message):
"""
Uses the system tray icon to notify the user about something.
:type message: str
:param message: Message to be shown to the user.
"""
# TODO: Would it be better if this were moved to manager.system_tray_icon?
self.tray_icon.showMessage(constant.TRAY_MESSAGE_TITLE, message,
QtGui.QSystemTrayIcon.Information, 5000)
@QtCore.pyqtSlot(str)
def update_ui(self, message):
"""
Updates the anime table in the main window.
Also, shows a message to the user using the system tray icon.
:type message: str
:param message: Message to be shown to the user.
"""
if self.window is not None:
self.window.update_anime_table()
self.show_tray_message(message)
class Nyaa():
"""
Search on Nyaa using HTML.
"""
def __init__(self, network):
self.log = LoggerManager().get_logger("Nyaa")
self.network = network
def search(self, text="", dic=None, category="1_37"): #"1_37" = English translated anime
"""
Returned dictionary struct:
dict[RESULT_NUMBER]["title"] = str
dict[RESULT_NUMBER]["link"] = str
dict[RESULT_NUMBER]["date"] = datetime
dict[RESULT_NUMBER]["downloads"] = int
:type text: str
:param text: search_terms
:type dic: dict
:param dic: where the data will be stored. If None, a new dict is created.
:type category: str
:param category: site specific filter to search only for anime.
"""
if dic is None:
dic = {}
search_terms = text.strip().split(" ")
url = self.__get_url(text, category)
html = self.network.get_data(url)
if "viewtorrentname" in html:
return self.__parse_html_1_result(html, dic, search_terms)
else:
return self.__parse_html(html, dic, search_terms)
@staticmethod
def __get_url(text, category):
url = ""
if text!="":
text = text.strip().replace(" ","+")
url = "https://www.nyaa.se/?page=search&term=%s&cats=%s" % (text,category)
return url
def __parse_html_1_result(self, html, dic, search_terms):
parser = HTMLparser1Result()
parser.dict = dic
parser.cont = len(dic)
parser.search_terms = search_terms
parser.debug = self
try:
parser.feed(html)
parser.close()
except CleanExit:
return parser.dict
except Exception as error:
self.log.print_traceback(error,self.log.error)
return parser.dict
def __parse_html(self,html, dic, search_terms):
parser = HTMLparser()
parser.dict = dic
parser.cont = len(dic)
parser.search_terms = search_terms
try:
parser.feed(html)
parser.close()
except CleanExit:
self.log.info("HTML successfully parsed (%i results)" % (len(parser.dict)))
return parser.dict
except Exception as error:
self.log.print_traceback(error,self.log.error)
else:
self.log.info("HTML successfully parsed (%i results)" % (len(parser.dict)))
#return dic
return parser.dict