class Network():
"""
Controls data requests.
"""
def __init__(self):
self.log = LoggerManager().get_logger("Network")
self.stopping_thread = False
def stop_thread(self):
"""
Data requests can't be stopped immediatelly, so this method warns the Network instance to stop as soon as possible.
"""
self.stopping_thread = True
def get_data(self, url, is_binary=False):
"""
Requests data, be it the html/rss of a site or a torrent file.
:type url: str
:param url: Link to the data to be downloaded.
:type is_binary: bool
:param is_binary: If the data is binary (torrent files) or not.
:rtype: str
:return: the data requested
"""
if url == "":
self.log.error("No link has been received (aka empty URL)!")
return None
resp = ""
if url.find("://") == -1:
url = "http://" + url
self.log.debug("link: %s" % url)
timeout = True
tries = 0
while timeout and not self.stopping_thread:
try:
if url.startswith("https"):
response = requests.get(url,
timeout=10,
verify=constant.CACERT_PATH)
else:
response = requests.get(url, timeout=10)
timeout = False
if is_binary:
resp = response.content
else:
resp = response.text
#resp = HTMLParser().unescape(response.text)
except (ReadTimeout, ConnectTimeout):
tries += 1
if tries == 3:
self.log.warning(
"Retried 3 times... Will try again later.")
return ""
self.log.debug("Retrying (%i)" % tries)
except Exception as error:
self.log.print_traceback(error, self.log.error)
return resp
def download_torrent(self, url, file_name, torrent_path, anime_folder):
"""
Requests download of the torrent file, saves it, and opens it with torrent application.
:type url: str
:param url: Link to the torrent file.
:type file_name: str
:param file_name: The name with which the torrent file will be saved.
:type torrent_path: str
:param torrent_path: The folder where the torrent file will be saved.
:type anime_folder: str
:param anime_folder: The folder where the torrent application should save the episode (uTorrent only).
:rtype: bool
:return: if the torrent was sent to the torrent application or not.
"""
if os.path.isdir(torrent_path):
try:
data = self.get_data(url, is_binary=True)
if self.__is_torrent(data):
self.__save_torrent(data, file_name, torrent_path,
anime_folder)
return True
return False
except Exception as error:
raise error
else:
self.log.error(
"The .torrent was NOT downloaded (does the specified folder (%s) exists?)"
% torrent_path)
return False
@staticmethod
def __is_torrent(file_data):
"""
Makes sure the file downloaded is indeed a ".torrent".
:param file_data: The file.
:rtype: bool
:return: If it's a torrent file or not.
"""
# seems like torrent files start with this. So yeah, it's POG, but for now it's working ^^
return str(file_data).find("d8:announce") == 0
def __save_torrent(self, data, file_name, torrent_path, anime_folder=""):
"""
Saves the torrent file and opens it with the torrent application selected.
:param data: The file data.
:type file_name: str or unicode
:param file_name: The name with which the torrent file will be saved.
:type torrent_path: str
:param torrent_path: The folder where the torrent file will be saved.
:type anime_folder: str
:param anime_folder: The folder where the torrent application should save the episode (uTorrent only).
"""
if os.path.isdir(torrent_path):
title = strings.remove_special_chars(file_name)
torrent_file_path = "%s\\%s.torrent" % (torrent_path, title)
with open(torrent_file_path, "wb") as torrent:
torrent.write(data)
self.log.info(".torrent saved")
try:
application_fullpath = torrent_application.fullpath()
self.log.debug("Opening '%s' with '%s'" %
(torrent_file_path, application_fullpath))
if torrent_application.is_utorrent() and os.path.isdir(
anime_folder):
self.log.debug("Using uTorrent, save in folder '%s'" %
strings.escape_unicode(anime_folder))
params = ['/DIRECTORY', anime_folder, torrent_file_path]
else:
params = [torrent_file_path]
# http://stackoverflow.com/questions/1910275/unicode-filenames-on-windows-with-python-subprocess-popen
# TLDR: Python 2.X's subprocess.Popen doesn't work well with unicode
QtCore.QProcess().startDetached(application_fullpath, params)
self.log.info(".torrent opened")
except Exception as error:
self.log.error("Error opening torrent application")
self.log.print_traceback(error, self.log.error)
raise error
else:
self.log.error(
"The .torrent was NOT saved. Apparently the specified folder (%s) does NOT exist."
% torrent_path)
class Downloader(QtCore.QObject):
"""
Processes the search requests from the [Add anime window].
Also responsible for actually searching for new episodes automatically according to the frequency chosen by the user.
"""
running = QtCore.pyqtSignal()
finish = QtCore.pyqtSignal()
restart = QtCore.pyqtSignal()
showMessage = QtCore.pyqtSignal(str)
update_ui = QtCore.pyqtSignal(str)
runningSearch = QtCore.pyqtSignal()
searchResult = QtCore.pyqtSignal(object) #Might be dict or None
def __init__(self, parent=None):
super(Downloader, self).__init__(parent)
self.log = None
self.dbManager = None
self.animeList = None
self.config = None
self.network = None
self.anime_index = None
self.anime_tosho = None
self.anirena = None
self.nyaa = None
self.tokyotosho = None
self.stopping_thread = None
self.timer = None
@QtCore.pyqtSlot()
def execute_once(self):
"""
Initializes variables necessary to search for new episodes.
"""
self.log = LoggerManager().get_logger("Downloader")
self.log.debug("#####################")
self.dbManager = DBManager()
self.animeList = self.dbManager.get_anime_list()
self.config = self.dbManager.get_config()
self.network = Network()
self.anime_index = AnimeIndex(self.network)
self.anime_tosho = AnimeTosho(self.network)
self.anirena = Anirena(self.network)
if self.config.prefer_rss:
self.nyaa = NyaaRSS(self.network)
else:
self.nyaa = Nyaa(self.network)
self.tokyotosho = Tokyotosho(self.network)
self.stopping_thread = False
self.timer = QtCore.QTimer()
self.running.emit()
self.log.debug("****************************")
number_of_downloaded_episodes = self.__search_new_episodes()
msg = "No" if number_of_downloaded_episodes==0 else str(number_of_downloaded_episodes)
if not self.stopping_thread:
self.log.info("%s episodes downloaded, sleeping for %s seconds" % (msg, self.config.sleep_time))
self.restart.emit()
else:
self.log.info("%s episodes downloaded, stopping downloader" % msg)
self.finish.emit()
def __search_new_episodes(self):
"""
Searches for new episodes for all enabled anime.
"""
downloaded_episodes = 0
current_anime = 0
for anime in self.animeList:
if anime.enabled and not self.stopping_thread:
current_anime += 1
self.log.info("(%d/%d) searching for episode %s of '%s'" %
(current_anime, self.dbManager.number_of_anime_enabled, anime.episode, escape_unicode(anime.name)))
dict_anime_index,dict_anime_tosho,dict_anirena,dict_nyaa,dict_tokyotosho = self.search(anime)
anime_dictionary = {}
if dict_anime_index is not None:
anime_dictionary = dict(anime_dictionary.items()+dict_anime_index.items())
if dict_anime_tosho is not None:
anime_dictionary = dict(anime_dictionary.items()+dict_anime_tosho.items())
if dict_anirena is not None:
anime_dictionary = dict(anime_dictionary.items()+dict_anirena.items())
if dict_nyaa is not None:
anime_dictionary = dict(anime_dictionary.items()+dict_nyaa.items())
if dict_tokyotosho is not None:
anime_dictionary = dict(anime_dictionary.items()+dict_tokyotosho.items())
title = "ERROR(No value)"
if not self.stopping_thread:
try:
for key in anime_dictionary.keys():
self.log.debug("'%s' is a match, checking episode number" % (escape_unicode(anime_dictionary[key]["title"])))
title = anime_dictionary[key]["title"]
if (".mkv" or ".mp4") in title:
self.log.debug("title = "+title)
regex_episode_and_version_number = re.compile("\\b[\s_~\-+.]?(%02d)[\s_~\-+.]*(\d*)[\s_]*v?(\d)?[\s\w.]*(\[|\(|.mkv|.mp4)" % anime.episode)
else:
regex_episode_and_version_number = re.compile("\\b[\s_~\-+.]?(%02d)[\s_~\-+.]*(\d*)[\s_]*v?(\d)?" % anime.episode)
result = regex_episode_and_version_number.findall(title)
self.log.debug("regex result = "+str(result))
#self.log.debug("REGEX result = '%s' (len(result)>0: %s)" % (result, len(result)>0))
if len(result)>0:
self.log.info("A torrent has been found")
try:
last_episode_number = int(result[0][1])
self.log.info("It's a double episode! The last episode was number %d" % last_episode_number)
except (TypeError,IndexError,ValueError):
last_episode_number = int(result[0][0])
self.log.info("It's a normal episode")
try:
last_version_number = int(result[0][2])
self.log.info("It's version %d" % last_version_number)
except (TypeError,IndexError,ValueError):
last_version_number = 1
if not self.stopping_thread:
result = False
try:
result = self.network.download_torrent(anime_dictionary[key]["link"],
"%s %d" % (anime.name,anime.episode),
constant.DEFAULT_TORRENTS_PATH,
anime.download_folder)
except Exception as error:
self.showMessage.emit("Error: %s (%s - %s)" % (type(error).__name__,anime.name,anime.episode))
if result:
downloaded_episodes+=1
#self.log.debug("Updating EpisodeNumber")
anime.update_episode(last_episode_number+1)
#self.log.debug("Updating LastVersionNumber")
anime.update_version(last_version_number)
#self.log.debug("Updating LastFileDownloaded")
anime.update_last_file_downloaded(anime_dictionary[key]["title"])
self.log.debug("Notifying user")
self.update_ui.emit("%s - %s" % (anime.name, anime.episode-1))
break
except Exception as error:
self.log.error("ERROR while analysing '%s'" % escape_unicode(title))
self.log.print_traceback(error,self.log.error)
if self.stopping_thread:
break
return downloaded_episodes
def search(self,anime):
"""
Searches for new episodes of a given anime.
:type anime: db.Anime
:param anime: Anime to search for new episode.
:rtype: dict or None,dict or None,dict or None,dict or None,dict or None
:return: results for each site: Anime Index, Anime Tosho, Anirena, Nyaa and Tokyotosho
"""
text = "%s %02d" % (anime.search_terms,anime.episode)
dict_anime_index = None
dict_anime_tosho = None
dict_anirena = None
dict_nyaa = None
dict_tokyotosho = None
if anime.check_anime_index and not self.stopping_thread:
dict_anime_index = self.anime_index.search(text)
if anime.check_anime_tosho and not self.stopping_thread:
dict_anime_tosho = self.anime_tosho.search(text)
if anime.check_anirena and not self.stopping_thread:
dict_anirena = self.anirena.search(text)
if anime.check_nyaa and not self.stopping_thread:
dict_nyaa = self.nyaa.search(text)
if anime.check_tokyotosho and not self.stopping_thread:
dict_tokyotosho = self.tokyotosho.search(text)
return dict_anime_index,dict_anime_tosho,dict_anirena,dict_nyaa,dict_tokyotosho
def execute_once_search_anime_index(self,anime):
"""
Executes search on this site.
:type anime: db.Anime
:param anime: Contains search terms.
"""
self.log = LoggerManager().get_logger("Downloader-Once-AI")
self.network = Network()
anime_index = AnimeIndex(self.network)
self.stopping_thread = False
self.runningSearch.emit()
dict_anime_index = None
if anime.check_anime_index and not self.stopping_thread:
dict_anime_index = anime_index.search(anime.search_terms)
self.searchResult.emit(dict_anime_index)
self.finish.emit()
def execute_once_search_anime_tosho(self,anime):
"""
Executes search on this site.
:type anime: db.Anime
:param anime: Contains search terms.
"""
self.log = LoggerManager().get_logger("Downloader-Once-AT")
self.network = Network()
anime_tosho = AnimeTosho(self.network)
self.stopping_thread = False
self.runningSearch.emit()
dict_anime_tosho = None
if anime.check_anime_tosho and not self.stopping_thread:
dict_anime_tosho = anime_tosho.search(anime.search_terms)
self.searchResult.emit(dict_anime_tosho)
self.finish.emit()
def execute_once_search_anirena(self,anime):
"""
Executes search on this site.
:type anime: db.Anime
:param anime: Contains search terms.
"""
self.log = LoggerManager().get_logger("Downloader-Once-AR")
self.network = Network()
anirena = Anirena(self.network)
self.stopping_thread = False
self.runningSearch.emit()
dict_anirena = None
if anime.check_anirena and not self.stopping_thread:
dict_anirena = anirena.search(anime.search_terms)
self.searchResult.emit(dict_anirena)
self.finish.emit()
def execute_once_search_nyaa(self,anime):
"""
Executes search on this site.
:type anime: db.Anime
:param anime: Contains search terms.
"""
self.log = LoggerManager().get_logger("Downloader-Once-NY")
self.network = Network()
if DBManager().get_config().prefer_rss:
nyaa = NyaaRSS(self.network)
else:
nyaa = Nyaa(self.network)
self.stopping_thread = False
self.runningSearch.emit()
dict_nyaa = None
if anime.check_nyaa and not self.stopping_thread:
dict_nyaa = nyaa.search(anime.search_terms)
self.searchResult.emit(dict_nyaa)
self.finish.emit()
def execute_once_search_tokyotosho(self,anime):
"""
Executes search on this site.
:type anime: db.Anime
:param anime: Contains search terms.
"""
self.log = LoggerManager().get_logger("Downloader-Once-TT")
self.network = Network()
tokyotosho = Tokyotosho(self.network)
self.stopping_thread = False
self.runningSearch.emit()
dict_tokyotosho = None
if anime.check_tokyotosho and not self.stopping_thread:
dict_tokyotosho = tokyotosho.search(anime.search_terms)
self.searchResult.emit(dict_tokyotosho)
self.finish.emit()
def stop_thread(self):
"""
Stops requests being executed to allow thread to finish gracefully.
"""
self.log.info("STOPPING DOWNLOADER THREAD")
self.stopping_thread = True
self.network.stop_thread()
class Downloader(QtCore.QObject):
"""
Processes the search requests from the [Add anime window].
Also responsible for actually searching for new episodes automatically according to the frequency chosen by the user.
"""
running = QtCore.pyqtSignal()
finish = QtCore.pyqtSignal()
restart = QtCore.pyqtSignal()
showMessage = QtCore.pyqtSignal(str)
update_ui = QtCore.pyqtSignal(str)
runningSearch = QtCore.pyqtSignal()
searchResult = QtCore.pyqtSignal(object) #Might be dict or None
def __init__(self, parent=None):
super(Downloader, self).__init__(parent)
self.log = None
self.dbManager = None
self.animeList = None
self.config = None
self.network = None
self.anime_index = None
self.anime_tosho = None
self.anirena = None
self.nyaa = None
self.tokyotosho = None
self.stopping_thread = None
self.timer = None
@QtCore.pyqtSlot()
def execute_once(self):
"""
Initializes variables necessary to search for new episodes.
"""
self.log = LoggerManager().get_logger("Downloader")
self.log.debug("#####################")
self.dbManager = DBManager()
self.animeList = self.dbManager.get_anime_list()
self.config = self.dbManager.get_config()
self.network = Network()
self.anime_index = AnimeIndex(self.network)
self.anime_tosho = AnimeTosho(self.network)
self.anirena = Anirena(self.network)
if self.config.prefer_rss:
self.nyaa = NyaaRSS(self.network)
else:
self.nyaa = Nyaa(self.network)
self.tokyotosho = Tokyotosho(self.network)
self.stopping_thread = False
self.timer = QtCore.QTimer()
self.running.emit()
self.log.debug("****************************")
number_of_downloaded_episodes = self.__search_new_episodes()
msg = "No" if number_of_downloaded_episodes == 0 else str(
number_of_downloaded_episodes)
if not self.stopping_thread:
self.log.info("%s episodes downloaded, sleeping for %s seconds" %
(msg, self.config.sleep_time))
self.restart.emit()
else:
self.log.info("%s episodes downloaded, stopping downloader" % msg)
self.finish.emit()
def __search_new_episodes(self):
"""
Searches for new episodes for all enabled anime.
"""
downloaded_episodes = 0
current_anime = 0
for anime in self.animeList:
if anime.enabled and not self.stopping_thread:
current_anime += 1
self.log.info(
"(%d/%d) searching for episode %s of '%s'" %
(current_anime, self.dbManager.number_of_anime_enabled,
anime.episode, escape_unicode(anime.name)))
dict_anime_index, dict_anime_tosho, dict_anirena, dict_nyaa, dict_tokyotosho = self.search(
anime)
anime_dictionary = {}
if dict_anime_index is not None:
anime_dictionary = dict(anime_dictionary.items() +
dict_anime_index.items())
if dict_anime_tosho is not None:
anime_dictionary = dict(anime_dictionary.items() +
dict_anime_tosho.items())
if dict_anirena is not None:
anime_dictionary = dict(anime_dictionary.items() +
dict_anirena.items())
if dict_nyaa is not None:
anime_dictionary = dict(anime_dictionary.items() +
dict_nyaa.items())
if dict_tokyotosho is not None:
anime_dictionary = dict(anime_dictionary.items() +
dict_tokyotosho.items())
title = "ERROR(No value)"
if not self.stopping_thread:
try:
for key in anime_dictionary.keys():
self.log.debug(
"'%s' is a match, checking episode number" %
(escape_unicode(
anime_dictionary[key]["title"])))
title = anime_dictionary[key]["title"]
if (".mkv" or ".mp4") in title:
self.log.debug("title = " + title)
regex_episode_and_version_number = re.compile(
"\\b[\s_~\-+.]?(%02d)[\s_~\-+.]*(\d*)[\s_]*v?(\d)?[\s\w.]*(\[|\(|.mkv|.mp4)"
% anime.episode)
else:
regex_episode_and_version_number = re.compile(
"\\b[\s_~\-+.]?(%02d)[\s_~\-+.]*(\d*)[\s_]*v?(\d)?"
% anime.episode)
result = regex_episode_and_version_number.findall(
title)
self.log.debug("regex result = " + str(result))
#self.log.debug("REGEX result = '%s' (len(result)>0: %s)" % (result, len(result)>0))
if len(result) > 0:
self.log.info("A torrent has been found")
try:
last_episode_number = int(result[0][1])
self.log.info(
"It's a double episode! The last episode was number %d"
% last_episode_number)
except (TypeError, IndexError, ValueError):
last_episode_number = int(result[0][0])
self.log.info("It's a normal episode")
try:
last_version_number = int(result[0][2])
self.log.info("It's version %d" %
last_version_number)
except (TypeError, IndexError, ValueError):
last_version_number = 1
if not self.stopping_thread:
result = False
try:
result = self.network.download_torrent(
anime_dictionary[key]["link"],
"%s %d" %
(anime.name, anime.episode),
constant.DEFAULT_TORRENTS_PATH,
anime.download_folder)
except Exception as error:
self.showMessage.emit(
"Error: %s (%s - %s)" %
(type(error).__name__, anime.name,
anime.episode))
if result:
downloaded_episodes += 1
#self.log.debug("Updating EpisodeNumber")
anime.update_episode(
last_episode_number + 1)
#self.log.debug("Updating LastVersionNumber")
anime.update_version(
last_version_number)
#self.log.debug("Updating LastFileDownloaded")
anime.update_last_file_downloaded(
anime_dictionary[key]["title"])
self.log.debug("Notifying user")
self.update_ui.emit(
"%s - %s" %
(anime.name, anime.episode - 1))
break
except Exception as error:
self.log.error("ERROR while analysing '%s'" %
escape_unicode(title))
self.log.print_traceback(error, self.log.error)
if self.stopping_thread:
break
return downloaded_episodes
def search(self, anime):
"""
Searches for new episodes of a given anime.
:type anime: db.Anime
:param anime: Anime to search for new episode.
:rtype: dict or None,dict or None,dict or None,dict or None,dict or None
:return: results for each site: Anime Index, Anime Tosho, Anirena, Nyaa and Tokyotosho
"""
text = "%s %02d" % (anime.search_terms, anime.episode)
dict_anime_index = None
dict_anime_tosho = None
dict_anirena = None
dict_nyaa = None
dict_tokyotosho = None
if anime.check_anime_index and not self.stopping_thread:
dict_anime_index = self.anime_index.search(text)
if anime.check_anime_tosho and not self.stopping_thread:
dict_anime_tosho = self.anime_tosho.search(text)
if anime.check_anirena and not self.stopping_thread:
dict_anirena = self.anirena.search(text)
if anime.check_nyaa and not self.stopping_thread:
dict_nyaa = self.nyaa.search(text)
if anime.check_tokyotosho and not self.stopping_thread:
dict_tokyotosho = self.tokyotosho.search(text)
return dict_anime_index, dict_anime_tosho, dict_anirena, dict_nyaa, dict_tokyotosho
def execute_once_search_anime_index(self, anime):
"""
Executes search on this site.
:type anime: db.Anime
:param anime: Contains search terms.
"""
self.log = LoggerManager().get_logger("Downloader-Once-AI")
self.network = Network()
anime_index = AnimeIndex(self.network)
self.stopping_thread = False
self.runningSearch.emit()
dict_anime_index = None
if anime.check_anime_index and not self.stopping_thread:
dict_anime_index = anime_index.search(anime.search_terms)
self.searchResult.emit(dict_anime_index)
self.finish.emit()
def execute_once_search_anime_tosho(self, anime):
"""
Executes search on this site.
:type anime: db.Anime
:param anime: Contains search terms.
"""
self.log = LoggerManager().get_logger("Downloader-Once-AT")
self.network = Network()
anime_tosho = AnimeTosho(self.network)
self.stopping_thread = False
self.runningSearch.emit()
dict_anime_tosho = None
if anime.check_anime_tosho and not self.stopping_thread:
dict_anime_tosho = anime_tosho.search(anime.search_terms)
self.searchResult.emit(dict_anime_tosho)
self.finish.emit()
def execute_once_search_anirena(self, anime):
"""
Executes search on this site.
:type anime: db.Anime
:param anime: Contains search terms.
"""
self.log = LoggerManager().get_logger("Downloader-Once-AR")
self.network = Network()
anirena = Anirena(self.network)
self.stopping_thread = False
self.runningSearch.emit()
dict_anirena = None
if anime.check_anirena and not self.stopping_thread:
dict_anirena = anirena.search(anime.search_terms)
self.searchResult.emit(dict_anirena)
self.finish.emit()
def execute_once_search_nyaa(self, anime):
"""
Executes search on this site.
:type anime: db.Anime
:param anime: Contains search terms.
"""
self.log = LoggerManager().get_logger("Downloader-Once-NY")
self.network = Network()
if DBManager().get_config().prefer_rss:
nyaa = NyaaRSS(self.network)
else:
nyaa = Nyaa(self.network)
self.stopping_thread = False
self.runningSearch.emit()
dict_nyaa = None
if anime.check_nyaa and not self.stopping_thread:
dict_nyaa = nyaa.search(anime.search_terms)
self.searchResult.emit(dict_nyaa)
self.finish.emit()
def execute_once_search_tokyotosho(self, anime):
"""
Executes search on this site.
:type anime: db.Anime
:param anime: Contains search terms.
"""
self.log = LoggerManager().get_logger("Downloader-Once-TT")
self.network = Network()
tokyotosho = Tokyotosho(self.network)
self.stopping_thread = False
self.runningSearch.emit()
dict_tokyotosho = None
if anime.check_tokyotosho and not self.stopping_thread:
dict_tokyotosho = tokyotosho.search(anime.search_terms)
self.searchResult.emit(dict_tokyotosho)
self.finish.emit()
def stop_thread(self):
"""
Stops requests being executed to allow thread to finish gracefully.
"""
self.log.info("STOPPING DOWNLOADER THREAD")
self.stopping_thread = True
self.network.stop_thread()