def check_url(configfile, dbfile, scraper=False): hostnames = RssConfig('Hostnames', configfile) sj = hostnames.get('sj') dj = hostnames.get('dj') sf = hostnames.get('sf') mb = hostnames.get('mb') hw = hostnames.get('hw') hs = hostnames.get('hs') fx = hostnames.get('fx') nk = hostnames.get('nk') dd = hostnames.get('dd') fc = hostnames.get('fc') if not scraper: scraper = cloudscraper.create_scraper() sj_url = 'https://' + sj dj_url = 'https://' + sj sf_url = 'https://' + sf mb_url = 'https://' + mb hw_url = 'https://' + hw fx_url = 'https://' + fx hs_url = 'https://' + hs + '/collection/neuerscheinungen/' nk_url = 'https://' + nk dd_url = 'https://' + dd fc_url = 'https://' + fc sj_blocked_proxy = False dj_blocked_proxy = False sf_blocked_proxy = False mb_blocked_proxy = False hw_blocked_proxy = False fx_blocked_proxy = False hs_blocked_proxy = False nk_blocked_proxy = False dd_blocked_proxy = False fc_blocked_proxy = False sj_blocked = False dj_blocked = False sf_blocked = False mb_blocked = False hw_blocked = False fx_blocked = False hs_blocked = False nk_blocked = False dd_blocked = False fc_blocked = False db = RssDb(dbfile, 'proxystatus') db.delete("SJ") db.delete("DJ") db.delete("SF") db.delete("MB") db.delete("HW") db.delete("FX") db.delete("HS") db.delete("NK") db.delete("DD") db.delete("FC") db_normal = RssDb(dbfile, 'normalstatus') db_normal.delete("SJ") db_normal.delete("DJ") db_normal.delete("SF") db_normal.delete("MB") db_normal.delete("HW") db_normal.delete("FX") db_normal.delete("HS") db_normal.delete("NK") db_normal.delete("DD") db_normal.delete("FC") proxy = RssConfig('RSScrawler', configfile).get('proxy') fallback = RssConfig('RSScrawler', configfile).get('fallback') if proxy: proxies = {'http': proxy, 'https': proxy} if not sj: db.store("SJ", "Blocked") else: try: if "block." in str( scraper.get( sj_url, proxies=proxies, timeout=30, allow_redirects=False).headers.get("location")): sj_blocked_proxy = True else: db.delete("SJ") except: sj_blocked_proxy = True if sj_blocked_proxy: print( u"Der Zugriff auf SJ ist mit der aktuellen Proxy-IP nicht möglich!" ) db.store("SJ", "Blocked") scraper = cloudscraper.create_scraper() if not dj: db.store("DJ", "Blocked") else: try: if "block." in str( scraper.get( dj_url, proxies=proxies, timeout=30, allow_redirects=False).headers.get("location")): dj_blocked_proxy = True else: db.delete("DJ") except: dj_blocked_proxy = True if dj_blocked_proxy: print( u"Der Zugriff auf DJ ist mit der aktuellen Proxy-IP nicht möglich!" ) db.store("DJ", "Blocked") scraper = cloudscraper.create_scraper() if not sf: db.store("SF", "Blocked") else: try: if "block." in str( scraper.get( sf_url, proxies=proxies, timeout=30, allow_redirects=False).headers.get("location")): sf_blocked_proxy = True else: db.delete("SF") except: sf_blocked_proxy = True if sf_blocked_proxy: print( u"Der Zugriff auf SF ist mit der aktuellen Proxy-IP nicht möglich!" ) db.store("SF", "Blocked") scraper = cloudscraper.create_scraper() if not mb: db.store("MB", "Blocked") else: try: if "<Response [403]>" in str( scraper.get(mb_url, proxies=proxies, timeout=30, allow_redirects=False)): mb_blocked_proxy = True else: db.delete("MB") except: mb_blocked_proxy = True if mb_blocked_proxy: print( u"Der Zugriff auf MB ist mit der aktuellen Proxy-IP nicht möglich!" ) db.store("MB", "Blocked") scraper = cloudscraper.create_scraper() if not hw: db.store("HW", "Blocked") else: try: if "<Response [403]>" in str( scraper.get(hw_url, proxies=proxies, timeout=30, allow_redirects=False)): hw_blocked_proxy = True else: db.delete("HW") except: hw_blocked_proxy = True if hw_blocked_proxy: print( u"Der Zugriff auf HW ist mit der aktuellen Proxy-IP nicht möglich!" ) db.store("HW", "Blocked") scraper = cloudscraper.create_scraper() if not fx: db.store("FX", "Blocked") else: try: if "<Response [403]>" in str( scraper.get(fx_url, proxies=proxies, timeout=30, allow_redirects=False)): fx_blocked_proxy = True else: db.delete("FX") except: fx_blocked_proxy = True session = requests.session() session.headers = scraper.headers session.cookies = scraper.cookies session.verify = False if "<Response [200]>" in str( session.get(fx_url, proxies=proxies, timeout=30, allow_redirects=False)): fx_blocked_proxy = False if fx_blocked_proxy: print( u"Der Zugriff auf FX ist mit der aktuellen Proxy-IP nicht möglich!" ) db.store("FX", "Blocked") scraper = cloudscraper.create_scraper() if not hs: db.store("HS", "Blocked") else: try: if "200" not in str( scraper.get(hs_url, timeout=30, allow_redirects=False).status_code): hs_blocked_proxy = True else: db.delete("HS") except: hs_blocked_proxy = True if hs_blocked_proxy: print( u"Der Zugriff auf HS ist mit der aktuellen Proxy-IP nicht möglich!" ) db.store("HS", "Blocked") scraper = cloudscraper.create_scraper() if not nk: db.store("NK", "Blocked") else: try: if "200" not in str( scraper.get(nk_url, timeout=30, allow_redirects=False).status_code): nk_blocked_proxy = True else: db.delete("NK") except: nk_blocked_proxy = True if nk_blocked_proxy: print( u"Der Zugriff auf NK ist mit der aktuellen Proxy-IP nicht möglich!" ) db.store("NK", "Blocked") scraper = cloudscraper.create_scraper() if not dd: db.store("DD", "Blocked") else: try: if "200" not in str( scraper.get(dd_url, timeout=30, allow_redirects=False).status_code): dd_blocked_proxy = True else: db.delete("DD") except: dd_blocked_proxy = True if dd_blocked_proxy: print( u"Der Zugriff auf DD ist mit der aktuellen Proxy-IP nicht möglich!" ) db.store("DD", "Blocked") scraper = cloudscraper.create_scraper() if not fc: db.store("FC", "Blocked") else: try: if "200" not in str( scraper.get(fc_url, timeout=30).status_code): fc_blocked_proxy = True else: db.delete("FC") except: fc_blocked_proxy = True if fc_blocked_proxy: print( u"Der Zugriff auf FC ist mit der aktuellen Proxy-IP nicht möglich!" ) db.store("FC", "Blocked") scraper = cloudscraper.create_scraper() if not proxy or (proxy and sj_blocked_proxy and fallback): if not sj: db.store("SJ", "Blocked") else: try: if "block." in str( scraper.get( sj_url, timeout=30, allow_redirects=False).headers.get("location")): sj_blocked = True except: sj_blocked = True if sj_blocked: db_normal.store("SJ", "Blocked") print( u"Der Zugriff auf SJ ist mit der aktuellen IP nicht möglich!" ) if not proxy or (proxy and dj_blocked_proxy and fallback): if not dj: db.store("DJ", "Blocked") else: try: if "block." in str( scraper.get( dj_url, timeout=30, allow_redirects=False).headers.get("location")): dj_blocked = True except: dj_blocked = True if dj_blocked: db_normal.store("DJ", "Blocked") print( u"Der Zugriff auf DJ ist mit der aktuellen IP nicht möglich!" ) if not sf: db.store("SF", "Blocked") else: try: if "block." in str( scraper.get( sf_url, timeout=30, allow_redirects=False).headers.get("location")): sf_blocked = True except: sf_blocked = True if sf_blocked: db_normal.store("SF", "Blocked") print( u"Der Zugriff auf SF ist mit der aktuellen IP nicht möglich!" ) if not proxy or (proxy and mb_blocked_proxy and fallback): if not mb: db.store("MB", "Blocked") else: try: if "<Response [403]>" in str( scraper.get(mb_url, timeout=30, allow_redirects=False)): mb_blocked = True except: mb_blocked = True if mb_blocked: db_normal.store("MB", "Blocked") print( u"Der Zugriff auf MB ist mit der aktuellen IP nicht möglich!" ) if not proxy or (proxy and hw_blocked_proxy and fallback): if not hw: db.store("HW", "Blocked") else: try: if "<Response [403]>" in str( scraper.get(hw_url, timeout=30, allow_redirects=False)): hw_blocked = True except: hw_blocked = True if hw_blocked: db_normal.store("HW", "Blocked") print( u"Der Zugriff auf HW ist mit der aktuellen IP nicht möglich!" ) if not proxy or (proxy and fx_blocked_proxy and fallback): if not fx: db.store("FX", "Blocked") else: try: if "<Response [403]>" in str( scraper.get(fx_url, timeout=30, allow_redirects=False)): fx_blocked = True except: fx_blocked = True session = requests.session() session.headers = scraper.headers session.cookies = scraper.cookies session.verify = False if "<Response [200]>" in str( session.get(fx_url, timeout=30, allow_redirects=False)): fx_blocked = False if fx_blocked: db_normal.store("FX", "Blocked") print( u"Der Zugriff auf FX ist mit der aktuellen IP nicht möglich!" ) if not proxy or (proxy and hs_blocked_proxy and fallback): if not hs: db.store("HS", "Blocked") else: try: if "200" not in str( scraper.get(hs_url, timeout=30, allow_redirects=False).status_code): hs_blocked = True except: hs_blocked = True if hs_blocked: db_normal.store("HS", "Blocked") print( u"Der Zugriff auf HS ist mit der aktuellen IP nicht möglich!" ) if not proxy or (proxy and nk_blocked_proxy and fallback): if not nk: db.store("NK", "Blocked") else: try: if "200" not in str( scraper.get(nk_url, timeout=30, allow_redirects=False).status_code): nk_blocked = True except: nk_blocked = True if nk_blocked: db_normal.store("NK", "Blocked") print( u"Der Zugriff auf NK ist mit der aktuellen IP nicht möglich!" ) if not proxy or (proxy and dd_blocked_proxy and fallback): if not dd: db.store("DD", "Blocked") else: try: if "200" not in str( scraper.get(dd_url, timeout=30, allow_redirects=False).status_code): dd_blocked = True except: dd_blocked = True if dd_blocked: db_normal.store("DD", "Blocked") print( u"Der Zugriff auf DD ist mit der aktuellen IP nicht möglich!" ) if not proxy or (proxy and fc_blocked_proxy and fallback): if not fc: db.store("FC", "Blocked") else: try: if "200" not in str( scraper.get(fc_url, timeout=30).status_code): fc_blocked = True except: fc_blocked = True if fc_blocked: db_normal.store("FC", "Blocked") print( u"Der Zugriff auf FC ist mit der aktuellen IP nicht möglich!" ) return scraper
class SF: def __init__(self, configfile, dbfile, device, logging, scraper, filename, internal_name): self._INTERNAL_NAME = internal_name self.configfile = configfile self.dbfile = dbfile self.device = device self.hostnames = RssConfig('Hostnames', self.configfile) self.sf = self.hostnames.get('sf') self.config = RssConfig(self._INTERNAL_NAME, self.configfile) self.rsscrawler = RssConfig("RSScrawler", self.configfile) self.hevc_retail = self.config.get("hevc_retail") self.retail_only = self.config.get("retail_only") self.hoster_fallback = self.config.get("hoster_fallback") self.hosters = RssConfig("Hosters", configfile).get_section() self.log_info = logging.info self.log_error = logging.error self.log_debug = logging.debug self.scraper = scraper self.filename = filename self.db = RssDb(self.dbfile, 'rsscrawler') self.quality = self.config.get("quality") self.cdc = RssDb(self.dbfile, 'cdc') self.last_set_sf = self.cdc.retrieve("SFSet-" + self.filename) self.last_sha_sf = self.cdc.retrieve("SF-" + self.filename) self.headers = { 'If-Modified-Since': str(self.cdc.retrieve("SFHeaders-" + self.filename)) } settings = [ "quality", "rejectlist", "regex", "hevc_retail", "retail_only", "hoster_fallback" ] self.settings = [] self.settings.append(self.rsscrawler.get("english")) self.settings.append(self.rsscrawler.get("surround")) self.settings.append(self.hosters) for s in settings: self.settings.append(self.config.get(s)) self.empty_list = False if self.filename == 'SJ_Staffeln_Regex': self.level = 3 elif self.filename == 'MB_Staffeln': self.seasonssource = self.config.get('seasonssource').lower() self.level = 2 elif self.filename == 'SJ_Serien_Regex': self.level = 1 else: self.level = 0 self.pattern = r'^(' + "|".join( self.get_series_list(self.filename, self.level)).lower() + ')' self.listtype = "" self.day = 0 def settings_hash(self, refresh): if refresh: settings = [ "quality", "rejectlist", "regex", "hevc_retail", "retail_only", "hoster_fallback" ] self.settings = [] self.settings.append(self.rsscrawler.get("english")) self.settings.append(self.rsscrawler.get("surround")) self.settings.append(self.hosters) for s in settings: self.settings.append(self.config.get(s)) self.pattern = r'^(' + "|".join( self.get_series_list(self.filename, self.level)).lower() + ')' set_sf = str(self.settings) + str(self.pattern) return hashlib.sha256(set_sf.encode('ascii', 'ignore')).hexdigest() def get_series_list(self, liste, series_type): if series_type == 1: self.listtype = " (RegEx)" elif series_type == 2: self.listtype = " (Staffeln)" elif series_type == 3: self.listtype = " (Staffeln/RegEx)" cont = ListDb(self.dbfile, liste).retrieve() titles = [] if cont: for title in cont: if title: title = title.replace(" ", ".") titles.append(title) if not titles: self.empty_list = True return titles def parse_download(self, series_url, title, language_id): if not check_valid_release(title, self.retail_only, self.hevc_retail, self.dbfile): self.log_debug( title + u" - Release ignoriert (Gleiche oder bessere Quelle bereits vorhanden)" ) return if self.filename == 'MB_Staffeln': if not self.config.get("seasonpacks"): staffelpack = re.search(r"s\d.*(-|\.).*s\d", title.lower()) if staffelpack: self.log_debug("%s - Release ignoriert (Staffelpaket)" % title) return if not re.search(self.seasonssource, title.lower()): self.log_debug(title + " - Release hat falsche Quelle") return try: if language_id == 2: lang = 'EN' else: lang = 'DE' epoch = str(datetime.datetime.now().timestamp()).replace('.', '')[:-3] api_url = series_url + '?lang=' + lang + '&_=' + epoch response = get_url(api_url, self.configfile, self.dbfile, self.scraper) info = json.loads(response) is_episode = re.findall(r'.*\.(s\d{1,3}e\d{1,3})\..*', title, re.IGNORECASE) if is_episode: episode_string = re.findall(r'.*S\d{1,3}(E\d{1,3}).*', is_episode[0])[0].lower() season_string = re.findall(r'.*(S\d{1,3})E\d{1,3}.*', is_episode[0])[0].lower() season_title = rreplace( title.lower().replace(episode_string, ''), "-", ".*", 1).lower() season_title = season_title.replace(".untouched", ".*").replace( ".dd+51", ".dd.51") episode = str(int(episode_string.replace("e", ""))) season = str(int(season_string.replace("s", ""))) episode_name = re.findall(r'.*\.s\d{1,3}(\..*).german', season_title, re.IGNORECASE) if episode_name: season_title = season_title.replace(episode_name[0], '') codec_tags = [".h264", ".x264"] for tag in codec_tags: season_title = season_title.replace(tag, ".*264") web_tags = [".web-rip", ".webrip", ".webdl", ".web-dl"] for tag in web_tags: season_title = season_title.replace(tag, ".web.*") else: season = False episode = False season_title = title multiple_episodes = re.findall(r'(e\d{1,3}-e*\d{1,3}\.)', season_title, re.IGNORECASE) if multiple_episodes: season_title = season_title.replace( multiple_episodes[0], '.*') content = BeautifulSoup(info['html'], 'lxml') releases = content.find( "small", text=re.compile(season_title, re.IGNORECASE)).parent.parent.parent links = releases.findAll("div", {'class': 'row'})[1].findAll('a') valid = False for link in links: download_link = link['href'] if check_hoster(link.text.replace('\n', ''), self.configfile): valid = True break if not valid and not self.hoster_fallback: storage = self.db.retrieve_all(title) if 'added' not in storage and 'notdl' not in storage: wrong_hoster = '[SF/Hoster fehlt] - ' + title if 'wrong_hoster' not in storage: self.log_info(wrong_hoster) self.db.store(title, 'wrong_hoster') notify([wrong_hoster], self.configfile) else: self.log_debug(wrong_hoster) else: return self.send_package(title, download_link, language_id, season, episode) except: print( u"SF hat die Serien-API angepasst. Breche Download-Prüfung ab!" ) def send_package(self, title, download_link, language_id, season, episode): englisch = "" if language_id == 2: englisch = "/Englisch" if self.filename == 'SJ_Serien_Regex': link_placeholder = '[Episode/RegEx' + englisch + '] - ' elif self.filename == 'SJ_Serien': link_placeholder = '[Episode' + englisch + '] - ' elif self.filename == 'SJ_Staffeln_Regex]': link_placeholder = '[Staffel/RegEx' + englisch + '] - ' else: link_placeholder = '[Staffel' + englisch + '] - ' try: storage = self.db.retrieve_all(title) except Exception as e: self.log_debug("Fehler bei Datenbankzugriff: %s, Grund: %s" % (e, title)) return if 'added' in storage or 'notdl' in storage: self.log_debug(title + " - Release ignoriert (bereits gefunden)") else: download_link = 'https://' + self.sf + download_link if season and episode: download_link = download_link.replace( '&_=', '&season=' + str(season) + '&episode=' + str(episode) + '&_=') download = add_decrypt(title, download_link, self.sf, self.dbfile) if download: self.db.store(title, 'added') log_entry = link_placeholder + title + ' - [SF]' self.log_info(log_entry) notify(["[Click'n'Load notwendig] - " + log_entry], self.configfile) return log_entry def periodical_task(self): if not self.sf: return self.device if self.filename == 'SJ_Serien_Regex': if not self.config.get('regex'): self.log_debug("Suche für SF-Regex deaktiviert!") return self.device elif self.filename == 'SJ_Staffeln_Regex': if not self.config.get('regex'): self.log_debug("Suche für SF-Regex deaktiviert!") return self.device elif self.filename == 'MB_Staffeln': if not self.config.get('crawlseasons'): self.log_debug("Suche für SF-Staffeln deaktiviert!") return self.device if self.empty_list: self.log_debug("Liste ist leer. Stoppe Suche für Serien!" + self.listtype) return self.device try: reject = self.config.get("rejectlist").replace( ",", "|").lower() if len( self.config.get("rejectlist")) > 0 else r"^unmatchable$" except TypeError: reject = r"^unmatchable$" set_sf = self.settings_hash(False) header = False response = False while self.day < 8: if self.last_set_sf == set_sf: try: delta = ( datetime.datetime.now() - datetime.timedelta(days=self.day)).strftime("%Y-%m-%d") response = get_url_headers( 'https://' + self.sf + '/updates/' + delta, self.configfile, self.dbfile, self.headers, self.scraper) self.scraper = response[1] response = response[0] if self.filename == "MB_Staffeln" or self.filename == "SJ_Staffeln_Regex": feed = sf_releases_to_feedparser_dict( response.text, "seasons", 'https://' + self.sf, True) else: feed = sf_releases_to_feedparser_dict( response.text, "episodes", 'https://' + self.sf, True) except: print(u"SF hat die Feed-API angepasst. Breche Suche ab!") feed = False if response: if response.status_code == 304: self.log_debug( "SF-Feed seit letztem Aufruf nicht aktualisiert - breche Suche ab!" ) return self.device header = True else: try: delta = ( datetime.datetime.now() - datetime.timedelta(days=self.day)).strftime("%Y-%m-%d") response = get_url( 'https://' + self.sf + '/updates/' + delta, self.configfile, self.dbfile, self.scraper) if self.filename == "MB_Staffeln" or self.filename == "SJ_Staffeln_Regex": feed = sf_releases_to_feedparser_dict( response, "seasons", 'https://' + self.sf, True) else: feed = sf_releases_to_feedparser_dict( response, "episodes", 'https://' + self.sf, True) except: print(u"SF hat die Feed-API angepasst. Breche Suche ab!") feed = False self.day += 1 if feed and feed.entries: first_post_sf = feed.entries[0] concat_sf = first_post_sf.title + first_post_sf.published + str( self.settings) + str(self.pattern) sha_sf = hashlib.sha256(concat_sf.encode( 'ascii', 'ignore')).hexdigest() else: self.log_debug("Feed ist leer - breche Suche ab!") return False for post in feed.entries: concat = post.title + post.published + \ str(self.settings) + str(self.pattern) sha = hashlib.sha256(concat.encode('ascii', 'ignore')).hexdigest() if sha == self.last_sha_sf: self.log_debug("Feed ab hier bereits gecrawlt (" + post.title + ") - breche Suche ab!") break series_url = post.series_url title = post.title.replace("-", "-") if self.filename == 'SJ_Serien_Regex': if self.config.get("regex"): if '.german.' in title.lower(): language_id = 1 elif self.rsscrawler.get('english'): language_id = 2 else: language_id = 0 if language_id: m = re.search(self.pattern, title.lower()) if not m and "720p" not in title and "1080p" not in title and "2160p" not in title: m = re.search( self.pattern.replace("480p", "."), title.lower()) self.quality = "480p" if m: if "720p" in title.lower(): self.quality = "720p" if "1080p" in title.lower(): self.quality = "1080p" if "2160p" in title.lower(): self.quality = "2160p" m = re.search(reject, title.lower()) if m: self.log_debug( title + " - Release durch Regex gefunden (trotz rejectlist-Einstellung)" ) title = re.sub(r'\[.*\] ', '', post.title) self.parse_download(series_url, title, language_id) else: self.log_debug( "%s - Englische Releases deaktiviert" % title) else: continue elif self.filename == 'SJ_Staffeln_Regex': if self.config.get("regex"): if '.german.' in title.lower(): language_id = 1 elif self.rsscrawler.get('english'): language_id = 2 else: language_id = 0 if language_id: m = re.search(self.pattern, title.lower()) if not m and "720p" not in title and "1080p" not in title and "2160p" not in title: m = re.search( self.pattern.replace("480p", "."), title.lower()) self.quality = "480p" if m: if "720p" in title.lower(): self.quality = "720p" if "1080p" in title.lower(): self.quality = "1080p" if "2160p" in title.lower(): self.quality = "2160p" m = re.search(reject, title.lower()) if m: self.log_debug( title + " - Release durch Regex gefunden (trotz rejectlist-Einstellung)" ) title = re.sub(r'\[.*\] ', '', post.title) self.parse_download(series_url, title, language_id) else: self.log_debug( "%s - Englische Releases deaktiviert" % title) else: continue else: if self.config.get("quality") != '480p': m = re.search(self.pattern, title.lower()) if m: if '.german.' in title.lower(): language_id = 1 elif self.rsscrawler.get('english'): language_id = 2 else: language_id = 0 if language_id: mm = re.search(self.quality, title.lower()) if mm: mmm = re.search(reject, title.lower()) if mmm: self.log_debug( title + " - Release ignoriert (basierend auf rejectlist-Einstellung)" ) continue if self.rsscrawler.get("surround"): if not re.match( r'.*\.(DTS|DD\+*51|DD\+*71|AC3\.5\.*1)\..*', title): self.log_debug( title + " - Release ignoriert (kein Mehrkanalton)" ) continue try: storage = self.db.retrieve_all(title) except Exception as e: self.log_debug( "Fehler bei Datenbankzugriff: %s, Grund: %s" % (e, title)) return self.device if 'added' in storage: self.log_debug( title + " - Release ignoriert (bereits gefunden)" ) continue self.parse_download( series_url, title, language_id) else: self.log_debug( "%s - Englische Releases deaktiviert" % title) else: m = re.search(self.pattern, title.lower()) if m: if '.german.' in title.lower(): language_id = 1 elif self.rsscrawler.get('english'): language_id = 2 else: language_id = 0 if language_id: if "720p" in title.lower( ) or "1080p" in title.lower( ) or "2160p" in title.lower(): continue mm = re.search(reject, title.lower()) if mm: self.log_debug( title + " Release ignoriert (basierend auf rejectlist-Einstellung)" ) continue if self.rsscrawler.get("surround"): if not re.match( r'.*\.(DTS|DD\+*51|DD\+*71|AC3\.5\.*1)\..*', title): self.log_debug( title + " - Release ignoriert (kein Mehrkanalton)" ) continue title = re.sub(r'\[.*\] ', '', post.title) try: storage = self.db.retrieve_all(title) except Exception as e: self.log_debug( "Fehler bei Datenbankzugriff: %s, Grund: %s" % (e, title)) return self.device if 'added' in storage: self.log_debug( title + " - Release ignoriert (bereits gefunden)" ) continue self.parse_download( series_url, title, language_id) else: self.log_debug( "%s - Englische Releases deaktiviert" % title) if set_sf: new_set_sf = self.settings_hash(True) if set_sf == new_set_sf: self.cdc.delete("SFSet-" + self.filename) self.cdc.store("SFSet-" + self.filename, set_sf) self.cdc.delete("SF-" + self.filename) self.cdc.store("SF-" + self.filename, sha_sf) if header and response: self.cdc.delete("SFHeaders-" + self.filename) self.cdc.store("SFHeaders-" + self.filename, response.headers['date']) return self.device
def download(configfile, dbfile, device, title, subdir, old_links, password, full_path=None, autostart=False): try: if not device or not is_device(device): device = get_device(configfile) if isinstance(old_links, list): links = [] for link in old_links: if link not in links: links.append(link) else: links = [old_links] links = str(links).replace(" ", "") crawljobs = RssConfig('Crawljobs', configfile) usesubdir = crawljobs.get("subdir") priority = "DEFAULT" if full_path: path = full_path else: if usesubdir: path = subdir + "/<jd:packagename>" else: path = "<jd:packagename>" if "Remux" in path: priority = "LOWER" try: device.linkgrabber.add_links( params=[{ "autostart": autostart, "links": links, "packageName": title, "extractPassword": password, "priority": priority, "downloadPassword": password, "destinationFolder": path, "comment": "RSScrawler by rix1337", "overwritePackagizerRules": False }]) except rsscrawler.myjdapi.TokenExpiredException: device = get_device(configfile) if not device or not is_device(device): return False device.linkgrabber.add_links( params=[{ "autostart": autostart, "links": links, "packageName": title, "extractPassword": password, "priority": priority, "downloadPassword": password, "destinationFolder": path, "comment": "RSScrawler by rix1337", "overwritePackagizerRules": False }]) db = RssDb(dbfile, 'crawldog') if db.retrieve(title): db.delete(title) db.store(title, 'retried') else: db.store(title, 'added') return device except rsscrawler.myjdapi.MYJDException as e: print(u"Fehler bei der Verbindung mit MyJDownloader: " + str(e)) return False
def crawldog(configfile, dbfile): disable_request_warnings(InsecureRequestWarning) crawljobs = RssConfig('Crawljobs', configfile) autostart = crawljobs.get("autostart") db = RssDb(dbfile, 'crawldog') grabber_was_collecting = False device = False while True: try: if not device or not is_device(device): device = get_device(configfile) myjd_packages = get_info(configfile, device) grabber_collecting = myjd_packages[2] if grabber_was_collecting or grabber_collecting: grabber_was_collecting = grabber_collecting time.sleep(5) else: packages_in_downloader_decrypted = myjd_packages[4][0] packages_in_linkgrabber_decrypted = myjd_packages[4][1] offline_packages = myjd_packages[4][2] encrypted_packages = myjd_packages[4][3] try: watched_titles = db.retrieve_all_titles() except: watched_titles = False notify_list = [] if packages_in_downloader_decrypted or packages_in_linkgrabber_decrypted or offline_packages or encrypted_packages: if watched_titles: for title in watched_titles: if packages_in_downloader_decrypted: for package in packages_in_downloader_decrypted: if title[0] in package[ 'name'] or title[0].replace( ".", " ") in package['name']: check = hoster_check( configfile, device, [package], title[0], [0]) device = check[0] if device: db.delete(title[0]) if packages_in_linkgrabber_decrypted: for package in packages_in_linkgrabber_decrypted: if title[0] in package[ 'name'] or title[0].replace( ".", " ") in package['name']: check = hoster_check( configfile, device, [package], title[0], [0]) device = check[0] episode = RssDb( dbfile, 'episode_remover').retrieve( title[0]) if episode: filenames = package['filenames'] if len(filenames) > 1: fname_episodes = [] for fname in filenames: try: if re.match( r'.*S\d{1,3}E\d{1,3}.*', fname, flags=re. IGNORECASE): fname = re.findall( r'S\d{1,3}E(\d{1,3})', fname, flags=re. IGNORECASE ).pop() else: fname = fname.replace( "hddl8", "").replace( "dd51", "" ).replace( "264", "" ).replace( "265", "") except: fname = fname.replace( "hddl8", "").replace( "dd51", "").replace( "264", "" ).replace( "265", "") fname_episode = "".join( re.findall( r'\d+', fname.split( ".part")[0])) try: fname_episodes.append( str( int(fname_episode ))) except: pass replacer = longest_substr( fname_episodes) new_fname_episodes = [] for new_ep_fname in fname_episodes: try: new_fname_episodes.append( str( int( new_ep_fname .replace( replacer, "")))) except: pass replacer = longest_substr( new_fname_episodes) newer_fname_episodes = [] for new_ep_fname in new_fname_episodes: try: newer_fname_episodes.append( str( int( re.sub( replacer, "", new_ep_fname, 1)))) except: pass replacer = longest_substr( newer_fname_episodes) even_newer_fname_episodes = [] for newer_ep_fname in newer_fname_episodes: try: even_newer_fname_episodes.append( str( int( re.sub( replacer, "", newer_ep_fname, 1)))) except: pass if even_newer_fname_episodes: fname_episodes = even_newer_fname_episodes elif newer_fname_episodes: fname_episodes = newer_fname_episodes elif new_fname_episodes: fname_episodes = new_fname_episodes delete_linkids = [] pos = 0 for delete_id in package[ 'linkids']: if str(episode) != str( fname_episodes[pos] ): delete_linkids.append( delete_id) pos += 1 if delete_linkids: delete_uuids = [ package['uuid'] ] RssDb( dbfile, 'episode_remover' ).delete(title[0]) device = remove_from_linkgrabber( configfile, device, delete_linkids, delete_uuids) if autostart: device = move_to_downloads( configfile, device, package['linkids'], [package['uuid']]) if device: db.delete(title[0]) if offline_packages: for package in offline_packages: if title[0] in package[ 'name'] or title[0].replace( ".", " ") in package['name']: notify_list.append("[Offline] - " + title[0]) print((u"[Offline] - " + title[0])) db.delete(title[0]) if encrypted_packages: for package in encrypted_packages: if title[0] in package[ 'name'] or title[0].replace( ".", " ") in package['name']: if title[1] == 'added': if retry_decrypt( configfile, dbfile, device, package['linkids'], [package['uuid']], package['urls']): db.delete(title[0]) db.store(title[0], 'retried') else: add_decrypt( package['name'], package['url'], "", dbfile) device = remove_from_linkgrabber( configfile, device, package['linkids'], [package['uuid']]) notify_list.append( "[Click'n'Load notwendig] - " + title[0]) print( u"[Click'n'Load notwendig] - " + title[0]) db.delete(title[0]) else: if not grabber_collecting: db.reset() if notify_list: notify(notify_list, configfile) time.sleep(30) except Exception: traceback.print_exc() time.sleep(30)
class SJ: def __init__(self, configfile, dbfile, device, logging, scraper, filename, internal_name): self._INTERNAL_NAME = internal_name self.configfile = configfile self.dbfile = dbfile self.device = device self.hostnames = RssConfig('Hostnames', self.configfile) self.sj = self.hostnames.get('sj') self.config = RssConfig(self._INTERNAL_NAME, self.configfile) self.rsscrawler = RssConfig("RSScrawler", self.configfile) self.hevc_retail = self.config.get("hevc_retail") self.retail_only = self.config.get("retail_only") self.hoster_fallback = self.config.get("hoster_fallback") self.hosters = RssConfig("Hosters", configfile).get_section() self.log_info = logging.info self.log_error = logging.error self.log_debug = logging.debug self.scraper = scraper self.filename = filename self.db = RssDb(self.dbfile, 'rsscrawler') self.quality = self.config.get("quality") self.cdc = RssDb(self.dbfile, 'cdc') self.last_set_sj = self.cdc.retrieve("SJSet-" + self.filename) self.last_sha_sj = self.cdc.retrieve("SJ-" + self.filename) self.headers = { 'If-Modified-Since': str(self.cdc.retrieve("SJHeaders-" + self.filename)) } settings = [ "quality", "rejectlist", "regex", "hevc_retail", "retail_only", "hoster_fallback" ] self.settings = [] self.settings.append(self.rsscrawler.get("english")) self.settings.append(self.rsscrawler.get("surround")) self.settings.append(self.hosters) for s in settings: self.settings.append(self.config.get(s)) self.empty_list = False if self.filename == 'SJ_Staffeln_Regex': self.level = 3 elif self.filename == 'MB_Staffeln': self.seasonssource = self.config.get('seasonssource').lower() self.level = 2 elif self.filename == 'SJ_Serien_Regex': self.level = 1 else: self.level = 0 self.pattern = r'^(' + "|".join( self.get_series_list(self.filename, self.level)).lower() + ')' self.listtype = "" self.day = 0 def settings_hash(self, refresh): if refresh: settings = [ "quality", "rejectlist", "regex", "hevc_retail", "retail_only", "hoster_fallback" ] self.settings = [] self.settings.append(self.rsscrawler.get("english")) self.settings.append(self.rsscrawler.get("surround")) self.settings.append(self.hosters) for s in settings: self.settings.append(self.config.get(s)) self.pattern = r'^(' + "|".join( self.get_series_list(self.filename, self.level)).lower() + ')' set_sj = str(self.settings) + str(self.pattern) return hashlib.sha256(set_sj.encode('ascii', 'ignore')).hexdigest() def get_series_list(self, liste, series_type): if series_type == 1: self.listtype = " (RegEx)" elif series_type == 2: self.listtype = " (Staffeln)" elif series_type == 3: self.listtype = " (Staffeln/RegEx)" cont = ListDb(self.dbfile, liste).retrieve() titles = [] if cont: for title in cont: if title: title = title.replace(" ", ".") titles.append(title) if not titles: self.empty_list = True return titles def parse_download(self, series_url, title, language_id): if not check_valid_release(title, self.retail_only, self.hevc_retail, self.dbfile): self.log_debug( title + u" - Release ignoriert (Gleiche oder bessere Quelle bereits vorhanden)" ) return if self.filename == 'MB_Staffeln': if not self.config.get("seasonpacks"): staffelpack = re.search(r"s\d.*(-|\.).*s\d", title.lower()) if staffelpack: self.log_debug("%s - Release ignoriert (Staffelpaket)" % title) return if not re.search(self.seasonssource, title.lower()): self.log_debug(title + " - Release hat falsche Quelle") return try: series_info = get_url(series_url, self.configfile, self.dbfile) series_id = re.findall(r'data-mediaid="(.*?)"', series_info)[0] api_url = 'https://' + self.sj + '/api/media/' + series_id + '/releases' response = get_url(api_url, self.configfile, self.dbfile, self.scraper) seasons = json.loads(response) for season in seasons: season = seasons[season] for item in season['items']: if item['name'] == title: valid = False for hoster in item['hoster']: if hoster: if check_hoster(hoster, self.configfile): valid = True if not valid and not self.hoster_fallback: storage = self.db.retrieve_all(title) if 'added' not in storage and 'notdl' not in storage: wrong_hoster = '[SJ/Hoster fehlt] - ' + title if 'wrong_hoster' not in storage: print(wrong_hoster) self.db.store(title, 'wrong_hoster') notify([wrong_hoster], self.configfile) else: self.log_debug(wrong_hoster) else: return self.send_package(title, series_url, language_id) except: print( u"SJ hat die Serien-API angepasst. Breche Download-Prüfung ab!" ) def send_package(self, title, series_url, language_id): englisch = "" if language_id == 2: englisch = "/Englisch" if self.filename == 'SJ_Serien_Regex': link_placeholder = '[Episode/RegEx' + englisch + '] - ' elif self.filename == 'SJ_Serien': link_placeholder = '[Episode' + englisch + '] - ' elif self.filename == 'SJ_Staffeln_Regex]': link_placeholder = '[Staffel/RegEx' + englisch + '] - ' else: link_placeholder = '[Staffel' + englisch + '] - ' try: storage = self.db.retrieve_all(title) except Exception as e: self.log_debug("Fehler bei Datenbankzugriff: %s, Grund: %s" % (e, title)) return if 'added' in storage or 'notdl' in storage: self.log_debug(title + " - Release ignoriert (bereits gefunden)") else: download = add_decrypt(title, series_url, self.sj, self.dbfile) if download: self.db.store(title, 'added') log_entry = link_placeholder + title + ' - [SJ]' self.log_info(log_entry) notify(["[Click'n'Load notwendig] - " + log_entry], self.configfile) return log_entry def periodical_task(self): if not self.sj: return self.device if self.filename == 'SJ_Serien_Regex': if not self.config.get('regex'): self.log_debug("Suche für SJ-Regex deaktiviert!") return self.device elif self.filename == 'SJ_Staffeln_Regex': if not self.config.get('regex'): self.log_debug("Suche für SJ-Regex deaktiviert!") return self.device elif self.filename == 'MB_Staffeln': if not self.config.get('crawlseasons'): self.log_debug("Suche für SJ-Staffeln deaktiviert!") return self.device if self.empty_list: self.log_debug("Liste ist leer. Stoppe Suche für Serien!" + self.listtype) return self.device try: reject = self.config.get("rejectlist").replace( ",", "|").lower() if len( self.config.get("rejectlist")) > 0 else r"^unmatchable$" except TypeError: reject = r"^unmatchable$" set_sj = self.settings_hash(False) header = False response = False while self.day < 8: if self.last_set_sj == set_sj: try: response = get_url_headers( 'https://' + self.sj + '/api/releases/latest/' + str(self.day), self.configfile, self.dbfile, self.headers, self.scraper) self.scraper = response[1] response = response[0] if self.filename == "MB_Staffeln" or self.filename == "SJ_Staffeln_Regex": feed = j_releases_to_feedparser_dict( response.text, "seasons", 'https://' + self.sj, True) else: feed = j_releases_to_feedparser_dict( response.text, "episodes", 'https://' + self.sj, True) except: print(u"SJ hat die Feed-API angepasst. Breche Suche ab!") feed = False if response: if response.status_code == 304: self.log_debug( "SJ-Feed seit letztem Aufruf nicht aktualisiert - breche Suche ab!" ) return self.device header = True else: try: response = get_url( 'https://' + self.sj + '/api/releases/latest/' + str(self.day), self.configfile, self.dbfile, self.scraper) if self.filename == "MB_Staffeln" or self.filename == "SJ_Staffeln_Regex": feed = j_releases_to_feedparser_dict( response, "seasons", 'https://' + self.sj, True) else: feed = j_releases_to_feedparser_dict( response, "episodes", 'https://' + self.sj, True) except: print(u"SJ hat die Feed-API angepasst. Breche Suche ab!") feed = False self.day += 1 if feed and feed.entries: first_post_sj = feed.entries[0] concat_sj = first_post_sj.title + first_post_sj.published + str( self.settings) + str(self.pattern) sha_sj = hashlib.sha256(concat_sj.encode( 'ascii', 'ignore')).hexdigest() else: self.log_debug("Feed ist leer - breche Suche ab!") return False for post in feed.entries: concat = post.title + post.published + \ str(self.settings) + str(self.pattern) sha = hashlib.sha256(concat.encode('ascii', 'ignore')).hexdigest() if sha == self.last_sha_sj: self.log_debug("Feed ab hier bereits gecrawlt (" + post.title + ") - breche Suche ab!") break series_url = post.series_url title = post.title.replace("-", "-") if self.filename == 'SJ_Serien_Regex': if self.config.get("regex"): if '.german.' in title.lower(): language_id = 1 elif self.rsscrawler.get('english'): language_id = 2 else: language_id = 0 if language_id: m = re.search(self.pattern, title.lower()) if not m and "720p" not in title and "1080p" not in title and "2160p" not in title: m = re.search( self.pattern.replace("480p", "."), title.lower()) self.quality = "480p" if m: if "720p" in title.lower(): self.quality = "720p" if "1080p" in title.lower(): self.quality = "1080p" if "2160p" in title.lower(): self.quality = "2160p" m = re.search(reject, title.lower()) if m: self.log_debug( title + " - Release durch Regex gefunden (trotz rejectlist-Einstellung)" ) title = re.sub(r'\[.*\] ', '', post.title) self.parse_download(series_url, title, language_id) else: self.log_debug( "%s - Englische Releases deaktiviert" % title) else: continue elif self.filename == 'SJ_Staffeln_Regex': if self.config.get("regex"): if '.german.' in title.lower(): language_id = 1 elif self.rsscrawler.get('english'): language_id = 2 else: language_id = 0 if language_id: m = re.search(self.pattern, title.lower()) if not m and "720p" not in title and "1080p" not in title and "2160p" not in title: m = re.search( self.pattern.replace("480p", "."), title.lower()) self.quality = "480p" if m: if "720p" in title.lower(): self.quality = "720p" if "1080p" in title.lower(): self.quality = "1080p" if "2160p" in title.lower(): self.quality = "2160p" m = re.search(reject, title.lower()) if m: self.log_debug( title + " - Release durch Regex gefunden (trotz rejectlist-Einstellung)" ) title = re.sub(r'\[.*\] ', '', post.title) self.parse_download(series_url, title, language_id) else: self.log_debug( "%s - Englische Releases deaktiviert" % title) else: continue else: if self.config.get("quality") != '480p': m = re.search(self.pattern, title.lower()) if m: if '.german.' in title.lower(): language_id = 1 elif self.rsscrawler.get('english'): language_id = 2 else: language_id = 0 if language_id: mm = re.search(self.quality, title.lower()) if mm: mmm = re.search(reject, title.lower()) if mmm: self.log_debug( title + " - Release ignoriert (basierend auf rejectlist-Einstellung)" ) continue if self.rsscrawler.get("surround"): if not re.match( r'.*\.(DTS|DD\+*51|DD\+*71|AC3\.5\.*1)\..*', title): self.log_debug( title + " - Release ignoriert (kein Mehrkanalton)" ) continue try: storage = self.db.retrieve_all(title) except Exception as e: self.log_debug( "Fehler bei Datenbankzugriff: %s, Grund: %s" % (e, title)) return self.device if 'added' in storage: self.log_debug( title + " - Release ignoriert (bereits gefunden)" ) continue self.parse_download( series_url, title, language_id) else: self.log_debug( "%s - Englische Releases deaktiviert" % title) else: m = re.search(self.pattern, title.lower()) if m: if '.german.' in title.lower(): language_id = 1 elif self.rsscrawler.get('english'): language_id = 2 else: language_id = 0 if language_id: if "720p" in title.lower( ) or "1080p" in title.lower( ) or "2160p" in title.lower(): continue mm = re.search(reject, title.lower()) if mm: self.log_debug( title + " Release ignoriert (basierend auf rejectlist-Einstellung)" ) continue if self.rsscrawler.get("surround"): if not re.match( r'.*\.(DTS|DD\+*51|DD\+*71|AC3\.5\.*1)\..*', title): self.log_debug( title + " - Release ignoriert (kein Mehrkanalton)" ) continue title = re.sub(r'\[.*\] ', '', post.title) try: storage = self.db.retrieve_all(title) except Exception as e: self.log_debug( "Fehler bei Datenbankzugriff: %s, Grund: %s" % (e, title)) return self.device if 'added' in storage: self.log_debug( title + " - Release ignoriert (bereits gefunden)" ) continue self.parse_download( series_url, title, language_id) else: self.log_debug( "%s - Englische Releases deaktiviert" % title) if set_sj: new_set_sj = self.settings_hash(True) if set_sj == new_set_sj: self.cdc.delete("SJSet-" + self.filename) self.cdc.store("SJSet-" + self.filename, set_sj) self.cdc.delete("SJ-" + self.filename) self.cdc.store("SJ-" + self.filename, sha_sj) if header and response: self.cdc.delete("SJHeaders-" + self.filename) self.cdc.store("SJHeaders-" + self.filename, response.headers['date']) return self.device