def check_url(configfile, dbfile, scraper=False):
hostnames = RssConfig('Hostnames', configfile)
sj = hostnames.get('sj')
dj = hostnames.get('dj')
sf = hostnames.get('sf')
mb = hostnames.get('mb')
hw = hostnames.get('hw')
hs = hostnames.get('hs')
fx = hostnames.get('fx')
nk = hostnames.get('nk')
dd = hostnames.get('dd')
fc = hostnames.get('fc')
if not scraper:
scraper = cloudscraper.create_scraper()
sj_url = 'https://' + sj
dj_url = 'https://' + sj
sf_url = 'https://' + sf
mb_url = 'https://' + mb
hw_url = 'https://' + hw
fx_url = 'https://' + fx
hs_url = 'https://' + hs + '/collection/neuerscheinungen/'
nk_url = 'https://' + nk
dd_url = 'https://' + dd
fc_url = 'https://' + fc
sj_blocked_proxy = False
dj_blocked_proxy = False
sf_blocked_proxy = False
mb_blocked_proxy = False
hw_blocked_proxy = False
fx_blocked_proxy = False
hs_blocked_proxy = False
nk_blocked_proxy = False
dd_blocked_proxy = False
fc_blocked_proxy = False
sj_blocked = False
dj_blocked = False
sf_blocked = False
mb_blocked = False
hw_blocked = False
fx_blocked = False
hs_blocked = False
nk_blocked = False
dd_blocked = False
fc_blocked = False
db = RssDb(dbfile, 'proxystatus')
db.delete("SJ")
db.delete("DJ")
db.delete("SF")
db.delete("MB")
db.delete("HW")
db.delete("FX")
db.delete("HS")
db.delete("NK")
db.delete("DD")
db.delete("FC")
db_normal = RssDb(dbfile, 'normalstatus')
db_normal.delete("SJ")
db_normal.delete("DJ")
db_normal.delete("SF")
db_normal.delete("MB")
db_normal.delete("HW")
db_normal.delete("FX")
db_normal.delete("HS")
db_normal.delete("NK")
db_normal.delete("DD")
db_normal.delete("FC")
proxy = RssConfig('RSScrawler', configfile).get('proxy')
fallback = RssConfig('RSScrawler', configfile).get('fallback')
if proxy:
proxies = {'http': proxy, 'https': proxy}
if not sj:
db.store("SJ", "Blocked")
else:
try:
if "block." in str(
scraper.get(
sj_url,
proxies=proxies,
timeout=30,
allow_redirects=False).headers.get("location")):
sj_blocked_proxy = True
else:
db.delete("SJ")
except:
sj_blocked_proxy = True
if sj_blocked_proxy:
print(
u"Der Zugriff auf SJ ist mit der aktuellen Proxy-IP nicht möglich!"
)
db.store("SJ", "Blocked")
scraper = cloudscraper.create_scraper()
if not dj:
db.store("DJ", "Blocked")
else:
try:
if "block." in str(
scraper.get(
dj_url,
proxies=proxies,
timeout=30,
allow_redirects=False).headers.get("location")):
dj_blocked_proxy = True
else:
db.delete("DJ")
except:
dj_blocked_proxy = True
if dj_blocked_proxy:
print(
u"Der Zugriff auf DJ ist mit der aktuellen Proxy-IP nicht möglich!"
)
db.store("DJ", "Blocked")
scraper = cloudscraper.create_scraper()
if not sf:
db.store("SF", "Blocked")
else:
try:
if "block." in str(
scraper.get(
sf_url,
proxies=proxies,
timeout=30,
allow_redirects=False).headers.get("location")):
sf_blocked_proxy = True
else:
db.delete("SF")
except:
sf_blocked_proxy = True
if sf_blocked_proxy:
print(
u"Der Zugriff auf SF ist mit der aktuellen Proxy-IP nicht möglich!"
)
db.store("SF", "Blocked")
scraper = cloudscraper.create_scraper()
if not mb:
db.store("MB", "Blocked")
else:
try:
if "<Response [403]>" in str(
scraper.get(mb_url,
proxies=proxies,
timeout=30,
allow_redirects=False)):
mb_blocked_proxy = True
else:
db.delete("MB")
except:
mb_blocked_proxy = True
if mb_blocked_proxy:
print(
u"Der Zugriff auf MB ist mit der aktuellen Proxy-IP nicht möglich!"
)
db.store("MB", "Blocked")
scraper = cloudscraper.create_scraper()
if not hw:
db.store("HW", "Blocked")
else:
try:
if "<Response [403]>" in str(
scraper.get(hw_url,
proxies=proxies,
timeout=30,
allow_redirects=False)):
hw_blocked_proxy = True
else:
db.delete("HW")
except:
hw_blocked_proxy = True
if hw_blocked_proxy:
print(
u"Der Zugriff auf HW ist mit der aktuellen Proxy-IP nicht möglich!"
)
db.store("HW", "Blocked")
scraper = cloudscraper.create_scraper()
if not fx:
db.store("FX", "Blocked")
else:
try:
if "<Response [403]>" in str(
scraper.get(fx_url,
proxies=proxies,
timeout=30,
allow_redirects=False)):
fx_blocked_proxy = True
else:
db.delete("FX")
except:
fx_blocked_proxy = True
session = requests.session()
session.headers = scraper.headers
session.cookies = scraper.cookies
session.verify = False
if "<Response [200]>" in str(
session.get(fx_url,
proxies=proxies,
timeout=30,
allow_redirects=False)):
fx_blocked_proxy = False
if fx_blocked_proxy:
print(
u"Der Zugriff auf FX ist mit der aktuellen Proxy-IP nicht möglich!"
)
db.store("FX", "Blocked")
scraper = cloudscraper.create_scraper()
if not hs:
db.store("HS", "Blocked")
else:
try:
if "200" not in str(
scraper.get(hs_url, timeout=30,
allow_redirects=False).status_code):
hs_blocked_proxy = True
else:
db.delete("HS")
except:
hs_blocked_proxy = True
if hs_blocked_proxy:
print(
u"Der Zugriff auf HS ist mit der aktuellen Proxy-IP nicht möglich!"
)
db.store("HS", "Blocked")
scraper = cloudscraper.create_scraper()
if not nk:
db.store("NK", "Blocked")
else:
try:
if "200" not in str(
scraper.get(nk_url, timeout=30,
allow_redirects=False).status_code):
nk_blocked_proxy = True
else:
db.delete("NK")
except:
nk_blocked_proxy = True
if nk_blocked_proxy:
print(
u"Der Zugriff auf NK ist mit der aktuellen Proxy-IP nicht möglich!"
)
db.store("NK", "Blocked")
scraper = cloudscraper.create_scraper()
if not dd:
db.store("DD", "Blocked")
else:
try:
if "200" not in str(
scraper.get(dd_url, timeout=30,
allow_redirects=False).status_code):
dd_blocked_proxy = True
else:
db.delete("DD")
except:
dd_blocked_proxy = True
if dd_blocked_proxy:
print(
u"Der Zugriff auf DD ist mit der aktuellen Proxy-IP nicht möglich!"
)
db.store("DD", "Blocked")
scraper = cloudscraper.create_scraper()
if not fc:
db.store("FC", "Blocked")
else:
try:
if "200" not in str(
scraper.get(fc_url, timeout=30).status_code):
fc_blocked_proxy = True
else:
db.delete("FC")
except:
fc_blocked_proxy = True
if fc_blocked_proxy:
print(
u"Der Zugriff auf FC ist mit der aktuellen Proxy-IP nicht möglich!"
)
db.store("FC", "Blocked")
scraper = cloudscraper.create_scraper()
if not proxy or (proxy and sj_blocked_proxy and fallback):
if not sj:
db.store("SJ", "Blocked")
else:
try:
if "block." in str(
scraper.get(
sj_url, timeout=30,
allow_redirects=False).headers.get("location")):
sj_blocked = True
except:
sj_blocked = True
if sj_blocked:
db_normal.store("SJ", "Blocked")
print(
u"Der Zugriff auf SJ ist mit der aktuellen IP nicht möglich!"
)
if not proxy or (proxy and dj_blocked_proxy and fallback):
if not dj:
db.store("DJ", "Blocked")
else:
try:
if "block." in str(
scraper.get(
dj_url, timeout=30,
allow_redirects=False).headers.get("location")):
dj_blocked = True
except:
dj_blocked = True
if dj_blocked:
db_normal.store("DJ", "Blocked")
print(
u"Der Zugriff auf DJ ist mit der aktuellen IP nicht möglich!"
)
if not sf:
db.store("SF", "Blocked")
else:
try:
if "block." in str(
scraper.get(
sf_url, timeout=30,
allow_redirects=False).headers.get("location")):
sf_blocked = True
except:
sf_blocked = True
if sf_blocked:
db_normal.store("SF", "Blocked")
print(
u"Der Zugriff auf SF ist mit der aktuellen IP nicht möglich!"
)
if not proxy or (proxy and mb_blocked_proxy and fallback):
if not mb:
db.store("MB", "Blocked")
else:
try:
if "<Response [403]>" in str(
scraper.get(mb_url, timeout=30,
allow_redirects=False)):
mb_blocked = True
except:
mb_blocked = True
if mb_blocked:
db_normal.store("MB", "Blocked")
print(
u"Der Zugriff auf MB ist mit der aktuellen IP nicht möglich!"
)
if not proxy or (proxy and hw_blocked_proxy and fallback):
if not hw:
db.store("HW", "Blocked")
else:
try:
if "<Response [403]>" in str(
scraper.get(hw_url, timeout=30,
allow_redirects=False)):
hw_blocked = True
except:
hw_blocked = True
if hw_blocked:
db_normal.store("HW", "Blocked")
print(
u"Der Zugriff auf HW ist mit der aktuellen IP nicht möglich!"
)
if not proxy or (proxy and fx_blocked_proxy and fallback):
if not fx:
db.store("FX", "Blocked")
else:
try:
if "<Response [403]>" in str(
scraper.get(fx_url, timeout=30,
allow_redirects=False)):
fx_blocked = True
except:
fx_blocked = True
session = requests.session()
session.headers = scraper.headers
session.cookies = scraper.cookies
session.verify = False
if "<Response [200]>" in str(
session.get(fx_url, timeout=30,
allow_redirects=False)):
fx_blocked = False
if fx_blocked:
db_normal.store("FX", "Blocked")
print(
u"Der Zugriff auf FX ist mit der aktuellen IP nicht möglich!"
)
if not proxy or (proxy and hs_blocked_proxy and fallback):
if not hs:
db.store("HS", "Blocked")
else:
try:
if "200" not in str(
scraper.get(hs_url, timeout=30,
allow_redirects=False).status_code):
hs_blocked = True
except:
hs_blocked = True
if hs_blocked:
db_normal.store("HS", "Blocked")
print(
u"Der Zugriff auf HS ist mit der aktuellen IP nicht möglich!"
)
if not proxy or (proxy and nk_blocked_proxy and fallback):
if not nk:
db.store("NK", "Blocked")
else:
try:
if "200" not in str(
scraper.get(nk_url, timeout=30,
allow_redirects=False).status_code):
nk_blocked = True
except:
nk_blocked = True
if nk_blocked:
db_normal.store("NK", "Blocked")
print(
u"Der Zugriff auf NK ist mit der aktuellen IP nicht möglich!"
)
if not proxy or (proxy and dd_blocked_proxy and fallback):
if not dd:
db.store("DD", "Blocked")
else:
try:
if "200" not in str(
scraper.get(dd_url, timeout=30,
allow_redirects=False).status_code):
dd_blocked = True
except:
dd_blocked = True
if dd_blocked:
db_normal.store("DD", "Blocked")
print(
u"Der Zugriff auf DD ist mit der aktuellen IP nicht möglich!"
)
if not proxy or (proxy and fc_blocked_proxy and fallback):
if not fc:
db.store("FC", "Blocked")
else:
try:
if "200" not in str(
scraper.get(fc_url, timeout=30).status_code):
fc_blocked = True
except:
fc_blocked = True
if fc_blocked:
db_normal.store("FC", "Blocked")
print(
u"Der Zugriff auf FC ist mit der aktuellen IP nicht möglich!"
)
return scraper
class SF:
def __init__(self, configfile, dbfile, device, logging, scraper, filename,
internal_name):
self._INTERNAL_NAME = internal_name
self.configfile = configfile
self.dbfile = dbfile
self.device = device
self.hostnames = RssConfig('Hostnames', self.configfile)
self.sf = self.hostnames.get('sf')
self.config = RssConfig(self._INTERNAL_NAME, self.configfile)
self.rsscrawler = RssConfig("RSScrawler", self.configfile)
self.hevc_retail = self.config.get("hevc_retail")
self.retail_only = self.config.get("retail_only")
self.hoster_fallback = self.config.get("hoster_fallback")
self.hosters = RssConfig("Hosters", configfile).get_section()
self.log_info = logging.info
self.log_error = logging.error
self.log_debug = logging.debug
self.scraper = scraper
self.filename = filename
self.db = RssDb(self.dbfile, 'rsscrawler')
self.quality = self.config.get("quality")
self.cdc = RssDb(self.dbfile, 'cdc')
self.last_set_sf = self.cdc.retrieve("SFSet-" + self.filename)
self.last_sha_sf = self.cdc.retrieve("SF-" + self.filename)
self.headers = {
'If-Modified-Since':
str(self.cdc.retrieve("SFHeaders-" + self.filename))
}
settings = [
"quality", "rejectlist", "regex", "hevc_retail", "retail_only",
"hoster_fallback"
]
self.settings = []
self.settings.append(self.rsscrawler.get("english"))
self.settings.append(self.rsscrawler.get("surround"))
self.settings.append(self.hosters)
for s in settings:
self.settings.append(self.config.get(s))
self.empty_list = False
if self.filename == 'SJ_Staffeln_Regex':
self.level = 3
elif self.filename == 'MB_Staffeln':
self.seasonssource = self.config.get('seasonssource').lower()
self.level = 2
elif self.filename == 'SJ_Serien_Regex':
self.level = 1
else:
self.level = 0
self.pattern = r'^(' + "|".join(
self.get_series_list(self.filename, self.level)).lower() + ')'
self.listtype = ""
self.day = 0
def settings_hash(self, refresh):
if refresh:
settings = [
"quality", "rejectlist", "regex", "hevc_retail", "retail_only",
"hoster_fallback"
]
self.settings = []
self.settings.append(self.rsscrawler.get("english"))
self.settings.append(self.rsscrawler.get("surround"))
self.settings.append(self.hosters)
for s in settings:
self.settings.append(self.config.get(s))
self.pattern = r'^(' + "|".join(
self.get_series_list(self.filename, self.level)).lower() + ')'
set_sf = str(self.settings) + str(self.pattern)
return hashlib.sha256(set_sf.encode('ascii', 'ignore')).hexdigest()
def get_series_list(self, liste, series_type):
if series_type == 1:
self.listtype = " (RegEx)"
elif series_type == 2:
self.listtype = " (Staffeln)"
elif series_type == 3:
self.listtype = " (Staffeln/RegEx)"
cont = ListDb(self.dbfile, liste).retrieve()
titles = []
if cont:
for title in cont:
if title:
title = title.replace(" ", ".")
titles.append(title)
if not titles:
self.empty_list = True
return titles
def parse_download(self, series_url, title, language_id):
if not check_valid_release(title, self.retail_only, self.hevc_retail,
self.dbfile):
self.log_debug(
title +
u" - Release ignoriert (Gleiche oder bessere Quelle bereits vorhanden)"
)
return
if self.filename == 'MB_Staffeln':
if not self.config.get("seasonpacks"):
staffelpack = re.search(r"s\d.*(-|\.).*s\d", title.lower())
if staffelpack:
self.log_debug("%s - Release ignoriert (Staffelpaket)" %
title)
return
if not re.search(self.seasonssource, title.lower()):
self.log_debug(title + " - Release hat falsche Quelle")
return
try:
if language_id == 2:
lang = 'EN'
else:
lang = 'DE'
epoch = str(datetime.datetime.now().timestamp()).replace('.',
'')[:-3]
api_url = series_url + '?lang=' + lang + '&_=' + epoch
response = get_url(api_url, self.configfile, self.dbfile,
self.scraper)
info = json.loads(response)
is_episode = re.findall(r'.*\.(s\d{1,3}e\d{1,3})\..*', title,
re.IGNORECASE)
if is_episode:
episode_string = re.findall(r'.*S\d{1,3}(E\d{1,3}).*',
is_episode[0])[0].lower()
season_string = re.findall(r'.*(S\d{1,3})E\d{1,3}.*',
is_episode[0])[0].lower()
season_title = rreplace(
title.lower().replace(episode_string, ''), "-", ".*",
1).lower()
season_title = season_title.replace(".untouched",
".*").replace(
".dd+51", ".dd.51")
episode = str(int(episode_string.replace("e", "")))
season = str(int(season_string.replace("s", "")))
episode_name = re.findall(r'.*\.s\d{1,3}(\..*).german',
season_title, re.IGNORECASE)
if episode_name:
season_title = season_title.replace(episode_name[0], '')
codec_tags = [".h264", ".x264"]
for tag in codec_tags:
season_title = season_title.replace(tag, ".*264")
web_tags = [".web-rip", ".webrip", ".webdl", ".web-dl"]
for tag in web_tags:
season_title = season_title.replace(tag, ".web.*")
else:
season = False
episode = False
season_title = title
multiple_episodes = re.findall(r'(e\d{1,3}-e*\d{1,3}\.)',
season_title, re.IGNORECASE)
if multiple_episodes:
season_title = season_title.replace(
multiple_episodes[0], '.*')
content = BeautifulSoup(info['html'], 'lxml')
releases = content.find(
"small", text=re.compile(season_title,
re.IGNORECASE)).parent.parent.parent
links = releases.findAll("div", {'class': 'row'})[1].findAll('a')
valid = False
for link in links:
download_link = link['href']
if check_hoster(link.text.replace('\n', ''), self.configfile):
valid = True
break
if not valid and not self.hoster_fallback:
storage = self.db.retrieve_all(title)
if 'added' not in storage and 'notdl' not in storage:
wrong_hoster = '[SF/Hoster fehlt] - ' + title
if 'wrong_hoster' not in storage:
self.log_info(wrong_hoster)
self.db.store(title, 'wrong_hoster')
notify([wrong_hoster], self.configfile)
else:
self.log_debug(wrong_hoster)
else:
return self.send_package(title, download_link, language_id,
season, episode)
except:
print(
u"SF hat die Serien-API angepasst. Breche Download-Prüfung ab!"
)
def send_package(self, title, download_link, language_id, season, episode):
englisch = ""
if language_id == 2:
englisch = "/Englisch"
if self.filename == 'SJ_Serien_Regex':
link_placeholder = '[Episode/RegEx' + englisch + '] - '
elif self.filename == 'SJ_Serien':
link_placeholder = '[Episode' + englisch + '] - '
elif self.filename == 'SJ_Staffeln_Regex]':
link_placeholder = '[Staffel/RegEx' + englisch + '] - '
else:
link_placeholder = '[Staffel' + englisch + '] - '
try:
storage = self.db.retrieve_all(title)
except Exception as e:
self.log_debug("Fehler bei Datenbankzugriff: %s, Grund: %s" %
(e, title))
return
if 'added' in storage or 'notdl' in storage:
self.log_debug(title + " - Release ignoriert (bereits gefunden)")
else:
download_link = 'https://' + self.sf + download_link
if season and episode:
download_link = download_link.replace(
'&_=', '&season=' + str(season) + '&episode=' +
str(episode) + '&_=')
download = add_decrypt(title, download_link, self.sf, self.dbfile)
if download:
self.db.store(title, 'added')
log_entry = link_placeholder + title + ' - [SF]'
self.log_info(log_entry)
notify(["[Click'n'Load notwendig] - " + log_entry],
self.configfile)
return log_entry
def periodical_task(self):
if not self.sf:
return self.device
if self.filename == 'SJ_Serien_Regex':
if not self.config.get('regex'):
self.log_debug("Suche für SF-Regex deaktiviert!")
return self.device
elif self.filename == 'SJ_Staffeln_Regex':
if not self.config.get('regex'):
self.log_debug("Suche für SF-Regex deaktiviert!")
return self.device
elif self.filename == 'MB_Staffeln':
if not self.config.get('crawlseasons'):
self.log_debug("Suche für SF-Staffeln deaktiviert!")
return self.device
if self.empty_list:
self.log_debug("Liste ist leer. Stoppe Suche für Serien!" +
self.listtype)
return self.device
try:
reject = self.config.get("rejectlist").replace(
",", "|").lower() if len(
self.config.get("rejectlist")) > 0 else r"^unmatchable$"
except TypeError:
reject = r"^unmatchable$"
set_sf = self.settings_hash(False)
header = False
response = False
while self.day < 8:
if self.last_set_sf == set_sf:
try:
delta = (
datetime.datetime.now() -
datetime.timedelta(days=self.day)).strftime("%Y-%m-%d")
response = get_url_headers(
'https://' + self.sf + '/updates/' + delta,
self.configfile, self.dbfile, self.headers,
self.scraper)
self.scraper = response[1]
response = response[0]
if self.filename == "MB_Staffeln" or self.filename == "SJ_Staffeln_Regex":
feed = sf_releases_to_feedparser_dict(
response.text, "seasons", 'https://' + self.sf,
True)
else:
feed = sf_releases_to_feedparser_dict(
response.text, "episodes", 'https://' + self.sf,
True)
except:
print(u"SF hat die Feed-API angepasst. Breche Suche ab!")
feed = False
if response:
if response.status_code == 304:
self.log_debug(
"SF-Feed seit letztem Aufruf nicht aktualisiert - breche Suche ab!"
)
return self.device
header = True
else:
try:
delta = (
datetime.datetime.now() -
datetime.timedelta(days=self.day)).strftime("%Y-%m-%d")
response = get_url(
'https://' + self.sf + '/updates/' + delta,
self.configfile, self.dbfile, self.scraper)
if self.filename == "MB_Staffeln" or self.filename == "SJ_Staffeln_Regex":
feed = sf_releases_to_feedparser_dict(
response, "seasons", 'https://' + self.sf, True)
else:
feed = sf_releases_to_feedparser_dict(
response, "episodes", 'https://' + self.sf, True)
except:
print(u"SF hat die Feed-API angepasst. Breche Suche ab!")
feed = False
self.day += 1
if feed and feed.entries:
first_post_sf = feed.entries[0]
concat_sf = first_post_sf.title + first_post_sf.published + str(
self.settings) + str(self.pattern)
sha_sf = hashlib.sha256(concat_sf.encode(
'ascii', 'ignore')).hexdigest()
else:
self.log_debug("Feed ist leer - breche Suche ab!")
return False
for post in feed.entries:
concat = post.title + post.published + \
str(self.settings) + str(self.pattern)
sha = hashlib.sha256(concat.encode('ascii',
'ignore')).hexdigest()
if sha == self.last_sha_sf:
self.log_debug("Feed ab hier bereits gecrawlt (" +
post.title + ") - breche Suche ab!")
break
series_url = post.series_url
title = post.title.replace("-", "-")
if self.filename == 'SJ_Serien_Regex':
if self.config.get("regex"):
if '.german.' in title.lower():
language_id = 1
elif self.rsscrawler.get('english'):
language_id = 2
else:
language_id = 0
if language_id:
m = re.search(self.pattern, title.lower())
if not m and "720p" not in title and "1080p" not in title and "2160p" not in title:
m = re.search(
self.pattern.replace("480p", "."),
title.lower())
self.quality = "480p"
if m:
if "720p" in title.lower():
self.quality = "720p"
if "1080p" in title.lower():
self.quality = "1080p"
if "2160p" in title.lower():
self.quality = "2160p"
m = re.search(reject, title.lower())
if m:
self.log_debug(
title +
" - Release durch Regex gefunden (trotz rejectlist-Einstellung)"
)
title = re.sub(r'\[.*\] ', '', post.title)
self.parse_download(series_url, title,
language_id)
else:
self.log_debug(
"%s - Englische Releases deaktiviert" % title)
else:
continue
elif self.filename == 'SJ_Staffeln_Regex':
if self.config.get("regex"):
if '.german.' in title.lower():
language_id = 1
elif self.rsscrawler.get('english'):
language_id = 2
else:
language_id = 0
if language_id:
m = re.search(self.pattern, title.lower())
if not m and "720p" not in title and "1080p" not in title and "2160p" not in title:
m = re.search(
self.pattern.replace("480p", "."),
title.lower())
self.quality = "480p"
if m:
if "720p" in title.lower():
self.quality = "720p"
if "1080p" in title.lower():
self.quality = "1080p"
if "2160p" in title.lower():
self.quality = "2160p"
m = re.search(reject, title.lower())
if m:
self.log_debug(
title +
" - Release durch Regex gefunden (trotz rejectlist-Einstellung)"
)
title = re.sub(r'\[.*\] ', '', post.title)
self.parse_download(series_url, title,
language_id)
else:
self.log_debug(
"%s - Englische Releases deaktiviert" % title)
else:
continue
else:
if self.config.get("quality") != '480p':
m = re.search(self.pattern, title.lower())
if m:
if '.german.' in title.lower():
language_id = 1
elif self.rsscrawler.get('english'):
language_id = 2
else:
language_id = 0
if language_id:
mm = re.search(self.quality, title.lower())
if mm:
mmm = re.search(reject, title.lower())
if mmm:
self.log_debug(
title +
" - Release ignoriert (basierend auf rejectlist-Einstellung)"
)
continue
if self.rsscrawler.get("surround"):
if not re.match(
r'.*\.(DTS|DD\+*51|DD\+*71|AC3\.5\.*1)\..*',
title):
self.log_debug(
title +
" - Release ignoriert (kein Mehrkanalton)"
)
continue
try:
storage = self.db.retrieve_all(title)
except Exception as e:
self.log_debug(
"Fehler bei Datenbankzugriff: %s, Grund: %s"
% (e, title))
return self.device
if 'added' in storage:
self.log_debug(
title +
" - Release ignoriert (bereits gefunden)"
)
continue
self.parse_download(
series_url, title, language_id)
else:
self.log_debug(
"%s - Englische Releases deaktiviert" %
title)
else:
m = re.search(self.pattern, title.lower())
if m:
if '.german.' in title.lower():
language_id = 1
elif self.rsscrawler.get('english'):
language_id = 2
else:
language_id = 0
if language_id:
if "720p" in title.lower(
) or "1080p" in title.lower(
) or "2160p" in title.lower():
continue
mm = re.search(reject, title.lower())
if mm:
self.log_debug(
title +
" Release ignoriert (basierend auf rejectlist-Einstellung)"
)
continue
if self.rsscrawler.get("surround"):
if not re.match(
r'.*\.(DTS|DD\+*51|DD\+*71|AC3\.5\.*1)\..*',
title):
self.log_debug(
title +
" - Release ignoriert (kein Mehrkanalton)"
)
continue
title = re.sub(r'\[.*\] ', '', post.title)
try:
storage = self.db.retrieve_all(title)
except Exception as e:
self.log_debug(
"Fehler bei Datenbankzugriff: %s, Grund: %s"
% (e, title))
return self.device
if 'added' in storage:
self.log_debug(
title +
" - Release ignoriert (bereits gefunden)"
)
continue
self.parse_download(
series_url, title, language_id)
else:
self.log_debug(
"%s - Englische Releases deaktiviert" %
title)
if set_sf:
new_set_sf = self.settings_hash(True)
if set_sf == new_set_sf:
self.cdc.delete("SFSet-" + self.filename)
self.cdc.store("SFSet-" + self.filename, set_sf)
self.cdc.delete("SF-" + self.filename)
self.cdc.store("SF-" + self.filename, sha_sf)
if header and response:
self.cdc.delete("SFHeaders-" + self.filename)
self.cdc.store("SFHeaders-" + self.filename,
response.headers['date'])
return self.device
def download(configfile,
dbfile,
device,
title,
subdir,
old_links,
password,
full_path=None,
autostart=False):
try:
if not device or not is_device(device):
device = get_device(configfile)
if isinstance(old_links, list):
links = []
for link in old_links:
if link not in links:
links.append(link)
else:
links = [old_links]
links = str(links).replace(" ", "")
crawljobs = RssConfig('Crawljobs', configfile)
usesubdir = crawljobs.get("subdir")
priority = "DEFAULT"
if full_path:
path = full_path
else:
if usesubdir:
path = subdir + "/<jd:packagename>"
else:
path = "<jd:packagename>"
if "Remux" in path:
priority = "LOWER"
try:
device.linkgrabber.add_links(
params=[{
"autostart": autostart,
"links": links,
"packageName": title,
"extractPassword": password,
"priority": priority,
"downloadPassword": password,
"destinationFolder": path,
"comment": "RSScrawler by rix1337",
"overwritePackagizerRules": False
}])
except rsscrawler.myjdapi.TokenExpiredException:
device = get_device(configfile)
if not device or not is_device(device):
return False
device.linkgrabber.add_links(
params=[{
"autostart": autostart,
"links": links,
"packageName": title,
"extractPassword": password,
"priority": priority,
"downloadPassword": password,
"destinationFolder": path,
"comment": "RSScrawler by rix1337",
"overwritePackagizerRules": False
}])
db = RssDb(dbfile, 'crawldog')
if db.retrieve(title):
db.delete(title)
db.store(title, 'retried')
else:
db.store(title, 'added')
return device
except rsscrawler.myjdapi.MYJDException as e:
print(u"Fehler bei der Verbindung mit MyJDownloader: " + str(e))
return False
def crawldog(configfile, dbfile):
disable_request_warnings(InsecureRequestWarning)
crawljobs = RssConfig('Crawljobs', configfile)
autostart = crawljobs.get("autostart")
db = RssDb(dbfile, 'crawldog')
grabber_was_collecting = False
device = False
while True:
try:
if not device or not is_device(device):
device = get_device(configfile)
myjd_packages = get_info(configfile, device)
grabber_collecting = myjd_packages[2]
if grabber_was_collecting or grabber_collecting:
grabber_was_collecting = grabber_collecting
time.sleep(5)
else:
packages_in_downloader_decrypted = myjd_packages[4][0]
packages_in_linkgrabber_decrypted = myjd_packages[4][1]
offline_packages = myjd_packages[4][2]
encrypted_packages = myjd_packages[4][3]
try:
watched_titles = db.retrieve_all_titles()
except:
watched_titles = False
notify_list = []
if packages_in_downloader_decrypted or packages_in_linkgrabber_decrypted or offline_packages or encrypted_packages:
if watched_titles:
for title in watched_titles:
if packages_in_downloader_decrypted:
for package in packages_in_downloader_decrypted:
if title[0] in package[
'name'] or title[0].replace(
".", " ") in package['name']:
check = hoster_check(
configfile, device, [package],
title[0], [0])
device = check[0]
if device:
db.delete(title[0])
if packages_in_linkgrabber_decrypted:
for package in packages_in_linkgrabber_decrypted:
if title[0] in package[
'name'] or title[0].replace(
".", " ") in package['name']:
check = hoster_check(
configfile, device, [package],
title[0], [0])
device = check[0]
episode = RssDb(
dbfile,
'episode_remover').retrieve(
title[0])
if episode:
filenames = package['filenames']
if len(filenames) > 1:
fname_episodes = []
for fname in filenames:
try:
if re.match(
r'.*S\d{1,3}E\d{1,3}.*',
fname,
flags=re.
IGNORECASE):
fname = re.findall(
r'S\d{1,3}E(\d{1,3})',
fname,
flags=re.
IGNORECASE
).pop()
else:
fname = fname.replace(
"hddl8",
"").replace(
"dd51", ""
).replace(
"264", ""
).replace(
"265", "")
except:
fname = fname.replace(
"hddl8",
"").replace(
"dd51",
"").replace(
"264", ""
).replace(
"265", "")
fname_episode = "".join(
re.findall(
r'\d+',
fname.split(
".part")[0]))
try:
fname_episodes.append(
str(
int(fname_episode
)))
except:
pass
replacer = longest_substr(
fname_episodes)
new_fname_episodes = []
for new_ep_fname in fname_episodes:
try:
new_fname_episodes.append(
str(
int(
new_ep_fname
.replace(
replacer,
""))))
except:
pass
replacer = longest_substr(
new_fname_episodes)
newer_fname_episodes = []
for new_ep_fname in new_fname_episodes:
try:
newer_fname_episodes.append(
str(
int(
re.sub(
replacer,
"",
new_ep_fname,
1))))
except:
pass
replacer = longest_substr(
newer_fname_episodes)
even_newer_fname_episodes = []
for newer_ep_fname in newer_fname_episodes:
try:
even_newer_fname_episodes.append(
str(
int(
re.sub(
replacer,
"",
newer_ep_fname,
1))))
except:
pass
if even_newer_fname_episodes:
fname_episodes = even_newer_fname_episodes
elif newer_fname_episodes:
fname_episodes = newer_fname_episodes
elif new_fname_episodes:
fname_episodes = new_fname_episodes
delete_linkids = []
pos = 0
for delete_id in package[
'linkids']:
if str(episode) != str(
fname_episodes[pos]
):
delete_linkids.append(
delete_id)
pos += 1
if delete_linkids:
delete_uuids = [
package['uuid']
]
RssDb(
dbfile,
'episode_remover'
).delete(title[0])
device = remove_from_linkgrabber(
configfile, device,
delete_linkids,
delete_uuids)
if autostart:
device = move_to_downloads(
configfile, device,
package['linkids'],
[package['uuid']])
if device:
db.delete(title[0])
if offline_packages:
for package in offline_packages:
if title[0] in package[
'name'] or title[0].replace(
".", " ") in package['name']:
notify_list.append("[Offline] - " +
title[0])
print((u"[Offline] - " + title[0]))
db.delete(title[0])
if encrypted_packages:
for package in encrypted_packages:
if title[0] in package[
'name'] or title[0].replace(
".", " ") in package['name']:
if title[1] == 'added':
if retry_decrypt(
configfile, dbfile, device,
package['linkids'],
[package['uuid']],
package['urls']):
db.delete(title[0])
db.store(title[0], 'retried')
else:
add_decrypt(
package['name'],
package['url'], "", dbfile)
device = remove_from_linkgrabber(
configfile, device,
package['linkids'],
[package['uuid']])
notify_list.append(
"[Click'n'Load notwendig] - " +
title[0])
print(
u"[Click'n'Load notwendig] - "
+ title[0])
db.delete(title[0])
else:
if not grabber_collecting:
db.reset()
if notify_list:
notify(notify_list, configfile)
time.sleep(30)
except Exception:
traceback.print_exc()
time.sleep(30)
class SJ:
def __init__(self, configfile, dbfile, device, logging, scraper, filename,
internal_name):
self._INTERNAL_NAME = internal_name
self.configfile = configfile
self.dbfile = dbfile
self.device = device
self.hostnames = RssConfig('Hostnames', self.configfile)
self.sj = self.hostnames.get('sj')
self.config = RssConfig(self._INTERNAL_NAME, self.configfile)
self.rsscrawler = RssConfig("RSScrawler", self.configfile)
self.hevc_retail = self.config.get("hevc_retail")
self.retail_only = self.config.get("retail_only")
self.hoster_fallback = self.config.get("hoster_fallback")
self.hosters = RssConfig("Hosters", configfile).get_section()
self.log_info = logging.info
self.log_error = logging.error
self.log_debug = logging.debug
self.scraper = scraper
self.filename = filename
self.db = RssDb(self.dbfile, 'rsscrawler')
self.quality = self.config.get("quality")
self.cdc = RssDb(self.dbfile, 'cdc')
self.last_set_sj = self.cdc.retrieve("SJSet-" + self.filename)
self.last_sha_sj = self.cdc.retrieve("SJ-" + self.filename)
self.headers = {
'If-Modified-Since':
str(self.cdc.retrieve("SJHeaders-" + self.filename))
}
settings = [
"quality", "rejectlist", "regex", "hevc_retail", "retail_only",
"hoster_fallback"
]
self.settings = []
self.settings.append(self.rsscrawler.get("english"))
self.settings.append(self.rsscrawler.get("surround"))
self.settings.append(self.hosters)
for s in settings:
self.settings.append(self.config.get(s))
self.empty_list = False
if self.filename == 'SJ_Staffeln_Regex':
self.level = 3
elif self.filename == 'MB_Staffeln':
self.seasonssource = self.config.get('seasonssource').lower()
self.level = 2
elif self.filename == 'SJ_Serien_Regex':
self.level = 1
else:
self.level = 0
self.pattern = r'^(' + "|".join(
self.get_series_list(self.filename, self.level)).lower() + ')'
self.listtype = ""
self.day = 0
def settings_hash(self, refresh):
if refresh:
settings = [
"quality", "rejectlist", "regex", "hevc_retail", "retail_only",
"hoster_fallback"
]
self.settings = []
self.settings.append(self.rsscrawler.get("english"))
self.settings.append(self.rsscrawler.get("surround"))
self.settings.append(self.hosters)
for s in settings:
self.settings.append(self.config.get(s))
self.pattern = r'^(' + "|".join(
self.get_series_list(self.filename, self.level)).lower() + ')'
set_sj = str(self.settings) + str(self.pattern)
return hashlib.sha256(set_sj.encode('ascii', 'ignore')).hexdigest()
def get_series_list(self, liste, series_type):
if series_type == 1:
self.listtype = " (RegEx)"
elif series_type == 2:
self.listtype = " (Staffeln)"
elif series_type == 3:
self.listtype = " (Staffeln/RegEx)"
cont = ListDb(self.dbfile, liste).retrieve()
titles = []
if cont:
for title in cont:
if title:
title = title.replace(" ", ".")
titles.append(title)
if not titles:
self.empty_list = True
return titles
def parse_download(self, series_url, title, language_id):
if not check_valid_release(title, self.retail_only, self.hevc_retail,
self.dbfile):
self.log_debug(
title +
u" - Release ignoriert (Gleiche oder bessere Quelle bereits vorhanden)"
)
return
if self.filename == 'MB_Staffeln':
if not self.config.get("seasonpacks"):
staffelpack = re.search(r"s\d.*(-|\.).*s\d", title.lower())
if staffelpack:
self.log_debug("%s - Release ignoriert (Staffelpaket)" %
title)
return
if not re.search(self.seasonssource, title.lower()):
self.log_debug(title + " - Release hat falsche Quelle")
return
try:
series_info = get_url(series_url, self.configfile, self.dbfile)
series_id = re.findall(r'data-mediaid="(.*?)"', series_info)[0]
api_url = 'https://' + self.sj + '/api/media/' + series_id + '/releases'
response = get_url(api_url, self.configfile, self.dbfile,
self.scraper)
seasons = json.loads(response)
for season in seasons:
season = seasons[season]
for item in season['items']:
if item['name'] == title:
valid = False
for hoster in item['hoster']:
if hoster:
if check_hoster(hoster, self.configfile):
valid = True
if not valid and not self.hoster_fallback:
storage = self.db.retrieve_all(title)
if 'added' not in storage and 'notdl' not in storage:
wrong_hoster = '[SJ/Hoster fehlt] - ' + title
if 'wrong_hoster' not in storage:
print(wrong_hoster)
self.db.store(title, 'wrong_hoster')
notify([wrong_hoster], self.configfile)
else:
self.log_debug(wrong_hoster)
else:
return self.send_package(title, series_url,
language_id)
except:
print(
u"SJ hat die Serien-API angepasst. Breche Download-Prüfung ab!"
)
def send_package(self, title, series_url, language_id):
englisch = ""
if language_id == 2:
englisch = "/Englisch"
if self.filename == 'SJ_Serien_Regex':
link_placeholder = '[Episode/RegEx' + englisch + '] - '
elif self.filename == 'SJ_Serien':
link_placeholder = '[Episode' + englisch + '] - '
elif self.filename == 'SJ_Staffeln_Regex]':
link_placeholder = '[Staffel/RegEx' + englisch + '] - '
else:
link_placeholder = '[Staffel' + englisch + '] - '
try:
storage = self.db.retrieve_all(title)
except Exception as e:
self.log_debug("Fehler bei Datenbankzugriff: %s, Grund: %s" %
(e, title))
return
if 'added' in storage or 'notdl' in storage:
self.log_debug(title + " - Release ignoriert (bereits gefunden)")
else:
download = add_decrypt(title, series_url, self.sj, self.dbfile)
if download:
self.db.store(title, 'added')
log_entry = link_placeholder + title + ' - [SJ]'
self.log_info(log_entry)
notify(["[Click'n'Load notwendig] - " + log_entry],
self.configfile)
return log_entry
def periodical_task(self):
if not self.sj:
return self.device
if self.filename == 'SJ_Serien_Regex':
if not self.config.get('regex'):
self.log_debug("Suche für SJ-Regex deaktiviert!")
return self.device
elif self.filename == 'SJ_Staffeln_Regex':
if not self.config.get('regex'):
self.log_debug("Suche für SJ-Regex deaktiviert!")
return self.device
elif self.filename == 'MB_Staffeln':
if not self.config.get('crawlseasons'):
self.log_debug("Suche für SJ-Staffeln deaktiviert!")
return self.device
if self.empty_list:
self.log_debug("Liste ist leer. Stoppe Suche für Serien!" +
self.listtype)
return self.device
try:
reject = self.config.get("rejectlist").replace(
",", "|").lower() if len(
self.config.get("rejectlist")) > 0 else r"^unmatchable$"
except TypeError:
reject = r"^unmatchable$"
set_sj = self.settings_hash(False)
header = False
response = False
while self.day < 8:
if self.last_set_sj == set_sj:
try:
response = get_url_headers(
'https://' + self.sj + '/api/releases/latest/' +
str(self.day), self.configfile, self.dbfile,
self.headers, self.scraper)
self.scraper = response[1]
response = response[0]
if self.filename == "MB_Staffeln" or self.filename == "SJ_Staffeln_Regex":
feed = j_releases_to_feedparser_dict(
response.text, "seasons", 'https://' + self.sj,
True)
else:
feed = j_releases_to_feedparser_dict(
response.text, "episodes", 'https://' + self.sj,
True)
except:
print(u"SJ hat die Feed-API angepasst. Breche Suche ab!")
feed = False
if response:
if response.status_code == 304:
self.log_debug(
"SJ-Feed seit letztem Aufruf nicht aktualisiert - breche Suche ab!"
)
return self.device
header = True
else:
try:
response = get_url(
'https://' + self.sj + '/api/releases/latest/' +
str(self.day), self.configfile, self.dbfile,
self.scraper)
if self.filename == "MB_Staffeln" or self.filename == "SJ_Staffeln_Regex":
feed = j_releases_to_feedparser_dict(
response, "seasons", 'https://' + self.sj, True)
else:
feed = j_releases_to_feedparser_dict(
response, "episodes", 'https://' + self.sj, True)
except:
print(u"SJ hat die Feed-API angepasst. Breche Suche ab!")
feed = False
self.day += 1
if feed and feed.entries:
first_post_sj = feed.entries[0]
concat_sj = first_post_sj.title + first_post_sj.published + str(
self.settings) + str(self.pattern)
sha_sj = hashlib.sha256(concat_sj.encode(
'ascii', 'ignore')).hexdigest()
else:
self.log_debug("Feed ist leer - breche Suche ab!")
return False
for post in feed.entries:
concat = post.title + post.published + \
str(self.settings) + str(self.pattern)
sha = hashlib.sha256(concat.encode('ascii',
'ignore')).hexdigest()
if sha == self.last_sha_sj:
self.log_debug("Feed ab hier bereits gecrawlt (" +
post.title + ") - breche Suche ab!")
break
series_url = post.series_url
title = post.title.replace("-", "-")
if self.filename == 'SJ_Serien_Regex':
if self.config.get("regex"):
if '.german.' in title.lower():
language_id = 1
elif self.rsscrawler.get('english'):
language_id = 2
else:
language_id = 0
if language_id:
m = re.search(self.pattern, title.lower())
if not m and "720p" not in title and "1080p" not in title and "2160p" not in title:
m = re.search(
self.pattern.replace("480p", "."),
title.lower())
self.quality = "480p"
if m:
if "720p" in title.lower():
self.quality = "720p"
if "1080p" in title.lower():
self.quality = "1080p"
if "2160p" in title.lower():
self.quality = "2160p"
m = re.search(reject, title.lower())
if m:
self.log_debug(
title +
" - Release durch Regex gefunden (trotz rejectlist-Einstellung)"
)
title = re.sub(r'\[.*\] ', '', post.title)
self.parse_download(series_url, title,
language_id)
else:
self.log_debug(
"%s - Englische Releases deaktiviert" % title)
else:
continue
elif self.filename == 'SJ_Staffeln_Regex':
if self.config.get("regex"):
if '.german.' in title.lower():
language_id = 1
elif self.rsscrawler.get('english'):
language_id = 2
else:
language_id = 0
if language_id:
m = re.search(self.pattern, title.lower())
if not m and "720p" not in title and "1080p" not in title and "2160p" not in title:
m = re.search(
self.pattern.replace("480p", "."),
title.lower())
self.quality = "480p"
if m:
if "720p" in title.lower():
self.quality = "720p"
if "1080p" in title.lower():
self.quality = "1080p"
if "2160p" in title.lower():
self.quality = "2160p"
m = re.search(reject, title.lower())
if m:
self.log_debug(
title +
" - Release durch Regex gefunden (trotz rejectlist-Einstellung)"
)
title = re.sub(r'\[.*\] ', '', post.title)
self.parse_download(series_url, title,
language_id)
else:
self.log_debug(
"%s - Englische Releases deaktiviert" % title)
else:
continue
else:
if self.config.get("quality") != '480p':
m = re.search(self.pattern, title.lower())
if m:
if '.german.' in title.lower():
language_id = 1
elif self.rsscrawler.get('english'):
language_id = 2
else:
language_id = 0
if language_id:
mm = re.search(self.quality, title.lower())
if mm:
mmm = re.search(reject, title.lower())
if mmm:
self.log_debug(
title +
" - Release ignoriert (basierend auf rejectlist-Einstellung)"
)
continue
if self.rsscrawler.get("surround"):
if not re.match(
r'.*\.(DTS|DD\+*51|DD\+*71|AC3\.5\.*1)\..*',
title):
self.log_debug(
title +
" - Release ignoriert (kein Mehrkanalton)"
)
continue
try:
storage = self.db.retrieve_all(title)
except Exception as e:
self.log_debug(
"Fehler bei Datenbankzugriff: %s, Grund: %s"
% (e, title))
return self.device
if 'added' in storage:
self.log_debug(
title +
" - Release ignoriert (bereits gefunden)"
)
continue
self.parse_download(
series_url, title, language_id)
else:
self.log_debug(
"%s - Englische Releases deaktiviert" %
title)
else:
m = re.search(self.pattern, title.lower())
if m:
if '.german.' in title.lower():
language_id = 1
elif self.rsscrawler.get('english'):
language_id = 2
else:
language_id = 0
if language_id:
if "720p" in title.lower(
) or "1080p" in title.lower(
) or "2160p" in title.lower():
continue
mm = re.search(reject, title.lower())
if mm:
self.log_debug(
title +
" Release ignoriert (basierend auf rejectlist-Einstellung)"
)
continue
if self.rsscrawler.get("surround"):
if not re.match(
r'.*\.(DTS|DD\+*51|DD\+*71|AC3\.5\.*1)\..*',
title):
self.log_debug(
title +
" - Release ignoriert (kein Mehrkanalton)"
)
continue
title = re.sub(r'\[.*\] ', '', post.title)
try:
storage = self.db.retrieve_all(title)
except Exception as e:
self.log_debug(
"Fehler bei Datenbankzugriff: %s, Grund: %s"
% (e, title))
return self.device
if 'added' in storage:
self.log_debug(
title +
" - Release ignoriert (bereits gefunden)"
)
continue
self.parse_download(
series_url, title, language_id)
else:
self.log_debug(
"%s - Englische Releases deaktiviert" %
title)
if set_sj:
new_set_sj = self.settings_hash(True)
if set_sj == new_set_sj:
self.cdc.delete("SJSet-" + self.filename)
self.cdc.store("SJSet-" + self.filename, set_sj)
self.cdc.delete("SJ-" + self.filename)
self.cdc.store("SJ-" + self.filename, sha_sj)
if header and response:
self.cdc.delete("SJHeaders-" + self.filename)
self.cdc.store("SJHeaders-" + self.filename,
response.headers['date'])
return self.device
