def tvdb(configfile, dbfile, tvdbid, tvd_user, tvd_userkey, tvd_api):
db = RssDb(dbfile, 'Ombi')
token = db.retrieve('tvdb_token')
if not token:
token = get_tvdb_token(configfile, dbfile, tvd_user, tvd_userkey, tvd_api)
get_info = get_url_headers('https://api.thetvdb.com/series/' + str(tvdbid), configfile, dbfile,
headers={'Authorization': 'Bearer ' + token, 'Content-Type': 'application/json',
'Accept': 'application/json', 'Accept-Language': 'de'})
if get_info.status_code == 401:
token = get_tvdb_token(configfile, dbfile, tvd_user, tvd_userkey, tvd_api)
get_info = get_url_headers('https://api.thetvdb.com/series/' + str(tvdbid), configfile, dbfile,
headers={'Authorization': 'Bearer ' + token, 'Content-Type': 'application/json',
'Accept': 'application/json', 'Accept-Language': 'de'})
raw_data = json.loads(get_info.text)
raw_info = raw_data.get('data')
raw_title = raw_info.get('seriesName')
if not raw_title:
get_info = get_url_headers('https://api.thetvdb.com/series/' + str(tvdbid), configfile, dbfile,
headers={'Authorization': 'Bearer ' + token, 'Content-Type': 'application/json',
'Accept': 'application/json', 'Accept-Language': 'en'})
raw_data = json.loads(get_info.text)
raw_info = raw_data.get('data')
raw_title = raw_info.get('seriesName')
title = sanitize(raw_title)
get_episodes = get_url_headers('https://api.thetvdb.com/series/' + str(tvdbid) + '/episodes', configfile, dbfile,
headers={'Authorization': 'Bearer ' + token, 'Content-Type': 'application/json',
'Accept': 'application/json', 'Accept-Language': 'de'})
raw_episode_data = json.loads(get_episodes.text)
episodes = raw_episode_data.get('data')
total_pages = raw_episode_data.get('links')
pages = total_pages.get('last')
if pages > 1:
page = 2
while page <= pages:
get_episodes = get_url_headers(
'https://api.thetvdb.com/series/' + str(tvdbid) + '/episodes?page=' + str(page), configfile, dbfile,
headers={'Authorization': 'Bearer ' + token,
'Content-Type': 'application/json',
'Accept': 'application/json', 'Accept-Language': 'de'})
raw_episode_data = json.loads(get_episodes.text)
more_episodes = raw_episode_data.get('data')
episodes = episodes + more_episodes
page += 1
eps = {}
for e in episodes:
season = e.get("airedSeason")
if season > 0:
episode = e.get("airedEpisodeNumber")
current = eps.get(season)
if current:
eps[season] = current + [episode]
else:
eps[season] = [episode]
return title, eps
def mdb(configfile, dbfile, tmdbid, mdb_api):
get_title = get_url_headers(
'https://api.themoviedb.org/3/movie/' + str(tmdbid) + '?api_key=' + mdb_api + '&language=de-DE', configfile,
dbfile, headers={'Content-Type': 'application/json'})
raw_title = json.loads(get_title.text).get("title")
title = sanitize(raw_title)
return title
def get_imdb(url, configfile, dbfile, scraper):
result = get_url_headers(url,
configfile,
dbfile,
scraper=scraper,
headers={'Accept-Language': 'de'})
output = result[0].text
scraper = result[1]
return output, scraper
def periodical_task(self):
if not self.sf:
return self.device
if self.filename == 'SJ_Serien_Regex':
if not self.config.get('regex'):
self.log_debug("Suche für SF-Regex deaktiviert!")
return self.device
elif self.filename == 'SJ_Staffeln_Regex':
if not self.config.get('regex'):
self.log_debug("Suche für SF-Regex deaktiviert!")
return self.device
elif self.filename == 'MB_Staffeln':
if not self.config.get('crawlseasons'):
self.log_debug("Suche für SF-Staffeln deaktiviert!")
return self.device
if self.empty_list:
self.log_debug("Liste ist leer. Stoppe Suche für Serien!" +
self.listtype)
return self.device
try:
reject = self.config.get("rejectlist").replace(
",", "|").lower() if len(
self.config.get("rejectlist")) > 0 else r"^unmatchable$"
except TypeError:
reject = r"^unmatchable$"
set_sf = self.settings_hash(False)
header = False
response = False
while self.day < 8:
if self.last_set_sf == set_sf:
try:
delta = (
datetime.datetime.now() -
datetime.timedelta(days=self.day)).strftime("%Y-%m-%d")
response = get_url_headers(
'https://' + self.sf + '/updates/' + delta,
self.configfile, self.dbfile, self.headers,
self.scraper)
self.scraper = response[1]
response = response[0]
if self.filename == "MB_Staffeln" or self.filename == "SJ_Staffeln_Regex":
feed = sf_releases_to_feedparser_dict(
response.text, "seasons", 'https://' + self.sf,
True)
else:
feed = sf_releases_to_feedparser_dict(
response.text, "episodes", 'https://' + self.sf,
True)
except:
print(u"SF hat die Feed-API angepasst. Breche Suche ab!")
feed = False
if response:
if response.status_code == 304:
self.log_debug(
"SF-Feed seit letztem Aufruf nicht aktualisiert - breche Suche ab!"
)
return self.device
header = True
else:
try:
delta = (
datetime.datetime.now() -
datetime.timedelta(days=self.day)).strftime("%Y-%m-%d")
response = get_url(
'https://' + self.sf + '/updates/' + delta,
self.configfile, self.dbfile, self.scraper)
if self.filename == "MB_Staffeln" or self.filename == "SJ_Staffeln_Regex":
feed = sf_releases_to_feedparser_dict(
response, "seasons", 'https://' + self.sf, True)
else:
feed = sf_releases_to_feedparser_dict(
response, "episodes", 'https://' + self.sf, True)
except:
print(u"SF hat die Feed-API angepasst. Breche Suche ab!")
feed = False
self.day += 1
if feed and feed.entries:
first_post_sf = feed.entries[0]
concat_sf = first_post_sf.title + first_post_sf.published + str(
self.settings) + str(self.pattern)
sha_sf = hashlib.sha256(concat_sf.encode(
'ascii', 'ignore')).hexdigest()
else:
self.log_debug("Feed ist leer - breche Suche ab!")
return False
for post in feed.entries:
concat = post.title + post.published + \
str(self.settings) + str(self.pattern)
sha = hashlib.sha256(concat.encode('ascii',
'ignore')).hexdigest()
if sha == self.last_sha_sf:
self.log_debug("Feed ab hier bereits gecrawlt (" +
post.title + ") - breche Suche ab!")
break
series_url = post.series_url
title = post.title.replace("-", "-")
if self.filename == 'SJ_Serien_Regex':
if self.config.get("regex"):
if '.german.' in title.lower():
language_id = 1
elif self.rsscrawler.get('english'):
language_id = 2
else:
language_id = 0
if language_id:
m = re.search(self.pattern, title.lower())
if not m and "720p" not in title and "1080p" not in title and "2160p" not in title:
m = re.search(
self.pattern.replace("480p", "."),
title.lower())
self.quality = "480p"
if m:
if "720p" in title.lower():
self.quality = "720p"
if "1080p" in title.lower():
self.quality = "1080p"
if "2160p" in title.lower():
self.quality = "2160p"
m = re.search(reject, title.lower())
if m:
self.log_debug(
title +
" - Release durch Regex gefunden (trotz rejectlist-Einstellung)"
)
title = re.sub(r'\[.*\] ', '', post.title)
self.parse_download(series_url, title,
language_id)
else:
self.log_debug(
"%s - Englische Releases deaktiviert" % title)
else:
continue
elif self.filename == 'SJ_Staffeln_Regex':
if self.config.get("regex"):
if '.german.' in title.lower():
language_id = 1
elif self.rsscrawler.get('english'):
language_id = 2
else:
language_id = 0
if language_id:
m = re.search(self.pattern, title.lower())
if not m and "720p" not in title and "1080p" not in title and "2160p" not in title:
m = re.search(
self.pattern.replace("480p", "."),
title.lower())
self.quality = "480p"
if m:
if "720p" in title.lower():
self.quality = "720p"
if "1080p" in title.lower():
self.quality = "1080p"
if "2160p" in title.lower():
self.quality = "2160p"
m = re.search(reject, title.lower())
if m:
self.log_debug(
title +
" - Release durch Regex gefunden (trotz rejectlist-Einstellung)"
)
title = re.sub(r'\[.*\] ', '', post.title)
self.parse_download(series_url, title,
language_id)
else:
self.log_debug(
"%s - Englische Releases deaktiviert" % title)
else:
continue
else:
if self.config.get("quality") != '480p':
m = re.search(self.pattern, title.lower())
if m:
if '.german.' in title.lower():
language_id = 1
elif self.rsscrawler.get('english'):
language_id = 2
else:
language_id = 0
if language_id:
mm = re.search(self.quality, title.lower())
if mm:
mmm = re.search(reject, title.lower())
if mmm:
self.log_debug(
title +
" - Release ignoriert (basierend auf rejectlist-Einstellung)"
)
continue
if self.rsscrawler.get("surround"):
if not re.match(
r'.*\.(DTS|DD\+*51|DD\+*71|AC3\.5\.*1)\..*',
title):
self.log_debug(
title +
" - Release ignoriert (kein Mehrkanalton)"
)
continue
try:
storage = self.db.retrieve_all(title)
except Exception as e:
self.log_debug(
"Fehler bei Datenbankzugriff: %s, Grund: %s"
% (e, title))
return self.device
if 'added' in storage:
self.log_debug(
title +
" - Release ignoriert (bereits gefunden)"
)
continue
self.parse_download(
series_url, title, language_id)
else:
self.log_debug(
"%s - Englische Releases deaktiviert" %
title)
else:
m = re.search(self.pattern, title.lower())
if m:
if '.german.' in title.lower():
language_id = 1
elif self.rsscrawler.get('english'):
language_id = 2
else:
language_id = 0
if language_id:
if "720p" in title.lower(
) or "1080p" in title.lower(
) or "2160p" in title.lower():
continue
mm = re.search(reject, title.lower())
if mm:
self.log_debug(
title +
" Release ignoriert (basierend auf rejectlist-Einstellung)"
)
continue
if self.rsscrawler.get("surround"):
if not re.match(
r'.*\.(DTS|DD\+*51|DD\+*71|AC3\.5\.*1)\..*',
title):
self.log_debug(
title +
" - Release ignoriert (kein Mehrkanalton)"
)
continue
title = re.sub(r'\[.*\] ', '', post.title)
try:
storage = self.db.retrieve_all(title)
except Exception as e:
self.log_debug(
"Fehler bei Datenbankzugriff: %s, Grund: %s"
% (e, title))
return self.device
if 'added' in storage:
self.log_debug(
title +
" - Release ignoriert (bereits gefunden)"
)
continue
self.parse_download(
series_url, title, language_id)
else:
self.log_debug(
"%s - Englische Releases deaktiviert" %
title)
if set_sf:
new_set_sf = self.settings_hash(True)
if set_sf == new_set_sf:
self.cdc.delete("SFSet-" + self.filename)
self.cdc.store("SFSet-" + self.filename, set_sf)
self.cdc.delete("SF-" + self.filename)
self.cdc.store("SF-" + self.filename, sha_sf)
if header and response:
self.cdc.delete("SFHeaders-" + self.filename)
self.cdc.store("SFHeaders-" + self.filename,
response.headers['date'])
return self.device
def periodical_task(self):
if self.filename == 'DJ_Dokus_Regex':
if not self.config.get('regex'):
self.log_debug("Suche für DJ-Regex deaktiviert!")
return self.device
if self.empty_list:
self.log_debug(
"Liste ist leer. Stoppe Suche für Dokus!" + self.listtype)
return self.device
try:
reject = self.config.get("rejectlist").replace(",", "|").lower() if len(
self.config.get("rejectlist")) > 0 else r"^unmatchable$"
except TypeError:
reject = r"^unmatchable$"
set_dj = self.settings_hash(False)
header = False
if self.last_set_dj == set_dj:
try:
response = get_url_headers(
decode_base64('aHR0cDovL2Rva3VqdW5raWVzLm9yZy8='),
self.configfile,
self.dbfile,
self.headers)
feed = dj_content_to_soup(response.content)
except:
response = False
feed = False
if response:
if response.status_code == 304:
self.log_debug(
"DJ-Feed seit letztem Aufruf nicht aktualisiert - breche Suche ab!")
return self.device
header = True
else:
feed = dj_content_to_soup(
get_url(decode_base64('aHR0cDovL2Rva3VqdW5raWVzLm9yZy8='), self.configfile, self.dbfile))
response = False
if feed and feed.entries:
first_post_dj = feed.entries[0]
concat_dj = first_post_dj.title + first_post_dj.published + str(self.settings) + str(self.pattern)
sha_dj = hashlib.sha256(concat_dj.encode(
'ascii', 'ignore')).hexdigest()
else:
self.log_debug(
"Feed ist leer - breche Suche ab!")
return False
for post in feed.entries:
if not post.link:
continue
concat = post.title + post.published + str(self.settings) + str(self.pattern)
sha = hashlib.sha256(concat.encode(
'ascii', 'ignore')).hexdigest()
if sha == self.last_sha_dj:
self.log_debug(
"Feed ab hier bereits gecrawlt (" + post.title + ") - breche Suche ab!")
break
link = post.link
title = post.title
genre = post.genre
if self.filename == 'DJ_Dokus_Regex':
if self.config.get("regex"):
if '[DEUTSCH]' in title or '[TV-FILM]' in title:
language_ok = 1
elif self.rsscrawler.get('english'):
language_ok = 2
else:
language_ok = 0
if language_ok:
m = re.search(self.pattern, title.lower())
if not m and "720p" not in title and "1080p" not in title and "2160p" not in title:
m = re.search(self.pattern.replace(
"480p", "."), title.lower())
self.quality = "480p"
if m:
if not re.match(self.genres, genre.lower()):
self.log_debug(
title + " - Release aufgrund unerwünschten Genres ignoriert (" + genre + ")")
continue
if "720p" in title.lower():
self.quality = "720p"
if "1080p" in title.lower():
self.quality = "1080p"
if "2160p" in title.lower():
self.quality = "2160p"
m = re.search(reject, title.lower())
if m:
self.log_debug(
title + " - Release durch Regex gefunden (trotz rejectlist-Einstellung)")
title = re.sub(r'\[.*\] ', '', post.title)
self.range_checkr(link, title, language_ok, genre)
else:
self.log_debug(
"%s - Englische Releases deaktiviert" % title)
else:
continue
else:
if self.config.get("quality") != '480p':
m = re.search(self.pattern, title.lower())
if m:
if not re.match(self.genres, genre.lower()):
self.log_debug(title + " - Release aufgrund unerwünschten Genres ignoriert (" + genre + ")")
continue
if 'german' in title.lower():
language_ok = 1
elif self.rsscrawler.get('english'):
language_ok = 2
else:
language_ok = 0
if language_ok:
mm = re.search(self.quality, title.lower())
if mm:
mmm = re.search(reject, title.lower())
if mmm:
self.log_debug(
title + " - Release ignoriert (basierend auf rejectlist-Einstellung)")
continue
if self.rsscrawler.get("surround"):
if not re.match(r'.*\.(DTS|DD\+*51|DD\+*71|AC3\.5\.*1)\..*', title):
self.log_debug(
title + " - Release ignoriert (kein Mehrkanalton)")
continue
title = re.sub(r'\[.*\] ', '', post.title)
try:
storage = self.db.retrieve(title)
except Exception as e:
self.log_debug(
"Fehler bei Datenbankzugriff: %s, Grund: %s" % (e, title))
return self.device
if storage == 'added':
self.log_debug(
title + " - Release ignoriert (bereits gefunden)")
continue
self.range_checkr(link, title, language_ok, genre)
else:
self.log_debug(
"%s - Englische Releases deaktiviert" % title)
else:
m = re.search(self.pattern, title.lower())
if m:
if '[DEUTSCH]' in title:
language_ok = 1
elif self.rsscrawler.get('english'):
language_ok = 2
else:
language_ok = 0
if language_ok:
if "720p" in title.lower() or "1080p" in title.lower() or "2160p" in title.lower():
continue
mm = re.search(reject, title.lower())
if mm:
self.log_debug(
title + " Release ignoriert (basierend auf rejectlist-Einstellung)")
continue
if self.rsscrawler.get("surround"):
if not re.match(r'.*\.(DTS|DD\+*51|DD\+*71|AC3\.5\.*1)\..*', title):
self.log_debug(
title + " - Release ignoriert (kein Mehrkanalton)")
continue
title = re.sub(r'\[.*\] ', '', post.title)
try:
storage = self.db.retrieve(title)
except Exception as e:
self.log_debug(
"Fehler bei Datenbankzugriff: %s, Grund: %s" % (e, title))
return self.device
if storage == 'added':
self.log_debug(
title + " - Release ignoriert (bereits gefunden)")
continue
self.range_checkr(link, title, language_ok, genre)
else:
self.log_debug(
"%s - Englische Releases deaktiviert" % title)
if set_dj:
new_set_dj = self.settings_hash(True)
if set_dj == new_set_dj:
self.cdc.delete("DJSet-" + self.filename)
self.cdc.store("DJSet-" + self.filename, set_dj)
self.cdc.delete("DJ-" + self.filename)
self.cdc.store("DJ-" + self.filename, sha_dj)
if header and response:
self.cdc.delete("DJHeaders-" + self.filename)
self.cdc.store("DJHeaders-" + self.filename, response.headers['Last-Modified'])
return self.device