def test_search(self):
self.url = "http://kickass.unblocked.li"
searchURL = "{}/usearch/American%20Dad%20S08/".format(self.url)
data = getURL(searchURL, session=requests.Session())
if not data:
return
with bs4_parser(data) as html:
torrent_table = html.find("table", attrs={"class": "data"})
# Continue only if one Release is found
torrent_rows = torrent_table.find_all("tr") if torrent_table else []
if len(torrent_rows) < 2:
print ("The data returned does not contain any torrents")
return
for tr in torrent_rows[1:]:
try:
link = urlparse.urljoin(self.url, (tr.find("div", {"class": "torrentname"}).find_all("a")[1])["href"])
id = tr.get("id")[-7:]
title = (tr.find("div", {"class": "torrentname"}).find_all("a")[1]).text or (
tr.find("div", {"class": "torrentname"}).find_all("a")[2]
).text
url = tr.find("a", "imagnet")["href"]
verified = True if tr.find("a", "iverify") else False
trusted = True if tr.find("img", {"alt": "verified"}) else False
seeders = int(tr.find_all("td")[-2].text)
leechers = int(tr.find_all("td")[-1].text)
except (AttributeError, TypeError):
continue
print title
def _xem_exceptions_fetcher():
if shouldRefresh('xem'):
for indexer in sickrage.srCore.INDEXER_API().indexers:
sickrage.srLogger.info("Checking for XEM scene exception updates for " + sickrage.srCore.INDEXER_API(indexer).name)
url = "http://thexem.de/map/allNames?origin=%s&seasonNumbers=1" % sickrage.srCore.INDEXER_API(indexer).config[
'xem_origin']
parsedJSON = getURL(url, timeout=90, json=True)
if not parsedJSON:
sickrage.srLogger.debug("Check scene exceptions update failed for " + sickrage.srCore.INDEXER_API(
indexer).name + ", Unable to get URL: " + url)
continue
if parsedJSON[b'result'] == 'failure':
continue
for indexerid, names in parsedJSON[b'data'].iteritems():
try:
xem_exception_dict[int(indexerid)] = names
except Exception as e:
sickrage.srLogger.warning("XEM: Rejected entry: indexerid:{0}; names:{1}".format(indexerid, names))
sickrage.srLogger.debug("XEM: Rejected entry error message:{0}".format(str(e)))
setLastRefresh('xem')
for xem_ex in xem_exception_dict:
if xem_ex in exception_dict:
exception_dict[xem_ex] = exception_dict[xem_ex] + xem_exception_dict[xem_ex]
else:
exception_dict[xem_ex] = xem_exception_dict[xem_ex]
return xem_exception_dict
def getURL(self, url, post_data=None, params=None, timeout=30, json=False, needBytes=False):
"""
By default this is just a simple urlopen call but this method should be overridden
for providers with special URL requirements (like cookies)
"""
return getURL(url, post_data=post_data, params=params, headers=self.headers, timeout=timeout,
session=self.session, json=json, needBytes=needBytes)
def update_network_dict():
"""Update timezone information from SR repositories"""
url = 'http://sickragetv.github.io/network_timezones/network_timezones.txt'
url_data = getURL(url)
if not url_data:
sickrage.srLogger.warning(
'Updating network timezones failed, this can happen from time to time. URL: %s'
% url)
load_network_dict()
return
d = {}
try:
for line in url_data.splitlines():
(key, val) = line.strip().rsplit(':', 1)
if key is None or val is None:
continue
d[key] = val
except (IOError, OSError):
pass
network_timezones = dict(
cache_db.CacheDB().select('SELECT * FROM network_timezones;'))
queries = []
for network, timezone in d.iteritems():
existing = network in network_timezones
if not existing:
queries.append([
'INSERT OR IGNORE INTO network_timezones VALUES (?,?);',
[network, timezone]
])
elif network_timezones[network] is not timezone:
queries.append([
'UPDATE OR IGNORE network_timezones SET timezone = ? WHERE network_name = ?;',
[timezone, network]
])
if existing:
del network_timezones[network]
if network_timezones:
purged = [x for x in network_timezones]
queries.append([
'DELETE FROM network_timezones WHERE network_name IN (%s);' %
','.join(['?'] * len(purged)), purged
])
if len(queries) > 0:
cache_db.CacheDB().mass_action(queries)
del queries # cleanup
def check_for_new_news(self, force=False):
"""
Checks GitHub for the latest news.
returns: unicode, a copy of the news
force: ignored
"""
news = ''
# Grab a copy of the news
sickrage.srLogger.debug(
'check_for_new_news: Checking GitHub for latest news.')
try:
news = getURL(sickrage.srConfig.NEWS_URL, session=self.session)
except:
sickrage.srLogger.warning(
'check_for_new_news: Could not load news from repo.')
if news:
dates = re.finditer(r'^####(\d{4}-\d{2}-\d{2})####$', news, re.M)
if not list(dates):
return news or ''
try:
last_read = datetime.strptime(sickrage.srConfig.NEWS_LAST_READ,
'%Y-%m-%d')
except:
last_read = 0
sickrage.srConfig.NEWS_UNREAD = 0
got_latest = False
for match in dates:
if not got_latest:
got_latest = True
sickrage.srConfig.NEWS_LATEST = match.group(1)
try:
if datetime.strptime(match.group(1),
'%Y-%m-%d') > last_read:
sickrage.srConfig.NEWS_UNREAD += 1
except Exception:
pass
return news
def getShowImage(url, imgNum=None):
if url is None:
return None
# if they provided a fanart number try to use it instead
tempURL = url
if imgNum:
tempURL = url.split('-')[0] + "-" + str(imgNum) + ".jpg"
sickrage.srLogger.debug("Fetching image from " + tempURL)
image_data = getURL(tempURL, needBytes=True)
if image_data is None:
sickrage.srLogger.warning("There was an error trying to retrieve the image, aborting")
return
return image_data
def getShowImage(url, imgNum=None):
if url is None:
return None
# if they provided a fanart number try to use it instead
tempURL = url
if imgNum:
tempURL = url.split('-')[0] + "-" + str(imgNum) + ".jpg"
sickrage.srLogger.debug("Fetching image from " + tempURL)
image_data = getURL(tempURL, needBytes=True)
if image_data is None:
sickrage.srLogger.warning(
"There was an error trying to retrieve the image, aborting")
return
return image_data
def update_network_dict():
"""Update timezone information from SR repositories"""
url = 'http://sickragetv.github.io/network_timezones/network_timezones.txt'
url_data = getURL(url)
if not url_data:
sickrage.srLogger.warning(
'Updating network timezones failed, this can happen from time to time. URL: %s' % url)
load_network_dict()
return
d = {}
try:
for line in url_data.splitlines():
(key, val) = line.strip().rsplit(':', 1)
if key is None or val is None:
continue
d[key] = val
except (IOError, OSError):
pass
network_timezones = dict(cache_db.CacheDB().select('SELECT * FROM network_timezones;'))
queries = []
for network, timezone in d.iteritems():
existing = network in network_timezones
if not existing:
queries.append(['INSERT OR IGNORE INTO network_timezones VALUES (?,?);', [network, timezone]])
elif network_timezones[network] is not timezone:
queries.append(['UPDATE OR IGNORE network_timezones SET timezone = ? WHERE network_name = ?;',
[timezone, network]])
if existing:
del network_timezones[network]
if network_timezones:
purged = [x for x in network_timezones]
queries.append(
['DELETE FROM network_timezones WHERE network_name IN (%s);' % ','.join(['?'] * len(purged)), purged])
if len(queries) > 0:
cache_db.CacheDB().mass_action(queries)
del queries # cleanup
def check_for_new_news(self, force=False):
"""
Checks GitHub for the latest news.
returns: unicode, a copy of the news
force: ignored
"""
news = ''
# Grab a copy of the news
sickrage.srLogger.debug('check_for_new_news: Checking GitHub for latest news.')
try:
news = getURL(sickrage.srConfig.NEWS_URL, session=self.session)
except:
sickrage.srLogger.warning('check_for_new_news: Could not load news from repo.')
if news:
dates = re.finditer(r'^####(\d{4}-\d{2}-\d{2})####$', news, re.M)
if not list(dates):
return news or ''
try:
last_read = datetime.strptime(sickrage.srConfig.NEWS_LAST_READ, '%Y-%m-%d')
except:
last_read = 0
sickrage.srConfig.NEWS_UNREAD = 0
got_latest = False
for match in dates:
if not got_latest:
got_latest = True
sickrage.srConfig.NEWS_LATEST = match.group(1)
try:
if datetime.strptime(match.group(1), '%Y-%m-%d') > last_read:
sickrage.srConfig.NEWS_UNREAD += 1
except Exception:
pass
return news
def _xem_exceptions_fetcher():
if shouldRefresh('xem'):
for indexer in sickrage.srCore.INDEXER_API().indexers:
sickrage.srLogger.info(
"Checking for XEM scene exception updates for " +
sickrage.srCore.INDEXER_API(indexer).name)
url = "http://thexem.de/map/allNames?origin=%s&seasonNumbers=1" % sickrage.srCore.INDEXER_API(
indexer).config['xem_origin']
parsedJSON = getURL(url, timeout=90, json=True)
if not parsedJSON:
sickrage.srLogger.debug(
"Check scene exceptions update failed for " +
sickrage.srCore.INDEXER_API(indexer).name +
", Unable to get URL: " + url)
continue
if parsedJSON[b'result'] == 'failure':
continue
for indexerid, names in parsedJSON[b'data'].iteritems():
try:
xem_exception_dict[int(indexerid)] = names
except Exception as e:
sickrage.srLogger.warning(
"XEM: Rejected entry: indexerid:{0}; names:{1}".format(
indexerid, names))
sickrage.srLogger.debug(
"XEM: Rejected entry error message:{0}".format(str(e)))
setLastRefresh('xem')
for xem_ex in xem_exception_dict:
if xem_ex in exception_dict:
exception_dict[
xem_ex] = exception_dict[xem_ex] + xem_exception_dict[xem_ex]
else:
exception_dict[xem_ex] = xem_exception_dict[xem_ex]
return xem_exception_dict
def test_search(self):
self.url = 'http://kickass.unblocked.li'
searchURL = '{}/usearch/American%20Dad%20S08/'.format(self.url)
data = getURL(searchURL, session=requests.Session())
if not data:
return
with bs4_parser(data) as html:
torrent_table = html.find('table', attrs={'class': 'data'})
# Continue only if one Release is found
torrent_rows = torrent_table.find_all('tr') if torrent_table else []
if len(torrent_rows) < 2:
print("The data returned does not contain any torrents")
return
for tr in torrent_rows[1:]:
try:
link = urlparse.urljoin(self.url,
(tr.find('div', {
'class': 'torrentname'
}).find_all('a')[1])['href'])
id = tr.get('id')[-7:]
title = (tr.find('div', {'class': 'torrentname'}).find_all('a')[1]).text \
or (tr.find('div', {'class': 'torrentname'}).find_all('a')[2]).text
url = tr.find('a', 'imagnet')['href']
verified = True if tr.find('a', 'iverify') else False
trusted = True if tr.find('img',
{'alt': 'verified'}) else False
seeders = int(tr.find_all('td')[-2].text)
leechers = int(tr.find_all('td')[-1].text)
except (AttributeError, TypeError):
continue
print title
def fetch_popular_shows(self):
"""Get popular show information from IMDB"""
popular_shows = []
data = getURL(self.url, session=self.session, params=self.params,
headers={'Referer': 'http://akas.imdb.com/'})
if not data:
return None
with bs4_parser(data) as soup:
results = soup.find("table", {"class": "results"})
rows = results.find_all("tr")
for row in rows:
show = {}
image_td = row.find("td", {"class": "image"})
if image_td:
image = image_td.find("img")
show[b'image_url_large'] = self.change_size(image[b'src'], 3)
show[b'image_path'] = os.path.join('images', 'imdb_popular',
os.path.basename(show[b'image_url_large']))
self.cache_image(show[b'image_url_large'])
td = row.find("td", {"class": "title"})
if td:
show[b'name'] = td.find("a").contents[0]
show[b'imdb_url'] = "http://www.imdb.com" + td.find("a")["href"]
show[b'imdb_tt'] = show[b'imdb_url'][-10:][0:9]
show[b'year'] = td.find("span", {"class": "year_type"}).contents[0].split(" ")[0][1:]
rating_all = td.find("div", {"class": "user_rating"})
if rating_all:
rating_string = rating_all.find("div", {"class": "rating rating-list"})
if rating_string:
rating_string = rating_string[b'title']
match = re.search(r".* (.*)\/10.*\((.*)\).*", rating_string)
if match:
matches = match.groups()
show[b'rating'] = matches[0]
show[b'votes'] = matches[1]
else:
show[b'rating'] = None
show[b'votes'] = None
else:
show[b'rating'] = None
show[b'votes'] = None
outline = td.find("span", {"class": "outline"})
if outline:
show[b'outline'] = outline.contents[0]
else:
show[b'outline'] = ''
popular_shows.append(show)
return popular_shows
def splitNZBResult(result):
"""
Split result into seperate episodes
:param result: search result object
:return: False upon failure, a list of episode objects otherwise
"""
urlData = getURL(result.url, session=requests.Session(), needBytes=True)
if urlData is None:
sickrage.srLogger.error("Unable to load url " + result.url +
", can't download season NZB")
return False
# parse the season ep name
try:
np = NameParser(False, showObj=result.show)
parse_result = np.parse(result.name)
except InvalidNameException:
sickrage.srLogger.debug("Unable to parse the filename " + result.name +
" into a valid episode")
return False
except InvalidShowException:
sickrage.srLogger.debug("Unable to parse the filename " + result.name +
" into a valid show")
return False
# bust it up
season = parse_result.season_number if parse_result.season_number is not None else 1
separateNZBs, xmlns = getSeasonNZBs(result.name, urlData, season)
resultList = []
for newNZB in separateNZBs:
sickrage.srLogger.debug("Split out " + newNZB + " from " + result.name)
# parse the name
try:
np = NameParser(False, showObj=result.show)
parse_result = np.parse(newNZB)
except InvalidNameException:
sickrage.srLogger.debug("Unable to parse the filename " + newNZB +
" into a valid episode")
return False
except InvalidShowException:
sickrage.srLogger.debug("Unable to parse the filename " + newNZB +
" into a valid show")
return False
# make sure the result is sane
if (parse_result.season_number is not None
and parse_result.season_number != season) or (
parse_result.season_number is None and season != 1):
sickrage.srLogger.warning(
"Found " + newNZB + " inside " + result.name +
" but it doesn't seem to belong to the same season, ignoring it"
)
continue
elif len(parse_result.episode_numbers) == 0:
sickrage.srLogger.warning(
"Found " + newNZB + " inside " + result.name +
" but it doesn't seem to be a valid episode NZB, ignoring it")
continue
wantEp = True
for epNo in parse_result.episode_numbers:
if not result.extraInfo[0].wantEpisode(season, epNo,
result.quality):
sickrage.srLogger.info(
"Ignoring result " + newNZB +
" because we don't want an episode that is " +
Quality.qualityStrings[result.quality])
wantEp = False
break
if not wantEp:
continue
# get all the associated episode objects
epObjList = []
for curEp in parse_result.episode_numbers:
epObjList.append(result.extraInfo[0].getEpisode(season, curEp))
# make a result
curResult = classes.NZBDataSearchResult(epObjList)
curResult.name = newNZB
curResult.provider = result.provider
curResult.quality = result.quality
curResult.extraInfo = [createNZBString(separateNZBs[newNZB], xmlns)]
resultList.append(curResult)
return resultList
def _check_for_new_version(self):
git_version_url = "https://raw.githubusercontent.com/{}/{}/master/sickrage/version.txt".format(
sickrage.srConfig.GIT_ORG, sickrage.srConfig.GIT_REPO)
git_version = getURL(git_version_url) or self._find_installed_version()
return git_version
def fetch_popular_shows(self):
"""Get popular show information from IMDB"""
popular_shows = []
data = getURL(self.url,
session=self.session,
params=self.params,
headers={'Referer': 'http://akas.imdb.com/'})
if not data:
return None
with bs4_parser(data) as soup:
results = soup.find("table", {"class": "results"})
rows = results.find_all("tr")
for row in rows:
show = {}
image_td = row.find("td", {"class": "image"})
if image_td:
image = image_td.find("img")
show[b'image_url_large'] = self.change_size(image[b'src'], 3)
show[b'image_path'] = os.path.join(
'images', 'imdb_popular',
os.path.basename(show[b'image_url_large']))
self.cache_image(show[b'image_url_large'])
td = row.find("td", {"class": "title"})
if td:
show[b'name'] = td.find("a").contents[0]
show[b'imdb_url'] = "http://www.imdb.com" + td.find(
"a")["href"]
show[b'imdb_tt'] = show[b'imdb_url'][-10:][0:9]
show[b'year'] = td.find("span", {
"class": "year_type"
}).contents[0].split(" ")[0][1:]
rating_all = td.find("div", {"class": "user_rating"})
if rating_all:
rating_string = rating_all.find(
"div", {"class": "rating rating-list"})
if rating_string:
rating_string = rating_string[b'title']
match = re.search(r".* (.*)\/10.*\((.*)\).*",
rating_string)
if match:
matches = match.groups()
show[b'rating'] = matches[0]
show[b'votes'] = matches[1]
else:
show[b'rating'] = None
show[b'votes'] = None
else:
show[b'rating'] = None
show[b'votes'] = None
outline = td.find("span", {"class": "outline"})
if outline:
show[b'outline'] = outline.contents[0]
else:
show[b'outline'] = ''
popular_shows.append(show)
return popular_shows
def splitNZBResult(result):
"""
Split result into seperate episodes
:param result: search result object
:return: False upon failure, a list of episode objects otherwise
"""
urlData = getURL(result.url, session=requests.Session(), needBytes=True)
if urlData is None:
sickrage.srLogger.error("Unable to load url " + result.url + ", can't download season NZB")
return False
# parse the season ep name
try:
np = NameParser(False, showObj=result.show)
parse_result = np.parse(result.name)
except InvalidNameException:
sickrage.srLogger.debug("Unable to parse the filename " + result.name + " into a valid episode")
return False
except InvalidShowException:
sickrage.srLogger.debug("Unable to parse the filename " + result.name + " into a valid show")
return False
# bust it up
season = parse_result.season_number if parse_result.season_number is not None else 1
separateNZBs, xmlns = getSeasonNZBs(result.name, urlData, season)
resultList = []
for newNZB in separateNZBs:
sickrage.srLogger.debug("Split out " + newNZB + " from " + result.name)
# parse the name
try:
np = NameParser(False, showObj=result.show)
parse_result = np.parse(newNZB)
except InvalidNameException:
sickrage.srLogger.debug("Unable to parse the filename " + newNZB + " into a valid episode")
return False
except InvalidShowException:
sickrage.srLogger.debug("Unable to parse the filename " + newNZB + " into a valid show")
return False
# make sure the result is sane
if (parse_result.season_number is not None and parse_result.season_number != season) or (
parse_result.season_number is None and season != 1):
sickrage.srLogger.warning(
"Found " + newNZB + " inside " + result.name + " but it doesn't seem to belong to the same season, ignoring it")
continue
elif len(parse_result.episode_numbers) == 0:
sickrage.srLogger.warning(
"Found " + newNZB + " inside " + result.name + " but it doesn't seem to be a valid episode NZB, ignoring it")
continue
wantEp = True
for epNo in parse_result.episode_numbers:
if not result.extraInfo[0].wantEpisode(season, epNo, result.quality):
sickrage.srLogger.info("Ignoring result " + newNZB + " because we don't want an episode that is " +
Quality.qualityStrings[result.quality])
wantEp = False
break
if not wantEp:
continue
# get all the associated episode objects
epObjList = []
for curEp in parse_result.episode_numbers:
epObjList.append(result.extraInfo[0].getEpisode(season, curEp))
# make a result
curResult = classes.NZBDataSearchResult(epObjList)
curResult.name = newNZB
curResult.provider = result.provider
curResult.quality = result.quality
curResult.extraInfo = [createNZBString(separateNZBs[newNZB], xmlns)]
resultList.append(curResult)
return resultList
def _check_for_new_version(self):
git_version_url = "https://raw.githubusercontent.com/{}/{}/master/sickrage/version.txt".format(
sickrage.srConfig.GIT_ORG, sickrage.srConfig.GIT_REPO)
git_version = getURL(git_version_url) or self._find_installed_version()
return git_version
def retrieve_exceptions(get_xem=True, get_anidb=True):
"""
Looks up the exceptions on github, parses them into a dict, and inserts them into the
scene_exceptions table in cache.db. Also clears the scene name cache.
"""
for indexer in sickrage.srCore.INDEXER_API().indexers:
indexer_name = sickrage.srCore.INDEXER_API(indexer).name
if shouldRefresh(indexer_name):
sickrage.srLogger.info(
"Checking for SiCKRAGE scene exception updates for {}".format(
indexer_name))
loc = sickrage.srCore.INDEXER_API(indexer).config[b'scene_loc']
try:
# each exception is on one line with the format indexer_id: 'show name 1', 'show name 2', etc
cur_line = None
for cur_line in getURL(loc).splitlines():
indexer_id, _, aliases = cur_line.partition(
':') # @UnusedVariable
if not aliases:
continue
# regex out the list of shows, taking \' into account
exception_dict[int(indexer_id)] = [{
re.sub(r'\\(.)', r'\1', x):
-1
} for x in re.findall(r"'(.*?)(?<!\\)',?", aliases)]
if cur_line is None:
sickrage.srLogger.debug(
"Check scene exceptions update failed. Unable to update from: {}"
.format(loc))
continue
# refreshed successfully
setLastRefresh(indexer_name)
except Exception:
continue
# XEM scene exceptions
if get_xem:
_xem_exceptions_fetcher()
# AniDB scene exceptions
if get_anidb:
_anidb_exceptions_fetcher()
sql_l = []
for cur_indexer_id in exception_dict:
sql_ex = cache_db.CacheDB().select(
"SELECT * FROM scene_exceptions WHERE indexer_id = ?;",
[cur_indexer_id])
existing_exceptions = [x[b"show_name"] for x in sql_ex]
if not cur_indexer_id in exception_dict:
continue
for cur_exception_dict in exception_dict[cur_indexer_id]:
for ex in cur_exception_dict.iteritems():
cur_exception, curSeason = ex
if cur_exception not in existing_exceptions:
sql_l.append([
"INSERT OR IGNORE INTO scene_exceptions (indexer_id, show_name, season) VALUES (?,?,?);",
[cur_indexer_id, cur_exception, curSeason]
])
if len(sql_l) > 0:
cache_db.CacheDB().mass_action(sql_l)
sickrage.srLogger.debug("Updated scene exceptions")
del sql_l # cleanup
else:
sickrage.srLogger.debug("No scene exceptions update needed")
# cleanup
exception_dict.clear()
anidb_exception_dict.clear()
xem_exception_dict.clear()
def xem_refresh(indexer_id, indexer, force=False):
"""
Refresh data from xem for a tv show
:param indexer_id: int
"""
if not indexer_id or indexer_id < 1:
return
indexer_id = int(indexer_id)
indexer = int(indexer)
xem_session = _setUpSession()
MAX_REFRESH_AGE_SECS = 86400 # 1 day
rows = main_db.MainDB().select(
"SELECT last_refreshed FROM xem_refresh WHERE indexer = ? AND indexer_id = ?",
[indexer, indexer_id])
if rows:
lastRefresh = int(rows[0][b'last_refreshed'])
refresh = int(time.mktime(
datetime.today().timetuple())) > lastRefresh + MAX_REFRESH_AGE_SECS
else:
refresh = True
if refresh or force:
sickrage.srLogger.debug(
'Looking up XEM scene mapping for show %s on %s' %
(indexer_id, sickrage.srCore.INDEXER_API(indexer).name))
# mark refreshed
main_db.MainDB().upsert(
"xem_refresh", {
'indexer': indexer,
'last_refreshed': int(time.mktime(
datetime.today().timetuple()))
}, {'indexer_id': indexer_id})
try:
# XEM MAP URL
url = "http://thexem.de/map/havemap?origin=%s" % sickrage.srCore.INDEXER_API(
indexer).config[b'xem_origin']
parsedJSON = getURL(url, session=xem_session, json=True)
if not parsedJSON or 'result' not in parsedJSON or 'success' not in parsedJSON[b'result'] \
or 'data' not in parsedJSON or str(indexer_id) not in parsedJSON[b'data']:
return
# XEM API URL
url = "http://thexem.de/map/all?id={}&origin={}&destination=scene".format(
indexer_id,
sickrage.srCore.INDEXER_API(indexer).config[b'xem_origin'])
parsedJSON = getURL(url, session=xem_session, json=True)
if not ((parsedJSON and 'result' in parsedJSON)
and 'success' in parsedJSON[b'result']):
sickrage.srLogger.info('No XEM data for show "%s on %s"' % (
indexer_id,
sickrage.srCore.INDEXER_API(indexer).name,
))
return
cl = []
for entry in parsedJSON[b'data']:
if 'scene' in entry:
cl.append([
"UPDATE tv_episodes SET scene_season = ?, scene_episode = ?, scene_absolute_number = ? WHERE showid = ? AND season = ? AND episode = ?",
[
entry[b'scene'][b'season'],
entry[b'scene'][b'episode'],
entry[b'scene'][b'absolute'], indexer_id,
entry[sickrage.srCore.INDEXER_API(
indexer).config[b'xem_origin']][b'season'],
entry[sickrage.srCore.INDEXER_API(
indexer).config[b'xem_origin']][b'episode']
]
])
if 'scene_2' in entry: # for doubles
cl.append([
"UPDATE tv_episodes SET scene_season = ?, scene_episode = ?, scene_absolute_number = ? WHERE showid = ? AND season = ? AND episode = ?",
[
entry[b'scene_2'][b'season'],
entry[b'scene_2'][b'episode'],
entry[b'scene_2'][b'absolute'], indexer_id,
entry[sickrage.srCore.INDEXER_API(
indexer).config[b'xem_origin']][b'season'],
entry[sickrage.srCore.INDEXER_API(
indexer).config[b'xem_origin']][b'episode']
]
])
if len(cl) > 0:
main_db.MainDB().mass_action(cl)
del cl # cleanup
except Exception as e:
sickrage.srLogger.warning(
"Exception while refreshing XEM data for show " +
str(indexer_id) + " on " +
sickrage.srCore.INDEXER_API(indexer).name +
": {}".format(e.message))
sickrage.srLogger.debug(traceback.format_exc())
def retrieve_exceptions(get_xem=True, get_anidb=True):
"""
Looks up the exceptions on github, parses them into a dict, and inserts them into the
scene_exceptions table in cache.db. Also clears the scene name cache.
"""
for indexer in sickrage.srCore.INDEXER_API().indexers:
indexer_name = sickrage.srCore.INDEXER_API(indexer).name
if shouldRefresh(indexer_name):
sickrage.srLogger.info("Checking for SiCKRAGE scene exception updates for {}".format(indexer_name))
loc = sickrage.srCore.INDEXER_API(indexer).config[b'scene_loc']
try:
# each exception is on one line with the format indexer_id: 'show name 1', 'show name 2', etc
cur_line = None
for cur_line in getURL(loc).splitlines():
indexer_id, _, aliases = cur_line.partition(':') # @UnusedVariable
if not aliases:
continue
# regex out the list of shows, taking \' into account
exception_dict[int(indexer_id)] = [{re.sub(r'\\(.)', r'\1', x): -1} for x in re.findall(r"'(.*?)(?<!\\)',?", aliases)]
if cur_line is None:
sickrage.srLogger.debug("Check scene exceptions update failed. Unable to update from: {}".format(loc))
continue
# refreshed successfully
setLastRefresh(indexer_name)
except Exception:
continue
# XEM scene exceptions
if get_xem:
_xem_exceptions_fetcher()
# AniDB scene exceptions
if get_anidb:
_anidb_exceptions_fetcher()
sql_l = []
for cur_indexer_id in exception_dict:
sql_ex = cache_db.CacheDB().select("SELECT * FROM scene_exceptions WHERE indexer_id = ?;", [cur_indexer_id])
existing_exceptions = [x[b"show_name"] for x in sql_ex]
if not cur_indexer_id in exception_dict:
continue
for cur_exception_dict in exception_dict[cur_indexer_id]:
for ex in cur_exception_dict.iteritems():
cur_exception, curSeason = ex
if cur_exception not in existing_exceptions:
sql_l.append(
["INSERT OR IGNORE INTO scene_exceptions (indexer_id, show_name, season) VALUES (?,?,?);",
[cur_indexer_id, cur_exception, curSeason]])
if len(sql_l) > 0:
cache_db.CacheDB().mass_action(sql_l)
sickrage.srLogger.debug("Updated scene exceptions")
else:
sickrage.srLogger.debug("No scene exceptions update needed")
# cleanup
exception_dict.clear()
anidb_exception_dict.clear()
xem_exception_dict.clear()
def xem_refresh(indexer_id, indexer, force=False):
"""
Refresh data from xem for a tv show
:param indexer_id: int
"""
if not indexer_id or indexer_id < 1:
return
indexer_id = int(indexer_id)
indexer = int(indexer)
xem_session = _setUpSession()
MAX_REFRESH_AGE_SECS = 86400 # 1 day
rows = main_db.MainDB().select("SELECT last_refreshed FROM xem_refresh WHERE indexer = ? AND indexer_id = ?",
[indexer, indexer_id])
if rows:
lastRefresh = int(rows[0][b'last_refreshed'])
refresh = int(time.mktime(datetime.today().timetuple())) > lastRefresh + MAX_REFRESH_AGE_SECS
else:
refresh = True
if refresh or force:
sickrage.srLogger.debug(
'Looking up XEM scene mapping for show %s on %s' % (indexer_id, sickrage.srCore.INDEXER_API(indexer).name))
# mark refreshed
main_db.MainDB().upsert("xem_refresh",
{'indexer': indexer,
'last_refreshed': int(time.mktime(datetime.today().timetuple()))},
{'indexer_id': indexer_id})
try:
# XEM MAP URL
url = "http://thexem.de/map/havemap?origin=%s" % sickrage.srCore.INDEXER_API(indexer).config[b'xem_origin']
parsedJSON = getURL(url, session=xem_session, json=True)
if not parsedJSON or 'result' not in parsedJSON or 'success' not in parsedJSON[b'result'] \
or 'data' not in parsedJSON or str(indexer_id) not in parsedJSON[b'data']:
return
# XEM API URL
url = "http://thexem.de/map/all?id={}&origin={}&destination=scene".format(
indexer_id, sickrage.srCore.INDEXER_API(indexer).config[b'xem_origin'])
parsedJSON = getURL(url, session=xem_session, json=True)
if not ((parsedJSON and 'result' in parsedJSON) and 'success' in parsedJSON[b'result']):
sickrage.srLogger.info(
'No XEM data for show "%s on %s"' % (indexer_id, sickrage.srCore.INDEXER_API(indexer).name,))
return
cl = []
for entry in parsedJSON[b'data']:
if 'scene' in entry:
cl.append([
"UPDATE tv_episodes SET scene_season = ?, scene_episode = ?, scene_absolute_number = ? WHERE showid = ? AND season = ? AND episode = ?",
[entry[b'scene'][b'season'],
entry[b'scene'][b'episode'],
entry[b'scene'][b'absolute'],
indexer_id,
entry[sickrage.srCore.INDEXER_API(indexer).config[b'xem_origin']][b'season'],
entry[sickrage.srCore.INDEXER_API(indexer).config[b'xem_origin']][b'episode']
]])
if 'scene_2' in entry: # for doubles
cl.append([
"UPDATE tv_episodes SET scene_season = ?, scene_episode = ?, scene_absolute_number = ? WHERE showid = ? AND season = ? AND episode = ?",
[entry[b'scene_2'][b'season'],
entry[b'scene_2'][b'episode'],
entry[b'scene_2'][b'absolute'],
indexer_id,
entry[sickrage.srCore.INDEXER_API(indexer).config[b'xem_origin']][b'season'],
entry[sickrage.srCore.INDEXER_API(indexer).config[b'xem_origin']][b'episode']
]])
if len(cl) > 0:
main_db.MainDB().mass_action(cl)
except Exception as e:
sickrage.srLogger.warning(
"Exception while refreshing XEM data for show " + str(
indexer_id) + " on " + sickrage.srCore.INDEXER_API(
indexer).name + ": {}".format(e.message))
sickrage.srLogger.debug(traceback.format_exc())
