def __init__(self, name):
self.name = name
self.anime_only = False
self.bt_cache_urls = [
'http://torcache.net/torrent/{torrent_hash}.torrent',
'http://thetorrent.org/torrent/{torrent_hash}.torrent',
'http://btdig.com/torrent/{torrent_hash}.torrent',
# 'http://torrage.com/torrent/{torrent_hash}.torrent',
# 'http://itorrents.org/torrent/{torrent_hash}.torrent',
]
self.cache = TVCache(self)
self.enable_backlog = False
self.enable_daily = False
self.enabled = False
self.headers = {'User-Agent': UA_POOL.random}
self.proper_strings = ['PROPER|REPACK|REAL']
self.provider_type = None
self.public = False
self.search_fallback = False
self.search_mode = None
self.session = make_session()
self.show = None
self.supports_absolute_numbering = False
self.supports_backlog = True
self.url = ''
self.urls = {}
shuffle(self.bt_cache_urls)
def __init__(self, name):
self.name = name
self.anime_only = False
self.bt_cache_urls = [
#'http://torcache.net/torrent/{torrent_hash}.torrent',
'http://torrentproject.se/torrent/{torrent_hash}.torrent',
'http://thetorrent.org/torrent/{torrent_hash}.torrent',
'http://btdig.com/torrent/{torrent_hash}.torrent',
# 'http://torrage.com/torrent/{torrent_hash}.torrent',
'http://itorrents.org/torrent/{torrent_hash}.torrent',
]
self.cache = TVCache(self)
self.enable_backlog = False
self.enable_daily = False
self.enabled = False
self.headers = {'User-Agent': UA_POOL.random}
self.proper_strings = ['PROPER|REPACK|REAL']
self.provider_type = None
self.public = False
self.search_fallback = False
self.search_mode = None
self.session = make_session()
self.show = None
self.supports_absolute_numbering = False
self.supports_backlog = True
self.url = ''
self.urls = {}
# Use and configure the attribute enable_cookies to show or hide the cookies input field per provider
self.enable_cookies = False
self.cookies = ''
self.rss_cookies = ''
shuffle(self.bt_cache_urls)
def __init__(self, name):
self.name = name
self.anime_only = False
self.bt_cache_urls = [
'http://torcache.net/torrent/{torrent_hash}.torrent',
'http://thetorrent.org/torrent/{torrent_hash}.torrent',
'http://btdig.com/torrent/{torrent_hash}.torrent',
# 'http://torrage.com/torrent/{torrent_hash}.torrent',
# 'http://itorrents.org/torrent/{torrent_hash}.torrent',
]
self.cache = TVCache(self)
self.enable_backlog = False
self.enable_daily = False
self.enabled = False
self.headers = {'User-Agent': UA_POOL.random}
self.proper_strings = ['PROPER|REPACK|REAL']
self.provider_type = None
self.public = False
self.search_fallback = False
self.search_mode = None
self.session = Session()
self.show = None
self.supports_absolute_numbering = False
self.supports_backlog = True
self.url = ''
self.urls = {}
shuffle(self.bt_cache_urls)
def test_womble(self):
RSSFeeds().clearCache()
result = RSSFeeds().getFeed('https://newshost.co.za/rss/?sec=tv-sd&fr=false')
self.assertTrue('entries' in result)
self.assertTrue('feed' in result)
for item in result['entries']:
self.assertTrue(TVCache._parseItem(item))
def test_womble(self):
RSSFeeds().clearCache()
result = RSSFeeds().getFeed(
'https://newshost.co.za/rss/?sec=tv-sd&fr=false')
self.assertTrue('entries' in result)
self.assertTrue('feed' in result)
for item in result['entries']:
self.assertTrue(TVCache._parseItem(item))
def __init__(self, name):
self.name = name
self.anime_only = False
self.bt_cache_urls = [
#'http://torcache.net/torrent/{torrent_hash}.torrent',
'http://torrentproject.se/torrent/{torrent_hash}.torrent',
'http://thetorrent.org/torrent/{torrent_hash}.torrent',
'http://btdig.com/torrent/{torrent_hash}.torrent',
# 'http://torrage.com/torrent/{torrent_hash}.torrent',
('https://t.torrage.info/download?h={torrent_hash}',
'https://torrage.info/torrent.php?h={torrent_hash}'),
"https://t.torrage.info/download?h=463E408429535139A0BBB5DD676DB10D5963BF05"
'http://itorrents.org/torrent/{torrent_hash}.torrent',
]
self.cache = TVCache(self)
self.enable_backlog = False
self.enable_daily = False
self.enabled = False
self.headers = {'User-Agent': UA_POOL.random}
self.proper_strings = ['PROPER|REPACK|REAL']
self.provider_type = None
self.public = False
self.search_fallback = False
self.search_mode = None
self.session = make_session()
self.show = None
self.supports_absolute_numbering = False
self.supports_backlog = True
self.url = ''
self.urls = {}
# Use and configure the attribute enable_cookies to show or hide the cookies input field per provider
self.enable_cookies = False
self.cookies = ''
self.rss_cookies = ''
self.ability_status = self.PROVIDER_OK
shuffle(self.bt_cache_urls)
class GenericProvider(object): # pylint: disable=too-many-instance-attributes
NZB = 'nzb'
NZBDATA = 'nzbdata'
TORRENT = 'torrent'
PROVIDER_BROKEN = 0
PROVIDER_DAILY = 1
PROVIDER_BACKLOG = 2
PROVIDER_OK = 3
ProviderStatus = {
PROVIDER_BROKEN: _("Not working"),
PROVIDER_DAILY: _("Daily/RSS only"),
PROVIDER_BACKLOG: _("Backlog/Manual Search only"),
PROVIDER_OK: _("Daily/RSS and Backlog/Manual Searches working")
}
def __init__(self, name):
self.name = name
self.anime_only = False
self.bt_cache_urls = [
#'http://torcache.net/torrent/{torrent_hash}.torrent',
'http://torrentproject.se/torrent/{torrent_hash}.torrent',
'http://thetorrent.org/torrent/{torrent_hash}.torrent',
'http://btdig.com/torrent/{torrent_hash}.torrent',
# 'http://torrage.com/torrent/{torrent_hash}.torrent',
'http://itorrents.org/torrent/{torrent_hash}.torrent',
]
self.cache = TVCache(self)
self.enable_backlog = False
self.enable_daily = False
self.enabled = False
self.headers = {'User-Agent': UA_POOL.random}
self.proper_strings = ['PROPER|REPACK|REAL']
self.provider_type = None
self.public = False
self.search_fallback = False
self.search_mode = None
self.session = make_session()
self.show = None
self.supports_absolute_numbering = False
self.supports_backlog = True
self.url = ''
self.urls = {}
# Use and configure the attribute enable_cookies to show or hide the cookies input field per provider
self.enable_cookies = False
self.cookies = ''
self.rss_cookies = ''
self.ability_status = self.PROVIDER_OK
shuffle(self.bt_cache_urls)
def download_result(self, result):
if not self.login():
return False
urls, filename = self._make_url(result)
for url in urls:
if 'NO_DOWNLOAD_NAME' in url:
continue
if url.startswith('http'):
self.headers.update(
{'Referer': '/'.join(url.split('/')[:3]) + '/'})
logger.log('Downloading a result from {0} at {1}'.format(
self.name, url))
downloaded_filename = download_file(
url,
filename,
session=self.session,
headers=self.headers,
hooks={'response': self.get_url_hook},
return_filename=True)
if downloaded_filename:
if self._verify_download(downloaded_filename):
logger.log(
'Saved result to {0}'.format(downloaded_filename),
logger.INFO)
return True
logger.log('Could not download {0}'.format(url),
logger.WARNING)
remove_file_failed(downloaded_filename)
if urls:
logger.log('Failed to download any results', logger.WARNING)
return False
def find_propers(self, search_date=None):
results = self.cache.list_propers(search_date)
return [
Proper(x[b'name'], x[b'url'], datetime.fromtimestamp(x[b'time']),
self.show) for x in results
]
def find_search_results(
self,
show,
episodes,
search_mode, # pylint: disable=too-many-branches,too-many-arguments,too-many-locals,too-many-statements
manual_search=False,
download_current_quality=False):
self._check_auth()
self.show = show
results = {}
items_list = []
searched_scene_season = None
for episode in episodes:
cache_result = self.cache.search_cache(
episode,
manual_search=manual_search,
down_cur_quality=download_current_quality)
if cache_result:
if episode.episode not in results:
results[episode.episode] = cache_result
else:
results[episode.episode].extend(cache_result)
continue
if len(
episodes
) > 1 and search_mode == 'sponly' and searched_scene_season == episode.scene_season:
continue
search_strings = []
searched_scene_season = episode.scene_season
if len(episodes) > 1 and search_mode == 'sponly':
search_strings = self.get_season_search_strings(episode)
elif search_mode == 'eponly':
search_strings = self.get_episode_search_strings(episode)
for search_string in search_strings:
items_list += self.search(search_string, ep_obj=episode)
if len(results) == len(episodes):
return results
if items_list:
items = {}
unknown_items = []
for item in items_list:
quality = self.get_quality(item, anime=show.is_anime)
if quality == Quality.UNKNOWN:
unknown_items.append(item)
elif quality == Quality.NONE:
pass # Skipping an HEVC when HEVC is not allowed by settings
else:
if quality not in items:
items[quality] = []
items[quality].append(item)
items_list = list(
chain(*[
v for (k_, v) in sorted(six.iteritems(items), reverse=True)
]))
items_list += unknown_items
cl = []
for item in items_list:
(title, url) = self._get_title_and_url(item)
try:
parse_result = NameParser(
parse_method=('normal',
'anime')[show.is_anime]).parse(title)
except (InvalidNameException, InvalidShowException) as error:
logger.log("{0}".format(error), logger.DEBUG)
continue
show_object = parse_result.show
quality = parse_result.quality
release_group = parse_result.release_group
version = parse_result.version
add_cache_entry = False
if not (show_object.air_by_date or show_object.sports):
if search_mode == 'sponly':
if parse_result.episode_numbers:
logger.log(
'This is supposed to be a season pack search but the result {0} is not a valid season pack, skipping it'
.format(title), logger.DEBUG)
add_cache_entry = True
elif not [
ep
for ep in episodes if parse_result.season_number ==
(ep.season, ep.scene_season)[ep.show.is_scene]
]:
logger.log(
'This season result {0} is for a season we are not searching for, skipping it'
.format(title), logger.DEBUG)
add_cache_entry = True
else:
if not all([
# pylint: disable=bad-continuation
parse_result.season_number is not None,
parse_result.episode_numbers,
[
ep for ep in episodes if
(ep.season, ep.scene_season)[ep.show.is_scene] ==
(parse_result.season_number,
parse_result.scene_season)[ep.show.is_scene] and
(ep.episode, ep.scene_episode)[ep.show.is_scene] in
parse_result.episode_numbers
]
]) and not all([
# fallback for anime on absolute numbering
parse_result.is_anime,
parse_result.ab_episode_numbers is not None,
[
ep for ep in episodes if ep.show.is_anime and ep.
absolute_number in parse_result.ab_episode_numbers
]
]):
logger.log(
'The result {0} doesn\'t seem to match an episode that we are currently trying to snatch, skipping it'
.format(title), logger.DEBUG)
add_cache_entry = True
if not add_cache_entry:
actual_season = parse_result.season_number
actual_episodes = parse_result.episode_numbers
else:
same_day_special = False
if not parse_result.is_air_by_date:
logger.log(
'This is supposed to be a date search but the result {0} didn\'t parse as one, skipping it'
.format(title), logger.DEBUG)
add_cache_entry = True
else:
air_date = parse_result.air_date.toordinal()
db = DBConnection()
sql_results = db.select(
'SELECT season, episode FROM tv_episodes WHERE showid = ? AND airdate = ?',
[show_object.indexerid, air_date])
if len(sql_results) == 2:
if int(sql_results[0][b'season']) == 0 and int(
sql_results[1][b'season']) != 0:
actual_season = int(sql_results[1][b'season'])
actual_episodes = [int(sql_results[1][b'episode'])]
same_day_special = True
elif int(sql_results[1][b'season']) == 0 and int(
sql_results[0][b'season']) != 0:
actual_season = int(sql_results[0][b'season'])
actual_episodes = [int(sql_results[0][b'episode'])]
same_day_special = True
elif len(sql_results) != 1:
logger.log(
'Tried to look up the date for the episode {0} but the database didn\'t give proper results, skipping it'
.format(title), logger.WARNING)
add_cache_entry = True
if not add_cache_entry and not same_day_special:
actual_season = int(sql_results[0][b'season'])
actual_episodes = [int(sql_results[0][b'episode'])]
if add_cache_entry:
logger.log(
'Adding item from search to cache: {0}'.format(title),
logger.DEBUG)
# pylint: disable=protected-access
# Access to a protected member of a client class
ci = self.cache._add_cache_entry(title,
url,
parse_result=parse_result)
if ci is not None:
cl.append(ci)
continue
episode_wanted = True
for episode_number in actual_episodes:
if not show_object.wantEpisode(actual_season, episode_number,
quality, manual_search,
download_current_quality):
episode_wanted = False
break
if not episode_wanted:
logger.log('Ignoring result {0}.'.format(title), logger.DEBUG)
continue
logger.log('Found result {0} at {1}'.format(title, url),
logger.DEBUG)
episode_object = []
for current_episode in actual_episodes:
episode_object.append(
show_object.getEpisode(actual_season, current_episode))
result = self.get_result(episode_object)
result.show = show_object
result.url = url
result.name = title
result.quality = quality
result.release_group = release_group
result.version = version
result.content = None
result.size = self._get_size(item)
if len(episode_object) == 1:
episode_number = episode_object[0].episode
logger.log('Single episode result.', logger.DEBUG)
elif len(episode_object) > 1:
episode_number = MULTI_EP_RESULT
logger.log(
'Separating multi-episode result to check for later - result contains episodes: {0}'
.format(parse_result.episode_numbers), logger.DEBUG)
elif len(episode_object) == 0:
episode_number = SEASON_RESULT
logger.log('Separating full season result to check for later',
logger.DEBUG)
if episode_number not in results:
results[episode_number] = [result]
else:
results[episode_number].append(result)
if cl:
# pylint: disable=protected-access
# Access to a protected member of a client class
cache_db = self.cache._get_db()
cache_db.mass_action(cl)
return results
def get_id(self, suffix=''):
return GenericProvider.make_id(self.name) + six.text_type(suffix)
def get_quality(self, item, anime=False):
(title, url_) = self._get_title_and_url(item)
quality = Quality.scene_quality(title, anime)
return quality
def get_result(self, episodes):
result = self._get_result(episodes)
result.provider = self
return result
@staticmethod
def get_url_hook(response, **kwargs_):
if response:
logger.log(
'{0} URL: {1} [Status: {2}]'.format(response.request.method,
response.request.url,
response.status_code),
logger.DEBUG)
if response.request.method == 'POST':
logger.log('With post data: {0}'.format(response.request.body),
logger.DEBUG)
def get_url(self, url, post_data=None, params=None, timeout=30, **kwargs): # pylint: disable=too-many-arguments,
kwargs['hooks'] = {'response': self.get_url_hook}
return getURL(url, post_data, params, self.headers, timeout,
self.session, **kwargs)
def image_name(self):
return self.get_id() + '.png'
@property
def is_active(self): # pylint: disable=no-self-use
return False
@property
def is_enabled(self):
return bool(self.enabled)
@property
def daily_enabled(self):
return int(self.enable_daily)
@property
def backlog_enabled(self):
return int(self.enable_backlog)
@property
def search_fallback_enabled(self):
return int(self.search_fallback)
@property
def can_daily(self):
return self.ability_status & self.PROVIDER_DAILY != 0
@property
def can_backlog(self):
return self.ability_status & self.PROVIDER_BACKLOG != 0 and self.supports_backlog
def status(self):
return self.ProviderStatus.get(self.ability_status)
@staticmethod
def make_id(name):
if not name:
return ''
return re.sub(r'[^\w\d_]', '_', str(name).strip().lower())
def search_rss(self, episodes):
return self.cache.find_needed_episodes(episodes)
def seed_ratio(self): # pylint: disable=no-self-use
return ''
def _check_auth(self): # pylint: disable=no-self-use
return True
def login(self): # pylint: disable=no-self-use
return True
def search(self, search_params, age=0, ep_obj=None): # pylint: disable=unused-argument,no-self-use
return []
def _get_result(self, episodes): # pylint: disable=no-self-use
return SearchResult(episodes)
def get_episode_search_strings(self, episode, add_string=''):
if not episode:
return []
search_string = {'Episode': []}
for show_name in allPossibleShowNames(episode.show,
season=episode.scene_season):
episode_string = show_name + ' '
episode_string_fallback = None
if episode.show.air_by_date:
episode_string += str(episode.airdate).replace('-', ' ')
elif episode.show.sports:
episode_string += str(episode.airdate).replace('-', ' ')
episode_string += ('|', ' ')[len(self.proper_strings) > 1]
episode_string += episode.airdate.strftime('%b')
elif episode.show.anime:
episode_string_fallback = episode_string + '{0:02d}'.format(
int(episode.scene_absolute_number))
episode_string += '{0:03d}'.format(
int(episode.scene_absolute_number))
else:
episode_string += sickbeard.config.naming_ep_type[2] % {
'seasonnumber': episode.scene_season,
'episodenumber': episode.scene_episode,
}
if add_string:
episode_string += ' ' + add_string
if episode_string_fallback:
episode_string_fallback += ' ' + add_string
search_string['Episode'].append(
episode_string.encode('utf-8').strip())
if episode_string_fallback:
search_string['Episode'].append(
episode_string_fallback.encode('utf-8').strip())
return [search_string]
def get_season_search_strings(self, episode):
search_string = {'Season': []}
for show_name in allPossibleShowNames(episode.show,
season=episode.scene_season):
season_string = show_name + ' '
if episode.show.air_by_date or episode.show.sports:
season_string += str(episode.airdate).split('-')[0]
elif episode.show.anime:
# use string below if you really want to search on season with number
# season_string += 'Season ' + '{0:d}'.format(int(episode.scene_season))
season_string += 'Season' # ignore season number to get all seasons in all formats
else:
season_string += 'S{0:02d}'.format(int(episode.scene_season))
search_string['Season'].append(
season_string.encode('utf-8').strip())
return [search_string]
def _get_size(self, item): # pylint: disable=unused-argument,no-self-use
try:
return item.get('size', -1)
except AttributeError:
return -1
def _get_storage_dir(self): # pylint: disable=no-self-use
return ''
def _get_title_and_url(self, item): # pylint: disable=no-self-use
if not item:
return '', ''
title = item.get('title', '')
url = item.get('link', '')
if title:
title = '' + title.replace(' ', '.')
else:
title = ''
if url:
url = url.replace('&', '&').replace('%26tr%3D', '&tr=')
else:
url = ''
return title, url
@staticmethod
def hash_from_magnet(magnet):
try:
torrent_hash = re.findall(r'urn:btih:([\w]{32,40})',
magnet)[0].upper()
if len(torrent_hash) == 32:
torrent_hash = b16encode(b32decode(torrent_hash)).upper()
return torrent_hash
except Exception:
logger.log(
'Unable to extract torrent hash or name from magnet: {0}'.
format(magnet), logger.ERROR)
return ''
def _make_url(self, result):
if not result:
return '', ''
filename = ''
urls = [result.url]
if result.url.startswith('magnet'):
torrent_hash = self.hash_from_magnet(result.url)
if not torrent_hash:
return urls, filename
try:
torrent_name = re.findall('dn=([^&]+)', result.url)[0]
except Exception:
torrent_name = 'NO_DOWNLOAD_NAME'
urls = [
x.format(torrent_hash=torrent_hash, torrent_name=torrent_name)
for x in self.bt_cache_urls
]
filename = ek(
join, self._get_storage_dir(),
sanitize_filename(result.name) + '.' + self.provider_type)
return urls, filename
def _verify_download(self, file_name=None): # pylint: disable=unused-argument,no-self-use
return True
def add_cookies_from_ui(self):
"""
Adds the cookies configured from UI to the providers requests session
:return: A tuple with the the (success result, and a descriptive message in str)
"""
# This is the generic attribute used to manually add cookies for provider authentication
if self.enable_cookies and self.cookies:
cookie_validator = re.compile(r'^(\w+=\w+)(;\w+=\w+)*$')
if not cookie_validator.match(self.cookies):
return False, 'Cookie is not correctly formatted: {0}'.format(
self.cookies)
add_dict_to_cookiejar(
self.session.cookies,
dict(x.rsplit('=', 1) for x in self.cookies.split(';')))
return True, 'torrent cookie'
return False, 'No Cookies added from ui for provider: {0}'.format(
self.name)
class GenericProvider(object): # pylint: disable=too-many-instance-attributes
NZB = 'nzb'
TORRENT = 'torrent'
def __init__(self, name):
self.name = name
self.anime_only = False
self.bt_cache_urls = [
'http://torcache.net/torrent/{torrent_hash}.torrent',
'http://thetorrent.org/torrent/{torrent_hash}.torrent',
'http://btdig.com/torrent/{torrent_hash}.torrent',
# 'http://torrage.com/torrent/{torrent_hash}.torrent',
# 'http://itorrents.org/torrent/{torrent_hash}.torrent',
]
self.cache = TVCache(self)
self.enable_backlog = False
self.enable_daily = False
self.enabled = False
self.headers = {'User-Agent': UA_POOL.random}
self.proper_strings = ['PROPER|REPACK|REAL']
self.provider_type = None
self.public = False
self.search_fallback = False
self.search_mode = None
self.session = make_session()
self.show = None
self.supports_absolute_numbering = False
self.supports_backlog = True
self.url = ''
self.urls = {}
shuffle(self.bt_cache_urls)
def download_result(self, result):
if not self.login():
return False
urls, filename = self._make_url(result)
for url in urls:
if 'NO_DOWNLOAD_NAME' in url:
continue
if url.startswith('http'):
self.headers.update(
{'Referer': '/'.join(url.split('/')[:3]) + '/'})
logger.log(u'Downloading a result from %s at %s' %
(self.name, url))
if url.endswith(GenericProvider.TORRENT) and filename.endswith(
GenericProvider.NZB):
filename = replace_extension(filename, GenericProvider.TORRENT)
if download_file(url,
filename,
session=self.session,
headers=self.headers,
hooks={'response': self.get_url_hook}):
if self._verify_download(filename):
logger.log(u'Saved result to %s' % filename, logger.INFO)
return True
logger.log(u'Could not download %s' % url, logger.WARNING)
remove_file_failed(filename)
if len(urls):
logger.log(u'Failed to download any results', logger.WARNING)
return False
def find_propers(self, search_date=None):
results = self.cache.listPropers(search_date)
return [
Proper(x['name'], x['url'], datetime.fromtimestamp(x['time']),
self.show) for x in results
]
def find_search_results(
self,
show,
episodes,
search_mode, # pylint: disable=too-many-branches,too-many-arguments,too-many-locals,too-many-statements
manual_search=False,
download_current_quality=False):
self._check_auth()
self.show = show
results = {}
items_list = []
searched_scene_season = None
for episode in episodes:
cache_result = self.cache.searchCache(
episode,
manualSearch=manual_search,
downCurQuality=download_current_quality)
if cache_result:
if episode.episode not in results:
results[episode.episode] = cache_result
else:
results[episode.episode].extend(cache_result)
continue
if len(
episodes
) > 1 and search_mode == 'sponly' and searched_scene_season == episode.scene_season:
continue
search_strings = []
searched_scene_season = episode.scene_season
if len(episodes) > 1 and search_mode == 'sponly':
search_strings = self._get_season_search_strings(episode)
elif search_mode == 'eponly':
search_strings = self._get_episode_search_strings(episode)
for search_string in search_strings:
items_list += self.search(search_string, ep_obj=episode)
if len(results) == len(episodes):
return results
if items_list:
items = {}
unknown_items = []
for item in items_list:
quality = self.get_quality(item, anime=show.is_anime)
if quality == Quality.UNKNOWN:
unknown_items.append(item)
else:
if quality not in items:
items[quality] = []
items[quality].append(item)
items_list = list(
chain(
*[v
for (_, v) in sorted(items.iteritems(), reverse=True)]))
items_list += unknown_items
cl = []
for item in items_list:
(title, url) = self._get_title_and_url(item)
try:
parse_result = NameParser(
parse_method=('normal',
'anime')[show.is_anime]).parse(title)
except (InvalidNameException, InvalidShowException) as error:
logger.log(u"{}".format(error), logger.DEBUG)
continue
show_object = parse_result.show
quality = parse_result.quality
release_group = parse_result.release_group
version = parse_result.version
add_cache_entry = False
if not (show_object.air_by_date or show_object.sports):
if search_mode == 'sponly':
if parse_result.episode_numbers:
logger.log(
u'This is supposed to be a season pack search but the result %s is not a valid season pack, skipping it'
% title, logger.DEBUG)
add_cache_entry = True
elif not [
ep
for ep in episodes if parse_result.season_number ==
(ep.season, ep.scene_season)[ep.show.is_scene]
]:
logger.log(
u'This season result %s is for a season we are not searching for, skipping it'
% title, logger.DEBUG)
add_cache_entry = True
else:
if not all([
# pylint: disable=bad-continuation
parse_result.season_number is not None,
parse_result.episode_numbers,
[
ep for ep in episodes
if (ep.season, ep.scene_season)[ep.show.is_scene]
== parse_result.season_number and (
ep.episode, ep.scene_episode
)[ep.show.is_scene] in parse_result.episode_numbers
]
]):
logger.log(
u'The result %s doesn\'t seem to match an episode that we are currently trying to snatch, skipping it'
% title, logger.DEBUG)
add_cache_entry = True
if not add_cache_entry:
actual_season = parse_result.season_number
actual_episodes = parse_result.episode_numbers
else:
same_day_special = False
if not parse_result.is_air_by_date:
logger.log(
u'This is supposed to be a date search but the result %s didn\'t parse as one, skipping it'
% title, logger.DEBUG)
add_cache_entry = True
else:
air_date = parse_result.air_date.toordinal()
db = DBConnection()
sql_results = db.select(
'SELECT season, episode FROM tv_episodes WHERE showid = ? AND airdate = ?',
[show_object.indexerid, air_date])
if len(sql_results) == 2:
if int(sql_results[0]['season']) == 0 and int(
sql_results[1]['season']) != 0:
actual_season = int(sql_results[1]['season'])
actual_episodes = [int(sql_results[1]['episode'])]
same_day_special = True
elif int(sql_results[1]['season']) == 0 and int(
sql_results[0]['season']) != 0:
actual_season = int(sql_results[0]['season'])
actual_episodes = [int(sql_results[0]['episode'])]
same_day_special = True
elif len(sql_results) != 1:
logger.log(
u'Tried to look up the date for the episode %s but the database didn\'t give proper results, skipping it'
% title, logger.WARNING)
add_cache_entry = True
if not add_cache_entry and not same_day_special:
actual_season = int(sql_results[0]['season'])
actual_episodes = [int(sql_results[0]['episode'])]
if add_cache_entry:
logger.log(u'Adding item from search to cache: %s' % title,
logger.DEBUG)
# pylint: disable=protected-access
# Access to a protected member of a client class
ci = self.cache._addCacheEntry(title,
url,
parse_result=parse_result)
if ci is not None:
cl.append(ci)
continue
episode_wanted = True
for episode_number in actual_episodes:
if not show_object.wantEpisode(actual_season, episode_number,
quality, manual_search,
download_current_quality):
episode_wanted = False
break
if not episode_wanted:
logger.log(u'Ignoring result %s.' % (title), logger.DEBUG)
continue
logger.log(u'Found result %s at %s' % (title, url), logger.DEBUG)
episode_object = []
for current_episode in actual_episodes:
episode_object.append(
show_object.getEpisode(actual_season, current_episode))
result = self.get_result(episode_object)
result.show = show_object
result.url = url
result.name = title
result.quality = quality
result.release_group = release_group
result.version = version
result.content = None
result.size = self._get_size(item)
if len(episode_object) == 1:
episode_number = episode_object[0].episode
logger.log(u'Single episode result.', logger.DEBUG)
elif len(episode_object) > 1:
episode_number = MULTI_EP_RESULT
logger.log(
u'Separating multi-episode result to check for later - result contains episodes: %s'
% str(parse_result.episode_numbers), logger.DEBUG)
elif len(episode_object) == 0:
episode_number = SEASON_RESULT
logger.log(u'Separating full season result to check for later',
logger.DEBUG)
if episode_number not in results:
results[episode_number] = [result]
else:
results[episode_number].append(result)
if len(cl) > 0:
# pylint: disable=protected-access
# Access to a protected member of a client class
db = self.cache._getDB()
db.mass_action(cl)
return results
def get_id(self):
return GenericProvider.make_id(self.name)
def get_quality(self, item, anime=False):
(title, _) = self._get_title_and_url(item)
quality = Quality.scene_quality(title, anime)
return quality
def get_result(self, episodes):
result = self._get_result(episodes)
result.provider = self
return result
@staticmethod
def get_url_hook(response, **kwargs):
logger.log(
u'{} URL: {} [Status: {}]'.format(response.request.method,
response.request.url,
response.status_code),
logger.DEBUG)
if response.request.method == 'POST':
logger.log(u'With post data: {}'.format(response.request.body),
logger.DEBUG)
def get_url(self, url, post_data=None, params=None, timeout=30, **kwargs): # pylint: disable=too-many-arguments,
kwargs['hooks'] = {'response': self.get_url_hook}
return getURL(url, post_data, params, self.headers, timeout,
self.session, **kwargs)
def image_name(self):
return self.get_id() + '.png'
def is_active(self): # pylint: disable=no-self-use
return False
def is_enabled(self):
return bool(self.enabled)
@staticmethod
def make_id(name):
if not name:
return ''
return re.sub(r'[^\w\d_]', '_', str(name).strip().lower())
def search_rss(self, episodes):
return self.cache.findNeededEpisodes(episodes)
def seed_ratio(self): # pylint: disable=no-self-use
return ''
def _check_auth(self): # pylint: disable=no-self-use
return True
def login(self): # pylint: disable=no-self-use
return True
def search(self, search_params, age=0, ep_obj=None): # pylint: disable=unused-argument,no-self-use
return []
def _get_result(self, episodes): # pylint: disable=no-self-use
return SearchResult(episodes)
def _get_episode_search_strings(self, episode, add_string=''):
if not episode:
return []
search_string = {'Episode': []}
for show_name in allPossibleShowNames(episode.show,
season=episode.scene_season):
episode_string = show_name + ' '
if episode.show.air_by_date:
episode_string += str(episode.airdate).replace('-', ' ')
elif episode.show.sports:
episode_string += str(episode.airdate).replace('-', ' ')
episode_string += ('|', ' ')[len(self.proper_strings) > 1]
episode_string += episode.airdate.strftime('%b')
elif episode.show.anime:
episode_string += '%02d' % int(episode.scene_absolute_number)
else:
episode_string += sickbeard.config.naming_ep_type[2] % {
'seasonnumber': episode.scene_season,
'episodenumber': episode.scene_episode,
}
if add_string:
episode_string += ' ' + add_string
search_string['Episode'].append(
episode_string.encode('utf-8').strip())
return [search_string]
def _get_season_search_strings(self, episode):
search_string = {'Season': []}
for show_name in allPossibleShowNames(episode.show,
season=episode.scene_season):
episode_string = show_name + ' '
if episode.show.air_by_date or episode.show.sports:
episode_string += str(episode.airdate).split('-')[0]
elif episode.show.anime:
episode_string += '%d' % int(episode.scene_absolute_number)
else:
episode_string += 'S%02d' % int(episode.scene_season)
search_string['Season'].append(
episode_string.encode('utf-8').strip())
return [search_string]
def _get_size(self, item): # pylint: disable=unused-argument,no-self-use
return -1
def _get_storage_dir(self): # pylint: disable=no-self-use
return ''
def _get_title_and_url(self, item): # pylint: disable=no-self-use
if not item:
return '', ''
title = item.get('title', '')
url = item.get('link', '')
if title:
title = u'' + title.replace(' ', '.')
else:
title = ''
if url:
url = url.replace('&', '&').replace('%26tr%3D', '&tr=')
else:
url = ''
return title, url
def _make_url(self, result):
if not result:
return '', ''
urls = []
filename = u''
if result.url.startswith('magnet'):
try:
torrent_hash = re.findall(r'urn:btih:([\w]{32,40})',
result.url)[0].upper()
try:
torrent_name = re.findall('dn=([^&]+)', result.url)[0]
except Exception:
torrent_name = 'NO_DOWNLOAD_NAME'
if len(torrent_hash) == 32:
torrent_hash = b16encode(b32decode(torrent_hash)).upper()
if not torrent_hash:
logger.log(
u'Unable to extract torrent hash from magnet: %s' %
ex(result.url), logger.ERROR)
return urls, filename
urls = [
x.format(torrent_hash=torrent_hash,
torrent_name=torrent_name)
for x in self.bt_cache_urls
]
except Exception:
logger.log(
u'Unable to extract torrent hash or name from magnet: %s' %
ex(result.url), logger.ERROR)
return urls, filename
else:
urls = [result.url]
filename = ek(
join, self._get_storage_dir(),
sanitize_filename(result.name) + '.' + self.provider_type)
return urls, filename
def _verify_download(self, file_name=None): # pylint: disable=unused-argument,no-self-use
return True
class GenericProvider(object): # pylint: disable=too-many-instance-attributes
NZB = 'nzb'
TORRENT = 'torrent'
def __init__(self, name):
self.name = name
self.anime_only = False
self.bt_cache_urls = [
#'http://torcache.net/torrent/{torrent_hash}.torrent',
'http://torrentproject.se/torrent/{torrent_hash}.torrent',
'http://thetorrent.org/torrent/{torrent_hash}.torrent',
'http://btdig.com/torrent/{torrent_hash}.torrent',
# 'http://torrage.com/torrent/{torrent_hash}.torrent',
'http://itorrents.org/torrent/{torrent_hash}.torrent',
]
self.cache = TVCache(self)
self.enable_backlog = False
self.enable_daily = False
self.enabled = False
self.headers = {'User-Agent': UA_POOL.random}
self.proper_strings = ['PROPER|REPACK|REAL']
self.provider_type = None
self.public = False
self.search_fallback = False
self.search_mode = None
self.session = make_session()
self.show = None
self.supports_absolute_numbering = False
self.supports_backlog = True
self.url = ''
self.urls = {}
# Use and configure the attribute enable_cookies to show or hide the cookies input field per provider
self.enable_cookies = False
self.cookies = ''
self.rss_cookies = ''
shuffle(self.bt_cache_urls)
def download_result(self, result):
if not self.login():
return False
urls, filename = self._make_url(result)
for url in urls:
if 'NO_DOWNLOAD_NAME' in url:
continue
if url.startswith('http'):
self.headers.update({
'Referer': '/'.join(url.split('/')[:3]) + '/'
})
logger.log(u'Downloading a result from {0} at {1}'.format(self.name, url))
if url.endswith(GenericProvider.TORRENT) and filename.endswith(GenericProvider.NZB):
filename = replace_extension(filename, GenericProvider.TORRENT)
if download_file(url, filename, session=self.session, headers=self.headers, hooks={'response': self.get_url_hook}):
if self._verify_download(filename):
logger.log(u'Saved result to {0}'.format(filename), logger.INFO)
return True
logger.log(u'Could not download {0}'.format(url), logger.WARNING)
remove_file_failed(filename)
if urls:
logger.log(u'Failed to download any results', logger.WARNING)
return False
def find_propers(self, search_date=None):
results = self.cache.listPropers(search_date)
return [Proper(x['name'], x['url'], datetime.fromtimestamp(x['time']), self.show) for x in results]
def find_search_results(self, show, episodes, search_mode, # pylint: disable=too-many-branches,too-many-arguments,too-many-locals,too-many-statements
manual_search=False, download_current_quality=False):
self._check_auth()
self.show = show
results = {}
items_list = []
searched_scene_season = None
for episode in episodes:
cache_result = self.cache.searchCache(episode, manualSearch=manual_search,
downCurQuality=download_current_quality)
if cache_result:
if episode.episode not in results:
results[episode.episode] = cache_result
else:
results[episode.episode].extend(cache_result)
continue
if len(episodes) > 1 and search_mode == 'sponly' and searched_scene_season == episode.scene_season:
continue
search_strings = []
searched_scene_season = episode.scene_season
if len(episodes) > 1 and search_mode == 'sponly':
search_strings = self._get_season_search_strings(episode)
elif search_mode == 'eponly':
search_strings = self._get_episode_search_strings(episode)
for search_string in search_strings:
items_list += self.search(search_string, ep_obj=episode)
if len(results) == len(episodes):
return results
if items_list:
items = {}
unknown_items = []
for item in items_list:
quality = self.get_quality(item, anime=show.is_anime)
if quality == Quality.UNKNOWN:
unknown_items.append(item)
else:
if quality not in items:
items[quality] = []
items[quality].append(item)
items_list = list(chain(*[v for (k_, v) in sorted(items.iteritems(), reverse=True)]))
items_list += unknown_items
cl = []
for item in items_list:
(title, url) = self._get_title_and_url(item)
try:
parse_result = NameParser(parse_method=('normal', 'anime')[show.is_anime]).parse(title)
except (InvalidNameException, InvalidShowException) as error:
logger.log(u"{0}".format(error), logger.DEBUG)
continue
show_object = parse_result.show
quality = parse_result.quality
release_group = parse_result.release_group
version = parse_result.version
add_cache_entry = False
if not (show_object.air_by_date or show_object.sports):
if search_mode == 'sponly':
if parse_result.episode_numbers:
logger.log(
u'This is supposed to be a season pack search but the result {0} is not a valid season pack, skipping it'.format(title),
logger.DEBUG
)
add_cache_entry = True
elif not [ep for ep in episodes if parse_result.season_number == (ep.season, ep.scene_season)[ep.show.is_scene]]:
logger.log(
u'This season result {0} is for a season we are not searching for, skipping it'.format(title),
logger.DEBUG
)
add_cache_entry = True
else:
if not all([
# pylint: disable=bad-continuation
parse_result.season_number is not None,
parse_result.episode_numbers,
[ep for ep in episodes if (ep.season, ep.scene_season)[ep.show.is_scene] ==
(parse_result.season_number, parse_result.scene_season)[ep.show.is_scene] and
(ep.episode, ep.scene_episode)[ep.show.is_scene] in parse_result.episode_numbers]
]):
logger.log(
u'The result {0} doesn\'t seem to match an episode that we are currently trying to snatch, skipping it'.format(title),
logger.DEBUG)
add_cache_entry = True
if not add_cache_entry:
actual_season = parse_result.season_number
actual_episodes = parse_result.episode_numbers
else:
same_day_special = False
if not parse_result.is_air_by_date:
logger.log(
u'This is supposed to be a date search but the result {0} didn\'t parse as one, skipping it'.format(title),
logger.DEBUG)
add_cache_entry = True
else:
air_date = parse_result.air_date.toordinal()
db = DBConnection()
sql_results = db.select(
'SELECT season, episode FROM tv_episodes WHERE showid = ? AND airdate = ?',
[show_object.indexerid, air_date]
)
if len(sql_results) == 2:
if int(sql_results[0]['season']) == 0 and int(sql_results[1]['season']) != 0:
actual_season = int(sql_results[1]['season'])
actual_episodes = [int(sql_results[1]['episode'])]
same_day_special = True
elif int(sql_results[1]['season']) == 0 and int(sql_results[0]['season']) != 0:
actual_season = int(sql_results[0]['season'])
actual_episodes = [int(sql_results[0]['episode'])]
same_day_special = True
elif len(sql_results) != 1:
logger.log(
u'Tried to look up the date for the episode {0} but the database didn\'t give proper results, skipping it'.format(title),
logger.WARNING)
add_cache_entry = True
if not add_cache_entry and not same_day_special:
actual_season = int(sql_results[0]['season'])
actual_episodes = [int(sql_results[0]['episode'])]
if add_cache_entry:
logger.log(u'Adding item from search to cache: {0}'.format(title), logger.DEBUG)
# pylint: disable=protected-access
# Access to a protected member of a client class
ci = self.cache._addCacheEntry(title, url, parse_result=parse_result)
if ci is not None:
cl.append(ci)
continue
episode_wanted = True
for episode_number in actual_episodes:
if not show_object.wantEpisode(actual_season, episode_number, quality, manual_search,
download_current_quality):
episode_wanted = False
break
if not episode_wanted:
logger.log(u'Ignoring result {0}.'.format(title), logger.DEBUG)
continue
logger.log(u'Found result {0} at {1}'.format(title, url), logger.DEBUG)
episode_object = []
for current_episode in actual_episodes:
episode_object.append(show_object.getEpisode(actual_season, current_episode))
result = self.get_result(episode_object)
result.show = show_object
result.url = url
result.name = title
result.quality = quality
result.release_group = release_group
result.version = version
result.content = None
result.size = self._get_size(item)
if len(episode_object) == 1:
episode_number = episode_object[0].episode
logger.log(u'Single episode result.', logger.DEBUG)
elif len(episode_object) > 1:
episode_number = MULTI_EP_RESULT
logger.log(u'Separating multi-episode result to check for later - result contains episodes: {0}'.format(
parse_result.episode_numbers), logger.DEBUG)
elif len(episode_object) == 0:
episode_number = SEASON_RESULT
logger.log(u'Separating full season result to check for later', logger.DEBUG)
if episode_number not in results:
results[episode_number] = [result]
else:
results[episode_number].append(result)
if cl:
# pylint: disable=protected-access
# Access to a protected member of a client class
cache_db = self.cache._getDB()
cache_db.mass_action(cl)
return results
def get_id(self):
return GenericProvider.make_id(self.name)
def get_quality(self, item, anime=False):
(title, url_) = self._get_title_and_url(item)
quality = Quality.scene_quality(title, anime)
return quality
def get_result(self, episodes):
result = self._get_result(episodes)
result.provider = self
return result
@staticmethod
def get_url_hook(response, **kwargs_):
logger.log(u'{0} URL: {1} [Status: {2}]'.format
(response.request.method, response.request.url, response.status_code), logger.DEBUG)
if response.request.method == 'POST':
logger.log(u'With post data: {0}'.format(response.request.body), logger.DEBUG)
def get_url(self, url, post_data=None, params=None, timeout=30, **kwargs): # pylint: disable=too-many-arguments,
kwargs['hooks'] = {'response': self.get_url_hook}
return getURL(url, post_data, params, self.headers, timeout, self.session, **kwargs)
def image_name(self):
return self.get_id() + '.png'
def is_active(self): # pylint: disable=no-self-use
return False
def is_enabled(self):
return bool(self.enabled)
@staticmethod
def make_id(name):
if not name:
return ''
return re.sub(r'[^\w\d_]', '_', str(name).strip().lower())
def search_rss(self, episodes):
return self.cache.findNeededEpisodes(episodes)
def seed_ratio(self): # pylint: disable=no-self-use
return ''
def _check_auth(self): # pylint: disable=no-self-use
return True
def login(self): # pylint: disable=no-self-use
return True
def search(self, search_params, age=0, ep_obj=None): # pylint: disable=unused-argument,no-self-use
return []
def _get_result(self, episodes): # pylint: disable=no-self-use
return SearchResult(episodes)
def _get_episode_search_strings(self, episode, add_string=''):
if not episode:
return []
search_string = {
'Episode': []
}
for show_name in allPossibleShowNames(episode.show, season=episode.scene_season):
episode_string = show_name + ' '
episode_string_fallback = None
if episode.show.air_by_date:
episode_string += str(episode.airdate).replace('-', ' ')
elif episode.show.sports:
episode_string += str(episode.airdate).replace('-', ' ')
episode_string += ('|', ' ')[len(self.proper_strings) > 1]
episode_string += episode.airdate.strftime('%b')
elif episode.show.anime:
episode_string_fallback = episode_string + '{0:02d}'.format(int(episode.scene_absolute_number))
episode_string += '{0:03d}'.format(int(episode.scene_absolute_number))
else:
episode_string += sickbeard.config.naming_ep_type[2] % {
'seasonnumber': episode.scene_season,
'episodenumber': episode.scene_episode,
}
if add_string:
episode_string += ' ' + add_string
if episode_string_fallback:
episode_string_fallback += ' ' + add_string
search_string['Episode'].append(episode_string.encode('utf-8').strip())
if episode_string_fallback:
search_string['Episode'].append(episode_string_fallback.encode('utf-8').strip())
return [search_string]
def _get_season_search_strings(self, episode):
search_string = {
'Season': []
}
for show_name in allPossibleShowNames(episode.show, season=episode.scene_season):
season_string = show_name + ' '
if episode.show.air_by_date or episode.show.sports:
season_string += str(episode.airdate).split('-')[0]
elif episode.show.anime:
# use string below if you really want to search on season with number
# season_string += 'Season ' + '{0:d}'.format(int(episode.scene_season))
season_string += 'Season' # ignore season number to get all seasons in all formats
else:
season_string += 'S{0:02d}'.format(int(episode.scene_season))
search_string['Season'].append(season_string.encode('utf-8').strip())
return [search_string]
def _get_size(self, item): # pylint: disable=unused-argument,no-self-use
try:
return item.get('size', -1)
except AttributeError:
return -1
def _get_storage_dir(self): # pylint: disable=no-self-use
return ''
def _get_title_and_url(self, item): # pylint: disable=no-self-use
if not item:
return '', ''
title = item.get('title', '')
url = item.get('link', '')
if title:
title = u'' + title.replace(' ', '.')
else:
title = ''
if url:
url = url.replace('&', '&').replace('%26tr%3D', '&tr=')
else:
url = ''
return title, url
@staticmethod
def hash_from_magnet(magnet):
try:
torrent_hash = re.findall(r'urn:btih:([\w]{32,40})', magnet)[0].upper()
if len(torrent_hash) == 32:
torrent_hash = b16encode(b32decode(torrent_hash)).upper()
return torrent_hash
except Exception:
logger.log(u'Unable to extract torrent hash or name from magnet: {0}'.format(magnet), logger.ERROR)
return ''
def _make_url(self, result):
if not result:
return '', ''
filename = u''
urls = [result.url]
if result.url.startswith('magnet'):
torrent_hash = self.hash_from_magnet(result.url)
if not torrent_hash:
return urls, filename
try:
torrent_name = re.findall('dn=([^&]+)', result.url)[0]
except Exception:
torrent_name = 'NO_DOWNLOAD_NAME'
urls = [x.format(torrent_hash=torrent_hash, torrent_name=torrent_name) for x in self.bt_cache_urls]
filename = ek(join, self._get_storage_dir(), sanitize_filename(result.name) + '.' + self.provider_type)
return urls, filename
def _verify_download(self, file_name=None): # pylint: disable=unused-argument,no-self-use
return True
def add_cookies_from_ui(self):
"""
Adds the cookies configured from UI to the providers requests session
:return: A tuple with the the (success result, and a descriptive message in str)
"""
# This is the generic attribute used to manually add cookies for provider authentication
if self.enable_cookies and self.cookies:
cookie_validator = re.compile(r'^(\w+=\w+)(;\w+=\w+)*$')
if not cookie_validator.match(self.cookies):
return False, 'Cookie is not correctly formatted: {0}'.format(self.cookies)
add_dict_to_cookiejar(self.session.cookies, dict(x.rsplit('=', 1) for x in self.cookies.split(';')))
return True, 'torrent cookie'
return False, 'No Cookies added from ui for provider: {0}'.format(self.name)
class GenericProvider(object): # pylint: disable=too-many-instance-attributes
NZB = 'nzb'
TORRENT = 'torrent'
def __init__(self, name):
self.name = name
self.anime_only = False
self.bt_cache_urls = [
'http://torcache.net/torrent/{torrent_hash}.torrent',
'http://thetorrent.org/torrent/{torrent_hash}.torrent',
'http://btdig.com/torrent/{torrent_hash}.torrent',
# 'http://torrage.com/torrent/{torrent_hash}.torrent',
# 'http://itorrents.org/torrent/{torrent_hash}.torrent',
]
self.cache = TVCache(self)
self.enable_backlog = False
self.enable_daily = False
self.enabled = False
self.headers = {'User-Agent': UA_POOL.random}
self.proper_strings = ['PROPER|REPACK|REAL']
self.provider_type = None
self.public = False
self.search_fallback = False
self.search_mode = None
self.session = Session()
self.show = None
self.supports_absolute_numbering = False
self.supports_backlog = True
self.url = ''
self.urls = {}
shuffle(self.bt_cache_urls)
def download_result(self, result):
if not self.login():
return False
urls, filename = self._make_url(result)
for url in urls:
if 'NO_DOWNLOAD_NAME' in url:
continue
if url.startswith('http'):
self.headers.update({
'Referer': '/'.join(url.split('/')[:3]) + '/'
})
logger.log(u'Downloading a result from %s at %s' % (self.name, url))
if url.endswith(GenericProvider.TORRENT) and filename.endswith(GenericProvider.NZB):
filename = replace_extension(filename, GenericProvider.TORRENT)
if download_file(url, filename, session=self.session, headers=self.headers):
if self._verify_download(filename):
logger.log(u'Saved result to %s' % filename, logger.INFO)
return True
logger.log(u'Could not download %s' % url, logger.WARNING)
remove_file_failed(filename)
if len(urls):
logger.log(u'Failed to download any results', logger.WARNING)
return False
def find_propers(self, search_date=None):
results = self.cache.listPropers(search_date)
return [Proper(x['name'], x['url'], datetime.fromtimestamp(x['time']), self.show) for x in results]
def find_search_results(self, show, episodes, search_mode, manual_search=False, download_current_quality=False): # pylint: disable=too-many-branches,too-many-arguments,too-many-locals,too-many-statements
self._check_auth()
self.show = show
results = {}
items_list = []
searched_scene_season = None
for episode in episodes:
cache_result = self.cache.searchCache(episode, manualSearch=manual_search,
downCurQuality=download_current_quality)
if cache_result:
if episode.episode not in results:
results[episode.episode] = cache_result
else:
results[episode.episode].extend(cache_result)
continue
if len(episodes) > 1 and search_mode == 'sponly' and searched_scene_season == episode.scene_season:
continue
search_strings = []
searched_scene_season = episode.scene_season
if len(episodes) > 1 and search_mode == 'sponly':
search_strings = self._get_season_search_strings(episode)
elif search_mode == 'eponly':
search_strings = self._get_episode_search_strings(episode)
first = search_strings and isinstance(search_strings[0], dict) and 'rid' in search_strings[0]
if first:
logger.log(u'First search_string has rid', logger.DEBUG)
for search_string in search_strings:
items_list += self.search(search_string, ep_obj=episode)
if first:
first = False
if items_list:
logger.log(u'First search_string had rid, and returned results, skipping query by string',
logger.DEBUG)
break
logger.log(u'First search_string had rid, but returned no results, searching with string query',
logger.DEBUG)
if len(results) == len(episodes):
return results
if items_list:
items = {}
unknown_items = []
for item in items_list:
quality = self.get_quality(item, anime=show.is_anime)
if quality == Quality.UNKNOWN:
unknown_items.append(item)
else:
if quality not in items:
items[quality] = []
items[quality].append(item)
items_list = list(chain(*[v for (_, v) in sorted(items.iteritems(), reverse=True)]))
items_list += unknown_items
cl = []
for item in items_list:
(title, url) = self._get_title_and_url(item)
try:
parser = NameParser(parse_method=('normal', 'anime')[show.is_anime])
parse_result = parser.parse(title)
except InvalidNameException:
logger.log(u'Unable to parse the filename %s into a valid episode' % title, logger.DEBUG)
continue
except InvalidShowException:
logger.log(u'Unable to parse the filename %s into a valid show' % title, logger.DEBUG)
continue
show_object = parse_result.show
quality = parse_result.quality
release_group = parse_result.release_group
version = parse_result.version
add_cache_entry = False
if not (show_object.air_by_date or show_object.sports):
if search_mode == 'sponly':
if len(parse_result.episode_numbers):
logger.log(
u'This is supposed to be a season pack search but the result %s is not a valid season pack, skipping it' % title,
logger.DEBUG
)
add_cache_entry = True
if len(parse_result.episode_numbers) and (
parse_result.season_number not in set([ep.season for ep in episodes]) or
not [ep for ep in episodes if ep.scene_episode in parse_result.episode_numbers]):
logger.log(
u'The result %s doesn\'t seem to be a valid episode that we are trying to snatch, ignoring' % title,
logger.DEBUG)
add_cache_entry = True
else:
if not len(parse_result.episode_numbers) and parse_result.season_number and not [ep for ep in
episodes if
ep.season == parse_result.season_number and ep.episode in parse_result.episode_numbers]:
logger.log(
u'The result %s doesn\'t seem to be a valid season that we are trying to snatch, ignoring' % title,
logger.DEBUG)
add_cache_entry = True
elif len(parse_result.episode_numbers) and not [ep for ep in episodes if
ep.season == parse_result.season_number and ep.episode in parse_result.episode_numbers]:
logger.log(
u'The result %s doesn\'t seem to be a valid episode that we are trying to snatch, ignoring' % title,
logger.DEBUG)
add_cache_entry = True
if not add_cache_entry:
actual_season = parse_result.season_number
actual_episodes = parse_result.episode_numbers
else:
same_day_special = False
if not parse_result.is_air_by_date:
logger.log(
u'This is supposed to be a date search but the result %s didn\'t parse as one, skipping it' % title,
logger.DEBUG)
add_cache_entry = True
else:
air_date = parse_result.air_date.toordinal()
db = DBConnection()
sql_results = db.select(
'SELECT season, episode FROM tv_episodes WHERE showid = ? AND airdate = ?',
[show_object.indexerid, air_date]
)
if len(sql_results) == 2:
if int(sql_results[0]['season']) == 0 and int(sql_results[1]['season']) != 0:
actual_season = int(sql_results[1]['season'])
actual_episodes = [int(sql_results[1]['episode'])]
same_day_special = True
elif int(sql_results[1]['season']) == 0 and int(sql_results[0]['season']) != 0:
actual_season = int(sql_results[0]['season'])
actual_episodes = [int(sql_results[0]['episode'])]
same_day_special = True
elif len(sql_results) != 1:
logger.log(
u'Tried to look up the date for the episode %s but the database didn\'t give proper results, skipping it' % title,
logger.WARNING)
add_cache_entry = True
if not add_cache_entry and not same_day_special:
actual_season = int(sql_results[0]['season'])
actual_episodes = [int(sql_results[0]['episode'])]
if add_cache_entry:
logger.log(u'Adding item from search to cache: %s' % title, logger.DEBUG)
# pylint: disable=protected-access
# Access to a protected member of a client class
ci = self.cache._addCacheEntry(title, url, parse_result=parse_result)
if ci is not None:
cl.append(ci)
continue
episode_wanted = True
for episode_number in actual_episodes:
if not show_object.wantEpisode(actual_season, episode_number, quality, manual_search,
download_current_quality):
episode_wanted = False
break
if not episode_wanted:
logger.log(u'Ignoring result %s.' % (title), logger.DEBUG)
continue
logger.log(u'Found result %s at %s' % (title, url), logger.DEBUG)
episode_object = []
for current_episode in actual_episodes:
episode_object.append(show_object.getEpisode(actual_season, current_episode))
result = self.get_result(episode_object)
result.show = show_object
result.url = url
result.name = title
result.quality = quality
result.release_group = release_group
result.version = version
result.content = None
result.size = self._get_size(item)
if len(episode_object) == 1:
episode_number = episode_object[0].episode
logger.log(u'Single episode result.', logger.DEBUG)
elif len(episode_object) > 1:
episode_number = MULTI_EP_RESULT
logger.log(u'Separating multi-episode result to check for later - result contains episodes: %s' % str(
parse_result.episode_numbers), logger.DEBUG)
elif len(episode_object) == 0:
episode_number = SEASON_RESULT
logger.log(u'Separating full season result to check for later', logger.DEBUG)
if episode_number not in results:
results[episode_number] = [result]
else:
results[episode_number].append(result)
if len(cl) > 0:
# pylint: disable=protected-access
# Access to a protected member of a client class
db = self.cache._getDB()
db.mass_action(cl)
return results
def get_id(self):
return GenericProvider.make_id(self.name)
def get_quality(self, item, anime=False):
(title, _) = self._get_title_and_url(item)
quality = Quality.sceneQuality(title, anime)
return quality
def get_result(self, episodes):
result = self._get_result(episodes)
result.provider = self
return result
def get_url(self, url, post_data=None, params=None, timeout=30, json=False, need_bytes=False): # pylint: disable=too-many-arguments,
return getURL(url, post_data=post_data, params=params, headers=self.headers, timeout=timeout,
session=self.session, json=json, need_bytes=need_bytes)
def image_name(self):
return self.get_id() + '.png'
def is_active(self): # pylint: disable=no-self-use
return False
def is_enabled(self):
return bool(self.enabled)
@staticmethod
def make_id(name):
if not name:
return ''
return re.sub(r'[^\w\d_]', '_', str(name).strip().lower())
def search_rss(self, episodes):
return self.cache.findNeededEpisodes(episodes)
def seed_ratio(self): # pylint: disable=no-self-use
return ''
def _check_auth(self): # pylint: disable=no-self-use
return True
def login(self): # pylint: disable=no-self-use
return True
def search(self, search_params, age=0, ep_obj=None): # pylint: disable=unused-argument,no-self-use
return []
def _get_result(self, episodes): # pylint: disable=no-self-use
return SearchResult(episodes)
def _get_episode_search_strings(self, episode, add_string=''):
if not episode:
return []
search_string = {
'Episode': []
}
for show_name in set(allPossibleShowNames(episode.show)):
episode_string = show_name + ' '
if episode.show.air_by_date:
episode_string += str(episode.airdate).replace('-', ' ')
elif episode.show.sports:
episode_string += str(episode.airdate).replace('-', ' ')
episode_string += ('|', ' ')[len(self.proper_strings) > 1]
episode_string += episode.airdate.strftime('%b')
elif episode.show.anime:
episode_string += '%02d' % int(episode.scene_absolute_number)
else:
episode_string += sickbeard.config.naming_ep_type[2] % {
'seasonnumber': episode.scene_season,
'episodenumber': episode.scene_episode,
}
if add_string:
episode_string += ' ' + add_string
search_string['Episode'].append(episode_string.encode('utf-8').strip())
return [search_string]
def _get_season_search_strings(self, episode):
search_string = {
'Season': []
}
for show_name in set(allPossibleShowNames(self.show)):
episode_string = show_name + ' '
if episode.show.air_by_date or episode.show.sports:
episode_string += str(episode.airdate).split('-')[0]
elif episode.show.anime:
episode_string += '%d' % int(episode.scene_absolute_number)
else:
episode_string += 'S%02d' % int(episode.scene_season)
search_string['Season'].append(episode_string.encode('utf-8').strip())
return [search_string]
def _get_size(self, item): # pylint: disable=unused-argument,no-self-use
return -1
def _get_storage_dir(self): # pylint: disable=no-self-use
return ''
def _get_title_and_url(self, item): # pylint: disable=no-self-use
if not item:
return '', ''
title = item.get('title', '')
url = item.get('link', '')
if title:
title = u'' + title.replace(' ', '.')
else:
title = ''
if url:
url = url.replace('&', '&').replace('%26tr%3D', '&tr=')
else:
url = ''
return title, url
def _make_url(self, result):
if not result:
return '', ''
urls = []
filename = u''
if result.url.startswith('magnet'):
try:
torrent_hash = re.findall(r'urn:btih:([\w]{32,40})', result.url)[0].upper()
try:
torrent_name = re.findall('dn=([^&]+)', result.url)[0]
except Exception:
torrent_name = 'NO_DOWNLOAD_NAME'
if len(torrent_hash) == 32:
torrent_hash = b16encode(b32decode(torrent_hash)).upper()
if not torrent_hash:
logger.log(u'Unable to extract torrent hash from magnet: %s' % ex(result.url), logger.ERROR)
return urls, filename
urls = [x.format(torrent_hash=torrent_hash, torrent_name=torrent_name) for x in self.bt_cache_urls]
except Exception:
logger.log(u'Unable to extract torrent hash or name from magnet: %s' % ex(result.url), logger.ERROR)
return urls, filename
else:
urls = [result.url]
filename = ek(join, self._get_storage_dir(), sanitize_filename(result.name) + '.' + self.provider_type)
return urls, filename
def _verify_download(self, file_name=None): # pylint: disable=unused-argument,no-self-use
return True
class GenericProvider(object): # pylint: disable=too-many-instance-attributes
NZB = 'nzb'
TORRENT = 'torrent'
def __init__(self, name):
self.name = name
self.anime_only = False
self.bt_cache_urls = [
'http://torcache.net/torrent/{torrent_hash}.torrent',
'http://thetorrent.org/torrent/{torrent_hash}.torrent',
'http://itorrents.org/torrent/{torrent_hash}.torrent',
# 'http://btdig.com/torrent/{torrent_hash}.torrent',
# 'http://torrage.com/torrent/{torrent_hash}.torrent',
]
self.cache = TVCache(self)
self.enable_backlog = False
self.enable_manualsearch = False
self.enable_daily = False
self.enabled = False
self.headers = {'User-Agent': UA_POOL.random}
self.proper_strings = ['PROPER|REPACK|REAL']
self.provider_type = None
self.public = False
self.search_fallback = False
self.search_mode = None
self.session = make_session()
self.show = None
self.supports_absolute_numbering = False
self.supports_backlog = True
self.url = ''
self.urls = {}
# Use and configure the attribute enable_cookies to show or hide the cookies input field per provider
self.enable_cookies = False
self.cookies = ''
# Paramaters for reducting the daily search results parsing
self.max_recent_items = 5
self.stop_at = 3
shuffle(self.bt_cache_urls)
def download_result(self, result):
if not self.login():
return False
urls, filename = self._make_url(result)
for url in urls:
if 'NO_DOWNLOAD_NAME' in url:
continue
if url.startswith('http'):
self.headers.update({
'Referer': '/'.join(url.split('/')[:3]) + '/'
})
logger.log('Downloading a result from %s at %s' % (self.name, url))
if url.endswith(GenericProvider.TORRENT) and filename.endswith(GenericProvider.NZB):
filename = replace_extension(filename, GenericProvider.TORRENT)
if download_file(url, filename, session=self.session, headers=self.headers, hooks={'response': self.get_url_hook}):
if self._verify_download(filename):
logger.log('Saved result to %s' % filename, logger.INFO)
return True
logger.log('Could not download %s' % url, logger.WARNING)
remove_file_failed(filename)
if urls:
logger.log('Failed to download any results', logger.WARNING)
return False
def find_propers(self, search_date=None):
results = self.cache.listPropers(search_date)
return [Proper(x[b'name'], x[b'url'], datetime.fromtimestamp(x[b'time']), self.show, x[b'seeders'],
x[b'leechers'], x[b'size'], x[b'pubdate'], x[b'hash']) for x in results]
def find_search_results(self, show, episodes, search_mode, forced_search=False,
download_current_quality=False, manual_search=False,
manual_search_type='episode'): # pylint: disable=too-many-branches,too-many-arguments,too-many-locals,too-many-statements
self._check_auth()
self.show = show
results = {}
items_list = []
searched_scene_season = None
for episode in episodes:
if not manual_search:
cache_result = self.cache.searchCache(episode, forced_search=forced_search,
downCurQuality=download_current_quality)
if cache_result:
if episode.episode not in results:
results[episode.episode] = cache_result
else:
results[episode.episode].extend(cache_result)
continue
search_strings = []
searched_scene_season = episode.scene_season
if (len(episodes) > 1 or manual_search_type == 'season') and search_mode == 'sponly':
search_strings = self._get_season_search_strings(episode)
elif search_mode == 'eponly':
search_strings = self._get_episode_search_strings(episode)
for search_string in search_strings:
# Find results from the provider
items_list += self.search(search_string, ep_obj=episode)
if len(results) == len(episodes):
return results
if items_list:
# categorize the items into lists by quality
items = defaultdict(list)
for item in items_list:
items[self.get_quality(item, anime=show.is_anime)].append(item)
# temporarily remove the list of items with unknown quality
unknown_items = items.pop(Quality.UNKNOWN, [])
# make a generator to sort the remaining items by descending quality
items_list = (items[quality] for quality in sorted(items, reverse=True))
# unpack all of the quality lists into a single sorted list
items_list = list(chain(*items_list))
# extend the list with the unknown qualities, now sorted at the bottom of the list
items_list.extend(unknown_items)
cl = []
for item in items_list:
(title, url) = self._get_title_and_url(item)
(seeders, leechers) = self._get_result_info(item)
size = self._get_size(item)
pubdate = self._get_pubdate(item)
torrent_hash = self._get_hash(item)
try:
parse_result = NameParser(parse_method=('normal', 'anime')[show.is_anime]).parse(title)
except (InvalidNameException, InvalidShowException) as error:
logger.log(u"{error}".format(error=error), logger.DEBUG)
continue
show_object = parse_result.show
quality = parse_result.quality
release_group = parse_result.release_group
version = parse_result.version
add_cache_entry = False
if not manual_search:
if not (show_object.air_by_date or show_object.sports):
if search_mode == 'sponly':
if parse_result.episode_numbers:
logger.log(
'This is supposed to be a season pack search but the result %s is not a valid season pack, skipping it' % title,
logger.DEBUG
)
add_cache_entry = True
elif not [ep for ep in episodes if parse_result.season_number == (ep.season, ep.scene_season)[ep.show.is_scene]]:
logger.log(
'This season result %s is for a season we are not searching for, skipping it' % title,
logger.DEBUG
)
add_cache_entry = True
else:
if not all([
# pylint: disable=bad-continuation
parse_result.season_number is not None,
parse_result.episode_numbers,
[ep for ep in episodes if (ep.season, ep.scene_season)[ep.show.is_scene] ==
parse_result.season_number and
(ep.episode, ep.scene_episode)[ep.show.is_scene] in
parse_result.episode_numbers]
]):
logger.log(
'The result %s doesn\'t seem to match an episode that we are currently trying to snatch, skipping it' % title,
logger.DEBUG)
add_cache_entry = True
if not add_cache_entry:
actual_season = parse_result.season_number
actual_episodes = parse_result.episode_numbers
else:
same_day_special = False
if not parse_result.is_air_by_date:
logger.log(
'This is supposed to be a date search but the result %s didn\'t parse as one, skipping it' % title,
logger.DEBUG)
add_cache_entry = True
else:
air_date = parse_result.air_date.toordinal()
db = DBConnection()
sql_results = db.select(
'SELECT season, episode FROM tv_episodes WHERE showid = ? AND airdate = ?',
[show_object.indexerid, air_date]
)
if len(sql_results) == 2:
if int(sql_results[0][b'season']) == 0 and int(sql_results[1][b'season']) != 0:
actual_season = int(sql_results[1][b'season'])
actual_episodes = [int(sql_results[1][b'episode'])]
same_day_special = True
elif int(sql_results[1][b'season']) == 0 and int(sql_results[0][b'season']) != 0:
actual_season = int(sql_results[0][b'season'])
actual_episodes = [int(sql_results[0][b'episode'])]
same_day_special = True
elif len(sql_results) != 1:
logger.log(
'Tried to look up the date for the episode %s but the database didn\'t give proper results, skipping it' % title,
logger.WARNING)
add_cache_entry = True
if not add_cache_entry and not same_day_special:
actual_season = int(sql_results[0][b'season'])
actual_episodes = [int(sql_results[0][b'episode'])]
else:
actual_season = parse_result.season_number
actual_episodes = parse_result.episode_numbers
if add_cache_entry:
logger.log('Adding item from search to cache: %s' % title, logger.DEBUG)
# pylint: disable=protected-access
# Access to a protected member of a client class
ci = self.cache._addCacheEntry(title, url, seeders, leechers, size, pubdate, torrent_hash)
if ci is not None:
cl.append(ci)
continue
episode_wanted = True
if not manual_search:
for episode_number in actual_episodes:
if not show_object.wantEpisode(actual_season, episode_number, quality, forced_search,
download_current_quality):
episode_wanted = False
break
if not episode_wanted:
logger.log('Ignoring result %s.' % (title), logger.DEBUG)
continue
logger.log('Found result %s at %s' % (title, url), logger.DEBUG)
episode_object = []
for current_episode in actual_episodes:
episode_object.append(show_object.getEpisode(actual_season, current_episode))
result = self.get_result(episode_object)
result.show = show_object
result.url = url
result.seeders = seeders
result.leechers = leechers
result.name = title
result.quality = quality
result.release_group = release_group
result.version = version
result.content = None
result.size = self._get_size(item)
result.pubdate = self._get_pubdate(item)
result.hash = self._get_hash(item)
if not episode_object:
episode_number = SEASON_RESULT
logger.log('Separating full season result to check for later', logger.DEBUG)
elif len(episode_object) == 1:
episode_number = episode_object[0].episode
logger.log('Single episode result.', logger.DEBUG)
else:
episode_number = MULTI_EP_RESULT
logger.log('Separating multi-episode result to check for later - result contains episodes: %s' % str(
parse_result.episode_numbers), logger.DEBUG)
if episode_number not in results:
results[episode_number] = [result]
else:
results[episode_number].append(result)
if cl:
# pylint: disable=protected-access
# Access to a protected member of a client class
db = self.cache._get_db()
db.mass_action(cl)
return results
def get_id(self):
return GenericProvider.make_id(self.name)
def get_quality(self, item, anime=False):
(title, _) = self._get_title_and_url(item)
quality = Quality.scene_quality(title, anime)
return quality
def get_result(self, episodes):
result = self._get_result(episodes)
result.provider = self
return result
@staticmethod
def get_url_hook(response, **kwargs):
logger.log('{} URL: {} [Status: {}]'.format
(response.request.method, response.request.url, response.status_code), logger.DEBUG)
if response.request.method == 'POST':
logger.log('With post data: {}'.format(response.request.body), logger.DEBUG)
def get_url(self, url, post_data=None, params=None, timeout=30, **kwargs): # pylint: disable=too-many-arguments,
kwargs['hooks'] = {'response': self.get_url_hook}
return getURL(url, post_data, params, self.headers, timeout, self.session, **kwargs)
def image_name(self):
return self.get_id() + '.png'
def is_active(self): # pylint: disable=no-self-use
return False
def is_enabled(self):
return bool(self.enabled)
@staticmethod
def make_id(name):
if not name:
return ''
return re.sub(r'[^\w\d_]', '_', str(name).strip().lower())
def search_rss(self, episodes):
return self.cache.findNeededEpisodes(episodes)
def seed_ratio(self): # pylint: disable=no-self-use
return ''
def _check_auth(self): # pylint: disable=no-self-use
return True
def login(self): # pylint: disable=no-self-use
return True
def search(self, search_params, age=0, ep_obj=None): # pylint: disable=unused-argument,no-self-use
return []
def _get_result(self, episodes): # pylint: disable=no-self-use
return SearchResult(episodes)
def _get_episode_search_strings(self, episode, add_string=''):
if not episode:
return []
search_string = {
'Episode': []
}
for show_name in allPossibleShowNames(episode.show, season=episode.scene_season):
episode_string = show_name + ' '
if episode.show.air_by_date:
episode_string += str(episode.airdate).replace('-', ' ')
elif episode.show.sports:
episode_string += str(episode.airdate).replace('-', ' ')
episode_string += ('|', ' ')[len(self.proper_strings) > 1]
episode_string += episode.airdate.strftime('%b')
elif episode.show.anime:
episode_string += '%02d' % int(episode.scene_absolute_number)
else:
episode_string += sickbeard.config.naming_ep_type[2] % {
'seasonnumber': episode.scene_season,
'episodenumber': episode.scene_episode,
}
if add_string:
episode_string += ' ' + add_string
search_string['Episode'].append(episode_string.strip())
return [search_string]
def _get_season_search_strings(self, episode):
search_string = {
'Season': []
}
for show_name in allPossibleShowNames(episode.show, season=episode.scene_season):
episode_string = show_name + ' '
if episode.show.air_by_date or episode.show.sports:
episode_string += str(episode.airdate).split('-')[0]
elif episode.show.anime:
episode_string += '%d' % int(episode.scene_absolute_number)
else:
episode_string += 'S%02d' % int(episode.scene_season)
search_string['Season'].append(episode_string.strip())
return [search_string]
def _get_size(self, item): # pylint: disable=unused-argument,no-self-use
return -1
def _get_storage_dir(self): # pylint: disable=no-self-use
return ''
def _get_result_info(self, item): # pylint: disable=no-self-use
return -1, -1
def _get_pubdate(self, item): # pylint: disable=unused-argument,no-self-use
"""
Return publish date of the item. If provider doesnt
have _get_pubdate function this will be used
"""
return None
def _get_hash(self, item): # pylint: disable=unused-argument,no-self-use
"""
Return hash of the item. If provider doesnt
have _get_hash function this will be used
"""
return None
def _get_title_and_url(self, item): # pylint: disable=no-self-use
if not item:
return '', ''
title = item.get('title', '')
url = item.get('link', '')
if title:
title = title.replace(' ', '.')
else:
title = ''
if url:
url = url.replace('&', '&').replace('%26tr%3D', '&tr=')
else:
url = ''
return title, url
def _make_url(self, result):
if not result:
return '', ''
urls = []
filename = ''
if result.url.startswith('magnet'):
try:
torrent_hash = re.findall(r'urn:btih:([\w]{32,40})', result.url)[0].upper()
try:
torrent_name = re.findall('dn=([^&]+)', result.url)[0]
except Exception:
torrent_name = 'NO_DOWNLOAD_NAME'
if len(torrent_hash) == 32:
torrent_hash = b16encode(b32decode(torrent_hash)).upper()
if not torrent_hash:
logger.log('Unable to extract torrent hash from magnet: %s' % ex(result.url), logger.ERROR)
return urls, filename
urls = [x.format(torrent_hash=torrent_hash, torrent_name=torrent_name) for x in self.bt_cache_urls]
except Exception:
logger.log('Unable to extract torrent hash or name from magnet: %s' % ex(result.url), logger.ERROR)
return urls, filename
else:
urls = [result.url]
filename = ek(join, self._get_storage_dir(), sanitize_filename(result.name) + '.' + self.provider_type)
return urls, filename
def _verify_download(self, file_name=None): # pylint: disable=unused-argument,no-self-use
return True
@property
def recent_results(self):
return recent_results.get(self.get_id(), [])
@recent_results.setter
def recent_results(self, items):
if not recent_results.get(self.get_id()):
recent_results.update({self.get_id(): []})
if items:
add_to_list = []
for item in items:
if item['link'] not in {cache_item['link'] for cache_item in recent_results[self.get_id()]}:
add_to_list += [item]
results = add_to_list + recent_results[self.get_id()]
recent_results[self.get_id()] = results[:self.max_recent_items]
def add_cookies_from_ui(self):
"""
Adds the cookies configured from UI to the providers requests session
:return: A dict with the the keys result as bool and message as string
"""
# This is the generic attribute used to manually add cookies for provider authentication
if self.enable_cookies:
if self.cookies:
cookie_validator = re.compile(r'^(\w+=\w+)(;\w+=\w+)*$')
if not cookie_validator.match(self.cookies):
ui.notifications.message('Failed to validate cookie for provider {provider}'.format(provider=self.name),
'Cookie is not correctly formatted: {0}'.format(self.cookies))
return {'result': False,
'message': 'Cookie is not correctly formatted: {0}'.format(self.cookies)}
# cookie_validator got at least one cookie key/value pair, let's return success
add_dict_to_cookiejar(self.session.cookies, dict(x.rsplit('=', 1) for x in self.cookies.split(';')))
return {'result': True,
'message': ''}
else: # Else is not needed, but placed it here for readability
ui.notifications.message('Failed to validate cookie for provider {provider}'.format(provider=self.name),
'No Cookies added from ui for provider: {0}'.format(self.name))
return {'result': False,
'message': 'No Cookies added from ui for provider: {0}'.format(self.name)}
return {'result': False,
'message': 'Adding cookies is not supported for provider: {0}'.format(self.name)}
class GenericProvider(object): # pylint: disable=too-many-instance-attributes
NZB = "nzb"
TORRENT = "torrent"
def __init__(self, name):
self.name = name
self.anime_only = False
self.bt_cache_urls = [
#'http://torcache.net/torrent/{torrent_hash}.torrent',
"http://torrentproject.se/torrent/{torrent_hash}.torrent",
"http://thetorrent.org/torrent/{torrent_hash}.torrent",
"http://btdig.com/torrent/{torrent_hash}.torrent",
# 'http://torrage.com/torrent/{torrent_hash}.torrent',
# 'http://itorrents.org/torrent/{torrent_hash}.torrent',
]
self.cache = TVCache(self)
self.enable_backlog = False
self.enable_daily = False
self.enabled = False
self.headers = {"User-Agent": UA_POOL.random}
self.proper_strings = ["PROPER|REPACK|REAL"]
self.provider_type = None
self.public = False
self.search_fallback = False
self.search_mode = None
self.session = make_session()
self.show = None
self.supports_absolute_numbering = False
self.supports_backlog = True
self.url = ""
self.urls = {}
# Use and configure the attribute enable_cookies to show or hide the cookies input field per provider
self.enable_cookies = False
self.cookies = ""
self.rss_cookies = ""
shuffle(self.bt_cache_urls)
def download_result(self, result):
if not self.login():
return False
urls, filename = self._make_url(result)
for url in urls:
if "NO_DOWNLOAD_NAME" in url:
continue
if url.startswith("http"):
self.headers.update({"Referer": "/".join(url.split("/")[:3]) + "/"})
logger.log(u"Downloading a result from {0} at {1}".format(self.name, url))
if url.endswith(GenericProvider.TORRENT) and filename.endswith(GenericProvider.NZB):
filename = replace_extension(filename, GenericProvider.TORRENT)
if download_file(
url, filename, session=self.session, headers=self.headers, hooks={"response": self.get_url_hook}
):
if self._verify_download(filename):
logger.log(u"Saved result to {0}".format(filename), logger.INFO)
return True
logger.log(u"Could not download {0}".format(url), logger.WARNING)
remove_file_failed(filename)
if urls:
logger.log(u"Failed to download any results", logger.WARNING)
return False
def find_propers(self, search_date=None):
results = self.cache.listPropers(search_date)
return [Proper(x["name"], x["url"], datetime.fromtimestamp(x["time"]), self.show) for x in results]
def find_search_results(
self,
show,
episodes,
search_mode, # pylint: disable=too-many-branches,too-many-arguments,too-many-locals,too-many-statements
manual_search=False,
download_current_quality=False,
):
self._check_auth()
self.show = show
results = {}
items_list = []
searched_scene_season = None
for episode in episodes:
cache_result = self.cache.searchCache(
episode, manualSearch=manual_search, downCurQuality=download_current_quality
)
if cache_result:
if episode.episode not in results:
results[episode.episode] = cache_result
else:
results[episode.episode].extend(cache_result)
continue
if len(episodes) > 1 and search_mode == "sponly" and searched_scene_season == episode.scene_season:
continue
search_strings = []
searched_scene_season = episode.scene_season
if len(episodes) > 1 and search_mode == "sponly":
search_strings = self._get_season_search_strings(episode)
elif search_mode == "eponly":
search_strings = self._get_episode_search_strings(episode)
for search_string in search_strings:
items_list += self.search(search_string, ep_obj=episode)
if len(results) == len(episodes):
return results
if items_list:
items = {}
unknown_items = []
for item in items_list:
quality = self.get_quality(item, anime=show.is_anime)
if quality == Quality.UNKNOWN:
unknown_items.append(item)
else:
if quality not in items:
items[quality] = []
items[quality].append(item)
items_list = list(chain(*[v for (k_, v) in sorted(items.iteritems(), reverse=True)]))
items_list += unknown_items
cl = []
for item in items_list:
(title, url) = self._get_title_and_url(item)
try:
parse_result = NameParser(parse_method=("normal", "anime")[show.is_anime]).parse(title)
except (InvalidNameException, InvalidShowException) as error:
logger.log(u"{0}".format(error), logger.DEBUG)
continue
show_object = parse_result.show
quality = parse_result.quality
release_group = parse_result.release_group
version = parse_result.version
add_cache_entry = False
if not (show_object.air_by_date or show_object.sports):
if search_mode == "sponly":
if parse_result.episode_numbers:
logger.log(
u"This is supposed to be a season pack search but the result {0} is not a valid season pack, skipping it".format(
title
),
logger.DEBUG,
)
add_cache_entry = True
elif not [
ep
for ep in episodes
if parse_result.season_number == (ep.season, ep.scene_season)[ep.show.is_scene]
]:
logger.log(
u"This season result {0} is for a season we are not searching for, skipping it".format(
title
),
logger.DEBUG,
)
add_cache_entry = True
else:
if not all(
[
# pylint: disable=bad-continuation
parse_result.season_number is not None,
parse_result.episode_numbers,
[
ep
for ep in episodes
if (ep.season, ep.scene_season)[ep.show.is_scene] == parse_result.season_number
and (ep.episode, ep.scene_episode)[ep.show.is_scene] in parse_result.episode_numbers
],
]
):
logger.log(
u"The result {0} doesn't seem to match an episode that we are currently trying to snatch, skipping it".format(
title
),
logger.DEBUG,
)
add_cache_entry = True
if not add_cache_entry:
actual_season = parse_result.season_number
actual_episodes = parse_result.episode_numbers
else:
same_day_special = False
if not parse_result.is_air_by_date:
logger.log(
u"This is supposed to be a date search but the result {0} didn't parse as one, skipping it".format(
title
),
logger.DEBUG,
)
add_cache_entry = True
else:
air_date = parse_result.air_date.toordinal()
db = DBConnection()
sql_results = db.select(
"SELECT season, episode FROM tv_episodes WHERE showid = ? AND airdate = ?",
[show_object.indexerid, air_date],
)
if len(sql_results) == 2:
if int(sql_results[0]["season"]) == 0 and int(sql_results[1]["season"]) != 0:
actual_season = int(sql_results[1]["season"])
actual_episodes = [int(sql_results[1]["episode"])]
same_day_special = True
elif int(sql_results[1]["season"]) == 0 and int(sql_results[0]["season"]) != 0:
actual_season = int(sql_results[0]["season"])
actual_episodes = [int(sql_results[0]["episode"])]
same_day_special = True
elif len(sql_results) != 1:
logger.log(
u"Tried to look up the date for the episode {0} but the database didn't give proper results, skipping it".format(
title
),
logger.WARNING,
)
add_cache_entry = True
if not add_cache_entry and not same_day_special:
actual_season = int(sql_results[0]["season"])
actual_episodes = [int(sql_results[0]["episode"])]
if add_cache_entry:
logger.log(u"Adding item from search to cache: {0}".format(title), logger.DEBUG)
# pylint: disable=protected-access
# Access to a protected member of a client class
ci = self.cache._addCacheEntry(title, url, parse_result=parse_result)
if ci is not None:
cl.append(ci)
continue
episode_wanted = True
for episode_number in actual_episodes:
if not show_object.wantEpisode(
actual_season, episode_number, quality, manual_search, download_current_quality
):
episode_wanted = False
break
if not episode_wanted:
logger.log(u"Ignoring result {0}.".format(title), logger.DEBUG)
continue
logger.log(u"Found result {0} at {1}".format(title, url), logger.DEBUG)
episode_object = []
for current_episode in actual_episodes:
episode_object.append(show_object.getEpisode(actual_season, current_episode))
result = self.get_result(episode_object)
result.show = show_object
result.url = url
result.name = title
result.quality = quality
result.release_group = release_group
result.version = version
result.content = None
result.size = self._get_size(item)
if len(episode_object) == 1:
episode_number = episode_object[0].episode
logger.log(u"Single episode result.", logger.DEBUG)
elif len(episode_object) > 1:
episode_number = MULTI_EP_RESULT
logger.log(
u"Separating multi-episode result to check for later - result contains episodes: {0}".format(
parse_result.episode_numbers
),
logger.DEBUG,
)
elif len(episode_object) == 0:
episode_number = SEASON_RESULT
logger.log(u"Separating full season result to check for later", logger.DEBUG)
if episode_number not in results:
results[episode_number] = [result]
else:
results[episode_number].append(result)
if cl:
# pylint: disable=protected-access
# Access to a protected member of a client class
cache_db = self.cache._getDB()
cache_db.mass_action(cl)
return results
def get_id(self):
return GenericProvider.make_id(self.name)
def get_quality(self, item, anime=False):
(title, url_) = self._get_title_and_url(item)
quality = Quality.scene_quality(title, anime)
return quality
def get_result(self, episodes):
result = self._get_result(episodes)
result.provider = self
return result
@staticmethod
def get_url_hook(response, **kwargs_):
logger.log(
u"{0} URL: {1} [Status: {2}]".format(response.request.method, response.request.url, response.status_code),
logger.DEBUG,
)
if response.request.method == "POST":
logger.log(u"With post data: {0}".format(response.request.body), logger.DEBUG)
def get_url(self, url, post_data=None, params=None, timeout=30, **kwargs): # pylint: disable=too-many-arguments,
kwargs["hooks"] = {"response": self.get_url_hook}
return getURL(url, post_data, params, self.headers, timeout, self.session, **kwargs)
def image_name(self):
return self.get_id() + ".png"
def is_active(self): # pylint: disable=no-self-use
return False
def is_enabled(self):
return bool(self.enabled)
@staticmethod
def make_id(name):
if not name:
return ""
return re.sub(r"[^\w\d_]", "_", str(name).strip().lower())
def search_rss(self, episodes):
return self.cache.findNeededEpisodes(episodes)
def seed_ratio(self): # pylint: disable=no-self-use
return ""
def _check_auth(self): # pylint: disable=no-self-use
return True
def login(self): # pylint: disable=no-self-use
return True
def search(self, search_params, age=0, ep_obj=None): # pylint: disable=unused-argument,no-self-use
return []
def _get_result(self, episodes): # pylint: disable=no-self-use
return SearchResult(episodes)
def _get_episode_search_strings(self, episode, add_string=""):
if not episode:
return []
search_string = {"Episode": []}
for show_name in allPossibleShowNames(episode.show, season=episode.scene_season):
episode_string = show_name + " "
if episode.show.air_by_date:
episode_string += str(episode.airdate).replace("-", " ")
elif episode.show.sports:
episode_string += str(episode.airdate).replace("-", " ")
episode_string += ("|", " ")[len(self.proper_strings) > 1]
episode_string += episode.airdate.strftime("%b")
elif episode.show.anime:
episode_string += "{0:02d}".format(int(episode.scene_absolute_number))
else:
episode_string += sickbeard.config.naming_ep_type[2] % {
"seasonnumber": episode.scene_season,
"episodenumber": episode.scene_episode,
}
if add_string:
episode_string += " " + add_string
search_string["Episode"].append(episode_string.encode("utf-8").strip())
return [search_string]
def _get_season_search_strings(self, episode):
search_string = {"Season": []}
for show_name in allPossibleShowNames(episode.show, season=episode.scene_season):
episode_string = show_name + " "
if episode.show.air_by_date or episode.show.sports:
episode_string += str(episode.airdate).split("-")[0]
elif episode.show.anime:
episode_string += "{0:d}".format(int(episode.scene_absolute_number))
else:
episode_string += "S{0:02d}".format(int(episode.scene_season))
search_string["Season"].append(episode_string.encode("utf-8").strip())
return [search_string]
def _get_size(self, item): # pylint: disable=unused-argument,no-self-use
try:
return item.get("size", -1)
except AttributeError:
return -1
def _get_storage_dir(self): # pylint: disable=no-self-use
return ""
def _get_title_and_url(self, item): # pylint: disable=no-self-use
if not item:
return "", ""
title = item.get("title", "")
url = item.get("link", "")
if title:
title = u"" + title.replace(" ", ".")
else:
title = ""
if url:
url = url.replace("&", "&").replace("%26tr%3D", "&tr=")
else:
url = ""
return title, url
@staticmethod
def hash_from_magnet(magnet):
try:
torrent_hash = re.findall(r"urn:btih:([\w]{32,40})", magnet)[0].upper()
if len(torrent_hash) == 32:
torrent_hash = b16encode(b32decode(torrent_hash)).upper()
return torrent_hash
except Exception:
logger.log(u"Unable to extract torrent hash or name from magnet: {0}".format(magnet), logger.ERROR)
return ""
def _make_url(self, result):
if not result:
return "", ""
filename = u""
urls = [result.url]
if result.url.startswith("magnet"):
torrent_hash = self.hash_from_magnet(result.url)
if not torrent_hash:
return urls, filename
try:
torrent_name = re.findall("dn=([^&]+)", result.url)[0]
except Exception:
torrent_name = "NO_DOWNLOAD_NAME"
urls = [x.format(torrent_hash=torrent_hash, torrent_name=torrent_name) for x in self.bt_cache_urls]
filename = ek(join, self._get_storage_dir(), sanitize_filename(result.name) + "." + self.provider_type)
return urls, filename
def _verify_download(self, file_name=None): # pylint: disable=unused-argument,no-self-use
return True
def add_cookies_from_ui(self):
"""
Adds the cookies configured from UI to the providers requests session
:return: A tuple with the the (success result, and a descriptive message in str)
"""
# This is the generic attribute used to manually add cookies for provider authentication
if self.enable_cookies and self.cookies:
cookie_validator = re.compile(r"^(\w+=\w+)(;\w+=\w+)*$")
if not cookie_validator.match(self.cookies):
return False, "Cookie is not correctly formatted: {0}".format(self.cookies)
add_dict_to_cookiejar(self.session.cookies, dict(x.rsplit("=", 1) for x in self.cookies.split(";")))
return True, "torrent cookie"
return False, "No Cookies added from ui for provider: {0}".format(self.name)
