def test_womble(self):
RSSFeeds().clearCache()
result = RSSFeeds().getFeed(
'https://newshost.co.za/rss/?sec=tv-sd&fr=false')
self.assertTrue('entries' in result)
self.assertTrue('feed' in result)
for item in result['entries']:
self.assertTrue(TVCache._parseItem(item))
def getRSSFeed(self, url, post_data=None, items=[]):
handlers = []
if self.provider.proxy.isEnabled():
self.provider.headers.update(
{'Referer': self.provider.proxy.getProxyURL()})
elif sickbeard.PROXY_SETTING:
logger.log("Using proxy for url: " + url, logger.DEBUG)
scheme, address = urllib2.splittype(sickbeard.PROXY_SETTING)
address = sickbeard.PROXY_SETTING if scheme else 'http://' + sickbeard.PROXY_SETTING
handlers = [
urllib2.ProxyHandler({
'http': address,
'https': address
})
]
self.provider.headers.update({'Referer': address})
elif 'Referer' in self.provider.headers:
self.provider.headers.pop('Referer')
return RSSFeeds(self.providerID).getFeed(
self.provider.proxy._buildURL(url),
post_data,
self.provider.headers,
items,
handlers=handlers)
def _search_provider(self, search_params, **kwargs):
self._authorised()
results = []
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
url = self.urls['browse'] % self.passkey
for mode in search_params.keys():
for search_string in search_params[mode]:
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
search_url = url + (self.urls['search'] % search_string, '')['Cache' == mode]
xml_data = RSSFeeds(self).get_feed(search_url)
cnt = len(items[mode])
if xml_data and 'entries' in xml_data:
for entry in xml_data['entries']:
try:
if entry['title'] and 'download' in entry['link']:
items[mode].append((entry['title'], entry['link'], None, None))
except KeyError:
continue
self._log_search(mode, len(items[mode]) - cnt, search_url)
results = list(set(results + items[mode]))
return results
def getRSSFeed(self, url, post_data=None, items=[]):
if self.provider.proxy.isEnabled():
self.provider.headers.update(
{'Referer': self.provider.proxy.getProxyURL()})
return RSSFeeds(self.providerID).getFeed(
self.provider.proxy._buildURL(url), post_data,
self.provider.headers, items)
def __init__(self,
name,
url,
cookies='',
search_mode='eponly',
search_fallback=False,
enable_recentsearch=False,
enable_backlog=False):
generic.TorrentProvider.__init__(self, name)
self.url = url.rstrip('/')
self.cookies = cookies
self.enable_recentsearch = enable_recentsearch
self.enable_backlog = enable_backlog
self.search_mode = search_mode
self.search_fallback = search_fallback
self.feeder = RSSFeeds(self)
self.cache = TorrentRssCache(self)
def __init__(self, name, url, cookies='', search_mode='eponly', search_fallback=False,
enable_recentsearch=False, enable_backlog=False):
generic.TorrentProvider.__init__(self, name)
self.url = url.rstrip('/')
self.cookies = cookies
self.enable_recentsearch = enable_recentsearch
self.enable_backlog = enable_backlog
self.search_mode = search_mode
self.search_fallback = search_fallback
self.feeder = RSSFeeds(self)
self.cache = TorrentRssCache(self)
def __init__(self,
name,
url,
cookies='',
search_mode='eponly',
search_fallback=False,
enable_recentsearch=False,
enable_backlog=False):
self.enable_backlog = bool(tryInt(enable_backlog))
generic.TorrentProvider.__init__(self,
name,
supports_backlog=self.enable_backlog,
cache_update_freq=15)
self.url = url.rstrip('/')
self.url_base = self.url
self.cookies = cookies
self.enable_recentsearch = bool(
tryInt(enable_recentsearch)) or not self.enable_backlog
self.search_mode = search_mode
self.search_fallback = bool(tryInt(search_fallback))
self.feeder = RSSFeeds(self)
def get_cache_data(self):
api_key = self._init_api()
if False is api_key:
return self.search_html()
if None is not api_key:
params = {
'user': self.username,
'api': api_key,
'eng': 1,
'catid': '19,20'
} # SD,HD
rss_url = self.urls['cache'] % urllib.urlencode(params)
logger.log(self.name + u' cache update URL: ' + rss_url,
logger.DEBUG)
data = RSSFeeds(self).get_feed(rss_url)
if data and 'entries' in data:
return data.entries
return []
def getRSSFeed(self, url, post_data=None, request_headers=None):
return RSSFeeds(self.providerID).getFeed(url, post_data,
request_headers)
class TorrentRssProvider(generic.TorrentProvider):
def __init__(self,
name,
url,
cookies='',
search_mode='eponly',
search_fallback=False,
enable_recentsearch=False,
enable_backlog=False):
generic.TorrentProvider.__init__(self, name)
self.url = url.rstrip('/')
self.cookies = cookies
self.enable_recentsearch = enable_recentsearch
self.enable_backlog = enable_backlog
self.search_mode = search_mode
self.search_fallback = search_fallback
self.feeder = RSSFeeds(self)
self.cache = TorrentRssCache(self)
def image_name(self):
return generic.GenericProvider.image_name(self, 'torrentrss')
def config_str(self):
return '%s|%s|%s|%d|%s|%d|%d|%d' % (
self.name or '', self.url or '', self.cookies
or '', self.enabled, self.search_mode or '', self.search_fallback,
self.enable_recentsearch, self.enable_backlog)
def _get_title_and_url(self, item):
title, url = None, None
if item.title:
title = re.sub(r'\s+', '.', u'' + item.title)
attempt_list = [
lambda: item.torrent_magneturi, lambda: item.enclosures[0].href,
lambda: item.link
]
for cur_attempt in attempt_list:
try:
url = cur_attempt()
except:
continue
if title and url:
break
return title, url
def validate_feed(self):
success, err_msg = self._check_cookie()
if not success:
return success, err_msg
try:
items = self.get_cache_data()
for item in items:
title, url = self._get_title_and_url(item)
if not (title and url):
continue
if url.startswith('magnet:'):
if re.search('urn:btih:([0-9a-f]{32,40})', url):
break
else:
torrent_file = self.get_url(url)
try:
bdecode(torrent_file)
break
except Exception:
pass
else:
return False, '%s fetched RSS feed data: %s' % \
(('Fail to validate', 'No items found in the')[0 == len(items)], self.url)
return True, None
except Exception as e:
return False, 'Error when trying to load RSS: ' + ex(e)
def get_cache_data(self):
logger.log(u'TorrentRssCache cache update URL: ' + self.url,
logger.DEBUG)
data = self.feeder.get_feed(self.url)
return [] if not (data and 'entries' in data) else data.entries
def test_newznab(self):
RSSFeeds().clearCache()
result = RSSFeeds().getFeed('http://lolo.sickbeard.com/api?t=caps')
self.assertTrue('entries' in result)
self.assertTrue('feed' in result)
self.assertTrue('categories' in result.feed)
class TorrentRssProvider(generic.TorrentProvider):
def __init__(self, name, url, cookies='', search_mode='eponly', search_fallback=False,
enable_recentsearch=False, enable_backlog=False):
generic.TorrentProvider.__init__(self, name)
self.url = url.rstrip('/')
self.cookies = cookies
self.enable_recentsearch = enable_recentsearch
self.enable_backlog = enable_backlog
self.search_mode = search_mode
self.search_fallback = search_fallback
self.feeder = RSSFeeds(self)
self.cache = TorrentRssCache(self)
def image_name(self):
return generic.GenericProvider.image_name(self, 'torrentrss')
def config_str(self):
return '%s|%s|%s|%d|%s|%d|%d|%d' % (self.name or '',
self.url or '',
self.cookies or '',
self.enabled,
self.search_mode or '',
self.search_fallback,
self.enable_recentsearch,
self.enable_backlog)
def _title_and_url(self, item):
title, url = None, None
if item.title:
title = re.sub(r'\s+', '.', u'' + item.title)
attempt_list = [lambda: item.torrent_magneturi,
lambda: item.enclosures[0].href,
lambda: item.link]
for cur_attempt in attempt_list:
try:
url = cur_attempt()
except:
continue
if title and url:
break
return title, url
def validate_feed(self):
success, err_msg = self._check_cookie()
if not success:
return success, err_msg
try:
items = self.cache_data()
for item in items:
title, url = self._title_and_url(item)
if not (title and url):
continue
if url.startswith('magnet:'):
if re.search('urn:btih:([0-9a-f]{32,40})', url):
break
else:
torrent_file = self.get_url(url)
try:
bdecode(torrent_file)
break
except Exception:
pass
else:
return False, '%s fetched RSS feed data: %s' % \
(('Fail to validate', 'No items found in the')[0 == len(items)], self.url)
return True, None
except Exception as e:
return False, 'Error when trying to load RSS: ' + ex(e)
def cache_data(self):
logger.log(u'TorrentRssCache cache update URL: ' + self.url, logger.DEBUG)
data = self.feeder.get_feed(self.url)
return [] if not (data and 'entries' in data) else data.entries
def getRSSFeed(self, url):
return RSSFeeds(self.provider).get_feed(url)
def get_rss(self, url, **kwargs):
return RSSFeeds(self.provider).get_feed(url, **kwargs)
class TorrentRssProvider(generic.TorrentProvider):
def __init__(self,
name,
url,
cookies='',
search_mode='eponly',
search_fallback=False,
enable_recentsearch=False,
enable_backlog=False):
self.enable_backlog = bool(tryInt(enable_backlog))
generic.TorrentProvider.__init__(self,
name,
supports_backlog=self.enable_backlog,
cache_update_freq=15)
self.url = url.rstrip('/')
self.url_base = self.url
self.cookies = cookies
self.enable_recentsearch = bool(
tryInt(enable_recentsearch)) or not self.enable_backlog
self.search_mode = search_mode
self.search_fallback = bool(tryInt(search_fallback))
self.feeder = RSSFeeds(self)
def image_name(self):
return generic.GenericProvider.image_name(self, 'torrentrss')
def config_str(self):
return '%s|%s|%s|%d|%s|%d|%d|%d' % (
self.name or '', self.url or '', self.cookies
or '', self.enabled, self.search_mode or '', self.search_fallback,
self.enable_recentsearch, self.enable_backlog)
def _title_and_url(self, item):
title, url = None, None
if item.title:
title = re.sub(r'\s+', '.', u'' + item.title)
attempt_list = [
lambda: item.torrent_magneturi, lambda: item.enclosures[0].href,
lambda: item.link
]
for cur_attempt in attempt_list:
try:
url = cur_attempt()
except (StandardError, Exception):
continue
if title and url:
break
return title, url
def validate_feed(self):
success, err_msg = self._check_cookie()
if not success:
return success, err_msg
try:
items = self._search_provider({'Validate': ['']})
for item in items:
title, url = self._title_and_url(item)
if not (title and url):
continue
if url.startswith('magnet:'):
if re.search('urn:btih:([0-9a-f]{32,40})', url):
break
else:
torrent_file = self.get_url(url)
try:
bdecode(torrent_file)
break
except (StandardError, Exception):
pass
else:
return False, '%s fetched RSS feed data: %s' % \
(('Fail to validate', 'No items found in the')[0 == len(items)], self.url)
return True, None
except Exception as e:
return False, 'Error when trying to load RSS: ' + ex(e)
def _search_provider(self, search_params, **kwargs):
result = []
for mode in search_params.keys():
data = self.feeder.get_feed(self.url)
result += (data and 'entries' in data) and data.entries or []
self.log_result(mode, count=len(result), url=self.url)
return result
