def search(self, search_strings, age=0, ep_obj=None):
results = []
for mode in search_strings:
items = []
logger.debug(_("Search Mode: {mode}".format(mode=mode)))
for search_string in {*search_strings[mode]}:
# Feed verified does not exist on this clone
# search_url = self.urls['verified'] if self.confirmed else self.urls['feed']
search_url = self.urls["feed"]
if mode != "RSS":
logger.debug(_("Search String: {search_string}".format(search_string=search_string)))
data = self.get_url(search_url, params={"f": search_string}, returns="text")
if not data:
logger.debug("No data returned from provider")
continue
if not data.startswith("<?xml"):
logger.info("Expected xml but got something else, is your mirror failing?")
continue
try:
with BS4Parser(data, "html5lib") as parser:
for item in parser("item"):
if item.category and "tv" not in item.category.get_text(strip=True).lower():
continue
title = item.title.get_text(strip=True)
t_hash = item.guid.get_text(strip=True).rsplit("/", 1)[-1]
if not all([title, t_hash]):
continue
download_url = "magnet:?xt=urn:btih:" + t_hash + "&dn=" + title + self._custom_trackers
torrent_size, seeders, leechers = self._split_description(item.find("description").text)
size = convert_size(torrent_size) or -1
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != "RSS":
logger.debug(
"Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})".format(
title, seeders, leechers
)
)
continue
result = {"title": title, "link": download_url, "size": size, "seeders": seeders, "leechers": leechers, "hash": t_hash}
items.append(result)
except Exception:
logger.exception("Failed parsing provider. Traceback: {0!r}".format(traceback.format_exc()))
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get("seeders", 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None):
results = []
for mode in search_strings:
items = []
logger.debug(_(f"Search Mode: {mode}"))
for search_string in search_strings[mode]:
if mode == 'Season':
search_string = re.sub(r'(.*)S0?', r'\1Saison ', search_string)
if mode != 'RSS':
logger.debug("Search string: {0}".format
(search_string))
search_url = self.url + '/recherche/' + search_string.replace('.', '-').replace(' ', '-') + '.html,trie-seeds-d'
else:
search_url = self.url + '/view_cat.php?categorie=series&trie=date-d'
data = self.get_url(search_url, returns='text')
if not data:
continue
with BS4Parser(data, 'html5lib') as html:
torrent_rows = html(class_=re.compile('ligne[01]'))
for result in torrent_rows:
try:
title = result.find(class_="titre").get_text(strip=True).replace("HDTV", "HDTV x264-CPasBien")
title = re.sub(r' Saison', ' Season', title, flags=re.I)
tmp = result.find("a")['href'].split('/')[-1].replace('.html', '.torrent').strip()
download_url = (self.url + '/telechargement/{0}'.format(tmp))
if not all([title, download_url]):
continue
seeders = try_int(result.find(class_="up").get_text(strip=True))
leechers = try_int(result.find(class_="down").get_text(strip=True))
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.debug("Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})".format
(title, seeders, leechers))
continue
torrent_size = result.find(class_="poid").get_text(strip=True)
units = ['o', 'Ko', 'Mo', 'Go', 'To', 'Po']
size = convert_size(torrent_size, units=units) or -1
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''}
if mode != 'RSS':
logger.debug("Found result: {0} with {1} seeders and {2} leechers".format(title, seeders, leechers))
items.append(item)
except Exception:
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals
results = []
for mode in search_strings:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode == 'Season':
search_string = re.sub(r'(.*)S0?', r'\1Saison ', search_string)
if mode != 'RSS':
logger.log("Search string: {0}".format
(search_string.decode("utf-8")), logger.DEBUG)
search_url = self.url + '/recherche/' + search_string.replace('.', '-').replace(' ', '-') + '.html,trie-seeds-d'
else:
search_url = self.url + '/view_cat.php?categorie=series&trie=date-d'
data = self.get_url(search_url, returns='text')
if not data:
continue
with BS4Parser(data, 'html5lib') as html:
torrent_rows = html(class_=re.compile('ligne[01]'))
for result in torrent_rows:
try:
title = result.find(class_="titre").get_text(strip=True).replace("HDTV", "HDTV x264-CPasBien")
title = re.sub(r' Saison', ' Season', title, flags=re.I)
tmp = result.find("a")['href'].split('/')[-1].replace('.html', '.torrent').strip()
download_url = (self.url + '/telechargement/{0}'.format(tmp))
if not all([title, download_url]):
continue
seeders = try_int(result.find(class_="up").get_text(strip=True))
leechers = try_int(result.find(class_="down").get_text(strip=True))
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.log("Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})".format
(title, seeders, leechers), logger.DEBUG)
continue
torrent_size = result.find(class_="poid").get_text(strip=True)
units = ['o', 'Ko', 'Mo', 'Go', 'To', 'Po']
size = convert_size(torrent_size, units=units) or -1
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''}
if mode != 'RSS':
logger.log("Found result: {0} with {1} seeders and {2} leechers".format(title, seeders, leechers), logger.DEBUG)
items.append(item)
except StandardError:
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None):
results = []
for mode in search_strings:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
# Feed verified does not exist on this clone
# search_url = self.urls['verified'] if self.confirmed else self.urls['feed']
search_url = self.urls['feed']
if mode != 'RSS':
logger.log("Search string: {0}".format
(search_string.decode("utf-8")), logger.DEBUG)
data = self.get_url(search_url, params={'f': search_string}, returns='text')
if not data:
logger.log("No data returned from provider", logger.DEBUG)
continue
if not data.startswith("<?xml"):
logger.log("Expected xml but got something else, is your mirror failing?", logger.INFO)
continue
try:
with BS4Parser(data, 'html5lib') as parser:
for item in parser('item'):
if item.category and 'tv' not in item.category.get_text(strip=True).lower():
continue
title = item.title.get_text(strip=True)
t_hash = item.guid.get_text(strip=True).rsplit('/', 1)[-1]
if not all([title, t_hash]):
continue
download_url = "magnet:?xt=urn:btih:" + t_hash + "&dn=" + title + self._custom_trackers
torrent_size, seeders, leechers = self._split_description(item.find('description').text)
size = convert_size(torrent_size) or -1
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.log("Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})".format
(title, seeders, leechers), logger.DEBUG)
continue
result = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': t_hash}
items.append(result)
except StandardError:
logger.log("Failed parsing provider. Traceback: {0!r}".format(traceback.format_exc()), logger.ERROR)
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals
results = []
for mode in search_strings:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
# Feed verified does not exist on this clone
# search_url = self.urls['verified'] if self.confirmed else self.urls['feed']
search_url = self.urls['feed']
if mode != 'RSS':
logger.log("Search string: {0}".format
(search_string.decode("utf-8")), logger.DEBUG)
data = self.get_url(search_url, params={'f': search_string}, returns='text')
if not data:
logger.log("No data returned from provider", logger.DEBUG)
continue
if not data.startswith("<?xml"):
logger.log("Expected xml but got something else, is your mirror failing?", logger.INFO)
continue
try:
with BS4Parser(data, 'html5lib') as parser:
for item in parser('item'):
if item.category and 'tv' not in item.category.get_text(strip=True).lower():
continue
title = item.title.get_text(strip=True)
t_hash = item.guid.get_text(strip=True).rsplit('/', 1)[-1]
if not all([title, t_hash]):
continue
download_url = "magnet:?xt=urn:btih:" + t_hash + "&dn=" + title + self._custom_trackers
torrent_size, seeders, leechers = self._split_description(item.find('description').text)
size = convert_size(torrent_size) or -1
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.log("Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})".format
(title, seeders, leechers), logger.DEBUG)
continue
result = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': t_hash}
items.append(result)
except StandardError:
logger.log("Failed parsing provider. Traceback: {0!r}".format(traceback.format_exc()), logger.ERROR)
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None):
results = []
# https://demonoid.is/files/?category=12&quality=58&seeded=0&external=2&sort=seeders&order=desc&query=SEARCH_STRING
search_params = {
"category": "12", # 12: TV
"seeded": 0, # 0: True
"external": 2, # 0: Demonoid (Only works if logged in), 1: External, 2: Both
"order": "desc",
"sort": self.sorting or "seeders"
}
for mode in search_strings:
items = []
logger.debug(_(f"Search Mode: {mode}"))
if mode == "RSS":
logger.info("Demonoid RSS search is not working through this provider yet, only string searches will work. Continuing")
continue
for search_string in search_strings[mode]:
search_params["query"] = search_string
logger.debug("Search string: {0}".format
(search_string))
time.sleep(cpu_presets[settings.CPU_PRESET])
data = self.get_url(self.urls['search'], params=search_params)
if not data:
logger.debug("No data returned from provider")
continue
with BS4Parser(data, "html5lib") as html:
for result in html("img", alt="Download torrent"):
try:
title = result.parent['title']
details_url = result.parent['href']
if not (title and details_url):
if mode != "RSS":
logger.debug("Discarding torrent because We could not parse the title and details")
continue
info = result.parent.parent.find_next_siblings("td")
size = convert_size(info[0].get_text(strip=True)) or -1
seeders = try_int(info[3].get_text(strip=True))
leechers = try_int(info[4].get_text(strip=True))
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != "RSS":
logger.debug("Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})".format
(title, seeders, leechers))
continue
download_url, magnet, torrent_hash = self.get_details(details_url)
if not all([download_url, magnet, torrent_hash]):
logger.info("Failed to get all required information from the details page. url:{}, magnet:{}, hash:{}".format(
bool(download_url), bool(magnet), bool(torrent_hash))
)
continue
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': torrent_hash}
if mode != "RSS":
logger.debug("Found result: {0} with {1} seeders and {2} leechers".format(title, seeders, leechers))
items.append(item)
except (AttributeError, TypeError, KeyError, ValueError, Exception):
logger.info(traceback.format_exc())
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches
results = []
if not self.login():
return results
# http://speed.cd/browse.php?c49=1&c50=1&c52=1&c41=1&c55=1&c2=1&c30=1&freeleech=on&search=arrow&d=on
# Search Params
search_params = {
'c30': 1, # Anime
'c41': 1, # TV/Packs
'c49': 1, # TV/HD
'c50': 1, # TV/Sports
'c52': 1, # TV/B-Ray
'c55': 1, # TV/Kids
'search': '',
}
# Units
units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']
def process_column_header(td):
result = ''
img = td.find('img')
if img:
result = img.get('alt')
if not result:
result = td.get_text(strip=True)
return result
if self.freeleech:
search_params['freeleech'] = 'on'
for mode in search_strings:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode != 'RSS':
logger.log("Search string: {0}".format
(search_string.decode("utf-8")), logger.DEBUG)
search_params['search'] = search_string.translate(None, string.punctuation)
data = self.get_url(self.urls['search'], params=search_params, returns='text')
if not data:
continue
with BS4Parser(data, 'html5lib') as html:
torrent_table = html.find('div', class_='boxContent')
torrent_table = torrent_table.find('table') if torrent_table else []
torrent_rows = torrent_table('tr') if torrent_table else []
# Continue only if at least one Release is found
if len(torrent_rows) < 2:
logger.log("Data returned from provider does not contain any torrents", logger.DEBUG)
continue
labels = [process_column_header(label) for label in torrent_rows[0]('th')]
# Skip column headers
for result in torrent_rows[1:]:
try:
cells = result('td')
title = cells[labels.index('Title')].find('a', class_='torrent').get_text()
download_url = urljoin(self.url, cells[labels.index('Download') - 1].a['href'])
if not all([title, download_url]):
continue
seeders = try_int(cells[labels.index('Seeders') - 1].get_text(strip=True))
leechers = try_int(cells[labels.index('Leechers') - 1].get_text(strip=True))
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.log(
"Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})".format(title, seeders, leechers), logger.DEBUG)
continue
torrent_size = cells[labels.index('Size') - 1].get_text()
torrent_size = torrent_size[:-2] + ' ' + torrent_size[-2:]
size = convert_size(torrent_size, units=units) or -1
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''}
if mode != 'RSS':
logger.log("Found result: {0} with {1} seeders and {2} leechers".format(title, seeders, leechers), logger.DEBUG)
items.append(item)
except StandardError:
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals,too-many-branches, too-many-statements
results = []
if not self.login():
return results
for mode in search_strings:
items = []
if mode != 'RSS':
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode != 'RSS':
logger.log("Search string: {0}".format
(search_string.decode("utf-8")), logger.DEBUG)
search_url = self.urls['search'] % (quote(search_string), self.categories[mode])
try:
data = self.get_url(search_url, returns='text')
time.sleep(cpu_presets[sickbeard.CPU_PRESET])
except Exception as e:
logger.log("Unable to fetch data. Error: {0}".format(repr(e)), logger.WARNING)
if not data:
continue
with BS4Parser(data, 'html5lib') as html:
torrent_table = html.find('table', id='torrents-table')
torrent_rows = torrent_table('tr') if torrent_table else []
# Continue only if at least one Release is found
if len(torrent_rows) < 2:
logger.log("Data returned from provider does not contain any torrents", logger.DEBUG)
continue
for result in torrent_table('tr')[1:]:
try:
link = result.find('td', class_='ttr_name').find('a')
url = result.find('td', class_='td_dl').find('a')
title = link.string
if re.search(r'\.\.\.', title):
data = self.get_url(urljoin(self.url, link['href']), returns='text')
if data:
with BS4Parser(data) as details_html:
title = re.search('(?<=").+(?<!")', details_html.title.string).group(0)
download_url = self.urls['download'] % url['href']
seeders = int(result.find('td', class_='ttr_seeders').string)
leechers = int(result.find('td', class_='ttr_leechers').string)
torrent_size = result.find('td', class_='ttr_size').contents[0]
size = convert_size(torrent_size) or -1
except (AttributeError, TypeError):
continue
if not all([title, download_url]):
continue
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.log(
"Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})".format(title, seeders, leechers), logger.DEBUG)
continue
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''}
if mode != 'RSS':
logger.log("Found result: {0} with {1} seeders and {2} leechers".format(title, seeders, leechers), logger.DEBUG)
items.append(item)
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None):
results = []
anime = (self.show
and self.show.anime) or (ep_obj and ep_obj.show
and ep_obj.show.anime) or False
search_params = {
"q": "",
"field": "seeders",
"sorder": "desc",
"rss": 1,
"category": ("tv", "anime")[anime]
}
for mode in search_strings:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
search_params["q"] = search_string if mode != "RSS" else ""
search_params[
"field"] = "seeders" if mode != "RSS" else "time_add"
if mode != "RSS":
logger.log(
"Search string: {0}".format(
search_string.decode("utf-8")), logger.DEBUG)
search_url = self.urls["search"] % ("usearch" if mode != "RSS"
else search_string)
if self.custom_url:
if not validators.url(self.custom_url):
logger.log(
"Invalid custom url: {0}".format(self.custom_url),
logger.WARNING)
return results
search_url = urljoin(self.custom_url,
search_url.split(self.url)[1])
data = self.get_url(search_url,
params=search_params,
returns="text")
if not data:
logger.log(
"URL did not return results/data, if the results are on the site maybe try a custom url, or a different one",
logger.DEBUG)
continue
if not data.startswith("<?xml"):
logger.log(
"Expected xml but got something else, is your mirror failing?",
logger.INFO)
continue
with BS4Parser(data, "html5lib") as html:
for item in html("item"):
try:
title = item.title.get_text(strip=True)
# Use the torcache link kat provides,
# unless it is not torcache or we are not using blackhole
# because we want to use magnets if connecting direct to client
# so that proxies work.
download_url = item.enclosure["url"]
if sickbeard.TORRENT_METHOD != "blackhole" or "torcache" not in download_url:
download_url = item.find(
"torrent:magneturi").next.replace(
"CDATA", "").strip(
"[!]") + self._custom_trackers
if not (title and download_url):
continue
seeders = try_int(
item.find("torrent:seeds").get_text(
strip=True))
leechers = try_int(
item.find("torrent:peers").get_text(
strip=True))
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != "RSS":
logger.log(
"Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})"
.format(title, seeders,
leechers), logger.DEBUG)
continue
verified = bool(
try_int(
item.find("torrent:verified").get_text(
strip=True)))
if self.confirmed and not verified:
if mode != "RSS":
logger.log(
"Found result " + title +
" but that doesn't seem like a verified result so I'm ignoring it",
logger.DEBUG)
continue
torrent_size = item.find(
"torrent:contentlength").get_text(strip=True)
size = convert_size(torrent_size) or -1
info_hash = item.find("torrent:infohash").get_text(
strip=True)
item = {
'title': title,
'link': download_url,
'size': size,
'seeders': seeders,
'leechers': leechers,
'hash': info_hash
}
if mode != "RSS":
logger.log(
"Found result: {0} with {1} seeders and {2} leechers"
.format(title, seeders,
leechers), logger.DEBUG)
items.append(item)
except (AttributeError, TypeError, KeyError,
ValueError):
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)),
reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-branches, too-many-locals, too-many-statements
results = []
for mode in search_strings:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode != "RSS":
logger.log("Search string: {0}".format
(search_string.decode("utf-8")), logger.DEBUG)
search = slugify(search_string.decode("utf-8"))
search_url = urljoin(self.url, '{}/{}/'.format(search[0], search))
else:
search_url = self.urls['rss']
if self.custom_url:
if not validators.url(self.custom_url):
logger.log("Invalid custom url: {0}".format(self.custom_url), logger.WARNING)
return results
search_url = urljoin(self.custom_url, search_url.split(self.url)[1])
data = self.get_url(search_url, returns="text")
if not data:
logger.log("URL did not return results/data, if the results are on the site maybe try a custom url, or a different one", logger.DEBUG)
continue
with BS4Parser(data, "html5lib") as html:
torrent_table = html.find("table", class_="download")
torrent_body = torrent_table.find('tbody') if torrent_table else []
torrent_rows = torrent_body("tr") if torrent_body else []
# Continue only if at least one Release is found
if not torrent_rows:
logger.log("Data returned from provider does not contain any torrents", logger.DEBUG)
continue
labels = [x.get_text(strip=True) for x in torrent_table.find('thead').find('tr')('th')]
# Skip column headers
for result in torrent_rows[0:-1:2]:
try:
if len(result("td")) < len(labels):
continue
title = result.find("td", class_="n").find("a")['title']
magnet = result.find("td", class_="m").find("a")['href']
seeders = try_int(result.find("td", class_="s").get_text(strip=True))
leechers = try_int(result.find("td", class_="l").get_text(strip=True))
size = convert_size(result("td")[labels.index('Size')].get_text(strip=True) or '') or -1
if not all([title, magnet]):
continue
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != "RSS":
logger.log("Discarding torrent because it doesn't meet the"
" minimum seeders or leechers: {0} (S:{1} L:{2})".format
(title, seeders, leechers), logger.DEBUG)
continue
item = {'title': title, 'link': magnet + self._custom_trackers, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''}
if mode != "RSS":
logger.log("Found result: {0} with {1} seeders and {2} leechers".format
(title, seeders, leechers), logger.DEBUG)
items.append(item)
except StandardError as e:
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: d.get('seeders', 0), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None):
results = []
for mode in search_strings:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode != 'RSS':
logger.log(
"Search string: {0}".format(
search_string.decode("utf-8")), logger.DEBUG)
try:
search_url = (self.urls['rss'], self.urls['search'] +
search_string + '/')[mode != 'RSS']
data = self.get_url(search_url, returns='text')
if not data:
logger.log("No data returned from provider",
logger.DEBUG)
continue
if not data.startswith('<?xml'):
logger.log(
'Expected xml but got something else, is your mirror failing?',
logger.INFO)
continue
data = BeautifulSoup(data, 'html5lib')
entries = data('item')
if not entries:
logger.log('Returned xml contained no results',
logger.INFO)
continue
for item in entries:
try:
title = item.title.text
download_url = item.enclosure['url']
torrent_hash = re.match(r"(.*)([A-F0-9]{40})(.*)",
download_url,
re.I).group(2)
if sickbeard.TORRENT_METHOD != "blackhole" and 'magnet:?' not in download_url:
download_url = "magnet:?xt=urn:btih:" + torrent_hash + "&dn=" + title + self._custom_trackers
if not (title and download_url):
continue
# seeders and leechers are presented diferently when doing a search and when looking for newly added
if mode == 'RSS':
# <![CDATA[
# Category: <a href="http://www.limetorrents.info/browse-torrents/TV-shows/">TV shows</a><br /> Seeds: 1<br />Leechers: 0<br />Size: 7.71 GB<br /><br /><a href="http://www.limetorrents.info/Owen-Hart-of-Gold-Djon91-torrent-7180661.html">More @ limetorrents.info</a><br />
# ]]>
description = item.find('description')
seeders = try_int(
description('br')
[0].next_sibling.strip().lstrip('Seeds: '))
leechers = try_int(
description('br')[1].next_sibling.strip().
lstrip('Leechers: '))
else:
# <description>Seeds: 6982 , Leechers 734</description>
description = item.find(
'description').text.partition(',')
seeders = try_int(
description[0].lstrip('Seeds: ').strip())
leechers = try_int(
description[2].lstrip('Leechers ').strip())
torrent_size = item.find('size').text
size = convert_size(torrent_size) or -1
except (AttributeError, TypeError, KeyError,
ValueError):
continue
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.log(
"Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})"
.format(title, seeders,
leechers), logger.DEBUG)
continue
item = {
'title': title,
'link': download_url,
'size': size,
'seeders': seeders,
'leechers': leechers,
'hash': torrent_hash
}
if mode != 'RSS':
logger.log(
"Found result: {0} with {1} seeders and {2} leechers"
.format(title, seeders,
leechers), logger.DEBUG)
items.append(item)
except (AttributeError, TypeError, KeyError, ValueError):
logger.log(
"Failed parsing provider. Traceback: {0!r}".format(
traceback.format_exc()), logger.ERROR)
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)),
reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals,too-many-branches,too-many-statements
results = []
if not self.login():
return results
for mode in search_strings:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode != 'RSS':
logger.log("Search string: {0}".format
(search_string.decode("utf-8")), logger.DEBUG)
self.search_params['searchstr'] = search_string
data = self.get_url(self.urls['search'], params=self.search_params, returns='text')
if not data:
logger.log('URL did not return data', logger.DEBUG)
continue
strTableStart = "<table class=\"torrent_table"
startTableIndex = data.find(strTableStart)
trimmedData = data[startTableIndex:]
if not trimmedData:
continue
try:
with BS4Parser(trimmedData, 'html5lib') as html:
result_table = html.find('table', {'id': 'torrent_table'})
if not result_table:
logger.log("Data returned from provider does not contain any torrents", logger.DEBUG)
continue
result_tbody = result_table.find('tbody')
entries = result_tbody.contents
del entries[1::2]
for result in entries[1:]:
torrent = result('td')
if len(torrent) <= 1:
break
allAs = (torrent[1])('a')
try:
notinternal = result.find('img', src='/static//common/user_upload.png')
if self.ranked and notinternal:
logger.log("Found a user uploaded release, Ignoring it..", logger.DEBUG)
continue
freeleech = result.find('img', src='/static//common/browse/freeleech.png')
if self.freeleech and not freeleech:
continue
title = allAs[2].string
download_url = self.urls['base_url'] + allAs[0].attrs['href']
torrent_size = result.find("td", class_="nobr").find_next_sibling("td").string
if torrent_size:
size = convert_size(torrent_size) or -1
seeders = try_int((result('td')[6]).text.replace(',', ''))
leechers = try_int((result('td')[7]).text.replace(',', ''))
except (AttributeError, TypeError):
continue
if not title or not download_url:
continue
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.log("Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})".format
(title, seeders, leechers), logger.DEBUG)
continue
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''}
if mode != 'RSS':
logger.log("Found result: {0} with {1} seeders and {2} leechers".format(title, seeders, leechers), logger.DEBUG)
items.append(item)
except Exception:
logger.log("Failed parsing provider. Traceback: {0}".format(traceback.format_exc()), logger.ERROR)
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_params, age=0, ep_obj=None): # pylint: disable=too-many-locals
results = []
search_url = self.urls['search']
download_url = self.urls['download']
if self.custom_url:
if not validators.url(self.custom_url):
logger.log("Invalid custom url: {0}".format(self.custom_url), logger.WARNING)
return results
search_url = urljoin(self.custom_url, search_url.split(self.url)[1])
download_url = urljoin(self.custom_url, download_url.split(self.url)[1])
if not self.login():
return results
for mode in search_params:
items = []
logger.log('Search Mode: {0}'.format(mode), logger.DEBUG)
for search_string in search_params[mode]:
if mode != 'RSS':
logger.log('Search string: {0}'.format
(search_string.decode('utf-8')), logger.DEBUG)
get_params = {}
get_params.update(self.categories[mode])
get_params["q"] = quote_plus(search_string.decode('utf-8', 'ignore'))
try:
torrents = self.get_url(search_url, params=get_params, returns='json')
# Handle empty string response or None #4304
if not torrents:
raise
# Make sure it is iterable #4304
iter(torrents)
except Exception:
logger.log('Data returned from provider does not contain any torrents', logger.DEBUG)
continue
for torrent in torrents:
title = re.sub(r'\[.*\=.*\].*\[/.*\]', '', torrent['name']) if torrent['name'] else None
torrent_url = urljoin(download_url, '{0}/{1}.torrent'.format(torrent['t'], torrent['name'])) if torrent['t'] and torrent['name'] else \
None
if not all([title, torrent_url]):
continue
seeders = try_int(torrent['seeders'])
leechers = try_int(torrent['leechers'])
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.log('Discarding torrent because it doesn\'t meet the minimum seeders or leechers: {0} (S:{1} L:{2})'.format(title, seeders, leechers), logger.DEBUG)
continue
torrent_size = torrent['size']
size = convert_size(torrent_size) or -1
item = {'title': title, 'link': torrent_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''}
if mode != 'RSS':
logger.log('Found result: {0} with {1} seeders and {2} leechers'.format
(title, seeders, leechers), logger.DEBUG)
items.append(item)
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals
results = []
if not self.login():
return results
for mode in search_strings:
items = []
logger.log(u"Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode != 'RSS':
logger.log(u"Search string: {0}".format
(search_string.decode("utf-8")), logger.DEBUG)
self.search_params['search'] = search_string
data = self.get_url(self.urls['search'], params=self.search_params, returns='text')
if not data:
continue
try:
with BS4Parser(data, "html.parser") as html:
torrent_table = html.find('table', class_='browsetable')
torrent_rows = torrent_table('tr') if torrent_table else []
# Continue only if one Release is found
if len(torrent_rows) < 2:
logger.log(u"Data returned from provider does not contain any torrents", logger.DEBUG)
continue
for result in torrent_rows[1:]:
cells = result('td')
link = cells[1].find('a')
download_url = self.urls['download'] % cells[2].find('a')['href']
try:
title = link.getText()
seeders = int(cells[10].getText().replace(',', ''))
leechers = int(cells[11].getText().replace(',', ''))
torrent_size = cells[8].getText()
size = convert_size(torrent_size) or -1
except (AttributeError, TypeError):
continue
if not all([title, download_url]):
continue
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.log(u"Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})".format
(title, seeders, leechers), logger.DEBUG)
continue
if seeders >= 32768 or leechers >= 32768:
continue
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''}
if mode != 'RSS':
logger.log(u"Found result: {0} with {1} seeders and {2} leechers".format(title, seeders, leechers), logger.DEBUG)
items.append(item)
except Exception:
logger.log(u"Failed parsing provider. Traceback: {0}".format(traceback.format_exc()), logger.WARNING)
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches
results = []
if not self.login():
return results
# Search Params
search_params = {
'user': self.username,
'passkey': self.passkey,
'search': '.', # Dummy query for RSS search, needs the search param sent.
'latest': 'true'
}
# Units
units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']
def process_column_header(td):
result = ''
if td.img:
result = td.img.get('title')
if not result:
result = td.get_text(strip=True)
return result.encode('utf-8')
for mode in search_strings:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode != 'RSS':
logger.log("Search string: {0}".format
(search_string.decode("utf-8")), logger.DEBUG)
search_params['latest'] = 'false'
search_params['search'] = search_string
data = self.get_url(self.urls['search'], params=search_params, returns='text')
if not data:
logger.log("No data returned from provider", logger.DEBUG)
continue
result = json.loads(data)
if 'results' in result:
for torrent in result['results']:
title = torrent['release_name']
download_url = torrent['download_url']
seeders = torrent['seeders']
leechers = torrent['leechers']
if seeders < self.minseed or leechers < self.minleech:
logger.log("Discarded {0} because with {1}/{2} seeders/leechers does not meet the requirement of {3}/{4} seeders/leechers".format(title, seeders, leechers, self.minseed, self.minleech))
continue
freeleech = torrent['freeleech']
if self.freeleech and not freeleech:
continue
size = torrent['size']
size = convert_size(size, units=units) or -1
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders,
'leechers': leechers, 'hash': ''}
logger.log("Found result: {0} with {1} seeders and {2} leechers".format
(title, seeders, leechers), logger.DEBUG)
items.append(item)
if 'error' in result:
logger.log(result['error'], logger.WARNING)
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals
results = []
if not self.show or not self.show.is_anime:
return results
for mode in search_strings:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode != 'RSS':
logger.log("Search string: {0}".format
(search_string.decode("utf-8")), logger.DEBUG)
search_params = {
"terms": search_string,
"type": 1, # get anime types
}
data = self.get_url(self.urls['search'], params=search_params, returns='text')
if not data:
continue
with BS4Parser(data, 'html5lib') as soup:
torrent_table = soup.find('table', class_='listing')
torrent_rows = torrent_table('tr') if torrent_table else []
# Continue only if one Release is found
if len(torrent_rows) < 2:
logger.log("Data returned from provider does not contain any torrents", logger.DEBUG)
continue
a = 1 if len(torrent_rows[0]('td')) < 2 else 0
for top, bot in zip(torrent_rows[a::2], torrent_rows[a + 1::2]):
try:
desc_top = top.find('td', class_='desc-top')
title = desc_top.get_text(strip=True)
download_url = desc_top.find('a')['href']
desc_bottom = bot.find('td', class_='desc-bot').get_text(strip=True)
size = convert_size(desc_bottom.split('|')[1].strip('Size: ')) or -1
stats = bot.find('td', class_='stats').get_text(strip=True)
sl = re.match(r'S:(?P<seeders>\d+)L:(?P<leechers>\d+)C:(?:\d+)ID:(?:\d+)', stats.replace(' ', ''))
seeders = try_int(sl.group('seeders')) if sl else 0
leechers = try_int(sl.group('leechers')) if sl else 0
except StandardError:
continue
if not all([title, download_url]):
continue
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.log("Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})".format
(title, seeders, leechers), logger.DEBUG)
continue
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''}
if mode != 'RSS':
logger.log("Found result: {0} with {1} seeders and {2} leechers".format(title, seeders, leechers), logger.DEBUG)
items.append(item)
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches
results = []
if not self.login():
return results
# Search Params
search_params = {
'tags_type': 1,
'order_by': 'time',
'order_way': 'desc',
'action': 'basic',
'searchsubmit': 1,
'searchstr': ''
}
# Units
units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']
def process_column_header(td):
result = ''
if td.a and td.a.img:
result = td.a.img.get('title', td.a.get_text(strip=True))
if not result:
result = td.get_text(strip=True)
return result
for mode in search_strings:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode != 'RSS':
logger.log("Search string: {0}".format
(search_string.decode("utf-8")), logger.DEBUG)
if mode == 'Season':
searchedSeason = re.match('.*\s(Season\s\d+|S\d+)', search_string).group(1)
search_params['searchstr'] = search_string
data = self.get_url(self.urls['search'], params=search_params, returns='text')
if not data:
logger.log("No data returned from provider", logger.DEBUG)
continue
with BS4Parser(data, 'html5lib') as html:
torrent_table = html.find('table', class_='torrent_table')
torrent_rows = torrent_table('tr') if torrent_table else []
# Continue only if at least one Release is found
if len(torrent_rows) < 2:
logger.log("Data returned from provider does not contain any torrents", logger.DEBUG)
continue
labels = [process_column_header(label) for label in torrent_rows[0]('td')]
# Skip column headers
for result in torrent_rows[1:]:
try:
# skip if torrent has been nuked due to poor quality
if result.find('img', alt='Nuked'):
continue
title = result.find('a', title='View torrent').get_text(strip=True)
if mode == 'Season':
# Skip if torrent isn't the right season, we can't search
# for an exact season on MTV, it returns all of them
if searchedSeason not in title:
continue
# If torrent is grouped, we need a folder name for title
if 'Season' in title:
torrentid = urljoin(self.url, result.find('span', title='Download').parent['href'])
torrentid = re.match('.*?id=([0-9]+)', torrentid).group(1)
group_params = {
'torrentid': torrentid
}
# Obtain folder name to use as title
torrentInfo = self.get_url(self.urls['search'], params=group_params,
returns='text').replace('\n', '')
releaseregex = '.*files_{0}.*?;">/(.+?(?=/))'.format(re.escape(torrentid))
releasename = re.search(releaseregex, torrentInfo).group(1)
title = releasename
download_url = urljoin(self.url, result.find('span', title='Download').parent['href'])
if not all([title, download_url]):
continue
cells = result('td')
seeders = try_int(cells[labels.index('Seeders')].get_text(strip=True))
leechers = try_int(cells[labels.index('Leechers')].get_text(strip=True))
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.log("Discarding torrent because it doesn't meet the"
" minimum seeders or leechers: {0} (S:{1} L:{2})".format
(title, seeders, leechers), logger.DEBUG)
continue
torrent_size = cells[labels.index('Size')].get_text(strip=True)
size = convert_size(torrent_size, units=units) or -1
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''}
if mode != 'RSS':
logger.log("Found result: {0} with {1} seeders and {2} leechers".format
(title, seeders, leechers), logger.DEBUG)
items.append(item)
except StandardError:
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches
results = []
if self.show and not self.show.is_anime:
return results
for mode in search_strings:
items = []
logger.log('Search Mode: {0}'.format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode != 'RSS':
logger.log('Search string: {0}'.format(search_string.decode('utf-8')), logger.DEBUG)
search_params = {
'page': 'rss',
'c': '1_0', # Category: All anime
's': 'id', # Sort by: 'id'=Date / 'size' / 'name' / 'seeders' / 'leechers' / 'downloads'
'o': 'desc', # Sort direction: asc / desc
'f': ('0', '2')[self.confirmed] # Quality filter: 0 = None / 1 = No Remakes / 2 = Trusted Only
}
if mode != 'RSS':
search_params['q'] = search_string
results = []
data = self.cache.get_rss_feed(self.url, params=search_params)['entries']
if not data:
logger.log('Data returned from provider does not contain any torrents', logger.DEBUG)
continue
for curItem in data:
try:
title = curItem['title']
download_url = curItem['link']
if not all([title, download_url]):
continue
seeders = try_int(curItem['nyaa_seeders'])
leechers = try_int(curItem['nyaa_leechers'])
torrent_size = curItem['nyaa_size']
info_hash = curItem['nyaa_infohash']
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.log('Discarding torrent because it doesn\'t meet the'
' minimum seeders or leechers: {0} (S:{1} L:{2})'.format
(title, seeders, leechers), logger.DEBUG)
continue
size = convert_size(torrent_size, units=['BYTES', 'KIB', 'MIB', 'GIB', 'TIB', 'PIB']) or -1
result = {'title': title, 'link': download_url, 'size': size,
'seeders': seeders, 'leechers': leechers, 'hash': info_hash}
if mode != 'RSS':
logger.log('Found result: {0} with {1} seeders and {2} leechers'.format
(title, seeders, leechers), logger.DEBUG)
items.append(result)
except StandardError:
continue
# For each search mode sort all the items by seeders
items.sort(key=lambda d: d.get('seeders', 0), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-branches, too-many-locals, too-many-statements
results = []
if not self.login():
return results
for mode in search_strings:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode != "RSS":
logger.log("Search string: {0}".format(search_string.decode("utf-8")), logger.DEBUG)
url = self.urls['search'] % (search_string)
data = self.get_url(url, returns="text")
try:
parsed_json = json.loads(data)
except ValueError as e:
continue
if not isinstance(parsed_json, dict):
logger.log("No data returned from provider", logger.DEBUG)
continue
torrent_results = parsed_json['total_results']
if not torrent_results:
logger.log("Data returned from provider does not contain any torrents", logger.DEBUG)
continue
logger.log('Number of torrents found on nCore = ' + str(torrent_results), logger.INFO)
for item in parsed_json['results']:
try:
title = item.pop("release_name")
download_url = item.pop("download_url")
if not all([title, download_url]):
continue
seeders = item.pop("seeders")
leechers = item.pop("leechers")
if seeders < self.minseed or leechers < self.minleech:
if mode != "RSS":
logger.log("Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})".format(title, seeders, leechers), logger.DEBUG)
continue
torrent_size = item.pop("size", -1)
size = convert_size(torrent_size) or -1
if mode != "RSS":
logger.log("Found result: {0} with {1} seeders and {2} leechers with a file size {3}".format(title, seeders, leechers, size), logger.DEBUG)
result = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''}
items.append(result)
except StandardError:
continue
# For each search mode sort all the items by seeders
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None):
results = []
if not self.login():
return results
# Search Params
search_params = {
'user': self.username,
'passkey': self.passkey,
'search':
'.', # Dummy query for RSS search, needs the search param sent.
'latest': 'true'
}
# Units
units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']
for mode in search_strings:
items = []
logger.debug(_("Search Mode: {mode}".format(mode=mode)))
for search_string in search_strings[mode]:
if mode != 'RSS':
logger.debug("Search string: {0}".format(search_string))
search_params['latest'] = 'false'
search_params['search'] = search_string
data = self.get_url(self.urls['search'],
params=search_params,
returns='text')
if not data:
logger.debug("No data returned from provider")
continue
result = json.loads(data)
if 'results' in result:
for torrent in result['results']:
title = torrent['release_name']
download_url = torrent['download_url']
seeders = torrent['seeders']
leechers = torrent['leechers']
if seeders < self.minseed or leechers < self.minleech:
logger.info(
"Discarded {0} because with {1}/{2} seeders/leechers does not meet the requirement of {3}/{4} seeders/leechers"
.format(title, seeders, leechers, self.minseed,
self.minleech))
continue
freeleech = torrent['freeleech']
if self.freeleech and not freeleech:
continue
size = torrent['size']
size = convert_size(size, units=units) or -1
item = {
'title': title,
'link': download_url,
'size': size,
'seeders': seeders,
'leechers': leechers,
'hash': ''
}
logger.debug(
"Found result: {0} with {1} seeders and {2} leechers"
.format(title, seeders, leechers))
items.append(item)
if 'error' in result:
logger.warning(result['error'])
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)),
reverse=True)
results += items
return results
def search(self, search_params, age=0, ep_obj=None): # pylint: disable=too-many-branches, too-many-locals
results = []
if not self.login():
return results
for mode in search_params:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_params[mode]:
if mode != 'RSS':
logger.log("Search string: {0}".format
(search_string.decode("utf-8")), logger.DEBUG)
query = { 'sec': 'jax', 'cata': 'yes', 'search': search_string }
query.update({"c"+str(i): 1 for i in self.categories})
data = self.get_url(self.urls['apisearch'], returns='text', post_data=query)
if not data:
continue
with BS4Parser(data, 'html5lib') as html:
torrent_table = html.find(id='torrenttable')
if torrent_table:
torrent_rows = torrent_table.findAll('tr')
else:
torrent_rows = []
# Continue only if one Release is found
if len(torrent_rows) < 2:
logger.log("Data returned from provider does not contain any torrents", logger.DEBUG)
continue
# Scenetime apparently uses different number of cells in #torrenttable based
# on who you are. This works around that by extracting labels from the first
# <tr> and using their index to find the correct download/seeders/leechers td.
labels = [label.get_text(strip=True) or label.img['title'] for label in torrent_rows[0]('td')]
for result in torrent_rows[1:]:
try:
cells = result('td')
link = cells[labels.index('Name')].find('a')
torrent_id = link['href'].replace('details.php?id=', '').split("&")[0]
title = link.get_text(strip=True)
download_url = self.urls['download'] % (torrent_id, "{0}.torrent".format(title.replace(" ", ".")))
seeders = try_int(cells[labels.index('Seeders')].get_text(strip=True))
leechers = try_int(cells[labels.index('Leechers')].get_text(strip=True))
torrent_size = cells[labels.index('Size')].get_text()
size = convert_size(torrent_size) or -1
except (AttributeError, TypeError, KeyError, ValueError):
continue
if not all([title, download_url]):
continue
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.log("Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})".format
(title, seeders, leechers), logger.DEBUG)
continue
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''}
if mode != 'RSS':
logger.log("Found result: {0} with {1} seeders and {2} leechers".format(title, seeders, leechers), logger.DEBUG)
items.append(item)
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None):
"""
Searches indexer using the params in search_strings, either for latest releases, or a string/id search
Returns: list of results in dict form
"""
results = []
if not self._check_auth():
return results
if "gingadaddy" not in self.url: # gingadaddy has no caps.
if not self.caps:
self.get_newznab_categories(just_caps=True)
if not self.caps:
return results
for mode in search_strings:
search_params = {
"t": ("search", "tvsearch")[bool(self.use_tv_search)],
"limit": 100,
"offset": 0,
"cat": self.catIDs.strip(", ") or "5030,5040",
"maxage": settings.USENET_RETENTION,
}
if self.needs_auth and self.key:
search_params["apikey"] = self.key
if mode != "RSS":
if self.use_tv_search:
if "tvdbid" in str(self.cap_tv_search):
search_params["tvdbid"] = ep_obj.show.indexerid
if ep_obj.show.air_by_date or ep_obj.show.sports:
# date_str = str(ep_obj.airdate)
# search_params['season'] = date_str.partition('-')[0]
# search_params['ep'] = date_str.partition('-')[2].replace('-', '/')
search_params["q"] = str(ep_obj.airdate)
elif ep_obj.show.is_anime:
search_params["ep"] = ep_obj.absolute_number
else:
search_params["season"] = ep_obj.scene_season
search_params["ep"] = ep_obj.scene_episode
if mode == "Season":
search_params.pop("ep", "")
if self.torznab:
search_params.pop("ep", "")
search_params.pop("season", "")
items = []
logger.debug("Search Mode: {0}".format(mode))
for search_string in {*search_strings[mode]}:
if mode != "RSS":
logger.debug(
_("Search String: {search_string}".format(
search_string=search_string)))
if "tvdbid" not in search_params:
search_params["q"] = search_string
time.sleep(cpu_presets[settings.CPU_PRESET])
data = self.get_url(urljoin(self.url, "api"),
params=search_params,
returns="text")
if not data:
break
with BS4Parser(data, "html5lib") as html:
if not self._check_auth_from_data(html):
break
# try:
# self.torznab = 'xmlns:torznab' in html.rss.attrs
# except AttributeError:
# self.torznab = False
for item in html("item"):
try:
title = item.title.get_text(strip=True)
download_url = None
if item.link:
if validators.url(
item.link.get_text(strip=True)):
download_url = item.link.get_text(
strip=True)
elif validators.url(item.link.next.strip()):
download_url = item.link.next.strip()
if (not download_url, item.enclosure
and validators.url(
item.enclosure.get("url",
"").strip())):
download_url = item.enclosure.get("url",
"").strip()
if not (title and download_url):
continue
seeders = leechers = None
if "gingadaddy" in self.url:
size_regex = re.search(r"\d*.?\d* [KMGT]B",
str(item.description))
item_size = size_regex.group(
) if size_regex else -1
else:
item_size = item.size.get_text(
strip=True) if item.size else -1
for attr in item.find_all(
["newznab:attr", "torznab:attr"]):
item_size = attr["value"] if attr[
"name"] == "size" else item_size
seeders = try_int(
attr["value"]
) if attr["name"] == "seeders" else seeders
leechers = try_int(
attr["value"]
) if attr["name"] == "peers" else leechers
if not item_size or (self.torznab and
(seeders is None
or leechers is None)):
continue
size = convert_size(item_size) or -1
result = {
"title": title,
"link": download_url,
"size": size,
"seeders": seeders,
"leechers": leechers
}
items.append(result)
except Exception:
continue
# Since we aren't using the search string,
# break out of the search string loop
if "tvdbid" in search_params:
break
if self.torznab:
results.sort(key=lambda d: try_int(d.get("seeders", 0)),
reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches, too-many-statements
results = []
"""
205 = SD, 208 = HD, 200 = All Videos
https://pirateproxy.pl/s/?q=Game of Thrones&type=search&orderby=7&page=0&category=200
"""
# oder_by is 7 in browse for seeders, but 8 in search!
search_params = {
"q": "",
"type": "search",
"orderby": 8,
"page": 0,
"category": 200
}
# Units
units = ["B", "KIB", "MIB", "GIB"]
def process_column_header(th):
text = ""
if th.a:
text = th.a.get_text(strip=True)
if not text:
text = th.get_text(strip=True)
return text
for mode in search_strings:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
search_urls = (self.urls["search"], self.urls["rss"])[mode == "RSS"]
if not isinstance(search_urls, list):
search_urls = [search_urls]
for search_url in search_urls:
if self.custom_url:
if not validators.url(self.custom_url):
logger.log("Invalid custom url: {0}".format(self.custom_url), logger.WARNING)
return results
search_url = urljoin(self.custom_url, search_url.split(self.url)[1])
if mode != "RSS":
search_params["q"] = search_string
logger.log("Search string: {}".format
(search_string.decode("utf-8")), logger.DEBUG)
# Prevents a 302 redirect, since there is always a 301 from .se to the best mirror having an extra
# redirect is excessive on the provider and spams the debug log unnecessarily
search_url, params = self.convert_url(search_url, search_params)
data = self.get_url(search_url, params=params, returns="text")
else:
data = self.get_url(search_url, returns="text")
if not data:
logger.log("URL did not return data, maybe try a custom url, or a different one", logger.DEBUG)
continue
with BS4Parser(data, "html5lib") as html:
torrent_table = html.find("table", id="searchResult")
torrent_rows = torrent_table("tr") if torrent_table else []
# Continue only if at least one Release is found
if len(torrent_rows) < 2:
logger.log("Data returned from provider does not contain any torrents", logger.DEBUG)
continue
labels = [process_column_header(label) for label in torrent_rows[0]("th")]
# Skip column headers
for result in torrent_rows[1:]:
try:
cells = result("td")
# Funky js on page messing up titles, this fixes that
title = result.find(class_="detLink")['title'].split('Details for ', 1)[-1]
download_url = result.find(title="Download this torrent using magnet")["href"] + self._custom_trackers
if not self.magnet_regex.match(download_url):
logger.log("Got an invalid magnet: {0}".format(download_url))
logger.log("Invalid ThePirateBay proxy please try another one", logger.DEBUG)
continue
if not all([title, download_url]):
continue
seeders = try_int(cells[labels.index("SE")].get_text(strip=True))
leechers = try_int(cells[labels.index("LE")].get_text(strip=True))
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != "RSS":
logger.log("Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})".format
(title, seeders, leechers), logger.DEBUG)
continue
# Accept Torrent only from Good People for every Episode Search
if self.confirmed and not result.find(alt=re.compile(r"VIP|Trusted")):
if mode != "RSS":
logger.log("Found result: {0} but that doesn't seem like a trusted result so I'm ignoring it".format(title), logger.DEBUG)
continue
# Convert size after all possible skip scenarios
torrent_size = re.sub(r".*Size ([\d.]+).+([KMGT]iB).*", r"\1 \2", result.find(class_="detDesc").get_text(strip=True))
size = convert_size(torrent_size, units=units) or -1
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''}
if mode != "RSS":
logger.log("Found result: {0} with {1} seeders and {2} leechers".format
(title, seeders, leechers), logger.DEBUG)
items.append(item)
except StandardError:
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None):
results = []
if self.show and not self.show.is_anime:
return results
for mode in search_strings:
items = []
logger.debug(_("Search Mode: {mode}".format(mode=mode)))
for search_string in {*search_strings[mode]}:
if mode != 'RSS':
logger.debug(
_("Search String: {search_string}".format(
search_string=search_string)))
search_params = {
'page': 'rss',
'c': '1_0', # Category: All anime
's':
'id', # Sort by: 'id'=Date / 'size' / 'name' / 'seeders' / 'leechers' / 'downloads'
'o': 'desc', # Sort direction: asc / desc
'f': (
'0', '2'
)[self.
confirmed] # Quality filter: 0 = None / 1 = No Remakes / 2 = Trusted Only
}
if mode != 'RSS':
search_params['q'] = search_string
results = []
data = self.cache.get_rss_feed(self.url,
params=search_params)['entries']
if not data:
logger.debug(
'Data returned from provider does not contain any torrents'
)
continue
for curItem in data:
try:
title = curItem['title']
download_url = curItem['link']
if not all([title, download_url]):
continue
seeders = try_int(curItem['nyaa_seeders'])
leechers = try_int(curItem['nyaa_leechers'])
torrent_size = curItem['nyaa_size']
info_hash = curItem['nyaa_infohash']
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.debug(
'Discarding torrent because it doesn\'t meet the'
' minimum seeders or leechers: {0} (S:{1} L:{2})'
.format(title, seeders, leechers))
continue
size = convert_size(
torrent_size,
units=['BYTES', 'KIB', 'MIB', 'GIB', 'TIB', 'PIB'
]) or -1
result = {
'title': title,
'link': download_url,
'size': size,
'seeders': seeders,
'leechers': leechers,
'hash': info_hash
}
if mode != 'RSS':
logger.debug(
_('Found result: {title} with {seeders} seeders and {leechers} leechers'
.format(title=title,
seeders=seeders,
leechers=leechers)))
items.append(result)
except Exception:
continue
# For each search mode sort all the items by seeders
items.sort(key=lambda d: d.get('seeders', 0), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches
results = []
if not self.login():
return results
# Search Params
search_params = {
"searchstr": "",
"filter_cat[1]": 1,
"filter_cat[2]": 1,
"filter_cat[3]": 1,
"filter_cat[4]": 1,
"filter_cat[5]": 1
}
# Units
units = ["B", "KB", "MB", "GB", "TB", "PB"]
def process_column_header(td):
result = ""
if td.a and td.a.img:
result = td.a.img.get("title", td.a.get_text(strip=True))
if not result:
result = td.get_text(strip=True)
return result
for mode in search_strings:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode != "RSS":
logger.log("Search string: {0}".format
(search_string.decode("utf-8")), logger.DEBUG)
search_params["searchstr"] = search_string
search_url = self.urls["search"]
data = self.get_url(search_url, params=search_params, returns="text")
if not data:
logger.log("No data returned from provider", logger.DEBUG)
continue
with BS4Parser(data, "html5lib") as html:
torrent_table = html.find("table", id="torrent_table")
torrent_rows = torrent_table("tr") if torrent_table else []
# Continue only if at least one Release is found
if len(torrent_rows) < 2:
logger.log("Data returned from provider does not contain any torrents", logger.DEBUG)
continue
# "", "", "Name /Year", "Files", "Time", "Size", "Snatches", "Seeders", "Leechers"
labels = [process_column_header(label) for label in torrent_rows[0]("td")]
# Skip column headers
for result in torrent_rows[1:]:
cells = result("td")
if len(cells) < len(labels):
continue
try:
title = cells[labels.index("Name /Year")].find("a", dir="ltr").get_text(strip=True)
download_url = urljoin(self.url, cells[labels.index("Name /Year")].find("a", title="Download")["href"])
if not all([title, download_url]):
continue
seeders = try_int(cells[labels.index("Seeders")].get_text(strip=True))
leechers = try_int(cells[labels.index("Leechers")].get_text(strip=True))
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != "RSS":
logger.log("Discarding torrent because it doesn't meet the"
" minimum seeders or leechers: {0} (S:{1} L:{2})".format
(title, seeders, leechers), logger.DEBUG)
continue
torrent_size = cells[labels.index("Size")].get_text(strip=True)
size = convert_size(torrent_size, units=units) or -1
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''}
if mode != "RSS":
logger.log("Found result: {0} with {1} seeders and {2} leechers".format
(title, seeders, leechers), logger.DEBUG)
items.append(item)
except StandardError:
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches
results = []
if not self.login():
return results
# https://www.thegft.org/browse.php?view=0&c26=1&c37=1&c19=1&c47=1&c17=1&c4=1&search=arrow
# Search Params
search_params = {
'view': 0, # BROWSE
'c4': 1, # TV/XVID
'c17': 1, # TV/X264
'c19': 1, # TV/DVDRIP
'c26': 1, # TV/BLURAY
'c37': 1, # TV/DVDR
'c47': 1, # TV/SD
'search': '',
}
# Units
units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']
def process_column_header(td):
result = ''
if td.a and td.a.img:
result = td.a.img.get('title', td.a.get_text(strip=True))
if not result:
result = td.get_text(strip=True)
return result
for mode in search_strings:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode != 'RSS':
logger.log("Search string: {0}".format
(search_string.decode("utf-8")), logger.DEBUG)
search_params['search'] = search_string
data = self.get_url(self.urls['search'], params=search_params, returns='text')
if not data:
logger.log("No data returned from provider", logger.DEBUG)
continue
with BS4Parser(data, 'html5lib') as html:
torrent_table = html.find('div', id='torrentBrowse')
torrent_rows = torrent_table('tr') if torrent_table else []
# Continue only if at least one Release is found
if len(torrent_rows) < 2:
logger.log("Data returned from provider does not contain any torrents", logger.DEBUG)
continue
labels = [process_column_header(label) for label in torrent_rows[0]('td')]
# Skip column headers
for result in torrent_rows[1:]:
try:
cells = result('td')
title = cells[labels.index('Name')].find('a').find_next('a')['title'] or cells[labels.index('Name')].find('a')['title']
download_url = self.url + cells[labels.index('DL')].find('a')['href']
if not all([title, download_url]):
continue
peers = cells[labels.index('S/L')].get_text(strip=True).split('/', 1)
seeders = try_int(peers[0])
leechers = try_int(peers[1])
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.log("Discarding torrent because it doesn't meet the"
" minimum seeders or leechers: {0} (S:{1} L:{2})".format
(title, seeders, leechers), logger.DEBUG)
continue
torrent_size = cells[labels.index('Size/Snatched')].get_text(strip=True).split('/', 1)[0]
size = convert_size(torrent_size, units=units) or -1
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''}
if mode != 'RSS':
logger.log("Found result: {0} with {1} seeders and {2} leechers".format
(title, seeders, leechers), logger.DEBUG)
items.append(item)
except StandardError:
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-branches, too-many-locals
results = []
url = "http://localhost:3000/"
if self.custom_url:
if not validators.url(self.custom_url):
logger.log("Invalid custom url set, please check your settings", logger.WARNING)
return results
url = self.custom_url
search_params = {}
anime = ep_obj and ep_obj.show and ep_obj.show.anime
search_params["category"] = ("tv", "anime")[bool(anime)]
if self.api_key:
search_params["apiKey"] = self.api_key
for mode in search_strings:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
search_params["q"] = search_string
if mode != "RSS":
logger.log("Search string: {0}".format
(search_string.decode('utf-8')), logger.DEBUG)
search_url = urljoin(url, "api/search")
parsed_json = self.get_url(search_url, params=search_params, returns="json")
if not parsed_json:
logger.log("No data returned from provider", logger.DEBUG)
continue
if not self._check_auth_from_data(parsed_json):
return results
for result in parsed_json.pop("torrents", {}):
try:
title = result.pop("title", "")
info_hash = result.pop("infoHash", "")
download_url = "magnet:?xt=urn:btih:" + info_hash
if not all([title, download_url, info_hash]):
continue
swarm = result.pop("swarm", None)
if swarm:
seeders = try_int(swarm.pop("seeders", 0))
leechers = try_int(swarm.pop("leechers", 0))
else:
seeders = leechers = 0
if seeders < self.minseed or leechers < self.minleech:
if mode != "RSS":
logger.log("Discarding torrent because it doesn't meet the "
"minimum seeders or leechers: {0} (S:{1} L:{2})".format
(title, seeders, leechers), logger.DEBUG)
continue
size = convert_size(result.pop("size", -1)) or -1
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''}
if mode != "RSS":
logger.log("Found result: {0} with {1} seeders and {2} leechers".format
(title, seeders, leechers), logger.DEBUG)
items.append(item)
except (AttributeError, TypeError, KeyError, ValueError):
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def _parse(self, data, mode):
"""
Parse search results for items.
:param data: The raw response from a search
:param mode: The current mode used to search, e.g. RSS
:return: A KV with a list of items found and if there's an next page to search
"""
def process_column_header(td):
ret = ''
if td.a and td.a.img:
ret = td.a.img.get('title', td.a.get_text(strip=True))
if not ret:
ret = td.get_text(strip=True)
return ret
items = []
has_next_page = False
with BS4Parser(data, 'html5lib') as html:
torrent_table = html.find('table', id='torrent_table')
torrent_rows = torrent_table('tr') if torrent_table else []
# ignore next page in RSS mode
has_next_page = mode != 'RSS' and html.find(
'a', class_='pager_next') is not None
logger.log('More Pages? {0}'.format(has_next_page), logger.DEBUG)
# Continue only if at least one Release is found
if len(torrent_rows) < 2:
logger.log(
'Data returned from provider does not contain any torrents',
logger.DEBUG)
return {'has_next_page': has_next_page, 'items': []}
# '', '', 'Name /Year', 'Files', 'Time', 'Size', 'Snatches', 'Seeders', 'Leechers'
labels = [
process_column_header(label) for label in torrent_rows[0]('td')
]
group_title = ''
# Skip column headers
for result in torrent_rows[1:]:
cells = result('td')
result_class = result.get('class')
# When "Grouping Torrents" is enabled, the structure of table change
group_index = -2 if 'group_torrent' in result_class else 0
try:
title = result.select(
'a[href^="torrents.php?id="]')[0].get_text()
title = re.sub(
'\s+', ' ',
title).strip() # clean empty lines and multiple spaces
if 'group' in result_class or 'torrent' in result_class:
# get international title if available
title = re.sub('.* \[(.*?)\](.*)', r'\1\2', title)
if 'group' in result_class:
group_title = title
continue
# Clean dash between title and season/episode
title = re.sub('- (S\d{2}(E\d{2,4})?)', r'\1', title)
for serie in self.absolute_numbering:
if serie in title:
# remove season from title when its in absolute format
title = re.sub('S\d{2}E(\d{2,4})', r'\1', title)
break
download_url = urljoin(
self.url,
result.select('a[href^="torrents.php?action=download"]'
)[0]['href'])
if not all([title, download_url]):
continue
seeders = try_int(cells[labels.index('Seeders') +
group_index].get_text(strip=True))
leechers = try_int(cells[labels.index('Leechers') +
group_index].get_text(strip=True))
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != "RSS":
logger.log(
"Discarding torrent because it doesn't meet the"
" minimum seeders or leechers: {0} (S:{1} L:{2})"
.format(title, seeders,
leechers), logger.DEBUG)
continue
torrent_details = None
if 'group_torrent' in result_class:
# torrents belonging to a group
torrent_details = title
title = group_title
elif 'torrent' in result_class:
# standalone/un grouped torrents
torrent_details = cells[labels.index('Nome/Ano')].find(
'div', class_='torrent_info').get_text()
torrent_details = torrent_details.replace(
'[', ' ').replace(']', ' ').replace('/', ' ')
torrent_details = torrent_details.replace(
'Full HD ', '1080p').replace('HD ', '720p')
torrent_size = cells[labels.index('Tamanho') +
group_index].get_text(strip=True)
size = convert_size(torrent_size) or -1
torrent_name = '{0} {1}'.format(
title, torrent_details.strip()).strip()
torrent_name = re.sub('\s+', ' ', torrent_name)
items.append({
'title': torrent_name,
'link': download_url,
'size': size,
'seeders': seeders,
'leechers': leechers,
'hash': ''
})
if mode != 'RSS':
logger.log(
'Found result: {0} with {1} seeders and {2} leechers'
.format(torrent_name, seeders,
leechers), logger.DEBUG)
except (AttributeError, TypeError, KeyError, ValueError,
IndexError):
logger.log('Failed parsing provider.', logger.ERROR)
return {'has_next_page': has_next_page, 'items': items}
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches, too-many-statements
results = []
if not self.login():
return results
# Search Params
search_params = {
'do': 'search',
'search_type': 't_name',
'category': 0,
'include_dead_torrents': 'no',
'submit': 'search'
}
# Units
units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']
for mode in search_strings:
items = []
logger.log('Search Mode: {0}'.format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode == 'Season':
search_string = re.sub(r'(.*)S0?', r'\1Series ',
search_string)
if mode != 'RSS':
logger.log(
'Search string: {0}'.format(
search_string.decode('utf-8')), logger.DEBUG)
search_params['keywords'] = search_string
data = self.get_url(self.urls['search'],
post_data=search_params,
returns='text')
if not data:
logger.log('No data returned from provider', logger.DEBUG)
continue
with BS4Parser(data, 'html5lib') as html:
torrent_table = html.find(id='sortabletable')
torrent_rows = torrent_table("tr") if torrent_table else []
# Continue only if at least one Release is found
if len(torrent_rows) < 2:
logger.log(
"Data returned from provider does not contain any torrents",
logger.DEBUG)
continue
labels = [
label.img['title'] if label.img else label.get_text(
strip=True) for label in torrent_rows[0]('td')
]
for torrent in torrent_rows[1:]:
try:
if self.freeleech and not torrent.find(
'img', alt=re.compile('Free Torrent')):
continue
title = torrent.find(
class_='tooltip-content').div.get_text(
strip=True)
download_url = torrent.find(
title='Click to Download this Torrent!'
).parent['href']
if not all([title, download_url]):
continue
seeders = try_int(
torrent.find(title='Seeders').get_text(
strip=True))
leechers = try_int(
torrent.find(title='Leechers').get_text(
strip=True))
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.log(
'Discarding torrent because it doesn\'t meet the'
' minimum seeders or leechers: {0} (S:{1} L:{2})'
.format(title, seeders,
leechers), logger.DEBUG)
continue
# Chop off tracker/channel prefix or we cant parse the result!
if mode != 'RSS' and search_params['keywords']:
show_name_first_word = re.search(
r'^[^ .]+',
search_params['keywords']).group()
if not title.startswith(show_name_first_word):
title = re.sub(
r'.*(' + show_name_first_word + '.*)',
r'\1', title)
# Change title from Series to Season, or we can't parse
if mode == 'Season':
title = re.sub(r'(.*)(?i)Series', r'\1Season',
title)
# Strip year from the end or we can't parse it!
title = re.sub(r'(.*)[\. ]?\(\d{4}\)', r'\1',
title)
title = re.sub(r'\s+', r' ', title)
torrent_size = torrent('td')[labels.index(
'Size')].get_text(strip=True)
size = convert_size(torrent_size,
units=units) or -1
if mode != 'RSS':
logger.log(
'Found result: {0} with {1} seeders and {2} leechers'
.format(title, seeders,
leechers), logger.DEBUG)
item = {
'title': title + '.hdtv.x264',
'link': download_url,
'size': size,
'seeders': seeders,
'leechers': leechers
}
items.append(item)
except StandardError:
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)),
reverse=True)
results += items
return results
def search(self, search_params, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches, too-many-statements
results = []
for mode in search_params:
items = []
logger.log(u'Search Mode: {0}'.format(mode), logger.DEBUG)
for search_string in search_params[mode]:
if search_string == '':
continue
search_string = six.text_type(search_string).replace('.', ' ')
logger.log(
u'Search string: {0}'.format(
search_string.decode('utf-8')), logger.DEBUG)
last_page = False
for page in range(0, self.max_pages):
if last_page:
break
logger.log('Processing page {0} of results'.format(page),
logger.DEBUG)
search_url = self.urls['search'].format(
search_string, page)
data = self.get_url(search_url, returns='text')
if not data:
logger.log(u'No data returned from provider',
logger.DEBUG)
continue
try:
with BS4Parser(data, 'html5lib') as html:
table_header = html.find('tr', class_='bordo')
torrent_table = table_header.find_parent(
'table') if table_header else None
if not torrent_table:
logger.log(u'Could not find table of torrents',
logger.ERROR)
continue
torrent_rows = torrent_table('tr')
# Continue only if one Release is found
if len(torrent_rows) < 6 or len(
torrent_rows[2]('td')) == 1:
logger.log(
u'Data returned from provider does not contain any torrents',
logger.DEBUG)
last_page = True
continue
if len(torrent_rows) < 45:
last_page = True
for result in torrent_rows[2:-3]:
result_cols = result('td')
if len(result_cols) == 1:
# Ignore empty rows in the middle of the table
continue
try:
info_link = result('td')[1].find(
'a')['href']
title = re.sub(
' +', ' ',
info_link.rsplit('/', 1)[-1].replace(
'_', ' '))
info_hash = result('td')[3].find(
'input',
class_='downarrow')['value'].upper()
download_url = self._magnet_from_result(
info_hash, title)
seeders = try_int(result('td')[5].text)
leechers = try_int(result('td')[6].text)
torrent_size = result('td')[2].string
size = convert_size(torrent_size) or -1
except (AttributeError, IndexError, TypeError):
continue
filename_qt = self._reverseQuality(
self._episodeQuality(result))
for text in self.hdtext:
title1 = title
title = title.replace(text, filename_qt)
if title != title1:
break
if Quality.nameQuality(
title) == Quality.UNKNOWN:
title += filename_qt
if not self._is_italian(
title) and not self.subtitle:
logger.log(
u'Torrent is subtitled, skipping: {0}'.
format(title), logger.DEBUG)
continue
if self.engrelease and not self._is_english(
title):
logger.log(
u'Torrent isn\'t english audio/subtitled, skipping: {0}'
.format(title), logger.DEBUG)
continue
search_show = re.split(r'([Ss][\d{1,2}]+)',
search_string)[0]
show_title = search_show
ep_params = ''
rindex = re.search(r'([Ss][\d{1,2}]+)', title)
if rindex:
show_title = title[:rindex.start()]
ep_params = title[rindex.start():]
if show_title.lower() != search_show.lower(
) and search_show.lower() in show_title.lower(
):
new_title = search_show + ep_params
title = new_title
if not all([title, download_url]):
continue
if self._is_season_pack(title):
title = re.sub(r'([Ee][\d{1,2}\-?]+)', '',
title)
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
logger.log(
u'Discarding torrent because it doesn\'t meet the minimum'
u' seeders or leechers: {0} (S:{1} L:{2})'
.format(title, seeders,
leechers), logger.DEBUG)
continue
item = {
'title': title,
'link': download_url,
'size': size,
'seeders': seeders,
'leechers': leechers,
'hash': info_hash
}
if mode != 'RSS':
logger.log(
u'Found result: {0} with {1} seeders and {2} leechers'
.format(title, seeders,
leechers), logger.DEBUG)
items.append(item)
except Exception as error:
logger.log(
u'Failed parsing provider. Error: {0}'.format(
error), logger.ERROR)
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)),
reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches
results = []
if not self.login():
return results
# Search Params
search_params = {"search": "", "cat": 0}
# Units
units = ["B", "KB", "MB", "GB", "TB", "PB"]
def process_column_header(td):
result = ""
if td.a and td.a.img:
result = td.a.img.get("title", td.a.get_text(strip=True))
if not result:
result = td.get_text(strip=True)
return result
for mode in search_strings:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode != "RSS":
logger.log(
"Search string: {search}".format(
search=search_string.decode("utf-8")),
logger.DEBUG)
search_params["search"] = search_string
search_url = self.urls["search"]
data = self.get_url(search_url,
params=search_params,
returns="text")
if not data:
logger.log("No data returned from provider", logger.DEBUG)
continue
with BS4Parser(data, "html5lib") as html:
torrent_rows = html.find_all("div", class_="torrentrow")
# Continue only if at least one Release is found
if not torrent_rows:
logger.log(
"Data returned from provider does not contain any torrents",
logger.DEBUG)
continue
# "Type", "Name", "Download", "Files", "Comments", "Added", "Size", "Snatched", "Seeders", "Leechers", "Upped by"
labels = []
columns = html.find_all("div", class_="colhead")
for index, column in enumerate(columns):
lbl = column.get_text(strip=True)
if lbl:
labels.append(str(lbl))
else:
lbl = column.find("img")
if lbl:
if lbl.has_attr("alt"):
lbl = lbl['alt']
labels.append(str(lbl))
else:
if index == 3:
lbl = "Download"
else:
lbl = str(index)
labels.append(lbl)
# Skip column headers
for result in torrent_rows:
cells = result.find_all("div", class_="torrenttable")
if len(cells) < len(labels):
continue
try:
title = cells[labels.index("Name")].find("a").find(
"b").get_text(strip=True)
download_url = urljoin(
self.url, cells[labels.index("Download")].find(
"a")["href"])
if not all([title, download_url]):
continue
seeders = try_int(cells[labels.index(
"Seeders")].find("span").get_text(strip=True))
leechers = try_int(cells[labels.index(
"Leechers")].find("span").get_text(strip=True))
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != "RSS":
logger.log(
"Discarding torrent because it doesn't meet the"
" minimum seeders or leechers: {0} (S:{1} L:{2})"
.format(title, seeders,
leechers), logger.DEBUG)
continue
torrent_size = cells[labels.index("Size")].find(
"span").get_text(strip=True)
size = convert_size(
torrent_size, units=units, sep='') or -1
item = {
'title': title,
'link': download_url,
'size': size,
'seeders': seeders,
'leechers': leechers,
'hash': None
}
if mode != "RSS":
logger.log(
"Found result: {0} with {1} seeders and {2} leechers"
.format(title, seeders,
leechers), logger.DEBUG)
items.append(item)
except StandardError:
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)),
reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches
results = []
if not self.login():
return results
freeleech = '2' if self.freeleech else '0'
# Search Params
# c59=1&c73=1&c5=1&c41=1&c60=1&c66=1&c65=1&c67=1&c62=1&c64=1&c61=1&search=Good+Behavior+S01E01&cat=0&incldead=0&freeleech=0&lang=0
search_params = {
'c5': '1', # Category: Series - DVDRip
'c41': '1', # Category: Series - HD
'c60': '1', # Category: Series - Pack TV
'c62': '1', # Category: Series - BDRip
'c64': '1', # Category: Series - VOSTFR
'c65': '1', # Category: Series - TV 720p
'c66': '1', # Category: Series - TV 1080p
'c67': '1', # Category: Series - Pack TV HD
'c73': '1', # Category: Anime
'incldead':
'0', # Include dead torrent - 0: off 1: yes 2: only dead
'freeleech':
freeleech, # Only freeleech torrent - 0: off 1: no freeleech 2: Only freeleech
'lang': '0' # Langugage - 0: off 1: English 2: French ....
}
# Units
units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']
for mode in search_strings:
items = []
logger.log('Search Mode: {0}'.format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
logger.log(
'Search String: {0} for mode {1}'.format(
search_strings[mode], mode), logger.DEBUG)
if mode != 'RSS':
logger.log(
'Search string: {0}'.format(
search_string.decode('utf-8')), logger.DEBUG)
search_params['search'] = re.sub(r'[()]', '', search_string)
data = self.get_url(self.urls['search'],
params=search_params,
returns='text')
if not data:
continue
with BS4Parser(data, 'html5lib') as html:
torrent_table = html.find(class_='ttable_headinner')
torrent_rows = torrent_table('tr') if torrent_table else []
# Continue only if at least one Release is found
if len(torrent_rows) < 2:
logger.log(
'Data returned from provider does not contain any torrents',
logger.DEBUG)
continue
# Catégorie, Release, Date, DL, Size, C, S, L
labels = [
label.get_text(strip=True)
for label in torrent_rows[0]('th')
]
# Skip column headers
for result in torrent_rows[1:]:
cells = result('td')
if len(cells) < len(labels):
continue
try:
id = re.search(
'id=([0-9]+)', cells[labels.index('Nom')].find(
'a')['href']).group(1)
title = cells[labels.index('Nom')].get_text(
strip=True)
download_url = urljoin(
self.urls['download'],
'?id={0}&name={1}'.format(id, title))
if not all([title, download_url]):
continue
seeders = try_int(
cells[labels.index('S')].get_text(strip=True))
leechers = try_int(
cells[labels.index('L')].get_text(strip=True))
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.log(
'Discarding torrent because it doesn\'t meet the minimum seeders or leechers: {0} (S:{1} L:{2})'
.format(title, seeders,
leechers), logger.DEBUG)
continue
size_index = labels.index(
'Size') if 'Size' in labels else labels.index(
'Taille')
torrent_size = cells[size_index].get_text()
size = convert_size(torrent_size,
units=units) or -1
item = {
'title': title,
'link': download_url,
'size': size,
'seeders': seeders,
'leechers': leechers,
'hash': ''
}
if mode != 'RSS':
logger.log(
'Found result: {0} with {1} seeders and {2} leechers'
.format(title, seeders,
leechers), logger.DEBUG)
items.append(item)
except StandardError:
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)),
reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None):
results = []
if not self.login():
return results
# Search Params
search_params = {
'cat[]': ['TV|SD|VOSTFR', 'TV|HD|VOSTFR', 'TV|SD|VF', 'TV|HD|VF', 'TV|PACK|FR', 'TV|PACK|VOSTFR', 'TV|EMISSIONS', 'ANIME'],
# Both ASC and DESC are available for sort direction
'way': 'DESC'
}
# Units
units = ['O', 'KO', 'MO', 'GO', 'TO', 'PO']
for mode in search_strings:
items = []
logger.debug(_("Search Mode: {mode}".format(mode=mode)))
for search_string in {*search_strings[mode]}:
if mode != 'RSS':
logger.debug('Search string: {0}'.format
(search_string))
# Sorting: Available parameters: ReleaseName, Seeders, Leechers, Snatched, Size
search_params['order'] = ('Seeders', 'Time')[mode == 'RSS']
search_params['search'] = re.sub(r'[()]', '', search_string)
data = self.get_url(self.urls['search'], params=search_params, returns='text')
if not data:
continue
with BS4Parser(data, 'html5lib') as html:
torrent_table = html.find(class_='torrent_table')
torrent_rows = torrent_table('tr') if torrent_table else []
# Continue only if at least one Release is found
if len(torrent_rows) < 2:
logger.debug('Data returned from provider does not contain any torrents')
continue
# Catégorie, Release, Date, DL, Size, C, S, L
labels = [label.get_text(strip=True) for label in torrent_rows[0]('td')]
# Skip column headers
for result in torrent_rows[1:]:
cells = result('td')
if len(cells) < len(labels):
continue
try:
title = cells[labels.index('Release')].get_text(strip=True)
download_url = urljoin(self.url, cells[labels.index('DL')].find('a', class_='tooltip')['href'])
if not all([title, download_url]):
continue
seeders = try_int(cells[labels.index('S')].get_text(strip=True))
leechers = try_int(cells[labels.index('L')].get_text(strip=True))
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.debug('Discarding torrent because it doesn\'t meet the minimum seeders or leechers: {0} (S:{1} L:{2})'.format
(title, seeders, leechers))
continue
size_index = labels.index('Size') if 'Size' in labels else labels.index('Taille')
torrent_size = cells[size_index].get_text()
size = convert_size(torrent_size, units=units) or -1
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''}
if mode != 'RSS':
logger.debug(_('Found result: {title} with {seeders} seeders and {leechers} leechers'.format(
title=title, seeders=seeders, leechers=leechers)))
items.append(item)
except Exception:
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches, too-many-statements
results = []
search_params = {
'out': 'json',
'filter': 2101,
'showmagnets': 'on',
'num': 50
}
for mode in search_strings: # Mode = RSS, Season, Episode
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode != 'RSS':
logger.log("Search string: {0}".format
(search_string.decode("utf-8")), logger.DEBUG)
search_params['s'] = search_string
if self.custom_url:
if not validators.url(self.custom_url):
logger.log("Invalid custom url set, please check your settings", logger.WARNING)
return results
search_url = self.custom_url
else:
search_url = self.url
torrents = self.get_url(search_url, params=search_params, returns='json')
if not (torrents and "total_found" in torrents and int(torrents["total_found"]) > 0):
logger.log("Data returned from provider does not contain any torrents", logger.DEBUG)
continue
del torrents["total_found"]
results = []
for i in torrents:
title = torrents[i]["title"]
seeders = try_int(torrents[i]["seeds"], 1)
leechers = try_int(torrents[i]["leechs"], 0)
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.log("Torrent doesn't meet minimum seeds & leechers not selecting : {0}".format(title), logger.DEBUG)
continue
t_hash = torrents[i]["torrent_hash"]
torrent_size = torrents[i]["torrent_size"]
if not all([t_hash, torrent_size]):
continue
download_url = torrents[i]["magnet"] + self._custom_trackers
size = convert_size(torrent_size) or -1
if not all([title, download_url]):
continue
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': t_hash}
if mode != 'RSS':
logger.log("Found result: {0} with {1} seeders and {2} leechers".format
(title, seeders, leechers), logger.DEBUG)
items.append(item)
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def _parse(self, data, mode):
"""
Parse search results for items.
:param data: The raw response from a search
:param mode: The current mode used to search, e.g. RSS
:return: A KV with a list of items found and if there's an next page to search
"""
def process_column_header(td):
ret = u''
if td.a and td.a.img:
ret = td.a.img.get('title', td.a.get_text(strip=True))
if not ret:
ret = td.get_text(strip=True)
return ret
items = []
has_next_page = False
with BS4Parser(data, 'html5lib') as html:
torrent_table = html.find('table', id='torrent_table')
torrent_rows = torrent_table('tr') if torrent_table else []
# ignore next page in RSS mode
has_next_page = mode != 'RSS' and html.find('a', class_='pager_next') is not None
logger.log(u'More Pages? {0}'.format(has_next_page), logger.DEBUG)
# Continue only if at least one Release is found
if len(torrent_rows) < 2:
logger.log('Data returned from provider does not contain any torrents', logger.DEBUG)
return {'has_next_page': has_next_page, 'items': []}
# '', '', 'Name /Year', 'Files', 'Time', 'Size', 'Snatches', 'Seeders', 'Leechers'
labels = [process_column_header(label) for label in torrent_rows[0]('td')]
group_title = u''
# Skip column headers
for result in torrent_rows[1:]:
cells = result('td')
result_class = result.get('class')
# When "Grouping Torrents" is enabled, the structure of table change
group_index = -2 if 'group_torrent' in result_class else 0
try:
title = result.select('a[href^="torrents.php?id="]')[0].get_text()
title = re.sub('\s+', ' ', title).strip() # clean empty lines and multiple spaces
if 'group' in result_class or 'torrent' in result_class:
# get international title if available
title = re.sub('.* \[(.*?)\](.*)', r'\1\2', title)
if 'group' in result_class:
group_title = title
continue
# Clean dash between title and season/episode
title = re.sub('- (S\d{2}(E\d{2,4})?)', r'\1', title)
for serie in self.absolute_numbering:
if serie in title:
# remove season from title when its in absolute format
title = re.sub('S\d{2}E(\d{2,4})', r'\1', title)
break
download_url = urljoin(self.url,
result.select('a[href^="torrents.php?action=download"]')[0]['href'])
if not all([title, download_url]):
continue
seeders = try_int(cells[labels.index('Seeders') + group_index].get_text(strip=True))
leechers = try_int(cells[labels.index('Leechers') + group_index].get_text(strip=True))
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != "RSS":
logger.log("Discarding torrent because it doesn't meet the"
" minimum seeders or leechers: {0} (S:{1} L:{2})".format
(title, seeders, leechers), logger.DEBUG)
continue
torrent_details = None
if 'group_torrent' in result_class:
# torrents belonging to a group
torrent_details = title
title = group_title
elif 'torrent' in result_class:
# standalone/un grouped torrents
torrent_details = cells[labels.index('Nome/Ano')].find('div', class_='torrent_info').get_text()
torrent_details = torrent_details.replace('[', ' ').replace(']', ' ').replace('/', ' ')
torrent_details = torrent_details.replace('Full HD ', '1080p').replace('HD ', '720p')
torrent_size = cells[labels.index('Tamanho') + group_index].get_text(strip=True)
size = convert_size(torrent_size) or -1
torrent_name = '{0} {1}'.format(title, torrent_details.strip()).strip()
torrent_name = re.sub('\s+', ' ', torrent_name)
items.append({
'title': torrent_name,
'link': download_url,
'size': size,
'seeders': seeders,
'leechers': leechers,
'hash': ''
})
if mode != 'RSS':
logger.log('Found result: {0} with {1} seeders and {2} leechers'.format
(torrent_name, seeders, leechers), logger.DEBUG)
except (AttributeError, TypeError, KeyError, ValueError, IndexError):
logger.log('Failed parsing provider.', logger.ERROR)
return {'has_next_page': has_next_page, 'items': items}
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches, too-many-statements
results = []
"""
205 = SD, 208 = HD, 200 = All Videos
https://pirateproxy.pl/s/?q=Game of Thrones&type=search&orderby=7&page=0&category=200
"""
search_params = {
"q": "",
"type": "search",
"orderby": 7, # order by seeders: most first
"page": 0, # first page of results
"category": 200 # All videos
}
# Units
units = ["B", "KIB", "MIB", "GIB"]
def process_column_header(th):
text = ""
if th.a:
text = th.a.get_text(strip=True)
if not text:
text = th.get_text(strip=True)
return text
for mode in search_strings:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
search_urls = (self.urls["search"],
self.urls["rss"])[mode == "RSS"]
if not isinstance(search_urls, list):
search_urls = [search_urls]
for search_url in search_urls:
if self.custom_url:
if not validators.url(self.custom_url):
logger.log(
"Invalid custom url: {0}".format(
self.custom_url), logger.WARNING)
return results
search_url = urljoin(self.custom_url,
search_url.split(self.url)[1])
if mode != "RSS":
search_params["q"] = search_string
logger.log(
"Search string: {}".format(
search_string.decode("utf-8")), logger.DEBUG)
# Prevents a 302 redirect, since there is always a 301 from .se to the best mirror having an extra
# redirect is excessive on the provider and spams the debug log unnecessarily
search_url, params = self.convert_url(
search_url, search_params)
data = self.get_url(search_url,
params=params,
returns="text")
else:
data = self.get_url(search_url, returns="text")
if not data:
logger.log(
"URL did not return data, maybe try a custom url, or a different one",
logger.DEBUG)
continue
with BS4Parser(data, "html5lib") as html:
torrent_table = html.find("table", id="searchResult")
torrent_rows = torrent_table(
"tr") if torrent_table else []
# Continue only if at least one Release is found
if len(torrent_rows) < 2:
logger.log(
"Data returned from provider does not contain any torrents",
logger.DEBUG)
continue
labels = [
process_column_header(label)
for label in torrent_rows[0]("th")
]
# Skip column headers
for result in torrent_rows[1:]:
try:
cells = result("td")
# Funky js on page messing up titles, this fixes that
title = result.find(
class_="detLink")['title'].split(
'Details for ', 1)[-1]
download_url = result.find(
title="Download this torrent using magnet"
)["href"] + self._custom_trackers
if not self.magnet_regex.match(download_url):
logger.log(
"Got an invalid magnet: {0}".format(
download_url))
logger.log(
"Invalid ThePirateBay proxy please try another one",
logger.DEBUG)
continue
if not all([title, download_url]):
continue
seeders = try_int(
cells[labels.index("SE")].get_text(
strip=True))
leechers = try_int(
cells[labels.index("LE")].get_text(
strip=True))
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != "RSS":
logger.log(
"Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})"
.format(title, seeders,
leechers), logger.DEBUG)
continue
# Accept Torrent only from Good People for every Episode Search
if self.confirmed and not result.find(
alt=re.compile(r"VIP|Trusted")):
if mode != "RSS":
logger.log(
"Found result: {0} but that doesn't seem like a trusted result so I'm ignoring it"
.format(title), logger.DEBUG)
continue
# Convert size after all possible skip scenarios
torrent_size = re.sub(
r".*Size ([\d.]+).+([KMGT]iB).*", r"\1 \2",
result.find(class_="detDesc").get_text(
strip=True))
size = convert_size(torrent_size,
units=units) or -1
item = {
'title': title,
'link': download_url,
'size': size,
'seeders': seeders,
'leechers': leechers,
'hash': ''
}
if mode != "RSS":
logger.log(
"Found result: {0} with {1} seeders and {2} leechers"
.format(title, seeders,
leechers), logger.DEBUG)
items.append(item)
except StandardError:
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)),
reverse=True)
results += items
return results
def search(self, search_params, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches, too-many-statements
results = []
for mode in search_params:
items = []
logger.log(u'Search Mode: {0}'.format(mode), logger.DEBUG)
for search_string in search_params[mode]:
if search_string == '':
continue
search_string = six.text_type(search_string).replace('.', ' ')
logger.log(u'Search string: {0}'.format(search_string.decode('utf-8')), logger.DEBUG)
last_page = False
for page in range(0, self.max_pages):
if last_page:
break
logger.log('Processing page {0} of results'.format(page), logger.DEBUG)
search_url = self.urls['search'].format(search_string, page)
data = self.get_url(search_url, returns='text')
if not data:
logger.log(u'No data returned from provider', logger.DEBUG)
continue
try:
with BS4Parser(data, 'html5lib') as html:
table_header = html.find('tr', class_='bordo')
torrent_table = table_header.find_parent('table') if table_header else None
if not torrent_table:
logger.log(u'Could not find table of torrents', logger.ERROR)
continue
torrent_rows = torrent_table('tr')
# Continue only if one Release is found
if len(torrent_rows) < 6 or len(torrent_rows[2]('td')) == 1:
logger.log(u'Data returned from provider does not contain any torrents', logger.DEBUG)
last_page = True
continue
if len(torrent_rows) < 45:
last_page = True
for result in torrent_rows[2:-3]:
result_cols = result('td')
if len(result_cols) == 1:
# Ignore empty rows in the middle of the table
continue
try:
info_link = result('td')[1].find('a')['href']
title = re.sub(' +', ' ', info_link.rsplit('/', 1)[-1].replace('_', ' '))
info_hash = result('td')[3].find('input', class_='downarrow')['value'].upper()
download_url = self._magnet_from_result(info_hash, title)
seeders = try_int(result('td')[5].text)
leechers = try_int(result('td')[6].text)
torrent_size = result('td')[2].string
size = convert_size(torrent_size) or -1
except (AttributeError, IndexError, TypeError):
continue
filename_qt = self._reverseQuality(self._episodeQuality(result))
for text in self.hdtext:
title1 = title
title = title.replace(text, filename_qt)
if title != title1:
break
if Quality.nameQuality(title) == Quality.UNKNOWN:
title += filename_qt
if not self._is_italian(title) and not self.subtitle:
logger.log(u'Torrent is subtitled, skipping: {0}'.format(title), logger.DEBUG)
continue
if self.engrelease and not self._is_english(title):
logger.log(u'Torrent isn\'t english audio/subtitled, skipping: {0}'.format(title),
logger.DEBUG)
continue
search_show = re.split(r'([Ss][\d{1,2}]+)', search_string)[0]
show_title = search_show
ep_params = ''
rindex = re.search(r'([Ss][\d{1,2}]+)', title)
if rindex:
show_title = title[:rindex.start()]
ep_params = title[rindex.start():]
if show_title.lower() != search_show.lower() and search_show.lower() in show_title.lower():
new_title = search_show + ep_params
title = new_title
if not all([title, download_url]):
continue
if self._is_season_pack(title):
title = re.sub(r'([Ee][\d{1,2}\-?]+)', '', title)
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
logger.log(u'Discarding torrent because it doesn\'t meet the minimum'
u' seeders or leechers: {0} (S:{1} L:{2})'.format(
title, seeders, leechers), logger.DEBUG)
continue
item = {'title': title, 'link': download_url, 'size': size,
'seeders': seeders, 'leechers': leechers, 'hash': info_hash}
if mode != 'RSS':
logger.log(u'Found result: {0} with {1} seeders and {2} leechers'.format(
title, seeders, leechers), logger.DEBUG)
items.append(item)
except Exception as error:
logger.log(u'Failed parsing provider. Error: {0}'.format(error), logger.ERROR)
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None):
results = []
if not (self.url and self.urls):
self.find_domain()
if not (self.url and self.urls):
return results
anime = (self.show
and self.show.anime) or (ep_obj and ep_obj.show
and ep_obj.show.anime) or False
search_params = {
"field": "seeders",
"sorder": "desc",
"category": ("tv", "anime")[anime]
}
for mode in search_strings:
items = []
logger.debug(_("Search Mode: {mode}".format(mode=mode)))
for search_string in search_strings[mode]:
# search_params["q"] = (search_string, None)[mode == "RSS"]
search_params["field"] = ("seeders", "time_add")[mode == "RSS"]
if mode != "RSS":
if anime:
continue
logger.debug(
_("Search String: {search_string}".format(
search_string=search_string)))
search_url = self.urls["search"].format(q=search_string)
else:
search_url = self.urls["rss"]
if self.custom_url:
if not validators.url(self.custom_url):
logger.warning("Invalid custom url: {0}".format(
self.custom_url))
return results
search_url = urljoin(self.custom_url,
search_url.split(self.url)[1])
data = self.get_url(search_url,
params=OrderedDict(
sorted(list(search_params.items()),
key=lambda x: x[0])),
returns="text")
if not data:
logger.info(
"{url} did not return any data, it may be disabled. Trying to get a new domain"
.format(url=self.url))
self.disabled_mirrors.append(self.url)
self.find_domain()
if self.url in self.disabled_mirrors:
logger.info("Could not find a better mirror to try.")
logger.info(
"The search did not return data, if the results are on the site maybe try a custom url, or a different one"
)
return results
# This will recurse a few times until all of the mirrors are exhausted if none of them work.
return self.search(search_strings, age, ep_obj)
with BS4Parser(data, "html5lib") as html:
labels = [
cell.get_text()
for cell in html.find(class_="firstr")("th")
]
logger.info("Found {} results".format(
len(html("tr", **self.rows_selector))))
for result in html("tr", **self.rows_selector):
try:
download_url = urllib.parse.unquote_plus(
result.find(
title="Torrent magnet link")["href"].split(
"url=")[1]) + self._custom_trackers
parsed_magnet = urllib.parse.parse_qs(download_url)
torrent_hash = self.hash_from_magnet(download_url)
title = result.find(class_="torrentname").find(
class_="cellMainLink").get_text(strip=True)
if title.endswith("..."):
title = parsed_magnet['dn'][0]
if not (title and download_url):
if mode != "RSS":
logger.debug(
"Discarding torrent because We could not parse the title and url"
)
continue
seeders = try_int(
result.find(class_="green").get_text(
strip=True))
leechers = try_int(
result.find(class_="red").get_text(strip=True))
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != "RSS":
logger.debug(
"Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})"
.format(title, seeders, leechers))
continue
if self.confirmed and not result.find(
class_="ka-green"):
if mode != "RSS":
logger.debug(
"Found result " + title +
" but that doesn't seem like a verified result so I'm ignoring it"
)
continue
torrent_size = result("td")[labels.index(
"size")].get_text(strip=True)
size = convert_size(torrent_size) or -1
item = {
'title': title,
'link': download_url,
'size': size,
'seeders': seeders,
'leechers': leechers,
'hash': torrent_hash
}
if mode != "RSS":
logger.debug(
"Found result: {0} with {1} seeders and {2} leechers"
.format(title, seeders, leechers))
items.append(item)
except (AttributeError, TypeError, KeyError,
ValueError, Exception):
logger.info(traceback.format_exc())
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)),
reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches
results = []
if not self.login():
return results
freeleech = '2' if self.freeleech else '0'
# Search Params
# c59=1&c73=1&c5=1&c41=1&c60=1&c66=1&c65=1&c67=1&c62=1&c64=1&c61=1&search=Good+Behavior+S01E01&cat=0&incldead=0&freeleech=0&lang=0
search_params = {
'c5': '1', # Category: Series - DVDRip
'c41': '1', # Category: Series - HD
'c60': '1', # Category: Series - Pack TV
'c62': '1', # Category: Series - BDRip
'c64': '1', # Category: Series - VOSTFR
'c65': '1', # Category: Series - TV 720p
'c66': '1', # Category: Series - TV 1080p
'c67': '1', # Category: Series - Pack TV HD
'c73': '1', # Category: Anime
'incldead': '0', # Include dead torrent - 0: off 1: yes 2: only dead
'freeleech': freeleech, # Only freeleech torrent - 0: off 1: no freeleech 2: Only freeleech
'lang': '0' # Langugage - 0: off 1: English 2: French ....
}
# Units
units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']
for mode in search_strings:
items = []
logger.log('Search Mode: {0}'.format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
logger.log('Search String: {0} for mode {1}'.format(search_strings[mode], mode), logger.DEBUG)
if mode != 'RSS':
logger.log('Search string: {0}'.format
(search_string.decode('utf-8')), logger.DEBUG)
search_params['search'] = re.sub(r'[()]', '', search_string)
data = self.get_url(self.urls['search'], params=search_params, returns='text')
if not data:
continue
with BS4Parser(data, 'html5lib') as html:
torrent_table = html.find(class_='ttable_headinner')
torrent_rows = torrent_table('tr') if torrent_table else []
# Continue only if at least one Release is found
if len(torrent_rows) < 2:
logger.log('Data returned from provider does not contain any torrents', logger.DEBUG)
continue
# Catégorie, Release, Date, DL, Size, C, S, L
labels = [label.get_text(strip=True) for label in torrent_rows[0]('th')]
# Skip column headers
for result in torrent_rows[1:]:
cells = result('td')
if len(cells) < len(labels):
continue
try:
id = re.search('id=([0-9]+)', cells[labels.index('Nom')].find('a')['href']).group(1)
title = cells[labels.index('Nom')].get_text(strip=True)
download_url = urljoin(self.urls['download'], '?id={0}&name={1}'.format(id, title))
if not all([title, download_url]):
continue
seeders = try_int(cells[labels.index('S')].get_text(strip=True))
leechers = try_int(cells[labels.index('L')].get_text(strip=True))
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.log('Discarding torrent because it doesn\'t meet the minimum seeders or leechers: {0} (S:{1} L:{2})'.format
(title, seeders, leechers), logger.DEBUG)
continue
size_index = labels.index('Size') if 'Size' in labels else labels.index('Taille')
torrent_size = cells[size_index].get_text()
size = convert_size(torrent_size, units=units) or -1
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''}
if mode != 'RSS':
logger.log('Found result: {0} with {1} seeders and {2} leechers'.format
(title, seeders, leechers), logger.DEBUG)
items.append(item)
except StandardError:
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None):
results = []
if not self.login():
return results
# http://speed.cd/browse/49/50/52/41/55/2/30/freeleech/deep/q/arrow
# Search Params
search_params = [
"browse",
"41", # TV/Packs
"2", # Episodes
"49", # TV/HD
"50", # TV/Sports
"52", # TV/B-Ray
"55", # TV/Kids
"30", # Anime
]
if self.freeleech:
search_params.append("freeleech")
search_params.append("deep")
# Units
units = ["B", "KB", "MB", "GB", "TB", "PB"]
def process_column_header(td):
result = ""
img = td.find("img")
if img:
result = img.get("alt")
if not result:
result = img.get("title")
if not result:
anchor = td.find("a")
if anchor:
result = anchor.get_text(strip=True)
if not result:
result = td.get_text(strip=True)
return result
for mode in search_strings:
items = []
logger.debug(_("Search Mode: {mode}".format(mode=mode)))
for search_string in {*search_strings[mode]}:
current_params = search_params
if mode != "RSS":
logger.debug(
_("Search String: {search_string}".format(
search_string=search_string)))
current_params += [
"q", re.sub(r"[^\w\s]", "", search_string)
]
data = self.get_url(urljoin(self.url,
"/".join(current_params)),
returns="text")
if not data:
continue
with BS4Parser(data, "html5lib") as html:
torrent_table = html.find("div", class_="boxContent")
torrent_table = torrent_table.find(
"table") if torrent_table else []
# noinspection PyCallingNonCallable
torrent_rows = torrent_table("tr") if torrent_table else []
# Continue only if at least one Release is found
if len(torrent_rows) < 2:
logger.debug(
"Data returned from provider does not contain any torrents"
)
continue
labels = [
process_column_header(label)
for label in torrent_rows[0]("th")
]
row_labels = [
process_column_header(label)
for label in torrent_rows[1]("td")
]
def label_index(name):
if name in labels:
return labels.index(name)
return row_labels.index(name)
# Skip column headers
for result in torrent_rows[1:]:
try:
cells = result("td")
title = cells[label_index("Title")].find(
"a").get_text()
download_url = urljoin(
self.url,
cells[label_index("Download")].a["href"])
if not all([title, download_url]):
continue
seeders = try_int(cells[label_index("Seeders") -
1].get_text(strip=True))
leechers = try_int(cells[label_index("Leechers") -
1].get_text(strip=True))
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != "RSS":
logger.debug(
"Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})"
.format(title, seeders, leechers))
continue
torrent_size = cells[label_index("Size") -
1].get_text()
size = convert_size(torrent_size,
units=units) or -1
item = {
"title": title,
"link": download_url,
"size": size,
"seeders": seeders,
"leechers": leechers,
"hash": ""
}
if mode != "RSS":
logger.debug(
"Found result: {0} with {1} seeders and {2} leechers"
.format(title, seeders, leechers))
items.append(item)
except Exception as error:
logger.debug(f"Speed.cd: {error}")
logger.debug(traceback.format_exc())
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get("seeders", 0)),
reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-branches, too-many-locals, too-many-statements
results = []
if not self.login():
return results
for mode in search_strings:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode != 'RSS':
search_url = self.urls['search'] % (quote_plus(search_string.replace('.', ' ')),)
else:
search_url = self.urls['search'] % ''
if mode != 'RSS':
logger.log("Search string: {0}".format
(search_string.decode("utf-8")), logger.DEBUG)
data = self.get_url(search_url, returns='text')
if not data or 'please try later' in data:
logger.log("No data returned from provider", logger.DEBUG)
continue
# Search result page contains some invalid html that prevents html parser from returning all data.
# We cut everything before the table that contains the data we are interested in thus eliminating
# the invalid html portions
try:
data = data.split('<div id="information"></div>')[1]
index = data.index('<table')
except ValueError:
logger.log("Could not find main torrent table", logger.ERROR)
continue
except IndexError:
logger.log("Could not parse data from provider", logger.DEBUG)
continue
html = BeautifulSoup(data[index:], 'html5lib')
if not html:
logger.log("No html data parsed from provider", logger.DEBUG)
continue
torrents = html('tr')
if not torrents:
continue
# Skip column headers
for result in torrents[1:]:
if len(result.contents) < 10:
# skip extraneous rows at the end
continue
try:
dl_href = result.find('a', attrs={'href': re.compile(r'download.php.*')})['href']
title = re.search('f=(.*).torrent', dl_href).group(1).replace('+', '.')
download_url = self.urls['base_url'] + dl_href
seeders = int(result.find('span', class_='seedy').find('a').text)
leechers = int(result.find('span', class_='leechy').find('a').text)
torrent_size = re.match(r'.*?([0-9]+,?\.?[0-9]* [KkMmGg]+[Bb]+).*', str(result), re.DOTALL).group(1)
size = convert_size(torrent_size) or -1
if not all([title, download_url]):
continue
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.log("Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})".format
(title, seeders, leechers), logger.DEBUG)
continue
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''}
if mode != 'RSS':
logger.log("Found result: {0} with {1} seeders and {2} leechers".format(title, seeders, leechers), logger.DEBUG)
items.append(item)
except (AttributeError, TypeError, KeyError, ValueError):
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None):
self.login()
results = []
for mode in search_strings:
items = []
logger.debug(_("Search Mode: {mode}".format(mode=mode)))
for search_string in search_strings[mode]:
if mode != 'RSS':
logger.debug(
_("Search String: {search_string}".format(
search_string=search_string)))
# search string needs to be normalized, single quotes are apparently not allowed on the site
# ç should also be replaced, people tend to use c instead
replace_chars = {"'": '', "ç": 'c'}
for k, v in replace_chars.items():
search_string = search_string.replace(k, v)
logger.debug('Sanitized string: {0}'.format(search_string))
try:
search_params = {
'category': '2145',
'sub_category': 'all',
'name': re.sub(r'[()]', '', search_string),
'do': 'search'
}
data = self.get_url(self.urls['search'],
params=search_params,
returns='text')
if not data:
continue
if 'logout' not in data:
logger.debug('Refreshing cookies')
self.login()
with BS4Parser(data, 'html5lib') as html:
torrent_table = html.find(class_='table')
torrent_rows = torrent_table(
'tr') if torrent_table else []
# Continue only if at least one Release is found
if len(torrent_rows) < 2:
logger.debug(
'Data returned from provider does not contain any torrents'
)
continue
# Skip column headers
for result in torrent_rows[1:]:
cells = result('td')
if len(cells) < 9:
continue
title = cells[1].find('a').get_text(strip=True)
id = cells[2].find('a')['target']
download_url = urljoin(
self.url, 'engine/download_torrent?id=' + id)
if not (title and download_url):
continue
seeders = try_int(cells[7].get_text(strip=True))
leechers = try_int(cells[8].get_text(strip=True))
torrent_size = cells[5].get_text()
size = convert_size(torrent_size) or -1
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.debug(
'Discarding torrent because it doesn\'t meet the minimum seeders or leechers: {0} (S:{1} L:{2})'
.format(title, seeders, leechers))
continue
item = {
'title': title,
'link': download_url,
'size': size,
'seeders': seeders,
'leechers': leechers,
'hash': ''
}
if mode != 'RSS':
logger.debug(
_('Found result: {title} with {seeders} seeders and {leechers} leechers'
.format(title=title,
seeders=seeders,
leechers=leechers)))
items.append(item)
except (AttributeError, TypeError, KeyError, ValueError):
logger.exception('Failed parsing provider {}.'.format(
self.name))
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)),
reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches, too-many-statements
results = []
if not self.login():
return results
for mode in search_strings:
items = []
logger.log("Search Mode: {}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode != 'RSS':
search_url = self.urls['search'] % quote_plus(search_string)
logger.log("Search string: {}".format(search_string), logger.DEBUG)
else:
search_url = self.urls['rss']
if self.freeleech:
search_url = search_url.replace('active=1', 'active=5')
logger.log("Search URL: {}".format(search_url), logger.DEBUG)
data = self.get_url(search_url)
if not data or 'Error' in data:
logger.log("No data returned from provider", logger.DEBUG)
continue
if data.find('Non abbiamo trovato nulla') != -1:
logger.log("Data returned from provider does not contain any torrents", logger.DEBUG)
continue
# Search result page contains some invalid html that prevents html parser from returning all data.
# We cut everything before the table that contains the data we are interested in thus eliminating
# the invalid html portions
try:
index = data.lower().index('<tbody id="highlighted"')
except ValueError:
logger.log("Could not find table of torrents highlighted", logger.DEBUG)
continue
# data = urllib.unquote(data[index:].encode('utf-8')).decode('utf-8').replace('\t', '')
data = data[index:]
with BS4Parser(data, 'html5lib') as html:
if not html:
logger.log("No html data parsed from provider", logger.DEBUG)
continue
torrent_rows = []
torrent_table = html.find('table', class_='highlighted')
if torrent_table:
torrent_rows = torrent_table.find_all('tr')
if not torrent_rows:
logger.log("Could not find results in returned data", logger.DEBUG)
continue
# Cat., Active, Filename, Dl, Wl, Added, Size, Uploader, S, L, C
labels = [label.a.get_text(strip=True) if label.a else label.get_text(strip=True) for label in torrent_rows[0].find_all('td')]
# Skip column headers
for result in torrent_rows[1:]:
try:
cells = result.findChildren('td')[:len(labels)]
if len(cells) < len(labels):
continue
title = cells[labels.index(1)].a.index(0).get_text(strip=True)
seeders = try_int(cells[labels.index(5)].a.index(0).get_text(strip=True))
leechers = try_int(cells[labels.index(5)].a.index(1).get_text(strip=True))
torrent_size = cells[labels.index(4)].get_text()
size = convert_size(torrent_size) or -1
download_url = self.url + '/' + cells[labels.index(1)].a.index(0)['href']
# title = cells[labels.index(u'Filename')].a.get_text(strip=True)
# seeders = try_int(cells[labels.index(u'S')].get_text(strip=True))
# leechers = try_int(cells[labels.index(u'L')].get_text(strip=True))
# torrent_size = cells[labels.index(u'Size')].get_text()
# size = convert_size(torrent_size) or -1
# download_url = self.url + '/' + cells[labels.index(u'Dl')].a['href']
except (AttributeError, TypeError, KeyError, ValueError, IndexError):
continue
if not all([title, download_url]):
continue
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.log(
"Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})".format(title, seeders, leechers), logger.DEBUG)
continue
item = title, download_url, size, seeders, leechers
if mode != 'RSS':
logger.log("Found result: {0} with {1} seeders and {2} leechers".format(title, seeders, leechers), logger.DEBUG)
items.append(item)
# For each search mode sort all the items by seeders if available
items.sort(key=lambda tup: tup[3], reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None):
results = []
if not self.login():
return results
for mode in search_strings:
items = []
logger.debug(_("Search Mode: {mode}".format(mode=mode)))
for search_string in {*search_strings[mode]}:
if mode != 'RSS':
logger.debug(
_("Search String: {search_string}".format(
search_string=search_string)))
self.search_params['searchstr'] = search_string
data = self.get_url(self.urls['search'],
params=self.search_params,
returns='text')
if not data:
logger.debug('URL did not return data')
continue
strTableStart = "<table class=\"torrent_table"
startTableIndex = data.find(strTableStart)
trimmedData = data[startTableIndex:]
if not trimmedData:
continue
try:
with BS4Parser(trimmedData, 'html5lib') as html:
result_table = html.find('table',
{'id': 'torrent_table'})
if not result_table:
logger.debug(
"Data returned from provider does not contain any torrents"
)
continue
result_tbody = result_table.find('tbody')
entries = result_tbody.contents
del entries[1::2]
for result in entries[1:]:
torrent = result('td')
if len(torrent) <= 1:
break
allAs = (torrent[1])('a')
try:
notinternal = result.find(
'img',
src='/static//common/user_upload.png')
if self.ranked and notinternal:
logger.debug(
"Found a user uploaded release, Ignoring it.."
)
continue
freeleech = result.find(
'img',
src='/static//common/browse/freeleech.png')
if self.freeleech and not freeleech:
continue
title = allAs[2].string
download_url = self.urls['base_url'] + allAs[
0].attrs['href']
torrent_size = result.find(
"td", class_="nobr").find_next_sibling(
"td").string
if torrent_size:
size = convert_size(torrent_size) or -1
seeders = try_int(
(result('td')[6]).text.replace(',', ''))
leechers = try_int(
(result('td')[7]).text.replace(',', ''))
except (AttributeError, TypeError):
continue
if not title or not download_url:
continue
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.debug(
"Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})"
.format(title, seeders, leechers))
continue
item = {
'title': title,
'link': download_url,
'size': size,
'seeders': seeders,
'leechers': leechers,
'hash': ''
}
if mode != 'RSS':
logger.debug(
"Found result: {0} with {1} seeders and {2} leechers"
.format(title, seeders, leechers))
items.append(item)
except Exception:
logger.exception(
"Failed parsing provider. Traceback: {0}".format(
traceback.format_exc()))
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)),
reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals
"""
Search query:
http://www.newpct.com/index.php?l=doSearch&q=fringe&category_=All&idioma_=1&bus_de_=All
q => Show name
category_ = Category 'Shows' (767)
idioma_ = Language Spanish (1), All
bus_de_ = Date from (All, mes, semana, ayer, hoy)
"""
results = []
# Only search if user conditions are true
lang_info = '' if not ep_obj or not ep_obj.show else ep_obj.show.lang
search_params = {
'l': 'doSearch',
'q': '',
'category_': 'All',
'idioma_': 1,
'bus_de_': 'All'
}
for mode in search_strings:
items = []
logger.log('Search Mode: {0}'.format(mode), logger.DEBUG)
if self.onlyspasearch:
search_params['idioma_'] = 1
else:
search_params['idioma_'] = 'All'
# Only search if user conditions are true
if self.onlyspasearch and lang_info != 'es' and mode != 'RSS':
logger.log('Show info is not spanish, skipping provider search', logger.DEBUG)
continue
search_params['bus_de_'] = 'All' if mode != 'RSS' else 'semana'
for search_string in search_strings[mode]:
if mode != 'RSS':
logger.log('Search string: {0}'.format
(search_string.decode('utf-8')), logger.DEBUG)
search_params['q'] = search_string
data = self.get_url(self.urls['search'], params=search_params, returns='text')
if not data:
continue
with BS4Parser(data, 'html5lib') as html:
torrent_table = html.find('table', id='categoryTable')
torrent_rows = torrent_table('tr') if torrent_table else []
# Continue only if at least one Release is found
if len(torrent_rows) < 3: # Headers + 1 Torrent + Pagination
logger.log('Data returned from provider does not contain any torrents', logger.DEBUG)
continue
# 'Fecha', 'Título', 'Tamaño', ''
# Date, Title, Size
labels = [label.get_text(strip=True) for label in torrent_rows[0]('th')]
for row in torrent_rows[1:-1]:
try:
cells = row('td')
torrent_row = row.find('a')
download_url = torrent_row.get('href', '')
title = self._processTitle(torrent_row.get('title', ''), download_url)
if not all([title, download_url]):
continue
# Provider does not provide seeders/leechers
seeders = 1
leechers = 0
#2 is the 'Tamaño' column.
torrent_size = cells[2].get_text(strip=True)
size = convert_size(torrent_size) or -1
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''}
if mode != 'RSS':
logger.log('Found result: {0}'.format(title), logger.DEBUG)
items.append(item)
except (AttributeError, TypeError):
continue
results += items
return results
def search(self, search_params, age=0, ep_obj=None):
results = []
if not self.login():
return results
for mode in search_params:
items = []
logger.debug(_("Search Mode: {mode}".format(mode=mode)))
for search_string in search_params[mode]:
if mode != 'RSS':
logger.debug(_("Search String: {search_string}".format(search_string=search_string)))
query = {'sec': 'jax', 'cata': 'yes', 'search': search_string}
query.update({"c" + str(i): 1 for i in self.categories})
data = self.get_url(self.urls['apisearch'], returns='text', post_data=query)
if not data:
continue
with BS4Parser(data, 'html5lib') as html:
torrent_table = html.find(id='torrenttable')
if torrent_table:
torrent_rows = torrent_table.findAll('tr')
else:
torrent_rows = []
# Continue only if one Release is found
if len(torrent_rows) < 2:
logger.debug("Data returned from provider does not contain any torrents")
continue
# Scenetime apparently uses different number of cells in #torrenttable based
# on who you are. This works around that by extracting labels from the first
# <tr> and using their index to find the correct download/seeders/leechers td.
labels = [label.get_text(strip=True) or label.img['title'] for label in torrent_rows[0]('td')]
for result in torrent_rows[1:]:
try:
cells = result('td')
link = cells[labels.index('Name')].find('a')
torrent_id = link['href'].replace('details.php?id=', '').split("&")[0]
title = link.get_text(strip=True)
download_url = self.urls['download'] % (torrent_id, "{0}.torrent".format(title.replace(" ", ".")))
seeders = try_int(cells[labels.index('Seeders')].get_text(strip=True))
leechers = try_int(cells[labels.index('Leechers')].get_text(strip=True))
torrent_size = cells[labels.index('Size')].get_text()
size = convert_size(torrent_size) or -1
except (AttributeError, TypeError, KeyError, ValueError):
continue
if not all([title, download_url]):
continue
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.debug("Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})".format
(title, seeders, leechers))
continue
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''}
if mode != 'RSS':
logger.debug("Found result: {0} with {1} seeders and {2} leechers".format(title, seeders, leechers))
items.append(item)
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches
results = []
if not self.login():
return results
# Search Params
search_params = {
'do': 'search',
'include_dead_torrents': 'no',
'search_type': 't_name',
'category': 0,
'keywords': ''
}
# Units
units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']
def process_column_header(td):
td_title = ''
if td.img:
td_title = td.img.get('title', td.get_text(strip=True))
if not td_title:
td_title = td.get_text(strip=True)
return td_title
for mode in search_strings:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode != 'RSS':
logger.log("Search string: {0}".format
(search_string.decode("utf-8")), logger.DEBUG)
search_params['keywords'] = search_string
data = self.get_url(self.urls['search'], params=search_params, returns='text')
if not data:
logger.log("No data returned from provider", logger.DEBUG)
continue
with BS4Parser(data, 'html5lib') as html:
torrent_table = html.find('table', id='sortabletable')
torrent_rows = torrent_table('tr') if torrent_table else []
# Continue only if at least one Release is found
if len(torrent_rows) < 2:
logger.log("Data returned from provider does not contain any torrents", logger.DEBUG)
continue
labels = [process_column_header(label) for label in torrent_rows[0]('td')]
# Skip column headers
for result in torrent_rows[1:]:
try:
title = result.find('div', class_='tooltip-target').get_text(strip=True)
# skip if torrent has been nuked due to poor quality
if title.startswith('Nuked.'):
continue
download_url = result.find(
'img', title='Click to Download this Torrent in SSL!').parent['href']
if not all([title, download_url]):
continue
cells = result('td')
seeders = try_int(cells[labels.index('Seeders')].get_text(strip=True))
leechers = try_int(cells[labels.index('Leechers')].get_text(strip=True))
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.log("Discarding torrent because it doesn't meet the"
" minimum seeders or leechers: {0} (S:{1} L:{2})".format
(title, seeders, leechers), logger.DEBUG)
continue
torrent_size = cells[labels.index('Size')].get_text(strip=True)
size = convert_size(torrent_size, units=units) or -1
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders,
'leechers': leechers, 'hash': ''}
if mode != 'RSS':
logger.log("Found result: {0} with {1} seeders and {2} leechers".format
(title, seeders, leechers), logger.DEBUG)
items.append(item)
except StandardError:
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches, too-many-statements
results = []
if not self.login():
return results
for mode in search_strings:
items = []
logger.log("Search Mode: {}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode != 'RSS':
search_url = self.urls['search'] % quote_plus(
search_string)
logger.log("Search string: {}".format(search_string),
logger.DEBUG)
else:
search_url = self.urls['rss']
if self.freeleech:
search_url = search_url.replace('active=1', 'active=5')
logger.log("Search URL: {}".format(search_url), logger.DEBUG)
data = self.get_url(search_url)
if not data or 'Error' in data:
logger.log("No data returned from provider", logger.DEBUG)
continue
if data.find('Non abbiamo trovato nulla') != -1:
logger.log(
"Data returned from provider does not contain any torrents",
logger.DEBUG)
continue
# Search result page contains some invalid html that prevents html parser from returning all data.
# We cut everything before the table that contains the data we are interested in thus eliminating
# the invalid html portions
try:
index = data.lower().index('<tbody id="highlighted"')
except ValueError:
logger.log("Could not find table of torrents highlighted",
logger.DEBUG)
continue
# data = urllib.unquote(data[index:].encode('utf-8')).decode('utf-8').replace('\t', '')
data = data[index:]
with BS4Parser(data, 'html5lib') as html:
if not html:
logger.log("No html data parsed from provider",
logger.DEBUG)
continue
torrent_rows = []
torrent_table = html.find('table', class_='highlighted')
if torrent_table:
torrent_rows = torrent_table.find_all('tr')
if not torrent_rows:
logger.log("Could not find results in returned data",
logger.DEBUG)
continue
# Cat., Active, Filename, Dl, Wl, Added, Size, Uploader, S, L, C
labels = [
label.a.get_text(strip=True)
if label.a else label.get_text(strip=True)
for label in torrent_rows[0].find_all('td')
]
# Skip column headers
for result in torrent_rows[1:]:
try:
cells = result.findChildren('td')[:len(labels)]
if len(cells) < len(labels):
continue
title = cells[labels.index(1)].a.index(0).get_text(
strip=True)
seeders = try_int(
cells[labels.index(5)].a.index(0).get_text(
strip=True))
leechers = try_int(
cells[labels.index(5)].a.index(1).get_text(
strip=True))
torrent_size = cells[labels.index(4)].get_text()
size = convert_size(torrent_size) or -1
download_url = self.url + '/' + cells[labels.index(
1)].a.index(0)['href']
# title = cells[labels.index(u'Filename')].a.get_text(strip=True)
# seeders = try_int(cells[labels.index(u'S')].get_text(strip=True))
# leechers = try_int(cells[labels.index(u'L')].get_text(strip=True))
# torrent_size = cells[labels.index(u'Size')].get_text()
# size = convert_size(torrent_size) or -1
# download_url = self.url + '/' + cells[labels.index(u'Dl')].a['href']
except (AttributeError, TypeError, KeyError,
ValueError, IndexError):
continue
if not all([title, download_url]):
continue
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.log(
"Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})"
.format(title, seeders,
leechers), logger.DEBUG)
continue
item = title, download_url, size, seeders, leechers
if mode != 'RSS':
logger.log(
"Found result: {0} with {1} seeders and {2} leechers"
.format(title, seeders,
leechers), logger.DEBUG)
items.append(item)
# For each search mode sort all the items by seeders if available
items.sort(key=lambda tup: tup[3], reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches
results = []
if not self.login():
return results
# Search Params
search_params = {
'cat[]': ['TV|SD|VOSTFR', 'TV|HD|VOSTFR', 'TV|SD|VF', 'TV|HD|VF', 'TV|PACK|FR', 'TV|PACK|VOSTFR', 'TV|EMISSIONS', 'ANIME'],
# Both ASC and DESC are available for sort direction
'way': 'DESC'
}
# Units
units = ['O', 'KO', 'MO', 'GO', 'TO', 'PO']
for mode in search_strings:
items = []
logger.log('Search Mode: {0}'.format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode != 'RSS':
logger.log('Search string: {0}'.format
(search_string.decode('utf-8')), logger.DEBUG)
# Sorting: Available parameters: ReleaseName, Seeders, Leechers, Snatched, Size
search_params['order'] = ('Seeders', 'Time')[mode == 'RSS']
search_params['search'] = re.sub(r'[()]', '', search_string)
data = self.get_url(self.urls['search'], params=search_params, returns='text')
if not data:
continue
with BS4Parser(data, 'html5lib') as html:
torrent_table = html.find(class_='torrent_table')
torrent_rows = torrent_table('tr') if torrent_table else []
# Continue only if at least one Release is found
if len(torrent_rows) < 2:
logger.log('Data returned from provider does not contain any torrents', logger.DEBUG)
continue
# Catégorie, Release, Date, DL, Size, C, S, L
labels = [label.get_text(strip=True) for label in torrent_rows[0]('td')]
# Skip column headers
for result in torrent_rows[1:]:
cells = result('td')
if len(cells) < len(labels):
continue
try:
title = cells[labels.index('Release')].get_text(strip=True)
download_url = urljoin(self.url, cells[labels.index('DL')].find('a', class_='tooltip')['href'])
if not all([title, download_url]):
continue
seeders = try_int(cells[labels.index('S')].get_text(strip=True))
leechers = try_int(cells[labels.index('L')].get_text(strip=True))
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.log('Discarding torrent because it doesn\'t meet the minimum seeders or leechers: {0} (S:{1} L:{2})'.format
(title, seeders, leechers), logger.DEBUG)
continue
size_index = labels.index('Size') if 'Size' in labels else labels.index('Taille')
torrent_size = cells[size_index].get_text()
size = convert_size(torrent_size, units=units) or -1
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''}
if mode != 'RSS':
logger.log('Found result: {0} with {1} seeders and {2} leechers'.format
(title, seeders, leechers), logger.DEBUG)
items.append(item)
except StandardError:
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None):
results = []
if self.show and not self.show.is_anime:
return results
for mode in search_strings:
items = []
logger.debug(_("Search Mode: {mode}".format(mode=mode)))
for search_string in {*search_strings[mode]}:
if mode != 'RSS':
logger.debug(
_("Search String: {search_string}".format(
search_string=search_string)))
search_params = {
"terms": search_string,
"type": 1, # get anime types
}
data = self.get_url(self.urls['search'],
params=search_params,
returns='text')
if not data:
continue
with BS4Parser(data, 'html5lib') as soup:
torrent_table = soup.find('table', class_='listing')
torrent_rows = torrent_table('tr') if torrent_table else []
# Continue only if one Release is found
if len(torrent_rows) < 2:
logger.debug(
"Data returned from provider does not contain any torrents"
)
continue
a = 1 if len(torrent_rows[0]('td')) < 2 else 0
for top, bot in zip(torrent_rows[a::2],
torrent_rows[a + 1::2]):
try:
desc_top = top.find('td', class_='desc-top')
title = desc_top.get_text(strip=True)
download_url = desc_top.find('a')['href']
desc_bottom = bot.find(
'td', class_='desc-bot').get_text(strip=True)
size = convert_size(
desc_bottom.split('|')[1].strip(
'Size: ')) or -1
stats = bot.find(
'td', class_='stats').get_text(strip=True)
sl = re.match(
r'S:(?P<seeders>\d+)L:(?P<leechers>\d+)C:(?:\d+)ID:(?:\d+)',
stats.replace(' ', ''))
seeders = try_int(sl.group('seeders')) if sl else 0
leechers = try_int(
sl.group('leechers')) if sl else 0
except Exception:
continue
if not all([title, download_url]):
continue
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.debug(
"Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})"
.format(title, seeders, leechers))
continue
item = {
'title': title,
'link': download_url,
'size': size,
'seeders': seeders,
'leechers': leechers,
'hash': ''
}
if mode != 'RSS':
logger.debug(
"Found result: {0} with {1} seeders and {2} leechers"
.format(title, seeders, leechers))
items.append(item)
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)),
reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals
results = []
for mode in search_strings:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode == 'Season':
search_string = re.sub(r'(.*)S0?', r'\1Saison ', search_string)
if mode != 'RSS':
logger.log("Search string: {0}".format
(search_string.decode("utf-8")), logger.DEBUG)
search_url = self.url + '/search_torrent/' + search_string.replace('.', '-').replace(' ', '-') + '.html'
else:
search_url = self.url + '/torrents_series.html'
data = self.get_url(search_url, returns='text')
if not data:
continue
with BS4Parser(data, 'html5lib') as html:
torrent_table = html.find('div', {'class': 'table-responsive'})
if torrent_table:
torrent_rows = torrent_table.findAll('tr')
else:
torrent_rows = None
if not torrent_rows:
continue
for result in torrent_rows:
try:
title = result.find('a').get_text(strip=False).replace("HDTV", "HDTV x264-Torrent9")
title = re.sub(r' Saison', ' Season', title, flags=re.I)
tmp = result.find("a")['href']
download_url = self._retrieve_dllink_from_url(self.url + tmp)
if not all([title, download_url]):
continue
seeders = try_int(result.find(class_="seed_ok").get_text(strip=True))
leechers = try_int(result.find_all('td')[3].get_text(strip=True))
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.log("Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})".format
(title, seeders, leechers), logger.DEBUG)
continue
torrent_size = result.find_all('td')[1].get_text(strip=True)
units = ['o', 'Ko', 'Mo', 'Go', 'To', 'Po']
size = convert_size(torrent_size, units=units) or -1
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''}
if mode != 'RSS':
logger.log("Found result: {0} with {1} seeders and {2} leechers".format(title, seeders, leechers), logger.DEBUG)
items.append(item)
except StandardError:
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals
results = []
for mode in search_strings:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode == 'Season':
search_string = re.sub(r'(.*)S0?', r'\1Saison ',
search_string)
if mode != 'RSS':
logger.log(
"Search string: {0}".format(
search_string.decode("utf-8")), logger.DEBUG)
search_url = self.url
post_data = {'torrentSearch': search_string}
else:
search_url = self.url + '/torrents_series.html'
post_data = None
data = self.get_url(search_url, post_data, returns='text')
if not data:
continue
with BS4Parser(data, 'html5lib') as html:
torrent_table = html.find('div',
{'class': 'table-responsive'})
if torrent_table:
torrent_rows = torrent_table.findAll('tr')
else:
torrent_rows = None
if not torrent_rows:
continue
for result in torrent_rows:
try:
title = result.find('a').get_text(
strip=False).replace("HDTV",
"HDTV x264-Torrent9")
title = re.sub(r' Saison',
' Season',
title,
flags=re.I)
tmp = result.find("a")['href']
download_url = urljoin(
self.url,
self._retrieve_dllink_from_url(
urljoin(self.url, tmp)))
if not all([title, download_url]):
continue
seeders = try_int(
result.find(class_="seed_ok").get_text(
strip=True))
leechers = try_int(
result.find_all('td')[3].get_text(strip=True))
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.log(
"Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})"
.format(title, seeders,
leechers), logger.DEBUG)
continue
torrent_size = result.find_all('td')[1].get_text(
strip=True)
units = ['o', 'Ko', 'Mo', 'Go', 'To', 'Po']
size = convert_size(torrent_size,
units=units) or -1
item = {
'title': title,
'link': download_url,
'size': size,
'seeders': seeders,
'leechers': leechers,
'hash': ''
}
if mode != 'RSS':
logger.log(
"Found result: {0} with {1} seeders and {2} leechers"
.format(title, seeders,
leechers), logger.DEBUG)
items.append(item)
except StandardError:
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)),
reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches, too-many-statements
results = []
if not self.login():
return results
search_params = {
"Episode": {"c33": 1, "c38": 1, "c32": 1, "c37": 1},
"Season": {"c41": 1}
}
for mode in search_strings:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode != "RSS":
logger.log("Search string: {0}".format
(search_string.decode("utf-8")), logger.DEBUG)
search_params[mode]["search"] = search_string
data = self.get_url(self.urls["search"], params=search_params[mode], returns="text")
if not data:
logger.log("No data returned from provider", logger.DEBUG)
continue
with BS4Parser(data, "html5lib") as html:
torrent_table = html.find("table", border="1")
torrent_rows = torrent_table("tr") if torrent_table else []
# Continue only if at least one Release is found
if len(torrent_rows) < 2:
logger.log("Data returned from provider does not contain any torrents", logger.DEBUG)
continue
# "Type", "Name", Files", "Comm.", "Added", "TTL", "Size", "Snatched", "Seeders", "Leechers"
labels = [label.get_text(strip=True) for label in torrent_rows[0]("td")]
for result in torrent_rows[1:]:
try:
cells = result("td")
download_url = urljoin(self.url, cells[labels.index("Name")].find("a", href=re.compile(r"download.php\?id="))["href"])
title_element = cells[labels.index("Name")].find("a", href=re.compile(r"details.php\?id="))
title = title_element.get("title", "") or title_element.get_text(strip=True)
if not all([title, download_url]):
continue
if self.freeleech:
# Free leech torrents are marked with green [F L] in the title (i.e. <font color=green>[F L]</font>)
freeleech = cells[labels.index("Name")].find("font", color="green")
if not freeleech or freeleech.get_text(strip=True) != "[F\xa0L]":
continue
seeders = try_int(cells[labels.index("Seeders")].get_text(strip=True))
leechers = try_int(cells[labels.index("Leechers")].get_text(strip=True))
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != "RSS":
logger.log("Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})".format
(title, seeders, leechers), logger.DEBUG)
continue
# Need size for failed downloads handling
torrent_size = cells[labels.index("Size")].get_text(strip=True)
size = convert_size(torrent_size) or -1
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''}
if mode != "RSS":
logger.log("Found result: {0} with {1} seeders and {2} leechers".format
(title, seeders, leechers), logger.DEBUG)
items.append(item)
except (AttributeError, TypeError, ValueError):
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-branches, too-many-locals, too-many-statements
results = []
"""
sorting
ss: relevance
ed: seeds desc
ea: seeds asc
pd: peers desc
pa: peers asc
sd: big > small
sa: small > big
ad: added desc (latest)
aa: added asc (oldest)
"""
for mode in search_strings:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode != "RSS":
logger.log(
"Search string: {0}".format(
search_string.decode("utf-8")), logger.DEBUG)
search_url = self.urls["search"].format(
sorting=("ed", "ad")[mode == "RSS"],
page=1,
search_string=search_string)
if self.custom_url:
if not validators.url(self.custom_url):
logger.log(
"Invalid custom url: {0}".format(self.custom_url),
logger.WARNING)
return results
search_url = urljoin(self.custom_url,
search_url.split(self.url)[1])
data = self.cache.get_rss_feed(search_url)['entries']
if not data:
logger.log(
'Data returned from provider does not contain any torrents',
logger.DEBUG)
continue
for item in data:
try:
title = item.title
download_url = item.link
if not (title and download_url):
continue
info = self.regex.search(item.description)
if not info:
continue
seeders = try_int(info.group("seeders"))
leechers = try_int(info.group("leechers"))
if seeders < self.minseed or leechers < self.minleech:
if mode != "RSS":
logger.log(
"Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})"
.format(title, seeders,
leechers), logger.DEBUG)
continue
category = item.category
if category != 'all':
logger.log(
'skytorrents.in has added categories! Please report this so it can be updated: Category={cat}, '
'Title={title}'.format(cat=category,
title=title),
logger.ERROR)
size = convert_size(info.group('size')) or -1
info_hash = item.guid.rsplit('/', 2)[1]
item = {
'title': title,
'link': download_url,
'size': size,
'seeders': seeders,
'leechers': leechers,
'hash': info_hash
}
if mode != "RSS":
logger.log(
"Found result: {0} with {1} seeders and {2} leechers"
.format(title, seeders,
leechers), logger.DEBUG)
items.append(item)
except (AttributeError, TypeError, KeyError, ValueError):
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)),
reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches
results = []
if not self._check_auth:
return results
search_params = {
'tv': 'true',
'username': self.username,
'apikey': self.api_key
}
for mode in search_strings:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if self.freeleech:
search_params['fl'] = 'true'
else:
search_params.pop('fl', '')
if mode != 'RSS':
logger.log("Search string: " + search_string.strip(), logger.DEBUG)
search_params['search'] = search_string
else:
search_params.pop('search', '')
try:
jdata = self.get_url(self.urls['search'], params=search_params, returns='json')
except ValueError:
logger.log("No data returned from provider", logger.DEBUG)
continue
if not jdata:
logger.log("No data returned from provider", logger.DEBUG)
continue
error = jdata.get('error')
if error:
logger.log("{}".format(error), logger.DEBUG)
return results
try:
if jdata['0']['total_results'] == 0:
logger.log("Provider has no results for this search", logger.DEBUG)
continue
except StandardError:
continue
for i in jdata:
try:
title = jdata[i]["release_name"]
download_url = jdata[i]["download_url"]
if not all([title, download_url]):
continue
seeders = jdata[i]["seeders"]
leechers = jdata[i]["leechers"]
if seeders < self.minseed or leechers < self.minleech:
if mode != 'RSS':
logger.log("Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})".format
(title, seeders, leechers), logger.DEBUG)
continue
torrent_size = str(jdata[i]["size"]) + ' MB'
size = convert_size(torrent_size) or -1
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''}
if mode != 'RSS':
logger.log("Found result: {0} with {1} seeders and {2} leechers".format(title, seeders, leechers), logger.DEBUG)
items.append(item)
except StandardError:
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None):
results = []
if self.show and not self.show.is_anime:
return results
for mode in search_strings:
items = []
logger.debug(_("Search Mode: {mode}".format(mode=mode)))
for search_string in {*search_strings[mode]}:
if mode != "RSS":
logger.debug(
_("Search String: {search_string}".format(
search_string=search_string)))
search_params = {
"terms": search_string,
"type": 1, # get anime types
}
data = self.get_url(self.urls["search"],
params=search_params,
returns="text")
if not data:
continue
with BS4Parser(data, "html5lib") as soup:
torrent_table = soup.find("table", class_="listing")
torrent_rows = torrent_table("tr") if torrent_table else []
# Continue only if one Release is found
if len(torrent_rows) < 2:
logger.debug(
"Data returned from provider does not contain any torrents"
)
continue
a = 1 if len(torrent_rows[0]("td")) < 2 else 0
for top, bot in zip(torrent_rows[a::2],
torrent_rows[a + 1::2]):
try:
desc_top = top.find("td", class_="desc-top")
title = desc_top.get_text(strip=True)
download_url = desc_top.find("a")["href"]
desc_bottom = bot.find(
"td", class_="desc-bot").get_text(strip=True)
size = convert_size(
desc_bottom.split("|")[1].strip(
"Size: ")) or -1
stats = bot.find(
"td", class_="stats").get_text(strip=True)
sl = re.match(
r"S:(?P<seeders>\d+)L:(?P<leechers>\d+)C:(?:\d+)ID:(?:\d+)",
stats.replace(" ", ""))
seeders = try_int(sl.group("seeders")) if sl else 0
leechers = try_int(
sl.group("leechers")) if sl else 0
except Exception:
continue
if not all([title, download_url]):
continue
# Filter unseeded torrent
if seeders < self.minseed or leechers < self.minleech:
if mode != "RSS":
logger.debug(
"Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})"
.format(title, seeders, leechers))
continue
item = {
"title": title,
"link": download_url,
"size": size,
"seeders": seeders,
"leechers": leechers,
"hash": ""
}
if mode != "RSS":
logger.debug(
"Found result: {0} with {1} seeders and {2} leechers"
.format(title, seeders, leechers))
items.append(item)
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get("seeders", 0)),
reverse=True)
results += items
return results
def test_convert_size(self):
# converts pretty file sizes to integers
self.assertEqual(convert_size('1 B'), 1)
self.assertEqual(convert_size('1 KB'), 1024)
self.assertEqual(convert_size('1 kb', use_decimal=True), 1000) # can use decimal units (e.g. KB = 1000 bytes instead of 1024)
# returns integer sizes for integers
self.assertEqual(convert_size(0, -1), 0)
self.assertEqual(convert_size(100, -1), 100)
self.assertEqual(convert_size(1.312, -1), 1) # returns integer sizes for floats too
# without a default value, failures return None
self.assertEqual(convert_size('pancakes'), None)
# default value can be anything
self.assertEqual(convert_size(None, -1), -1)
self.assertEqual(convert_size('', 3.14), 3.14)
self.assertEqual(convert_size('elephant', 'frog'), 'frog')
# negative sizes return 0
self.assertEqual(convert_size(-1024, -1), 0)
self.assertEqual(convert_size('-1 GB', -1), 0)
# can also use `or` for a default value
self.assertEqual(convert_size(None) or 100, 100)
self.assertEqual(convert_size(None) or 1.61803, 1.61803) # default doesn't have to be integer
self.assertEqual(convert_size(None) or '100', '100') # default doesn't have to be numeric either
self.assertEqual(convert_size('-1 GB') or -1, -1) # can use `or` to provide a default when size evaluates to 0
# default units can be kwarg'd
self.assertEqual(convert_size('1', default_units='GB'), convert_size('1 GB'))
# separator can be kwarg'd
self.assertEqual(convert_size('1?GB', sep='?'), convert_size('1 GB'))
# can use custom dictionary to support internationalization
french = ['O', 'KO', 'MO', 'GO', 'TO', 'PO']
self.assertEqual(convert_size('1 o', units=french), 1)
self.assertEqual(convert_size('1 go', use_decimal=True, units=french), 1000000000)
self.assertEqual(convert_size('1 o'), None) # Wrong units so result is None
# custom units need to be uppercase or they won't match
oops = ['b', 'kb', 'Mb', 'Gb', 'tB', 'Pb']
self.assertEqual(convert_size('1 b', units=oops), None)
self.assertEqual(convert_size('1 B', units=oops), None)
self.assertEqual(convert_size('1 Mb', units=oops), None)
self.assertEqual(convert_size('1 MB', units=oops), None)
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-arguments, too-many-locals, too-many-branches, too-many-statements
"""
Searches indexer using the params in search_strings, either for latest releases, or a string/id search
Returns: list of results in dict form
"""
results = []
if not self._check_auth():
return results
if 'gingadaddy' not in self.url: # gingadaddy has no caps.
if not self.caps:
self.get_newznab_categories(just_caps=True)
if not self.caps:
return results
for mode in search_strings:
search_params = {
't': ('search', 'tvsearch')[bool(self.use_tv_search)],
'limit': 100,
'offset': 0,
'cat': self.catIDs.strip(', ') or '5030,5040',
'maxage': sickbeard.USENET_RETENTION
}
if self.needs_auth and self.key:
search_params['apikey'] = self.key
if mode != 'RSS':
if self.use_tv_search:
if 'tvdbid' in str(self.cap_tv_search):
search_params['tvdbid'] = ep_obj.show.indexerid
if ep_obj.show.air_by_date or ep_obj.show.sports:
date_str = str(ep_obj.airdate)
search_params['season'] = date_str.partition('-')[0]
search_params['ep'] = date_str.partition('-')[2].replace('-', '/')
elif ep_obj.show.is_anime:
search_params['ep'] = ep_obj.absolute_number
else:
search_params['season'] = ep_obj.scene_season
search_params['ep'] = ep_obj.scene_episode
if mode == 'Season':
search_params.pop('ep', '')
if self.torznab:
search_params.pop('ep', '')
search_params.pop('season', '')
items = []
logger.log('Search Mode: {0}'.format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode != 'RSS':
logger.log('Search string: {0}'.format
(search_string.decode('utf-8')), logger.DEBUG)
if 'tvdbid' not in search_params:
search_params['q'] = search_string
time.sleep(cpu_presets[sickbeard.CPU_PRESET])
data = self.get_url(urljoin(self.url, 'api'), params=search_params, returns='text')
if not data:
break
with BS4Parser(data, 'html5lib') as html:
if not self._check_auth_from_data(html):
break
# try:
# self.torznab = 'xmlns:torznab' in html.rss.attrs
# except AttributeError:
# self.torznab = False
for item in html('item'):
try:
title = item.title.get_text(strip=True)
download_url = None
if item.link:
if validators.url(item.link.get_text(strip=True)):
download_url = item.link.get_text(strip=True)
elif validators.url(item.link.next.strip()):
download_url = item.link.next.strip()
if (not download_url, item.enclosure and
validators.url(item.enclosure.get('url', '').strip())):
download_url = item.enclosure.get('url', '').strip()
if not (title and download_url):
continue
seeders = leechers = None
if 'gingadaddy' in self.url:
size_regex = re.search(r'\d*.?\d* [KMGT]B', str(item.description))
item_size = size_regex.group() if size_regex else -1
else:
item_size = item.size.get_text(strip=True) if item.size else -1
for attr in item.find_all(['newznab:attr','torznab:attr']):
item_size = attr['value'] if attr['name'] == 'size' else item_size
seeders = try_int(attr['value']) if attr['name'] == 'seeders' else seeders
leechers = try_int(attr['value']) if attr['name'] == 'peers' else leechers
if not item_size or (self.torznab and (seeders is None or leechers is None)):
continue
size = convert_size(item_size) or -1
result = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers}
items.append(result)
except StandardError:
continue
# Since we aren't using the search string,
# break out of the search string loop
if 'tvdbid' in search_params:
break
if self.torznab:
results.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-branches, too-many-locals, too-many-statements
results = []
"""
sorting
ss: relevance
ed: seeds desc
ea: seeds asc
pd: peers desc
pa: peers asc
sd: big > small
sa: small > big
ad: added desc (latest)
aa: added asc (oldest)
"""
for mode in search_strings:
items = []
logger.log("Search Mode: {0}".format(mode), logger.DEBUG)
for search_string in search_strings[mode]:
if mode != "RSS":
logger.log("Search string: {0}".format
(search_string.decode("utf-8")), logger.DEBUG)
search_url = self.urls["search"].format(sorting=("ed", "ad")[mode == "RSS"], page=1, search_string=search_string)
if self.custom_url:
if not validators.url(self.custom_url):
logger.log("Invalid custom url: {0}".format(self.custom_url), logger.WARNING)
return results
search_url = urljoin(self.custom_url, search_url.split(self.url)[1])
data = self.cache.get_rss_feed(search_url)['entries']
if not data:
logger.log('Data returned from provider does not contain any torrents', logger.DEBUG)
continue
for item in data:
try:
title = item.title
download_url = item.link
if not (title and download_url):
continue
info = self.regex.search(item.description)
if not info:
continue
seeders = try_int(info.group("seeders"))
leechers = try_int(info.group("leechers"))
if seeders < self.minseed or leechers < self.minleech:
if mode != "RSS":
logger.log("Discarding torrent because it doesn't meet the minimum seeders or leechers: {0} (S:{1} L:{2})".format
(title, seeders, leechers), logger.DEBUG)
continue
category = item.category
if category != 'all':
logger.log('skytorrents.in has added categories! Please report this so it can be updated: Category={cat}, '
'Title={title}'.format(cat=category, title=title), logger.ERROR)
size = convert_size(info.group('size')) or -1
info_hash = item.guid.rsplit('/', 2)[1]
item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': info_hash}
if mode != "RSS":
logger.log("Found result: {0} with {1} seeders and {2} leechers".format(title, seeders, leechers), logger.DEBUG)
items.append(item)
except (AttributeError, TypeError, KeyError, ValueError):
continue
# For each search mode sort all the items by seeders if available
items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True)
results += items
return results
