def url_rewrite(self, task, entry):
url = entry['url']
page = None
for (scheme, netloc) in EZTV_MIRRORS:
try:
_, _, path, params, query, fragment = urlparse(url)
url = urlunparse((scheme, netloc, path, params, query, fragment))
page = task.requests.get(url).content
except RequestException as e:
log.debug('Eztv mirror `%s` seems to be down', url)
continue
break
if not page:
raise UrlRewritingError('No mirrors found for url %s' % entry['url'])
log.debug('Eztv mirror `%s` chosen', url)
try:
soup = get_soup(page)
mirrors = soup.find_all('a', attrs={'class': re.compile(r'download_\d')})
except Exception as e:
raise UrlRewritingError(e)
log.debug('%d torrent mirrors found', len(mirrors))
if not mirrors:
raise UrlRewritingError('Unable to locate download link from url %s' % url)
entry['urls'] = [m.get('href') for m in mirrors]
entry['url'] = mirrors[0].get('href')
def url_rewrite(self, task, entry):
soup = self._get_soup(task, entry['url'])
link_re = re.compile('rarefile\.net.*\.rar$')
# grab links from the main entry:
blog_entry = soup.find('div', class_="entry")
num_links = 0
link_list = None
for paragraph in blog_entry.find_all('p'):
links = paragraph.find_all('a', href=link_re)
if len(links) > num_links:
link_list = links
num_links = len(links)
if 'urls' in entry:
urls = list(entry['urls'])
else:
urls = []
if link_list is not None:
for link in link_list:
urls.append(normalize_unicode(link['href']))
else:
raise UrlRewritingError('No useable links found at %s' %
entry['url'])
num_links = len(urls)
log.verbose('Found %d links at %s.', num_links, entry['url'])
if num_links:
entry['urls'] = urls
entry['url'] = urls[0]
else:
raise UrlRewritingError('No useable links found at %s' %
entry['url'])
def parse_downloads(self, series_url, search_title):
page = requests.get(series_url).content
try:
soup = get_soup(page)
except Exception as e:
raise UrlRewritingError(e)
urls = []
# find all titles
episode_titles = self.find_all_titles(search_title)
if not episode_titles:
raise UrlRewritingError('Unable to find episode')
for ep_title in episode_titles:
# find matching download
episode_title = soup.find('strong',
text=re.compile(ep_title, re.I))
if not episode_title:
continue
# find download container
episode = episode_title.parent
if not episode:
continue
# find episode language
episode_lang = episode.find_previous(
'strong', text=re.compile('Sprache')).next_sibling
if not episode_lang:
log.warning('No language found for: %s', series_url)
continue
# filter language
if not self.check_language(episode_lang):
log.warning('languages not matching: %s <> %s',
self.config['language'], episode_lang)
continue
# find download links
links = episode.find_all('a')
if not links:
log.warning('No links found for: %s', series_url)
continue
for link in links:
if not link.has_attr('href'):
continue
url = link['href']
pattern = (
r'http:\/\/download\.serienjunkies\.org.*%s_.*\.html' %
self.config['hoster'])
if re.match(pattern, url) or self.config['hoster'] == 'all':
urls.append(url)
else:
continue
return urls
def _get_soup(self, task, url):
try:
page = task.requests.get(url)
except RequestException as e:
raise UrlRewritingError(str(e))
try:
return get_soup(page.text)
except Exception as e:
raise UrlRewritingError(str(e))
def parse_download_page(self, url, requests):
txheaders = {'User-agent': 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'}
page = requests.get(url, headers=txheaders)
try:
soup = get_soup(page.text)
except Exception as e:
raise UrlRewritingError(e)
down_link = soup.find('a', attrs={'href': re.compile(r"download/\d+/.*\.torrent")})
if not down_link:
raise UrlRewritingError('Unable to locate download link from url %s' % url)
return 'http://www.deadfrog.us/' + down_link.get('href')
def parse_download_page(self, url, requests):
txheaders = {
'User-agent': 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
}
page = requests.get(url, headers=txheaders)
try:
soup = get_soup(page.text)
except Exception as e:
raise UrlRewritingError(e)
tag_a = soup.select_one('a[href^="magnet:"]')
if not tag_a:
raise UrlRewritingError(
f"Unable to locate download link from url {url}")
return tag_a.get('href')
def parse_download_page(self, url, requests):
page = requests.get(url).content
try:
soup = get_soup(page)
tag_div = soup.find('div', attrs={'class': 'download'})
if not tag_div:
raise UrlRewritingError('Unable to locate download link from url %s' % url)
tag_a = tag_div.find('a')
torrent_url = tag_a.get('href')
# URL is sometimes missing the schema
if torrent_url.startswith('//'):
torrent_url = urlparse(url).scheme + ':' + torrent_url
return torrent_url
except Exception as e:
raise UrlRewritingError(e)
def parse_download_page(self, url, requests):
txheaders = {
'User-agent': 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
}
page = requests.get(url, headers=txheaders)
try:
soup = get_soup(page.text)
except Exception as e:
raise UrlRewritingError(e)
tag_a = soup.find('a', attrs={'class': 'download_link'})
if not tag_a:
raise UrlRewritingError(
'Unable to locate download link from url %s' % url)
torrent_url = 'https://bakabt.me/' + tag_a.get('href')
return torrent_url
def url_rewrite(self, task, entry):
"""
Gets the download information for 1337x result
"""
url = entry['url']
logger.info('1337x rewriting download url: {}', url)
try:
page = task.requests.get(url)
logger.debug('requesting: {}', page.url)
except RequestException as e:
logger.error('1337x request failed: {}', e)
raise UrlRewritingError('1337x request failed: %s' % e)
soup = get_soup(page.content)
magnet_url = str(
soup.find('a',
href=re.compile(r'^magnet:\?')).get('href')).lower()
torrent_url = str(
soup.find('a',
href=re.compile(r'\.torrent$')).get('href')).lower()
entry['url'] = torrent_url
entry.setdefault('urls', []).append(torrent_url)
entry['urls'].append(magnet_url)
def url_from_page(self, url):
"""Parses torrent url from newtorrents download page"""
try:
page = requests.get(url)
data = page.text
except requests.RequestException:
raise UrlRewritingError('URLerror when retrieving page')
p = re.compile(r"copy\(\'(.*)\'\)", re.IGNORECASE)
f = p.search(data)
if not f:
# the link in which plugin relies is missing!
raise UrlRewritingError(
'Failed to get url from download page. Plugin may need a update.'
)
else:
return f.group(1)
def url_rewrite(self, task, entry):
"""
Gets torrent information for topic from rutracker api
"""
url = entry['url']
logger.info('rewriting download url: {}', url)
topic_id = parse_qs(urlparse(url).query)['t'][0]
api_url = f"{self.base_url}/v1/get_tor_topic_data"
api_params = {
'by': 'topic_id',
'val': topic_id,
}
try:
topic_request = task.requests.get(api_url, params=api_params)
except RequestException as e:
raise UrlRewritingError(f'rutracker request failed: {e}')
topic = topic_request.json()['result'][topic_id]
magnet = {
'xt': f"urn:btih:{topic['info_hash']}",
'tr': [f'http://bt{i}.t-ru.org/ann?magnet' for i in ['', '2', '3', '4']],
'dn': topic['topic_title']
}
magnet_qs = urlencode(magnet, doseq=True, safe=':')
magnet_uri = f"magnet:?{magnet_qs}"
entry['url'] = magnet_uri
def url_rewrite(self, task, entry):
logger.debug('Requesting {}', entry['url'])
page = requests.get(entry['url'])
soup = get_soup(page.text)
for link in soup.findAll('a', attrs={'href': re.compile(r'^/url')}):
# Extract correct url from google internal link
href = 'http://google.com' + link['href']
args = parse_qs(urlparse(href).query)
href = args['q'][0]
# import IPython; IPython.embed()
# import sys
# sys.exit(1)
# href = link['href'].lstrip('/url?q=').split('&')[0]
# Test if entry with this url would be recognized by some urlrewriter
logger.trace('Checking if {} is known by some rewriter', href)
fake_entry = {'title': entry['title'], 'url': href}
urlrewriting = plugin.get('urlrewriting', self)
if urlrewriting.url_rewritable(task, fake_entry):
logger.debug('--> rewriting {} (known url pattern)', href)
entry['url'] = href
return
else:
logger.debug('<-- ignoring {} (unknown url pattern)', href)
raise UrlRewritingError('Unable to resolve')
def url_rewrite(self, task, entry):
url = entry['url']
if url.startswith('http://www.newtorrents.info/?q=') or url.startswith(
'http://www.newtorrents.info/search'):
results = self.entries_from_search(entry['title'], url=url)
if not results:
raise UrlRewritingError("No matches for %s" % entry['title'])
url = results[0]['url']
else:
url = self.url_from_page(url)
if url:
entry['url'] = url
self.resolved.append(url)
else:
raise UrlRewritingError('Bug in newtorrents urlrewriter')
def url_rewrite(self, task, entry):
try:
page = task.requests.get(entry['url'])
except RequestException as e:
raise UrlRewritingError(str(e))
try:
soup = get_soup(page.text)
except Exception as e:
raise UrlRewritingError(str(e))
link_elements = soup.find_all('pre', class_='links')
if 'urls' in entry:
urls = list(entry['urls'])
else:
urls = []
for element in link_elements:
urls.extend(element.text.splitlines())
regexps = self.config.get('filehosters_re', [])
filtered_urls = []
for i, url in enumerate(urls):
urls[i] = normalize_unicode(url)
for regexp in regexps:
if re.search(regexp, urls[i]):
filtered_urls.append(urls[i])
log.debug('Url: "%s" matched filehoster filter: %s',
urls[i], regexp)
break
else:
if regexps:
log.debug(
'Url: "%s" does not match any of the given filehoster filters: %s',
urls[i],
str(regexps),
)
if regexps:
log.debug('Using filehosters_re filters: %s', str(regexps))
urls = filtered_urls
else:
log.debug(
'No filehoster filters configured, using all found links.')
num_links = len(urls)
log.verbose('Found %d links at %s.', num_links, entry['url'])
if num_links:
entry['urls'] = urls
entry['url'] = urls[0]
else:
raise UrlRewritingError('No useable links found at %s' %
entry['url'])
def parse_download_page(self, url, requests):
txheaders = {'User-agent': 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'}
try:
page = requests.get(url, headers=txheaders)
except requests.exceptions.RequestException as e:
msg = 'Cannot open "%s" : %s' % (url, str(e))
logger.error(msg)
raise UrlRewritingError(msg)
try:
soup = get_soup(page.text)
except Exception as e:
raise UrlRewritingError(str(e))
down_link = soup.find('a', attrs={'href': re.compile(r"down\.php\?.*")})
if not down_link:
raise UrlRewritingError('Unable to locate download link from url "%s"' % url)
return 'http://bt.hliang.com/' + down_link.get('href')
def url_rewrite(self, task, entry):
if 'url' not in entry:
log.error('Didn\'t actually get a URL...')
else:
log.debug('Got the URL: %s', entry['url'])
if entry['url'].startswith('https://www.torrentday.com/browse'):
# use search
results = self.search(task, entry)
if not results:
raise UrlRewritingError('No search results found')
entry['url'] = results[0]['url']
def parse_download_page(self, page_url, requests):
page = requests.get(page_url)
try:
soup = get_soup(page.text)
except Exception as e:
raise UrlRewritingError(e)
tag_a = soup.find("a", {"class": "dl_link"})
if not tag_a:
if soup.findAll(text="Connexion ?"):
raise UrlRewritingError('You are not logged in,\
check if your cookie for\
authentication is up to date')
else:
raise UrlRewritingError('You have reached your download\
limit per 24hours, so I cannot\
get the torrent')
torrent_url = "http://www.frenchtorrentdb.com" + tag_a.get(
'href') + "&js=1"
logger.debug('TORRENT URL is : {}', torrent_url)
return torrent_url
def url_rewrite(self, task, entry):
try:
# need to fake user agent
txheaders = {'User-agent': 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'}
page = task.requests.get(entry['url'], headers=txheaders)
soup = get_soup(page.text)
results = soup.find_all('a', attrs={'class': 'l'})
if not results:
raise UrlRewritingError('No results')
for res in results:
url = res.get('href')
url = url.replace('/interstitial?url=', '')
# generate match regexp from google search result title
regexp = '.*'.join([x.contents[0] for x in res.find_all('em')])
if re.match(regexp, entry['title']):
logger.debug('resolved, found with {}', regexp)
entry['url'] = url
return
raise UrlRewritingError('Unable to resolve')
except Exception as e:
raise UrlRewritingError(e)
def url_rewrite(self, task, entry):
if 'url' not in entry:
logger.error("Didn't actually get a URL...")
else:
logger.debug('Got the URL: {}', entry['url'])
if entry['url'].startswith('https://www.torrentleech.org/torrents/browse/list/query/'):
# use search
results = self.search(task, entry)
if not results:
raise UrlRewritingError("No search results found")
# TODO: Search doesn't enforce close match to title, be more picky
entry['url'] = results[0]['url']
def url_rewrite(self, task, entry):
if 'url' not in entry:
logger.error("Didn't actually get a URL...")
else:
logger.debug('Got the URL: {}', entry['url'])
if entry['url'].startswith(SEARCH_URL):
# use search
results = self.search(task, entry)
if not results:
raise UrlRewritingError("No search results found")
# TODO: Search doesn't enforce close match to title, be more picky
entry['url'] = results[0]['url']
def url_rewrite(self, task, entry):
if 'url' not in entry:
logger.error("Didn't actually get a URL...")
else:
logger.debug('Got the URL: {}', entry['url'])
if self.url_search.match(entry['url']):
# use search
results = self.search(task, entry)
if not results:
raise UrlRewritingError("No search results found")
# TODO: Close matching was taken out of search methods, this may need to be fixed to be more picky
entry['url'] = results[0]['url']
else:
torrent_id = self.url_match(entry['url']).group(1)
url = f"{self.url}/api/v1/torrents/{torrent_id}"
logger.debug('Getting info for torrent ID {}', torrent_id)
json_result = task.requests.get(url).json()
# if json_result['error'] == '404':
if not 'result' in json_result:
raise UrlRewritingError("Torrent with ID %s does not exist." %
torrent_id)
entry['url'] = json_result['result']['rating']['magnet']
def url_rewrite(self, task, entry):
if 'url' not in entry:
logger.error("Didn't actually get a URL...")
else:
logger.debug('Got the URL: {}', entry['url'])
if self.url_search.match(entry['url']):
# use search
results = self.search(task, entry)
if not results:
raise UrlRewritingError("No search results found")
# TODO: Close matching was taken out of search methods, this may need to be fixed to be more picky
entry['url'] = results[0]['url']
else:
torrent_id = self.url_match.match(entry['url']).group(1)
url = f"{self.url}/t.php?id={torrent_id}"
logger.debug('Getting info for torrent ID {}', torrent_id)
json_result = task.requests.get(url).json()
if json_result['id'] == '0':
raise UrlRewritingError(
f"Torrent with ID {torrent_id} does not exist.")
entry['url'] = self.info_hash_to_magnet(json_result['info_hash'],
json_result['name'])
def parse_download_page(self, url, task):
log.verbose('Descargas2020 URL: %s', url)
try:
page = self.requests.get(url)
except requests.RequestException as e:
raise UrlRewritingError(e)
try:
soup = get_soup(page.text)
except Exception as e:
raise UrlRewritingError(e)
torrent_id = None
url_format = DESCARGAS2020_TORRENT_FORMAT
torrent_id_prog = re.compile(
r"(?:parametros\s*=\s*\n?)\s*{\s*\n(?:\s*'\w+'\s*:.*\n)+\s*'(?:torrentID|id)"
"'\s*:\s*'(\d+)'")
torrent_ids = soup.findAll(text=torrent_id_prog)
if torrent_ids:
match = torrent_id_prog.search(torrent_ids[0])
if match:
torrent_id = match.group(1)
if not torrent_id:
log.debug('torrent ID not found, searching openTorrent script')
match = re.search(
r'function openTorrent.*\n.*\{.*(\n.*)+window\.location\.href =\s*\".*\/(\d+.*)\";',
page.text,
re.MULTILINE,
)
if match:
torrent_id = match.group(2).rstrip('/')
if not torrent_id:
raise UrlRewritingError('Unable to locate torrent ID from url %s' %
url)
return url_format.format(torrent_id)
def url_rewrite(self, task, entry):
if 'url' not in entry:
log.error("Didn't actually get a URL...")
else:
log.debug("Got the URL: %s" % entry['url'])
if self.url_search.match(entry['url']):
# use search
results = self.search(task, entry)
if not results:
raise UrlRewritingError("No search results found")
# TODO: Close matching was taken out of search methods, this may need to be fixed to be more picky
entry['url'] = results[0]['url']
else:
# parse download page
entry['url'] = self.parse_download_page(entry['url'], task.requests)
def url_rewrite(self, task, entry):
for name, config in self.resolves.get(task.name, {}).items():
regexp = config['regexp_compiled']
format = config['format']
if regexp.search(entry['url']):
log.debug('Regexp resolving %s with %s' % (entry['url'], name))
# run the regexp
entry['url'] = regexp.sub(format, entry['url'])
if regexp.match(entry['url']):
entry.fail('urlrewriting')
raise UrlRewritingError(
'Regexp %s result should NOT continue to match!' % name
)
return
def url_rewrite(self, task, entry):
soup = self._get_soup(task, entry['url'])
# grab links from the main post:
link_elements = []
log.debug(
'Searching %s for a tags where the text matches one of: %s',
entry['url'],
str(self.config.get('link_text_re')),
)
for regexp in self.config.get('link_text_re'):
link_elements.extend(soup.find_all('a', string=re.compile(regexp)))
log.debug('Original urls: %s', str(entry['urls']))
if 'urls' in entry:
urls = list(entry['urls'])
log.debug('Original urls: %s', str(entry['urls']))
else:
urls = []
log.debug('Found link elements: %s', str(link_elements))
for element in link_elements:
if re.search('nfo1.rlsbb.(ru|com)', element['href']):
# grab multipart links
urls.extend(self.grab_multilinks(task, element['href']))
else:
urls.append(element['href'])
# grab links from comments
regexps = self.config.get('filehosters_re', [])
if self.config.get('parse_comments'):
comments = soup.find_all('div', id=re.compile("commentbody"))
log.debug('Comment parsing enabled: found %d comments.',
len(comments))
if comments and not regexps:
log.warn(
'You have enabled comment parsing but you did not define any filehoster_re filter. You may get a lot of unwanted and potentially dangerous links from the comments.'
)
for comment in comments:
links = comment.find_all('a')
for link in links:
urls.append(link['href'])
# filter urls:
filtered_urls = []
for i, url in enumerate(urls):
urls[i] = normalize_unicode(url)
for regexp in regexps:
if re.search(regexp, urls[i]):
filtered_urls.append(urls[i])
log.debug('Url: "%s" matched filehoster filter: %s',
urls[i], regexp)
break
else:
if regexps:
log.debug(
'Url: "%s" was discarded because it does not match any of the given filehoster filters: %s',
urls[i],
str(regexps),
)
if regexps:
log.debug('Using filehosters_re filters: %s', str(regexps))
urls = filtered_urls
else:
log.debug(
'No filehoster filters configured, using all found links.')
num_links = len(urls)
log.verbose('Found %d links at %s.', num_links, entry['url'])
if num_links:
entry['urls'] = urls
entry['url'] = urls[0]
else:
raise UrlRewritingError('No useable links found at %s' %
entry['url'])