def search(self, query, comparator=StringComparator(), config=None):
"""
Search for name from piratebay.
"""
if not isinstance(config, dict):
config = {}
sort = SORT.get(config.get('sort_by', 'seeds'))
if config.get('sort_reverse'):
sort += 1
if isinstance(config.get('category'), int):
category = config['category']
else:
category = CATEGORIES.get(config.get('category', 'all'))
filter_url = '/0/%d/%d' % (sort, category)
comparator.set_seq1(query)
query = comparator.search_string()
# urllib.quote will crash if the unicode string has non ascii characters, so encode in utf-8 beforehand
url = 'http://thepiratebay.se/search/' + urllib.quote(query.encode('utf-8')) + filter_url
log.debug('Using %s as piratebay search url' % url)
page = requests.get(url).content
soup = get_soup(page)
entries = []
for link in soup.find_all('a', attrs={'class': 'detLink'}):
comparator.set_seq2(link.contents[0])
log.debug('name: %s' % comparator.a)
log.debug('found name: %s' % comparator.b)
log.debug('confidence: %s' % comparator.ratio())
if not comparator.matches():
continue
entry = Entry()
entry['title'] = link.contents[0]
entry['url'] = 'http://thepiratebay.se' + link.get('href')
tds = link.parent.parent.parent.find_all('td')
entry['torrent_seeds'] = int(tds[-2].contents[0])
entry['torrent_leeches'] = int(tds[-1].contents[0])
entry['search_ratio'] = comparator.ratio()
entry['search_sort'] = torrent_availability(entry['torrent_seeds'], entry['torrent_leeches'])
# Parse content_size
size = link.find_next(attrs={'class': 'detDesc'}).contents[0]
size = re.search('Size ([\.\d]+)\xa0([GMK])iB', size)
if size:
if size.group(2) == 'G':
entry['content_size'] = int(float(size.group(1)) * 1000 ** 3 / 1024 ** 2)
elif size.group(2) == 'M':
entry['content_size'] = int(float(size.group(1)) * 1000 ** 2 / 1024 ** 2)
else:
entry['content_size'] = int(float(size.group(1)) * 1000 / 1024 ** 2)
entries.append(entry)
if not entries:
dashindex = query.rfind('-')
if dashindex != -1:
return self.search(query[:dashindex], comparator=comparator)
else:
raise PluginWarning('No close matches for %s' % query, log, log_once=True)
entries.sort(reverse=True, key=lambda x: x.get('search_sort'))
return entries
def get_first_result(tmdb_function, value):
if isinstance(value, unicode):
value = value.encode('utf-8')
if isinstance(value, basestring):
value = quote(value, safe=b'')
print "TMDB_FUNCT" + tmdb_function
if tmdb_function == 'imdbLookup':
url = '%s/3/movie/%s?&api_key=%s' % (server, value, api_key)
elif tmdb_function == 'search':
url = '%s/3/search/movie?query=%s&api_key=%s&page=1' % (server, value, api_key)
elif tmdb_function == 'getInfo':
url = '%s/3/movie/%s?&api_key=%s' % (server, value, api_key)
print url
else:
print value
try:
data = requests.get(url)
except requests.RequestException:
log.warning('Request failed %s' % url)
return
try:
result = data.json()
except ValueError:
log.warning('TMDb returned invalid json.')
return
# Make sure there is a valid result to return
if len(result) and result.get('id'):
return result
def query_member_id(api_key, user_token, login_name):
"""
Get the member id of a member identified by its login name.
:param string api_key: Api key requested at https://www.betaseries.com/api
:param string user_token: obtained with a call to create_token()
:param string login_name: The login name of the member
:return: Id of the member identified by its login name or `None` if not found
"""
r = requests.get(
API_URL_PREFIX + 'members/search',
params={'login': login_name},
headers={
'Accept': 'application/json',
'X-BetaSeries-Version': '2.1',
'X-BetaSeries-Key': api_key,
'X-BetaSeries-Token': user_token,
},
)
assert r.status_code == 200, "Bad HTTP status code: %s" % r.status_code
j = r.json()
error_list = j['errors']
for err in error_list:
log.error(str(err))
found_id = None
if not error_list:
for candidate in j['users']:
if candidate['login'] == login_name:
found_id = candidate['id']
break
return found_id
def on_task_input(self, task, config):
try:
r = requests.post('http://www.pogdesign.co.uk/cat/', data={'username': config['username'], 'password': config['password'], 'sub_login': '******'}, allow_redirects=True)
if 'U / P Invalid' in r.text:
raise plugin.PluginError('Invalid username/password for pogdesign.')
page = requests.get('http://www.pogdesign.co.uk/cat/showselect.php', cookies=r.cookies)
except requests.RequestException as e:
raise plugin.PluginError('Error retrieving source: %s' % e)
soup = BeautifulSoup(page.text)
entries = []
for row in soup.find_all('label', {'class': 'label_check'}):
if row.find(attrs={'checked': 'checked'}):
t = row.text
if t.endswith('[The]'):
t = 'The ' + t[:-6]
# Make certain names friendlier
if t in self.name_map:
t = self.name_map[t]
e = Entry()
e['title'] = t
url = row.find_next('a', {'class': 'selectsummary'})
e['url'] = 'http://www.pogdesign.co.uk' + url['href']
entries.append(e)
return entries
def url_rewrite(self, task, entry):
url = entry['url']
for (scheme, netloc) in EZTV_MIRRORS:
try:
_, _, path, params, query, fragment = urlparse(url)
url = urlunparse((scheme, netloc, path, params, query, fragment))
page = requests.get(url).content
except RequestException as e:
log.debug('Eztv mirror `%s` seems to be down', url)
continue
break
if not page:
raise UrlRewritingError('No mirrors found for url %s' % entry['url'])
log.debug('Eztv mirror `%s` chosen', url)
try:
soup = get_soup(page)
mirrors = soup.find('a', attrs={'class': re.compile(r'download_\d')})
if not mirrors:
raise UrlRewritingError('Unable to locate download link from url %s'
% url)
entry['urls'] = [m.get('href') for m in mirrors]
entry['url'] = mirrors[0].get('href')
except Exception as e:
raise UrlRewritingError(e)
def get_http_seeds(url, info_hash):
url = get_scrape_url(url, info_hash)
if not url:
log.debug('if not url is true returning 0')
return 0
log.debug('Checking for seeds from %s' % url)
try:
data = bdecode(requests.get(url).text.encode('utf-8')).get('files')
except RequestException as e:
log.debug('Error scraping: %s' % e)
return 0
except SyntaxError as e:
log.warning('Error decoding tracker response: %s' % e)
return 0
except BadStatusLine as e:
log.warning('Error BadStatusLine: %s' % e)
return 0
except IOError as e:
log.warning('Server error: %s' % e)
return 0
if not data:
log.debug('No data received from tracker scrape.')
return 0
log.debug('get_http_seeds is returning: %s' % list(data.values())[0]['complete'])
return list(data.values())[0]['complete']
def query_member_id(api_key, user_token, login_name):
"""
Get the member id of a member identified by its login name.
:param string api_key: Api key requested at http://www.betaseries.com/api
:param string user_token: obtained with a call to create_token()
:param string login_name: The login name of the member
:return: Id of the member identified by its login name or `None` if not found
"""
r = requests.get(
API_URL_PREFIX + "members/search",
params={"login": login_name},
headers={
"Accept": "application/json",
"X-BetaSeries-Version": "2.1",
"X-BetaSeries-Key": api_key,
"X-BetaSeries-Token": user_token,
},
)
assert r.status_code == 200, "Bad HTTP status code: %s" % r.status_code
j = r.json()
error_list = j["errors"]
for err in error_list:
log.error(str(err))
found_id = None
if not error_list:
for candidate in j["users"]:
if candidate["login"] == login_name:
found_id = candidate["id"]
break
return found_id
def entries_from_search(self, name, url=None):
"""Parses torrent download url from search results"""
name = normalize_unicode(name)
if not url:
url = 'http://www.newtorrents.info/search/%s' % quote(
name.encode('utf-8'), safe=b':/~?=&%'
)
logger.debug('search url: {}', url)
html = requests.get(url).text
# fix </SCR'+'IPT> so that BS does not crash
# TODO: should use beautifulsoup massage
html = re.sub(r'(</SCR.*?)...(.*?IPT>)', r'\1\2', html)
soup = get_soup(html)
# saving torrents in dict
torrents = []
for link in soup.find_all('a', attrs={'href': re.compile('down.php')}):
torrent_url = 'http://www.newtorrents.info%s' % link.get('href')
release_name = link.parent.next.get('title')
# quick dirty hack
seed = link.find_next('td', attrs={'class': re.compile('s')}).renderContents()
if seed == 'n/a':
seed = 0
else:
try:
seed = int(seed)
except ValueError:
logger.warning(
'Error converting seed value ({}) from newtorrents to integer.', seed
)
seed = 0
# TODO: also parse content_size and peers from results
torrents.append(
Entry(
title=release_name,
url=torrent_url,
torrent_seeds=seed,
torrent_availability=torrent_availability(seed, 0),
)
)
# sort with seed number Reverse order
torrents.sort(reverse=True, key=lambda x: x.get('torrent_availability', 0))
# choose the torrent
if not torrents:
dashindex = name.rfind('-')
if dashindex != -1:
return self.entries_from_search(name[:dashindex])
else:
return torrents
else:
if len(torrents) == 1:
logger.debug('found only one matching search result.')
else:
logger.debug(
'search result contains multiple matches, sorted {} by most seeders', torrents
)
return torrents
def search(self, query, comparator=StringComparator(), config=None):
"""
Search for name from prelist.
"""
if not isinstance(config, dict):
config = {}
comparator.set_seq1(query)
query = comparator.search_string()
# urllib.quote will crash if the unicode string has non ascii characters, so encode in utf-8 beforehand
url = 'http://www.prelist.ws/?search=' + urllib.quote(query.encode('utf-8'))
log.debug('Using %s as prelist search url' % url)
page = requests.get(url).content
soup = get_soup(page)
entries = []
for link in soup.find_all('a',attrs={'href' : re.compile('search=')}):
log.debug('name: %s' % link)
comparator.set_seq2(link.contents[0])
log.debug('name: %s' % comparator.a)
log.debug('found name: %s' % comparator.b)
log.debug('confidence: %s' % comparator.ratio())
if not comparator.matches():
continue
entry = Entry()
entry['title'] = link.contents[0]
entry['url'] = 'http://www.prelist.ws/' + link.get('href')
entries.append(entry)
return entries
def entries_from_search(self, name, url=None):
"""Parses torrent download url from search results"""
name = normalize_unicode(name)
if not url:
url = 'http://www.newtorrents.info/search/%s' % quote(
name.encode('utf-8'), safe=b':/~?=&%'
)
log.debug('search url: %s' % url)
html = requests.get(url).text
# fix </SCR'+'IPT> so that BS does not crash
# TODO: should use beautifulsoup massage
html = re.sub(r'(</SCR.*?)...(.*?IPT>)', r'\1\2', html)
soup = get_soup(html)
# saving torrents in dict
torrents = []
for link in soup.find_all('a', attrs={'href': re.compile('down.php')}):
torrent_url = 'http://www.newtorrents.info%s' % link.get('href')
release_name = link.parent.next.get('title')
# quick dirty hack
seed = link.find_next('td', attrs={'class': re.compile('s')}).renderContents()
if seed == 'n/a':
seed = 0
else:
try:
seed = int(seed)
except ValueError:
log.warning(
'Error converting seed value (%s) from newtorrents to integer.' % seed
)
seed = 0
# TODO: also parse content_size and peers from results
torrents.append(
Entry(
title=release_name,
url=torrent_url,
torrent_seeds=seed,
torrent_availability=torrent_availability(seed, 0),
)
)
# sort with seed number Reverse order
torrents.sort(reverse=True, key=lambda x: x.get('torrent_availability', 0))
# choose the torrent
if not torrents:
dashindex = name.rfind('-')
if dashindex != -1:
return self.entries_from_search(name[:dashindex])
else:
return torrents
else:
if len(torrents) == 1:
log.debug('found only one matching search result.')
else:
log.debug(
'search result contains multiple matches, sorted %s by most seeders' % torrents
)
return torrents
def search(self, task, entry, config=None):
"""
Search for name from iptorrents
"""
categories = config.get('category', 'All')
# Make sure categories is a list
if not isinstance(categories, list):
categories = [categories]
# If there are any text categories, turn them into their id number
categories = [c if isinstance(c, int) else CATEGORIES[c]
for c in categories]
filter_url = '&'.join((str(c) + '=') for c in categories)
entries = set()
for search_string in entry.get('search_strings', [entry['title']]):
query = normalize_unicode(search_string)
query = quote_plus(query.encode('utf8'))
url = "{base_url}/t?{filter}&q={query}&qf=".format(base_url=BASE_URL, filter=filter_url, query=query)
log.debug('searching with url: %s' % url)
req = requests.get(url, cookies={'uid': str(config['uid']), 'pass': config['password']})
if '/u/' + str(config['uid']) not in req.text:
raise plugin.PluginError("Invalid cookies (user not logged in)...")
soup = get_soup(req.content, parser="html.parser")
torrents = soup.find('table', {'id': 'torrents'})
results = torrents.findAll('tr')
for torrent in results:
if torrent.th and 'ac' in torrent.th.get('class'):
# Header column
continue
if torrent.find('td', {'colspan': '99'}):
log.debug('No results found for search %s', search_string)
return
entry = Entry()
link = torrent.find('a', href=re.compile('download'))['href']
entry['url'] = "{base}{link}?torrent_pass={key}".format(
base=BASE_URL, link=link, key=config.get('rss_key'))
entry['title'] = torrent.find('a', href=re.compile('details')).text
seeders = torrent.findNext('td', {'class': 'ac t_seeders'}).text
leechers = torrent.findNext('td', {'class': 'ac t_leechers'}).text
entry['torrent_seeds'] = int(seeders)
entry['torrent_leeches'] = int(leechers)
entry['search_sort'] = torrent_availability(entry['torrent_seeds'],
entry['torrent_leeches'])
size = torrent.findNext(text=re.compile('^([\.\d]+) ([GMK]?)B$'))
size = re.search('^([\.\d]+) ([GMK]?)B$', size)
entry['content_size'] = parse_filesize(size.group(0))
log.debug('Found entry %s', entry)
entries.add(entry)
return entries
def retrieve_tracker_config(cls, tracker_config_file):
"""
Will attempt to retrieve the .tracker file from disk or github. Returns the parsed XML.
:param tracker_config_file: URL or path to .tracker file
:return: parsed XML
"""
tracker_config_file = os.path.expanduser(tracker_config_file)
if os.path.exists(tracker_config_file):
return cls.read_tracker_config(tracker_config_file)
elif re.match('^(?![\\\/]).+\.tracker$', tracker_config_file):
tracker_basepath = os.path.abspath(os.path.join(manager.config_base, 'trackers'))
save_path = os.path.join(tracker_basepath, tracker_config_file)
if not os.path.exists(tracker_basepath):
try:
os.mkdir(tracker_basepath)
except IOError as e:
raise TrackerFileError(e)
if not os.path.exists(save_path):
log.info('Tracker file not found on disk. Attempting to fetch tracker config file from Github.')
url = 'https://raw.githubusercontent.com/autodl-community/autodl-trackers/master/' + tracker_config_file
try:
r = requests.get(url)
with io.open(save_path, 'wb') as tracker_file:
for chunk in r.iter_content(8192):
tracker_file.write(chunk)
except (requests.RequestException, IOError) as e:
raise TrackerFileError(e)
return cls.read_tracker_config(save_path)
else:
raise TrackerFileError('Unable to open tracker config file %s', tracker_config_file)
def get_file(self, only_cached=False):
"""Makes sure the poster is downloaded to the local cache (in userstatic folder) and
returns the path split into a list of directory and file components"""
from flexget.manager import manager
base_dir = os.path.join(manager.config_base, 'userstatic')
if self.file and os.path.isfile(os.path.join(base_dir, self.file)):
return self.file.split(os.sep)
elif only_cached:
return
# If we don't already have a local copy, download one.
log.debug('Downloading poster %s' % self.url)
dirname = os.path.join('tmdb', 'posters', str(self.movie_id))
# Create folders if they don't exist
fullpath = os.path.join(base_dir, dirname)
if not os.path.isdir(fullpath):
os.makedirs(fullpath)
filename = os.path.join(dirname, posixpath.basename(self.url))
thefile = file(os.path.join(base_dir, filename), 'wb')
thefile.write(requests.get(self.url).content)
self.file = filename
# If we are detached from a session, update the db
if not Session.object_session(self):
session = Session()
try:
poster = session.query(TMDBPoster).filter(TMDBPoster.db_id == self.db_id).first()
if poster:
poster.file = filename
finally:
session.close()
return filename.split(os.sep)
def query_series(api_key, user_token, member_name=None):
"""
Get the list of series followed by the authenticated user
:param string api_key: Api key requested at http://www.betaseries.com/api
:param string user_token: Obtained with a call to create_token()
:param string member_name: [optional] A member name to get the list of series from. If None, will query the member
for whom the user_token was for
:return: List of serie titles or empty list
"""
params = {}
if member_name:
member_id = query_member_id(api_key, user_token, member_name)
if member_id:
params = {'id': member_id}
else:
log.error("member %r not found" % member_name)
return []
r = requests.get(API_URL_PREFIX + 'members/infos', params=params, headers={
'Accept': 'application/json',
'X-BetaSeries-Version': '2.1',
'X-BetaSeries-Key': api_key,
'X-BetaSeries-Token': user_token,
})
assert r.status_code == 200, "Bad HTTP status code: %s" % r.status_code
j = r.json()
error_list = j['errors']
for err in error_list:
log.error(str(err))
if not error_list:
return [x['title'] for x in j['member']['shows'] if x['user']['archived'] is False]
else:
return []
def plex_get_accesstoken(self, config, globalaccesstoken=""):
accesstoken = None
if not globalaccesstoken:
globalaccesstoken = self.plex_get_globalaccesstoken(config)
if config['server'] in ('localhost', '127.0.0.1'):
log.debug('Server using localhost. Global Token will be used')
return globalaccesstoken
try:
r = requests.get(
"https://my.plexapp.com/pms/servers?X-Plex-Token=%s" %
globalaccesstoken)
except requests.RequestException as e:
raise plugin.PluginError(
"Could not get servers from my.plexapp.com using "
"authentication-token: %s. (%s)" % (globalaccesstoken, e))
dom = parseString(r.text)
for node in dom.getElementsByTagName('Server'):
if config['server'] in (node.getAttribute('address'),
node.getAttribute('localAddresses')):
accesstoken = node.getAttribute('accessToken')
log.debug("Got plextoken: %s" % accesstoken)
if not accesstoken:
raise plugin.PluginError('Could not retrieve accesstoken for %s.' %
config['server'])
else:
return accesstoken
def on_task_input(self, task, config):
try:
r = requests.post('http://www.pogdesign.co.uk/cat/',
data={
'username': config['username'],
'password': config['password'],
'sub_login': '******'
},
allow_redirects=True)
if 'U / P Invalid' in r.text:
raise plugin.PluginError(
'Invalid username/password for pogdesign.')
page = requests.get(
'http://www.pogdesign.co.uk/cat/showselect.php',
cookies=r.cookies)
except requests.RequestException as e:
raise plugin.PluginError('Error retrieving source: %s' % e)
soup = BeautifulSoup(page.text)
entries = []
for row in soup.find_all('label', {'class': 'label_check'}):
if row.find(attrs={'checked': 'checked'}):
t = row.text
if t.endswith('[The]'):
t = 'The ' + t[:-6]
# Make certain names friendlier
if t in self.name_map:
t = self.name_map[t]
e = Entry()
e['title'] = t
url = row.find_next('a', {'class': 'selectsummary'})
e['url'] = 'http://www.pogdesign.co.uk' + url['href']
entries.append(e)
return entries
def query_series(api_key, user_token, member_name=None):
"""
Get the list of series followed by the authenticated user
:param string api_key: Api key requested at http://www.betaseries.com/api
:param string user_token: Obtained with a call to create_token()
:param string member_name: [optional] A member name to get the list of series from. If None, will query the member
for whom the user_token was for
:return: List of serie titles or empty list
"""
params = {}
if member_name:
member_id = query_member_id(api_key, user_token, member_name)
if member_id:
params = {'id': member_id}
else:
log.error("member %r not found" % member_name)
return []
r = requests.get(API_URL_PREFIX + 'members/infos', params=params, headers={
'Accept': 'application/json',
'X-BetaSeries-Version': '2.1',
'X-BetaSeries-Key': api_key,
'X-BetaSeries-Token': user_token,
})
assert r.status_code == 200, "Bad HTTP status code: %s" % r.status_code
j = r.json()
error_list = j['errors']
for err in error_list:
log.error(str(err))
if not error_list:
return [x['title'] for x in j['member']['shows'] if x['user']['archived'] is False]
else:
return []
def retrieve_tracker_config(cls, tracker_config_file):
"""
Will attempt to retrieve the .tracker file from disk or github. Returns the parsed XML.
:param tracker_config_file: URL or path to .tracker file
:return: parsed XML
"""
tracker_config_file = os.path.expanduser(tracker_config_file)
if os.path.exists(tracker_config_file):
return cls.read_tracker_config(tracker_config_file)
elif re.match('^(?![\\\/]).+\.tracker$', tracker_config_file):
tracker_basepath = os.path.abspath(os.path.join(manager.config_base, 'trackers'))
save_path = os.path.join(tracker_basepath, tracker_config_file)
if not os.path.exists(tracker_basepath):
try:
os.mkdir(tracker_basepath)
except IOError as e:
raise TrackerFileError(e)
if not os.path.exists(save_path):
log.info('Tracker file not found on disk. Attempting to fetch tracker config file from Github.')
url = 'https://raw.githubusercontent.com/autodl-community/autodl-trackers/master/' + tracker_config_file
try:
r = requests.get(url)
with io.open(save_path, 'wb') as tracker_file:
for chunk in r.iter_content(8192):
tracker_file.write(chunk)
except (requests.RequestException, IOError) as e:
raise TrackerFileError(e)
return cls.read_tracker_config(save_path)
else:
raise TrackerFileError('Unable to open tracker config file %s', tracker_config_file)
def plex_get_accesstoken(self, config, globalaccesstoken=""):
accesstoken = None
if not globalaccesstoken:
globalaccesstoken = self.plex_get_globalaccesstoken(config)
if config['server'] in ('localhost', '127.0.0.1'):
log.debug('Server using localhost. Global Token will be used')
return globalaccesstoken
try:
r = requests.get(
"https://my.plexapp.com/pms/servers?X-Plex-Token=%s" % globalaccesstoken
)
except requests.RequestException as e:
raise plugin.PluginError(
"Could not get servers from my.plexapp.com using "
"authentication-token: %s. (%s)" % (globalaccesstoken, e)
)
dom = parseString(r.text)
for node in dom.getElementsByTagName('Server'):
if config['server'] in (
node.getAttribute('address'),
node.getAttribute('localAddresses'),
):
accesstoken = node.getAttribute('accessToken')
log.debug("Got plextoken: %s", accesstoken)
if not accesstoken:
raise plugin.PluginError('Could not retrieve accesstoken for %s.' % config['server'])
else:
return accesstoken
def search(self, task, entry, config=None):
"""
Search for name from iptorrents
"""
categories = config.get('category', 'All')
# Make sure categories is a list
if not isinstance(categories, list):
categories = [categories]
# If there are any text categories, turn them into their id number
categories = [c if isinstance(c, int) else CATEGORIES[c]
for c in categories]
filter_url = '&'.join((str(c) + '=') for c in categories)
entries = set()
for search_string in entry.get('search_strings', [entry['title']]):
query = normalize_unicode(search_string)
query = quote_plus(query.encode('utf8'))
url = "{base_url}/t?{filter}&q={query}&qf=".format(base_url=BASE_URL, filter=filter_url, query=query)
log.debug('searching with url: %s' % url)
req = requests.get(url, cookies={'uid': str(config['uid']), 'pass': config['password']})
if '/u/' + str(config['uid']) not in req.text:
raise plugin.PluginError("Invalid cookies (user not logged in)...")
soup = get_soup(req.content, parser="html.parser")
torrents = soup.find('table', {'id': 'torrents'})
results = torrents.findAll('tr')
for torrent in results:
if torrent.th and 'ac' in torrent.th.get('class'):
# Header column
continue
if torrent.find('td', {'colspan': '99'}):
log.debug('No results found for search %s', search_string)
break
entry = Entry()
link = torrent.find('a', href=re.compile('download'))['href']
entry['url'] = "{base}{link}?torrent_pass={key}".format(
base=BASE_URL, link=link, key=config.get('rss_key'))
entry['title'] = torrent.find('a', href=re.compile('details')).text
seeders = torrent.findNext('td', {'class': 'ac t_seeders'}).text
leechers = torrent.findNext('td', {'class': 'ac t_leechers'}).text
entry['torrent_seeds'] = int(seeders)
entry['torrent_leeches'] = int(leechers)
entry['search_sort'] = torrent_availability(entry['torrent_seeds'],
entry['torrent_leeches'])
size = torrent.findNext(text=re.compile('^([\.\d]+) ([GMK]?)B$'))
size = re.search('^([\.\d]+) ([GMK]?)B$', size)
entry['content_size'] = parse_filesize(size.group(0))
log.debug('Found entry %s', entry)
entries.add(entry)
return entries
def query_member_id(api_key, user_token, login_name):
"""
Get the member id of a member identified by its login name.
:param string api_key: Api key requested at http://www.betaseries.com/api
:param string user_token: obtained with a call to create_token()
:param string login_name: The login name of the member
:return: Id of the member identified by its login name or `None` if not found
"""
r = requests.get(API_URL_PREFIX + 'members/search', params={
'login': login_name
}, headers={
'Accept': 'application/json',
'X-BetaSeries-Version': '2.1',
'X-BetaSeries-Key': api_key,
'X-BetaSeries-Token': user_token,
})
assert r.status_code == 200, "Bad HTTP status code: %s" % r.status_code
j = r.json()
error_list = j['errors']
for err in error_list:
log.error(str(err))
found_id = None
if not error_list:
for candidate in j['users']:
if candidate['login'] == login_name:
found_id = candidate['id']
break
return found_id
def get_file(self, only_cached=False):
"""Makes sure the poster is downloaded to the local cache (in userstatic folder) and
returns the path split into a list of directory and file components"""
from flexget.manager import manager
base_dir = os.path.join(manager.config_base, 'userstatic')
if self.file and os.path.isfile(os.path.join(base_dir, self.file)):
return self.file.split(os.sep)
elif only_cached:
return
# If we don't already have a local copy, download one.
log.debug('Downloading poster %s' % self.url)
dirname = os.path.join('tmdb', 'posters', str(self.movie_id))
# Create folders if they don't exist
fullpath = os.path.join(base_dir, dirname)
if not os.path.isdir(fullpath):
os.makedirs(fullpath)
filename = os.path.join(dirname, posixpath.basename(self.url))
thefile = file(os.path.join(base_dir, filename), 'wb')
thefile.write(requests.get(self.url).content)
self.file = filename
# If we are detached from a session, update the db
if not Session.object_session(self):
session = Session()
poster = session.query(TMDBPoster).filter(TMDBPoster.db_id == self.db_id).first()
if poster:
poster.file = filename
session.commit()
session.close()
return filename.split(os.sep)
def get_http_seeds(url, info_hash):
url = get_scrape_url(url, info_hash)
if not url:
log.debug('if not url is true returning 0')
return 0
log.debug('Checking for seeds from %s' % url)
try:
data = bdecode(requests.get(url).content).get('files')
except RequestException as e:
log.debug('Error scraping: %s', e)
return 0
except SyntaxError as e:
log.warning('Error decoding tracker response: %s', e)
return 0
except BadStatusLine as e:
log.warning('Error BadStatusLine: %s', e)
return 0
except IOError as e:
log.warning('Server error: %s', e)
return 0
if not data:
log.debug('No data received from tracker scrape.')
return 0
log.debug('get_http_seeds is returning: %s', list(data.values())[0]['complete'])
return list(data.values())[0]['complete']
def search(self, task, entry, config=None):
"""
Search for name from piratebay.
"""
if not isinstance(config, dict):
config = {}
sort = SORT.get(config.get('sort_by', 'seeds'))
if config.get('sort_reverse'):
sort += 1
if isinstance(config.get('category'), int):
category = config['category']
else:
category = CATEGORIES.get(config.get('category', 'all'))
filter_url = '/0/%d/%d' % (sort, category)
entries = set()
for search_string in entry.get('search_strings', [entry['title']]):
query = normalize_unicode(search_string)
# TPB search doesn't like dashes
query = query.replace('-', ' ')
# urllib.quote will crash if the unicode string has non ascii characters, so encode in utf-8 beforehand
url = 'http://thepiratebay.%s/search/%s%s' % (
CUR_TLD, quote(query.encode('utf-8')), filter_url)
log.debug('Using %s as piratebay search url' % url)
page = requests.get(url).content
soup = get_soup(page)
for link in soup.find_all('a', attrs={'class': 'detLink'}):
entry = Entry()
entry['title'] = self.extract_title(link)
if not entry['title']:
log.error(
'Malformed search result. No title or url found. Skipping.'
)
continue
entry['url'] = 'http://thepiratebay.%s%s' % (CUR_TLD,
link.get('href'))
tds = link.parent.parent.parent.find_all('td')
entry['torrent_seeds'] = int(tds[-2].contents[0])
entry['torrent_leeches'] = int(tds[-1].contents[0])
entry['search_sort'] = torrent_availability(
entry['torrent_seeds'], entry['torrent_leeches'])
# Parse content_size
size = link.find_next(attrs={'class': 'detDesc'}).contents[0]
size = re.search('Size ([\.\d]+)\xa0([GMK])iB', size)
if size:
if size.group(2) == 'G':
entry['content_size'] = int(
float(size.group(1)) * 1000**3 / 1024**2)
elif size.group(2) == 'M':
entry['content_size'] = int(
float(size.group(1)) * 1000**2 / 1024**2)
else:
entry['content_size'] = int(
float(size.group(1)) * 1000 / 1024**2)
entries.add(entry)
return sorted(entries,
reverse=True,
key=lambda x: x.get('search_sort'))
def bluray_request(endpoint, **params) -> Any:
full_url = BASE_URL + endpoint
response = requests.get(full_url, params=params)
if response.content:
try:
return response.json(strict=False)
except JSONDecodeError as e:
raise PluginError('Could decode json from response blu-ray api')
def on_task_input(self, task, config):
try:
r = requests.post('http://www.pogdesign.co.uk/cat/', data={'username': config['username'], 'password': config['password'], 'sub_login': '******'}, allow_redirects=True)
if 'U / P Invalid' in r.text:
raise plugin.PluginError('Invalid username/password for pogdesign.')
page = requests.get('http://www.pogdesign.co.uk/cat/showselect.php', cookies=r.cookies)
except requests.RequestException, e:
raise plugin.PluginError('Error retrieving source: %s' % e)
def parse_downloads(self, series_url, search_title):
page = requests.get(series_url).content
try:
soup = get_soup(page)
except Exception as e:
raise UrlRewritingError(e)
urls = []
# find all titles
episode_titles = self.find_all_titles(search_title)
if not episode_titles:
raise UrlRewritingError('Unable to find episode')
for ep_title in episode_titles:
# find matching download
episode_title = soup.find('strong',
text=re.compile(ep_title, re.I))
if not episode_title:
continue
# find download container
episode = episode_title.parent
if not episode:
continue
# find episode language
episode_lang = episode.find_previous(
'strong', text=re.compile('Sprache')).next_sibling
if not episode_lang:
log.warning('No language found for: %s', series_url)
continue
# filter language
if not self.check_language(episode_lang):
log.warning('languages not matching: %s <> %s',
self.config['language'], episode_lang)
continue
# find download links
links = episode.find_all('a')
if not links:
log.warning('No links found for: %s', series_url)
continue
for link in links:
if not link.has_attr('href'):
continue
url = link['href']
pattern = (
r'http:\/\/download\.serienjunkies\.org.*%s_.*\.html' %
self.config['hoster'])
if re.match(pattern, url) or self.config['hoster'] == 'all':
urls.append(url)
else:
continue
return urls
def search(self, task, entry, config=None):
"""
Search for name from iptorrents
"""
categories = config.get('category', 'all')
# Make sure categories is a list
if not isinstance(categories, list):
categories = [categories]
# If there are any text categories, turn them into their id number
categories = [c if isinstance(c, int) else CATEGORIES[c]
for c in categories]
filter_url = '&'.join((str(c) + '=') for c in categories)
entries = set()
for search_string in entry.get('search_strings', [entry['title']]):
query = normalize_unicode(search_string)
query = quote_plus(query.encode('utf8'))
url = "{base_url}/t?{filter}&q={query}&qf=".format(base_url=BASE_URL, filter=filter_url, query=query)
log.debug('searching with url: %s' % url)
req = requests.get(url, cookies={'uid': str(config['uid']), 'pass': config['password']})
if '/u/' + str(config.get('uid')) not in req.content:
raise plugin.PluginError("Invalid cookies (user not logged in)...")
soup = get_soup(req.content, parser="html.parser")
torrents = soup.find('table', {'class': 'torrents'})
for torrent in torrents.findAll('a', href=re.compile('\.torrent$')):
entry = Entry()
entry['url'] = "{base}{link}?torrent_pass={key}".format(
base=BASE_URL, link=torrent['href'], key=config.get('rss_key'))
entry['title'] = torrent.findPrevious("a", attrs={'class': 't_title'}).text
seeders = torrent.findNext('td', {'class': 'ac t_seeders'}).text
leechers = torrent.findNext('td', {'class': 'ac t_leechers'}).text
entry['torrent_seeds'] = int(seeders)
entry['torrent_leeches'] = int(leechers)
entry['search_sort'] = torrent_availability(entry['torrent_seeds'],
entry['torrent_leeches'])
size = torrent.findNext(text=re.compile('^([\.\d]+) ([GMK]?)B$'))
size = re.search('^([\.\d]+) ([GMK]?)B$', size)
if size:
if size.group(2) == 'G':
entry['content_size'] = int(float(size.group(1)) * 1000 ** 3 / 1024 ** 2)
elif size.group(2) == 'M':
entry['content_size'] = int(float(size.group(1)) * 1000 ** 2 / 1024 ** 2)
elif size.group(2) == 'K':
entry['content_size'] = int(float(size.group(1)) * 1000 / 1024 ** 2)
else:
entry['content_size'] = int(float(size.group(1)) / 1024 ** 2)
entries.add(entry)
return entries
def search(self, task, entry, config=None):
"""
Search for entries on PublicHD
"""
categories = config.get('category', 'all')
# Ensure categories a list
if not isinstance(categories, list):
categories = [categories]
# Convert named category to its respective category id number
categories = [c if isinstance(c, int) else CATEGORIES[c] for c in categories]
category_url_fragment = '&category=%s' % urllib.quote(';'.join(str(c) for c in categories))
base_url = 'http://publichd.se/index.php?page=torrents&active=0'
entries = set()
for search_string in entry.get('search_strings', [entry['title']]):
query = normalize_unicode(search_string)
query_url_fragment = '&search=' + urllib.quote(query.encode('utf8'))
# http://publichd.se/index.php?page=torrents&active=0&category=5;15&search=QUERY
url = (base_url + category_url_fragment + query_url_fragment)
log.debug('PublicHD search url: %s' % url)
page = requests.get(url).content
soup = get_soup(page)
for result in soup.findAll('a', href=re.compile('page=torrent-details')):
entry = Entry()
entry['title'] = result.text
# Expand the selection to whole row
result = result.findPrevious('tr')
download_url = result.find('a', href=re.compile('\.torrent$'))['href']
torrent_hash = re.search(r'/([0-9a-fA-F]{5,40})/', download_url).group(1)
entry['url'] = 'http://publichd.se/download.php?id=%s' % torrent_hash
seeds, leeches = result.findAll('td', text=re.compile('^\d+$'))
entry['torrent_seeds'] = int(seeds.text)
entry['torrent_leeches'] = int(leeches.text)
entry['search_sort'] = torrent_availability(entry['torrent_seeds'], entry['torrent_leeches'])
size = result.find("td", text=re.compile('(\d+(?:[.,]\d+)*)\s?([KMG]B)')).text
size = re.search('(\d+(?:[.,]\d+)*)\s?([KMG]B)', size)
if size:
if size.group(2) == 'GB':
entry['content_size'] = int(float(size.group(1).replace(',', '')) * 1000 ** 3 / 1024 ** 2)
elif size.group(2) == 'MB':
entry['content_size'] = int(float(size.group(1).replace(',', '')) * 1000 ** 2 / 1024 ** 2)
elif size.group(2) == 'KB':
entry['content_size'] = int(float(size.group(1).replace(',', '')) * 1000 / 1024 ** 2)
else:
entry['content_size'] = int(float(size.group(1).replace(',', '')) / 1024 ** 2)
entries.add(entry)
return entries
def search(self, task, entry, config=None):
"""
Search for entries on PublicHD
"""
categories = config.get('category', 'all')
# Ensure categories a list
if not isinstance(categories, list):
categories = [categories]
# Convert named category to its respective category id number
categories = [c if isinstance(c, int) else CATEGORIES[c] for c in categories]
category_url_fragment = '&category=%s' % urllib.parse.quote(';'.join(str(c) for c in categories))
base_url = 'http://publichd.se/index.php?page=torrents&active=0'
entries = set()
for search_string in entry.get('search_strings', [entry['title']]):
query = normalize_unicode(search_string)
query_url_fragment = '&search=' + urllib.parse.quote(query.encode('utf8'))
# http://publichd.se/index.php?page=torrents&active=0&category=5;15&search=QUERY
url = (base_url + category_url_fragment + query_url_fragment)
log.debug('PublicHD search url: %s' % url)
page = requests.get(url).content
soup = get_soup(page)
for result in soup.findAll('a', href=re.compile('page=torrent-details')):
entry = Entry()
entry['title'] = result.text
# Expand the selection to whole row
result = result.findPrevious('tr')
download_url = result.find('a', href=re.compile('\.torrent$'))['href']
torrent_hash = re.search(r'/([0-9a-fA-F]{5,40})/', download_url).group(1)
entry['url'] = 'http://publichd.se/download.php?id=%s' % torrent_hash
seeds, leeches = result.findAll('td', text=re.compile('^\d+$'))
entry['torrent_seeds'] = int(seeds.text)
entry['torrent_leeches'] = int(leeches.text)
entry['search_sort'] = torrent_availability(entry['torrent_seeds'], entry['torrent_leeches'])
size = result.find("td", text=re.compile('(\d+(?:[.,]\d+)*)\s?([KMG]B)')).text
size = re.search('(\d+(?:[.,]\d+)*)\s?([KMG]B)', size)
if size:
if size.group(2) == 'GB':
entry['content_size'] = int(float(size.group(1).replace(',', '')) * 1000 ** 3 / 1024 ** 2)
elif size.group(2) == 'MB':
entry['content_size'] = int(float(size.group(1).replace(',', '')) * 1000 ** 2 / 1024 ** 2)
elif size.group(2) == 'KB':
entry['content_size'] = int(float(size.group(1).replace(',', '')) * 1000 / 1024 ** 2)
else:
entry['content_size'] = int(float(size.group(1).replace(',', '')) / 1024 ** 2)
entries.add(entry)
return entries
def parse_download_page(self, url):
try:
page = requests.get(url).content
soup = get_soup(page, "html.parser")
download_link = soup.findAll(href=re.compile("redirect|redirectlink"))
download_href = download_link[0]["href"]
return download_href
except Exception:
raise UrlRewritingError("Unable to locate torrent from url %s" % url)
def get_token(self):
# Don't use a session as tokens are not affected by domain limit
try:
r = get(self.base_url, params={'get_token': 'get_token', 'format': 'json'}).json()
token = r.get('token')
log.debug('RarBG token: %s' % token)
return token
except RequestException as e:
log.debug('Could not retrieve RarBG token: %s', e.args[0])
def search(self, task, entry, config=None):
"""
Search for name from torrentday.
"""
categories = config.get('category', 'all')
# Make sure categories is a list
if not isinstance(categories, list):
categories = [categories]
# If there are any text categories, turn them into their id number
categories = [c if isinstance(c, int) else CATEGORIES[c] for c in categories]
params = { 'cata': 'yes', 'c%s' % ','.join(str(c) for c in categories): 1, 'clear-new': 1}
entries = set()
for search_string in entry.get('search_strings', [entry['title']]):
url = 'https://www.torrentday.com/browse.php'
params['search'] = normalize_unicode(search_string).replace(':', '')
cookies = { 'uid': config['uid'], 'pass': config['passkey'], '__cfduid': config['cfduid'] }
try:
page = requests.get(url, params=params, cookies=cookies).content
except RequestException as e:
raise PluginError('Could not connect to torrentday: %s' % e)
soup = get_soup(page)
for tr in soup.find_all('tr', { 'class': 'browse' }):
entry = Entry()
# find the torrent names
title = tr.find('a', { 'class': 'torrentName' })
entry['title'] = title.contents[0]
log.debug('title: %s', title.contents[0])
# find download link
torrent_url = tr.find('td', { 'class': 'dlLinksInfo' })
torrent_url = torrent_url.find('a').get('href')
# construct download URL
torrent_url = ( 'https://www.torrentday.com/' + torrent_url + '?torrent_pass='******'rss_key'] )
log.debug('RSS-ified download link: %s', torrent_url)
entry['url'] = torrent_url
# us tr object for seeders/leechers
seeders, leechers = tr.find_all('td', { 'class': ['seedersInfo', 'leechersInfo']})
entry['torrent_seeds'] = int(seeders.contents[0].replace(',', ''))
entry['torrent_leeches'] = int(leechers.contents[0].replace(',', ''))
entry['search_sort'] = torrent_availability(entry['torrent_seeds'], entry['torrent_leeches'])
# use tr object for size
size = tr.find('td', text=re.compile('([\.\d]+) ([TGMKk]?)B')).contents[0]
size = re.search('([\.\d]+) ([TGMKk]?)B', str(size))
entry['content_size'] = parse_filesize(size.group(0))
entries.add(entry)
return sorted(entries, reverse=True, key=lambda x: x.get('search_sort'))
def get_token(self):
# Don't use a session as tokens are not affected by domain limit
r = get('https://torrentapi.org/pubapi.php', params={'get_token': 'get_token', 'format': 'json'})
token = None
try:
token = r.json().get('token')
except ValueError:
log.error('Could not retrieve RARBG token.')
log.debug('RarBG token: %s' % token)
return token
def search(self, task, entry, config=None):
"""
Search for entries on Serienjunkies
"""
base_url = 'http://serienjunkies.org/search/'
mull = {
"Dauer:", "Download:", "Uploader:", u"Größe:", u"Tonhöhe:",
"Sprache:", "Format:", "HQ-Cover:"
}
self.config = task.config.get('searchSerienjunkies') or {}
self.config.setdefault('hoster', DEFHOS)
self.config.setdefault('language', DEFLANG)
entries = set()
for search_string in entry.get('search_strings', [entry['title']]):
query = normalize_unicode(search_string)
query_url_fragment = urllib.quote(query.encode('utf8'))
# http://serienjunkies.org/search/QUERY
url = (base_url + query_url_fragment)
log.debug('Serienjunkies search url: %s' % url)
page = requests.get(url).content
soup = get_soup(page)
hoster = self.config['hoster']
if self.config['language'] == 'english':
english = True
else:
english = None
for p in soup.find_all('p'):
entry = Entry()
if p.strong is not None and p.strong.text not in mull:
if english:
try:
if not p.strong.find(text=re.compile(
"german", flags=re.IGNORECASE)):
link = p.find(
text=re.compile(hoster)).find_previous('a')
entry['title'] = p.strong.text
entry['url'] = link.get('href')
entries.add(entry)
except:
pass
else:
try:
if p.strong.find(text=re.compile(
"german", flags=re.IGNORECASE)):
link = p.find(
text=re.compile(hoster)).find_previous('a')
entry['title'] = p.strong.text
entry['url'] = link.get('href')
entries.add(entry)
except:
pass
return entries
def parse_download(self, series_url, search_title, config, entry):
page = requests.get(series_url).content
try:
soup = get_soup(page)
except Exception as e:
raise UrlRewritingError(e)
config = config or {}
config.setdefault('hoster', 'ul')
config.setdefault('language', 'en')
# find matching download
episode_title = soup.find('strong', text=search_title)
if not episode_title:
raise UrlRewritingError('Unable to find episode')
# find download container
episode = episode_title.parent
if not episode:
raise UrlRewritingError('Unable to find episode container')
# find episode language
episode_lang = episode.find_previous('strong', text=re.compile('Sprache')).next_sibling
if not episode_lang:
raise UrlRewritingError('Unable to find episode language')
# filter language
if config['language'] in ['de', 'both']:
if not re.search('german|deutsch', episode_lang, flags=re.IGNORECASE):
entry.reject('Language doesn\'t match selected')
return series_url
elif config['language'] in ['en', 'both']:
if not re.search('englisc?h', episode_lang, flags=re.IGNORECASE):
entry.reject('Language doesn\'t match selected')
return series_url
# find download links
links = episode.find_all('a')
if not links:
raise UrlRewritingError('Unable to find download links')
for link in links:
if not link.has_attr('href'):
continue
url = link['href']
pattern = 'http:\/\/download\.serienjunkies\.org.*%s_.*\.html' % config['hoster']
if re.match(pattern, url):
return url
else:
log.verbose('Hoster doesn\'t match selected')
continue
raise UrlRewritingError('URL-Rewriting failed, enable verbose logging for details.')
def parse_download_page(self, url):
try:
page = requests.get(url).content
soup = get_soup(page, 'html.parser')
download_link = soup.findAll(href=re.compile('redirect.php'))
download_href = download_link[0]['href']
return download_href[download_href.index('url=') + 4:]
except Exception:
raise UrlRewritingError(
'Unable to locate torrent from url %s' % url
)
def parse_download_page(self, url):
try:
page = requests.get(url).content
soup = get_soup(page, 'html.parser')
download_link = soup.findAll(
href=re.compile('redirect|redirectlink'))
download_href = download_link[0]['href']
return download_href
except Exception:
raise UrlRewritingError('Unable to locate torrent from url %s' %
url)
def search(self, arg_entry, config=None):
"""
Search for name from piratebay.
"""
if not isinstance(config, dict):
config = {}
sort = SORT.get(config.get('sort_by', 'seeds'))
if config.get('sort_reverse'):
sort += 1
if isinstance(config.get('category'), int):
category = config['category']
else:
category = CATEGORIES.get(config.get('category', 'all'))
filter_url = '/0/%d/%d' % (sort, category)
entries = set()
for search_string in arg_entry.get('search_string',
[arg_entry['title']]):
query = normalize_unicode(search_string)
# TPB search doesn't like dashes
query = query.replace('-', ' ')
# urllib.quote will crash if the unicode string has non ascii characters, so encode in utf-8 beforehand
url = 'http://thepiratebay.%s/search/%s%s' % (
CUR_TLD, urllib.quote(query.encode('utf-8')), filter_url)
log.debug('Using %s as piratebay search url' % url)
page = requests.get(url).content
soup = get_soup(page)
for link in soup.find_all('a', attrs={'class': 'detLink'}):
entry = Entry()
entry['title'] = link.contents[0]
entry['url'] = 'http://thepiratebay.%s%s' % (CUR_TLD,
link.get('href'))
tds = link.parent.parent.parent.find_all('td')
entry['torrent_seeds'] = int(tds[-2].contents[0])
entry['torrent_leeches'] = int(tds[-1].contents[0])
entry['search_sort'] = torrent_availability(
entry['torrent_seeds'], entry['torrent_leeches'])
# Parse content_size
size = link.find_next(attrs={'class': 'detDesc'}).contents[0]
size = re.search('Size ([\.\d]+)\xa0([GMK])iB', size)
if size:
if size.group(2) == 'G':
entry['content_size'] = int(
float(size.group(1)) * 1000**3 / 1024**2)
elif size.group(2) == 'M':
entry['content_size'] = int(
float(size.group(1)) * 1000**2 / 1024**2)
else:
entry['content_size'] = int(
float(size.group(1)) * 1000 / 1024**2)
entries.add(entry)
return sorted(
entries, reverse=True, key=lambda x: x.get('search_sort'))
def parse_download_page(self, url):
page = requests.get(url).content
try:
soup = get_soup(page)
tag_div = soup.find('div', attrs={'class': 'download'})
if not tag_div:
raise UrlRewritingError('Unable to locate download link from url %s' % url)
tag_a = tag_div.find('a')
torrent_url = tag_a.get('href')
return torrent_url
except Exception, e:
raise UrlRewritingError(e)
def get_json(url):
try:
log.debug('fetching json at %s' % url)
data = requests.get(url)
result = data.json()
except URLError as e:
log.warning('Request failed %s' % url)
return
except ValueError:
log.warning('Rotten Tomatoes returned invalid json at: %s' % url)
return
return result
def get_json(url):
try:
log.debug('fetching json at %s' % url)
data = requests.get(url)
result = data.json()
except URLError as e:
log.warning('Request failed %s' % url)
return
except ValueError:
log.warning('Rotten Tomatoes returned invalid json at: %s' % url)
return
return result
def parse_downloads(self, series_url, search_title):
page = requests.get(series_url).content
try:
soup = get_soup(page)
except Exception as e:
raise UrlRewritingError(e)
urls = []
# find all titles
episode_titles = self.find_all_titles(search_title)
if not episode_titles:
raise UrlRewritingError("Unable to find episode")
for ep_title in episode_titles:
# find matching download
episode_title = soup.find("strong", text=re.compile(ep_title, re.I))
if not episode_title:
continue
# find download container
episode = episode_title.parent
if not episode:
continue
# find episode language
episode_lang = episode.find_previous("strong", text=re.compile("Sprache")).next_sibling
if not episode_lang:
log.warning("No language found for: %s", series_url)
continue
# filter language
if not self.check_language(episode_lang):
log.warning("languages not matching: %s <> %s", self.config["language"], episode_lang)
continue
# find download links
links = episode.find_all("a")
if not links:
log.warning("No links found for: %s", series_url)
continue
for link in links:
if not link.has_attr("href"):
continue
url = link["href"]
pattern = "http:\/\/download\.serienjunkies\.org.*%s_.*\.html" % self.config["hoster"]
if re.match(pattern, url) or self.config["hoster"] == "all":
urls.append(url)
else:
continue
return urls
def search(self, entry, config=None):
config = self.process_config(config)
feed = REPUTATIONS[config['reputation']]
entries = set()
for search_string in entry.get('search_strings', [entry['title']]):
query = normalize_unicode(search_string +
config.get('extra_terms', ''))
for domain in ['eu', 'me', 'ch', 'in']:
# urllib.quote will crash if the unicode string has non ascii characters, so encode in utf-8 beforehand
url = 'http://torrentz.%s/%s?q=%s' % (
domain, feed, urllib.quote(query.encode('utf-8')))
log.debug('requesting: %s' % url)
try:
r = requests.get(url)
break
except requests.RequestException as err:
log.warning('torrentz.%s failed. Error: %s' %
(domain, err))
else:
raise plugin.PluginError(
'Error getting torrentz search results')
if not r.content.strip():
raise plugin.PluginError(
'No data from %s. Maybe torrentz is blocking the FlexGet User-Agent'
% url)
rss = feedparser.parse(r.content)
if rss.get('bozo_exception'):
raise plugin.PluginError('Got bozo_exception (bad rss feed)')
for item in rss.entries:
m = re.search(
r'Size: ([\d]+) Mb Seeds: ([,\d]+) Peers: ([,\d]+) Hash: ([a-f0-9]+)',
item.description, re.IGNORECASE)
if not m:
log.debug('regexp did not find seeds / peer data')
continue
entry = Entry()
entry['title'] = item.title
entry['url'] = item.link
entry['content_size'] = int(m.group(1))
entry['torrent_seeds'] = int(m.group(2).replace(',', ''))
entry['torrent_leeches'] = int(m.group(3).replace(',', ''))
entry['torrent_info_hash'] = m.group(4).upper()
entry['search_sort'] = torrent_availability(
entry['torrent_seeds'], entry['torrent_leeches'])
entries.add(entry)
log.debug('Search got %d results' % len(entries))
return entries
def parse_downloads(self, series_url, search_title):
page = requests.get(series_url).content
try:
soup = get_soup(page)
except Exception as e:
raise UrlRewritingError(e)
urls = []
# find all titles
episode_titles = self.find_all_titles(search_title)
if not episode_titles:
raise UrlRewritingError('Unable to find episode')
for ep_title in episode_titles:
# find matching download
episode_title = soup.find('strong', text=re.compile(ep_title, re.I))
if not episode_title:
continue
# find download container
episode = episode_title.parent
if not episode:
continue
# find episode language
episode_lang = episode.find_previous('strong', text=re.compile('Sprache')).next_sibling
if not episode_lang:
log.warning('No language found for: %s' % series_url)
continue
# filter language
if not self.check_language(episode_lang):
log.warning('languages not matching: %s <> %s' % (self.config['language'], episode_lang))
continue
# find download links
links = episode.find_all('a')
if not links:
log.warning('No links found for: %s' % series_url)
continue
for link in links:
if not link.has_attr('href'):
continue
url = link['href']
pattern = 'http:\/\/download\.serienjunkies\.org.*%s_.*\.html' % self.config['hoster']
if re.match(pattern, url) or self.config['hoster'] == 'all':
urls.append(url)
else:
continue
return urls
def search(self, task, entry, config=None):
entries = set()
search_strings = [
normalize_unicode(s)
for s in entry.get('search_strings', [entry['title']])
]
for search_string in search_strings:
url = 'https://yts.am/api/v2/list_movies.json?query_term=%s' % (
quote(search_string.encode('utf-8')))
log.debug('requesting: %s' % url)
try:
result = requests.get(url)
try:
data = result.json()
except ValueError:
log.debug('Could not decode json from response: %s',
result.text)
raise plugin.PluginError('Error getting result from yts.')
except requests.RequestException as e:
raise plugin.PluginError(
'Could not retrieve query from yts (%s)' % e.args[0])
if not data['status'] == 'ok':
raise plugin.PluginError('failed to query YTS')
try:
if data['data']['movie_count'] > 0:
for item in data['data']['movies']:
for torrent in item['torrents']:
entry = Entry()
entry['title'] = item['title']
entry['year'] = item['year']
entry['url'] = torrent['url']
entry['content_size'] = parse_filesize(
str(torrent['size_bytes']) + "b")
entry['torrent_seeds'] = torrent['seeds']
entry['torrent_leeches'] = torrent['peers']
entry['torrent_info_hash'] = torrent['hash']
entry[
'torrent_availability'] = torrent_availability(
entry['torrent_seeds'],
entry['torrent_leeches'])
entry['quality'] = torrent['quality']
entry['imdb_id'] = item['imdb_code']
if entry.isvalid():
entries.add(entry)
except Exception:
log.debug('invalid return structure from YTS')
log.debug('Search got %d results' % len(entries))
return entries
def get_url_content(self, url, params={}, method = "GET", json = False):
if method is "GET":
result = requests.get(url, params=params)
elif method is "POST":
result = requests.post(url, data=params)
if json:
if result.content:
return result.json()
else:
return {}
else:
return BeautifulSoup(result.content, "html5lib")
def get_url_content(self, url, params={}, method="GET", json=False):
if method is "GET":
result = requests.get(url, params=params)
elif method is "POST":
result = requests.post(url, data=params)
if json:
if result.content:
return result.json()
else:
return {}
else:
return BeautifulSoup(result.content, "html5lib")
def on_task_metainfo(self, task, config): # NOQA
if not config:
return
for entry in task.entries:
if entry.get('series_id_type') == 'ep':
if 'tvdb_id' in entry:
log.debug('The entry has a tvdb_id and will be used for mapping')
else:
log.debug('The entry doesn\'t have tvdb_id, will check xem\'s list of shows')
response = requests.get('http://thexem.de/map/allNames?origin=tvdb&defaultNames=1&season=eq%s' % entry['series_season'])
shownames = json.loads(response.content)
for tvdb_id in shownames['data'].keys():
if entry['series_name'] in shownames['data'][tvdb_id]:
entry['tvdb_id'] = tvdb_id
log.debug('The tvdb_id for %s is %s', entry['series_name'], entry['tvdb_id'])
break
if 'tvdb_id' not in entry:
log.debug('An tvdb_id was not found, will search the xem\'s site')
response = requests.get('http://thexem.de/search?q=%s' % entry['series_name'])
if response.url.startswith('http://thexem.de/xem/show/'):
soup = get_soup(response.content)
try:
entry['tvdb_id'] = soup.findAll('a', {'href': re.compile('^http://thetvdb.com/\?tab=series')})[0].next
log.debug('The tvdb_id for %s is %s', entry['series_name'], entry['tvdb_id'])
except:
pass
if 'tvdb_id' not in entry:
log.error('Unable to find a tvdb_id for %s, manually specify a tvdb_id using set', entry['series_name'])
response = requests.get('http://thexem.de/map/all?id=%s&origin=tvdb' % entry['tvdb_id'])
episode_map = json.loads(response.content)
for episode_entry in episode_map['data']:
if episode_entry[config['source']]['season'] == entry['series_season'] and episode_entry[config['source']]['episode'] == entry['series_episode']:
log.debug('An XEM entry was found for %s, %s episode S%02dE%02d maps to %s episode S%02dE%02d' % (entry['series_name'], config['source'], entry['series_season'], entry['series_episode'], config['destination'], episode_entry[config['destination']]['season'], episode_entry[config['destination']]['episode']))
if 'description' in entry:
entry['description'] = entry['description'].replace('Season: %s; Episode: %s' % (entry['series_season'], entry['series_episode']), 'Season: %s; Episode: %s' % (episode_entry[config['destination']]['season'], episode_entry[config['destination']]['episode']))
entry['series_season'] = episode_entry[config['destination']]['season']
entry['series_episode'] = episode_entry[config['destination']]['episode']
entry['series_id'] = 'S%02dE%02d' % (episode_entry[config['destination']]['season'], episode_entry[config['destination']]['episode'])
break
def get_token(self):
# Don't use a session as tokens are not affected by domain limit
r = get(self.base_url,
params={
'get_token': 'get_token',
'format': 'json'
})
token = None
try:
token = r.json().get('token')
except ValueError:
log.error('Could not retrieve RARBG token.')
log.debug('RarBG token: %s' % token)
return token
def parse_download_page(self, url):
page = requests.get(url,verify=False).content
try:
soup = get_soup(page)
tag_div = soup.find('div', attrs={'class': 'download'})
if not tag_div:
raise UrlRewritingError('Unable to locate download link from url %s' % url)
tag_a = tag_div.find('a')
torrent_url = tag_a.get('href')
# URL is sometimes missing the schema
if torrent_url.startswith('//'):
torrent_url = 'http:' + torrent_url
return torrent_url
except Exception as e:
raise UrlRewritingError(e)
def tvmaze_lookup(lookup_type, lookup_values):
"""
Build the URL and return the reply from TVMaze API
:param lookup_type: Selects the endpoint that will be used
:param lookup_values: A list of values to be used in the URL
:return: A JSON reply from the API
"""
lookup_url = BASE_URL + TVMAZE_ENDPOINTS[lookup_type].format(
*lookup_values)
log.debug('querying tvmaze API with the following URL: %s', lookup_url)
try:
result = requests.get(lookup_url).json()
except HTTPError as e:
raise LookupError(e)
return result
def __call__(self, parser, namespace, values, option_string=None):
# Print the version number
console('%s' % self.version)
# Check for latest version from server
try:
page = requests.get('http://download.flexget.com/latestversion')
except requests.RequestException:
console('Error getting latest version number from download.flexget.com')
else:
ver = page.text.strip()
if self.version == ver:
console('You are on the latest release.')
else:
console('Latest release: %s' % ver)
parser.exit()
def tvmaze_lookup(lookup_url, **kwargs):
"""
Build the URL and return the reply from TVMaze API
:param lookup_type: Selects the endpoint that will be used
:param lookup_values: A list of values to be used in the URL
:return: A JSON reply from the API
"""
url = BASE_URL + lookup_url
log.debug('querying tvmaze API with the following URL: %s', url)
try:
result = requests.get(url, **kwargs).json()
except RequestException as e:
raise LookupError(e.args[0])
return result
def url_from_page(self, url):
"""Parses torrent url from newtorrents download page"""
try:
page = requests.get(url)
data = page.text
except requests.RequestException:
raise UrlRewritingError('URLerror when retrieving page')
p = re.compile(r"copy\(\'(.*)\'\)", re.IGNORECASE)
f = p.search(data)
if not f:
# the link in which plugin relies is missing!
raise UrlRewritingError(
'Failed to get url from download page. Plugin may need a update.'
)
else:
return f.group(1)
def on_task_input(self, task, config):
try:
r = requests.post('http://www.pogdesign.co.uk/cat/',
data={
'username': config['username'],
'password': config['password'],
'sub_login': '******'
},
allow_redirects=True)
if 'U / P Invalid' in r.text:
raise plugin.PluginError(
'Invalid username/password for pogdesign.')
page = requests.get(
'http://www.pogdesign.co.uk/cat/showselect.php',
cookies=r.cookies)
except requests.RequestException, e:
raise plugin.PluginError('Error retrieving source: %s' % e)
def parse_download_page(self, url, task):
log.verbose('Descargas2020 URL: %s', url)
try:
page = requests.get(url)
except requests.exceptions.RequestException as e:
raise UrlRewritingError(e)
try:
soup = get_soup(page.text)
except Exception as e:
raise UrlRewritingError(e)
torrent_id = None
url_format = DESCARGAS2020_TORRENT_FORMAT
torrent_id_prog = re.compile(
"(?:parametros\s*=\s*\n?)\s*{\s*\n(?:\s*'\w+'\s*:.*\n)+\s*'(?:torrentID|id)"
"'\s*:\s*'(\d+)'")
torrent_ids = soup.findAll(text=torrent_id_prog)
if torrent_ids:
match = torrent_id_prog.search(torrent_ids[0])
if match:
torrent_id = match.group(1)
if not torrent_id:
log.debug('torrent ID not found, searching openTorrent script')
torrent_id_prog = re.compile(
'function openTorrent.*\n.*\{.*(\n.*)+window\.location\.href =\s*\"(.*\/\d+_-.*[^\/])\/?\";'
)
torrent_ids = soup.findAll(text=torrent_id_prog)
if torrent_ids:
match = torrent_id_prog.search(torrent_ids[0])
if match:
torrent_id = match.group(2)
return torrent_id.replace('descargar-torrent',
'download') + '.torrent'
if not torrent_id:
raise UrlRewritingError('Unable to locate torrent ID from url %s' %
url)
return url_format.format(torrent_id)