def inject_series(self, release_name):
self.execute_task(
'inject_series',
options={'inject': [Entry(title=release_name, url='')]})
def search(self, task, entry, config=None):
"""
Search for name from torrentleech.
"""
request_headers = {'User-Agent': 'curl/7.54.0'}
rss_key = config['rss_key']
# build the form request:
data = {'username': config['username'], 'password': config['password']}
# POST the login form:
try:
login = task.requests.post(
'https://www.torrentleech.org/user/account/login/',
data=data,
headers=request_headers,
allow_redirects=True)
except RequestException as e:
raise PluginError('Could not connect to torrentleech: %s', str(e))
if not isinstance(config, dict):
config = {}
# sort = SORT.get(config.get('sort_by', 'seeds'))
# if config.get('sort_reverse'):
# sort += 1
categories = config.get('category', 'all')
# Make sure categories is a list
if not isinstance(categories, list):
categories = [categories]
# If there are any text categories, turn them into their id number
categories = [
c if isinstance(c, int) else CATEGORIES[c] for c in categories
]
filter_url = '/categories/{}'.format(','.join(
str(c) for c in categories))
entries = set()
for search_string in entry.get('search_strings', [entry['title']]):
query = normalize_unicode(search_string).replace(":", "")
# urllib.quote will crash if the unicode string has non ascii characters,
# so encode in utf-8 beforehand
url = ('https://www.torrentleech.org/torrents/browse/list/query/' +
quote(query.encode('utf-8')) + filter_url)
log.debug('Using %s as torrentleech search url', url)
results = task.requests.get(url,
headers=request_headers,
cookies=login.cookies).json()
for torrent in results['torrentList']:
entry = Entry()
entry['download_headers'] = request_headers
entry['title'] = torrent['name']
# construct download URL
torrent_url = 'https://www.torrentleech.org/rss/download/{}/{}/{}'.format(
torrent['fid'], rss_key, torrent['filename'])
log.debug('RSS-ified download link: %s', torrent_url)
entry['url'] = torrent_url
# seeders/leechers
entry['torrent_seeds'] = torrent['seeders']
entry['torrent_leeches'] = torrent['leechers']
entry['search_sort'] = torrent_availability(
entry['torrent_seeds'], entry['torrent_leeches'])
entry['content_size'] = parse_filesize(
str(torrent['size']) + ' b')
entries.add(entry)
return sorted(entries,
reverse=True,
key=lambda x: x.get('search_sort'))
def search(self, task, entry, config):
"""
Search for entries on Limetorrents
"""
if not isinstance(config, dict):
config = {'category': config}
order_by = ''
if isinstance(config.get('order_by'), str):
if config['order_by'] != 'date':
order_by = '{0}/1'.format(config['order_by'])
category = 'all'
if isinstance(config.get('category'), str):
category = '{0}'.format(config['category'])
entries = set()
for search_string in entry.get('search_strings', [entry['title']]):
# No special characters - use dashes instead of %20
cleaned_search_string = clean_symbols(search_string).replace(
' ', '-')
query = 'search/{0}/{1}/{2}'.format(
category, cleaned_search_string.encode('utf8'), order_by)
logger.debug(
'Using search: {}; category: {}; ordering: {}',
cleaned_search_string,
category,
order_by or 'default',
)
try:
page = task.requests.get(self.base_url + query)
logger.debug('requesting: {}', page.url)
except RequestException as e:
logger.error('Limetorrents request failed: {}', e)
continue
soup = get_soup(page.content)
if soup.find('a', attrs={'class': 'csprite_dl14'}) is not None:
for link in soup.findAll('a', attrs={'class': 'csprite_dl14'}):
row = link.find_parent('tr')
info_url = str(link.get('href'))
# Get the title from the URL as it's complete versus the actual Title text which gets cut off
title = str(link.next_sibling.get('href'))
title = title[:title.rfind('-torrent')].replace('-', ' ')
title = title[1:]
data = row.findAll('td', attrs={'class': 'tdnormal'})
size = str(data[1].text).replace(',', '')
seeds = int(
row.find('td', attrs={
'class': 'tdseed'
}).text.replace(',', ''))
leeches = int(
row.find('td', attrs={
'class': 'tdleech'
}).text.replace(',', ''))
size = parse_filesize(size)
e = Entry()
e['url'] = info_url
e['title'] = title
e['torrent_seeds'] = seeds
e['torrent_leeches'] = leeches
e['torrent_availability'] = torrent_availability(
e['torrent_seeds'], e['torrent_leeches'])
e['content_size'] = size
entries.add(e)
return entries
def post(self, session=None):
""" Execute task and stream results """
data = request.json
for task in data.get('tasks'):
if task.lower() not in [
t.lower() for t in self.manager.user_config.get('tasks', {}).keys()
]:
raise NotFoundError('task %s does not exist' % task)
queue = ExecuteLog()
output = queue if data.get('loglevel') else None
stream = (
True
if any(
arg[0] in ['progress', 'summary', 'loglevel', 'entry_dump']
for arg in data.items()
if arg[1]
)
else False
)
loglevel = data.pop('loglevel', None)
# This emulates the CLI command of using `--now` and `no-cache`
options = {
'interval_ignore': data.pop('now', None),
'nocache': data.pop('no_cache', None),
'allow_manual': True,
}
for option, value in data.items():
options[option] = value
if data.get('inject'):
entries = []
for item in data.get('inject'):
entry = Entry()
entry['url'] = item['url']
if not item.get('title'):
try:
value, params = cgi.parse_header(
requests.head(item['url']).headers['Content-Disposition']
)
entry['title'] = params['filename']
except KeyError:
raise BadRequest(
'No title given, and couldn\'t get one from the URL\'s HTTP response'
)
else:
entry['title'] = item.get('title')
if item.get('force'):
entry['immortal'] = True
if item.get('accept'):
entry.accept(reason='accepted by API inject')
if item.get('fields'):
for key, value in item.get('fields').items():
entry[key] = value
entries.append(entry)
options['inject'] = entries
executed_tasks = self.manager.execute(options=options, output=output, loglevel=loglevel)
tasks_queued = []
for task_id, task_name, task_event in executed_tasks:
tasks_queued.append({'id': task_id, 'name': task_name, 'event': task_event})
_streams[task_id] = {'queue': queue, 'last_update': datetime.now(), 'args': data}
if not stream:
return jsonify(
{'tasks': [{'id': task['id'], 'name': task['name']} for task in tasks_queued]}
)
def stream_response():
# First return the tasks to execute
yield '{"stream": ['
yield json.dumps(
{'tasks': [{'id': task['id'], 'name': task['name']} for task in tasks_queued]}
) + ',\n'
while True:
try:
yield queue.get(timeout=1) + ',\n'
continue
except Empty:
pass
if queue.empty() and all([task['event'].is_set() for task in tasks_queued]):
for task in tasks_queued:
del _streams[task['id']]
break
yield '{}]}'
return Response(stream_response(), mimetype='text/event-stream')
def entries_from_search(self,
name,
url=None,
comparator=StringComparator(cutoff=0.9)):
"""Parses torrent download url from search results"""
comparator.set_seq1(name)
name = comparator.search_string()
if not url:
url = 'http://www.newtorrents.info/search/%s' % urllib.quote(
name.encode('utf-8'), safe=':/~?=&%')
log.debug('search url: %s' % url)
html = urlopener(url, log).read()
# fix </SCR'+'IPT> so that BS does not crash
# TODO: should use beautifulsoup massage
html = re.sub(r'(</SCR.*?)...(.*?IPT>)', r'\1\2', html)
soup = get_soup(html)
# saving torrents in dict
torrents = []
for link in soup.find_all('a', attrs={'href': re.compile('down.php')}):
torrent_url = 'http://www.newtorrents.info%s' % link.get('href')
release_name = link.parent.next.get('title')
# quick dirty hack
seed = link.find_next('td', attrs={
'class': re.compile('s')
}).renderContents()
if seed == 'n/a':
seed = 0
else:
try:
seed = int(seed)
except ValueError:
log.warning(
'Error converting seed value (%s) from newtorrents to integer.'
% seed)
seed = 0
#TODO: also parse content_size and peers from results
if comparator.matches(release_name):
torrents.append(
Entry(title=release_name,
url=torrent_url,
torrent_seeds=seed,
search_ratio=comparator.ratio(),
search_sort=torrent_availability(seed, 0)))
else:
log.debug('rejecting search result: %s !~ %s' %
(release_name, name))
# sort with seed number Reverse order
torrents.sort(reverse=True, key=lambda x: x.get('search_sort', 0))
# choose the torrent
if not torrents:
dashindex = name.rfind('-')
if dashindex != -1:
return self.entries_from_search(name[:dashindex],
comparator=comparator)
else:
raise PluginWarning('No matches for %s' % name,
log,
log_once=True)
else:
if len(torrents) == 1:
log.debug('found only one matching search result.')
else:
log.debug(
'search result contains multiple matches, sorted %s by most seeders'
% torrents)
return torrents
def search(self, task, entry, config=None):
"""
Search for name from torrent411.
"""
url_base = 'http://www.t411.in'
if not isinstance(config, dict):
config = {}
category = config.get('category')
if category in list(CATEGORIES):
category = CATEGORIES[category]
sub_categories = config.get('sub_category')
if not isinstance(sub_categories, list):
sub_categories = [sub_categories]
filter_url = ''
if isinstance(category, int):
filter_url = '&cat=%s' % str(category)
if sub_categories[0] is not None:
sub_categories = [SUB_CATEGORIES[c] for c in sub_categories]
filter_url = filter_url + '&' + '&'.join([
urllib.quote_plus('term[%s][]' % c[0]).encode('utf-8') +
'=' + str(c[1]) for c in sub_categories
])
entries = set()
for search_string in entry.get('search_strings', [entry['title']]):
query = normalize_unicode(search_string)
url_search = ('/torrents/search/?search=%40name+' +
urllib.quote_plus(query.encode('utf-8')) +
filter_url)
opener = urllib2.build_opener()
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
response = opener.open(url_base + url_search)
data = response.read()
soup = get_soup(data)
tb = soup.find("table", class_="results")
if not tb:
continue
for tr in tb.findAll('tr')[1:][:-1]:
entry = Entry()
nfo_link_res = re.search('torrents/nfo/\?id=(\d+)', str(tr))
if nfo_link_res is not None:
tid = nfo_link_res.group(1)
title_res = re.search(
'<a href=\"//www.t411.in/torrents/([-A-Za-z0-9+&@#/%|?=~_|!:,.;]+)\" title="([^"]*)">',
str(tr))
if title_res is not None:
entry['title'] = title_res.group(2).decode('utf-8')
size = tr('td')[5].contents[0]
entry[
'url'] = 'http://www.t411.in/torrents/download/?id=%s' % tid
entry['torrent_seeds'] = tr('td')[7].contents[0]
entry['torrent_leeches'] = tr('td')[8].contents[0]
entry['search_sort'] = torrent_availability(
entry['torrent_seeds'], entry['torrent_leeches'])
size = re.search('([\.\d]+) ([GMK]?)B', size)
if size:
if size.group(2) == 'G':
entry['content_size'] = int(
float(size.group(1)) * 1000**3 / 1024**2)
elif size.group(2) == 'M':
entry['content_size'] = int(
float(size.group(1)) * 1000**2 / 1024**2)
elif size.group(2) == 'K':
entry['content_size'] = int(
float(size.group(1)) * 1000 / 1024**2)
else:
entry['content_size'] = int(
float(size.group(1)) / 1024**2)
auth_handler = t411Auth(config['username'], config['password'])
entry['download_auth'] = auth_handler
entries.add(entry)
return sorted(entries,
reverse=True,
key=lambda x: x.get('search_sort'))
def test_next_series_episodes_with_uncompleted_season(self, execute_task):
execute_task(
'inject_series',
options={'inject': [Entry(title='My Show 1 S02 480p', url='')]})
task = execute_task('test_next_series_episodes_with_unaccepted_season')
assert task.find_entry(title='My Show 1 S02E01')
def search(self, task, entry, config):
"""
Search for entries on RarBG
"""
categories = config.get('category', 'all')
# Ensure categories a list
if not isinstance(categories, list):
categories = [categories]
# Convert named category to its respective category id number
categories = [c if isinstance(c, int) else CATEGORIES[c] for c in categories]
category_url_fragment = ';'.join(str(c) for c in categories)
entries = set()
token = self.get_token()
if not token:
log.error('Could not retrieve token. Abandoning search.')
return entries
params = {'mode': 'search', 'token': token, 'ranked': int(config['ranked']),
'min_seeders': config['min_seeders'], 'min_leechers': config['min_leechers'],
'sort': config['sorted_by'], 'category': category_url_fragment, 'format': 'json_extended',
'app_id': 'flexget'}
for search_string in entry.get('search_strings', [entry['title']]):
params.pop('search_string', None)
params.pop('search_imdb', None)
params.pop('search_tvdb', None)
if entry.get('movie_name'):
params['search_imdb'] = entry.get('imdb_id')
else:
query = normalize_scene(search_string)
query_url_fragment = query.encode('utf8')
params['search_string'] = query_url_fragment
if config['use_tvdb']:
plugin.get_plugin_by_name('thetvdb_lookup').instance.lazy_series_lookup(entry)
params['search_tvdb'] = entry.get('tvdb_id')
log.debug('Using tvdb id %s', entry.get('tvdb_id'))
try:
page = requests.get(self.base_url, params=params)
log.debug('requesting: %s', page.url)
except RequestException as e:
log.error('RarBG request failed: %s' % e.args[0])
continue
r = page.json()
# error code 20 just means no results were found
if r.get('error_code') == 20:
searched_string = params.get('search_string') or 'imdb={0}'.format(params.get('search_imdb')) or \
'tvdb={0}'.format(params.get('tvdb_id'))
log.debug('No results found for %s', searched_string)
continue
elif r.get('error'):
log.error('Error code %s: %s', r.get('error_code'), r.get('error'))
continue
else:
for result in r.get('torrent_results'):
e = Entry()
e['title'] = result.get('title')
e['url'] = result.get('download')
e['torrent_seeds'] = int(result.get('seeders'))
e['torrent_leeches'] = int(result.get('leechers'))
e['content_size'] = int(result.get('size')) / 1024 / 1024
episode_info = result.get('episode_info')
if episode_info:
e['imdb_id'] = episode_info.get('imdb')
e['tvdb_id'] = episode_info.get('tvdb')
e['tvrage_id'] = episode_info.get('tvrage')
entries.add(e)
return entries
def on_task_input(self, task, config):
# Let details plugin know that it is ok if this task doesn't produce any entries
task.no_entries_ok = True
filename = os.path.expanduser(config['file'])
encoding = config.get('encoding', 'utf-8')
with Session() as session:
db_pos = (session.query(TailPosition).filter(
TailPosition.task == task.name).filter(
TailPosition.filename == filename).first())
if db_pos:
last_pos = db_pos.position
else:
last_pos = 0
with open(filename, 'r', encoding=encoding,
errors='replace') as file:
if task.options.tail_reset == filename or task.options.tail_reset == task.name:
if last_pos == 0:
logger.info('Task {} tail position is already zero',
task.name)
else:
logger.info('Task {} tail position ({}) reset to zero',
task.name, last_pos)
last_pos = 0
if os.path.getsize(filename) < last_pos:
logger.info(
'File size is smaller than in previous execution, resetting to beginning of the file'
)
last_pos = 0
file.seek(last_pos)
logger.debug('continuing from last position {}', last_pos)
entry_config = config.get('entry')
format_config = config.get('format', {})
# keep track what fields have been found
used = {}
entries = []
entry = Entry()
# now parse text
for line in file:
if not line:
break
for field, regexp in entry_config.items():
# log.debug('search field: %s regexp: %s' % (field, regexp))
match = re.search(regexp, line)
if match:
# check if used field detected, in such case start with new entry
if field in used:
if entry.isvalid():
logger.info(
'Found field {} again before entry was completed. Adding current incomplete, but valid entry and moving to next.',
field,
)
self.format_entry(entry, format_config)
entries.append(entry)
else:
logger.info(
'Invalid data, entry field {} is already found once. Ignoring entry.',
field,
)
# start new entry
entry = Entry()
used = {}
# add field to entry
entry[field] = match.group(1)
used[field] = True
logger.debug('found field: {} value: {}', field,
entry[field])
# if all fields have been found
if len(used) == len(entry_config):
# check that entry has at least title and url
if not entry.isvalid():
logger.info(
'Invalid data, constructed entry is missing mandatory fields (title or url)'
)
else:
self.format_entry(entry, format_config)
entries.append(entry)
logger.debug('Added entry {}', entry)
# start new entry
entry = Entry()
used = {}
last_pos = file.tell()
if db_pos:
db_pos.position = last_pos
else:
session.add(
TailPosition(task=task.name,
filename=filename,
position=last_pos))
return entries
def get(self, entry):
with Session() as session:
match = self._entry_query(session=session,
entry=entry,
approved=True)
return Entry(match.entry) if match else None
def test_encoding(self):
e = Entry('title', 'url')
with pytest.raises(EntryUnicodeError):
e['invalid'] = b'\x8e'
def getter(self):
return Entry(json.loads(getattr(self, name), decode_datetime=True))
def search(self, task, entry, config):
"""
Search for entries on AwesomeHD
"""
# need lxml to parse xml
try:
import lxml # noqa
except ImportError as e:
log.debug('Error importing lxml: %s', e)
raise plugin.DependencyError(
'awesomehd', 'lxml',
'lxml module required. ImportError: %s' % e)
config = self.prepare_config(config)
# set a domain limit, but allow the user to overwrite it
if 'awesome-hd.me' not in task.requests.domain_limiters:
task.requests.add_domain_limiter(
TimedLimiter('awesome-hd.me', '5 seconds'))
entries = set()
# Can only search for imdb
if not entry.get('imdb_id'):
log.debug('Skipping entry %s because of missing imdb id',
entry['title'])
return entries
# Standard search params
params = {
'passkey': config['passkey'],
'internal': int(config['only_internal']),
'action': 'imdbsearch',
'imdb': entry['imdb_id']
}
try:
response = task.requests.get(self.base_url + 'searchapi.php',
params=params).content
except RequestException as e:
log.error('Failed to search for imdb id %s: %s', entry['imdb_id'],
e)
return entries
try:
soup = get_soup(response, 'xml')
if soup.find('error'):
log.error(soup.find('error').get_text())
return entries
except Exception as e:
log.error('Failed to parse xml result for imdb id %s: %s',
entry['imdb_id'], e)
return entries
authkey = soup.find('authkey').get_text()
for result in soup.find_all('torrent'):
# skip audio releases for now
if not result.find('resolution').get_text():
log.debug('Skipping audio release')
continue
e = Entry()
e['imdb_id'] = result.find('imdb').get_text()
e['torrent_id'] = int(result.find('id').get_text())
e['uploaded_at'] = dateutil_parse(result.find('time').get_text())
e['content_size'] = parse_filesize('{} b'.format(
result.find('size').get_text()))
e['torrent_snatches'] = int(result.find('snatched').get_text())
e['torrent_seeds'] = int(result.find('seeders').get_text())
e['torrent_leeches'] = int(result.find('leechers').get_text())
e['release_group'] = result.find('releasegroup').get_text()
e['freeleech_percent'] = int(
(1 - float(result.find('freeleech').get_text())) * 100)
e['encode_status'] = result.find('encodestatus').get_text()
e['subtitles'] = result.find('subtitles').get_text().split(', ')
e['url'] = self.base_url + 'torrents.php?action=download&id={}&authkey={}&torrent_pass={}'\
.format(e['torrent_id'], authkey, config['passkey'])
# Generate a somewhat sensible title
audio = result.find('audioformat').get_text().replace(
'AC-3', 'AC3') # normalize a bit
source = result.find('media').get_text()
encoder = result.find('encoding').get_text()
# calling a WEB-DL a remux is pretty redundant
if 'WEB' in source.upper():
encoder = re.sub('REMUX', '', encoder,
flags=re.IGNORECASE).strip()
e['title'] = '{movie_name} {year} {resolution} {source} {audio} {encoder}-{release_group}'\
.format(movie_name=result.find('name').get_text(), year=result.find('year').get_text(),
resolution=result.find('resolution').get_text(), source=source, audio=audio, encoder=encoder,
release_group=e['release_group'])
entries.add(e)
return entries
def on_task_input(self, task, config):
if not task.requests.cookies:
username = config['username']
password = config['password']
log.debug("Logging in to %s ..." % URL)
params = {
'username': username,
'password': password,
'action': 'Login'
}
try:
loginsrc = task.requests.post(URL + 'login.php', data=params)
if 'login' in loginsrc.url:
raise plugin.PluginWarning(
('Login to myepisodes.com failed, please check '
'your account data or see if the site is down.'),
log,
)
except RequestException as e:
raise plugin.PluginError("Error logging in to myepisodes: %s" %
e)
page = task.requests.get(URL + "myshows/manage/").content
try:
soup = get_soup(page)
except Exception as e:
raise plugin.PluginError(
"Unable to parse myepisodes.com page: %s" % (e, ))
entries = []
def show_list(select_id):
return soup.find('select', {'id': select_id}).findAll('option')
options = show_list('shows')
if config['include_ignored']:
options = chain(options, show_list('ignored_shows'))
for option in options:
name = option.text
if config.get('strip_dates'):
# Remove year from end of name if present
name = re.sub(r'\s+\(\d{4}\)$', '', name)
showid = option.get('value')
url = '%sviews.php?type=epsbyshow&showid=%s' % (URL, showid)
entry = Entry()
entry['title'] = name
entry['url'] = url
entry['series_name'] = name
entry['myepisodes_id'] = showid
if entry.isvalid():
entries.append(entry)
else:
log.debug('Invalid entry created? %s' % entry)
if not entries:
log.warning(
"No shows found on myepisodes.com list. Maybe you need to add some first?"
)
return entries
def _parse_tiles(self, task, config, tiles, series_info):
max_age = config.get('max_episode_age_days')
download_premium = config.get('download_premium')
entries = []
if tiles is not None:
for tile in tiles:
# there is only one list_item per tile
for list_item in get_soup(tile).findAll(
'div', class_='npo-asset-tile-container'):
episode_id = list_item['data-id']
premium = 'npo-premium-content' in list_item['class']
log.debug('Parsing episode: %s', episode_id)
url = list_item.find('a')['href']
# Check if the URL found to the episode matches the expected pattern
if len(url.split('/')) != 6:
log.verbose(
'Skipping %s, the URL has an unexpected pattern: %s',
episode_id, url)
continue # something is wrong; skip this episode
episode_name = list_item.find('h2')
if episode_name:
title = '{} ({})'.format(
next(episode_name.stripped_strings), episode_id)
else:
title = '{}'.format(episode_id)
timer = '0'
timerdiv = list_item.find('div',
class_='npo-asset-tile-timer')
if timerdiv: # some entries are missing a running time
timer = next(timerdiv.stripped_strings)
remove_url = list_item.find('div',
class_='npo-asset-tile-delete')
if premium and not download_premium:
log.debug(
'Skipping %s, no longer available without premium',
title)
continue
entry_date = url.split('/')[-2]
entry_date = self._parse_date(entry_date)
if max_age >= 0 and date.today() - entry_date > timedelta(
days=max_age):
log.debug('Skipping %s, aired on %s', title,
entry_date)
continue
e = Entry(series_info)
e['url'] = url
e['title'] = title
e['series_name'] = series_info['npo_name']
e['series_name_plain'] = self._convert_plain(
series_info['npo_name'])
e['series_date'] = entry_date
e['series_id_type'] = 'date'
e['npo_id'] = episode_id
e['npo_premium'] = premium
e['npo_runtime'] = timer.strip('min').strip()
e['language'] = series_info['npo_language']
if remove_url and remove_url['data-link']:
e['remove_url'] = remove_url['data-link']
if config.get('remove_accepted'):
e.on_complete(self.entry_complete, task=task)
entries.append(e)
return entries
def wrapped_func(*args, **kwargs):
# get task from method parameters
task = args[1]
# detect api version
api_ver = 1
if len(args) == 3:
api_ver = 2
if api_ver == 1:
# get name for a cache from tasks configuration
if not self.name in task.config:
raise Exception(
'@cache config name %s is not configured in task %s' %
(self.name, task.name))
hash = config_hash(task.config[self.name])
else:
hash = config_hash(args[2])
log.trace('self.name: %s' % self.name)
log.trace('hash: %s' % hash)
cache_name = self.name + '_' + hash
log.debug('cache name: %s (has: %s)' %
(cache_name, ', '.join(self.cache.keys())))
if cache_name in self.cache:
# return from the cache
log.trace('cache hit')
entries = []
for entry in self.cache[cache_name]:
fresh = copy.deepcopy(entry)
entries.append(fresh)
if entries:
log.verbose('Restored %s entries from cache' %
len(entries))
return entries
else:
if self.persist and not task.manager.options.nocache:
# Check database cache
db_cache = task.session.query(InputCache).filter(InputCache.name == self.name).\
filter(InputCache.hash == hash).\
filter(InputCache.added > datetime.now() - self.persist).\
first()
if db_cache:
entries = [Entry(e.entry) for e in db_cache.entries]
log.verbose('Restored %s entries from db cache' %
len(entries))
# Store to in memory cache
self.cache[cache_name] = copy.deepcopy(entries)
return entries
# Nothing was restored from db or memory cache, run the function
log.trace('cache miss')
# call input event
try:
response = func(*args, **kwargs)
except PluginError, e:
# If there was an error producing entries, but we have valid entries in the db cache, return those.
if self.persist and not task.manager.options.nocache:
db_cache = task.session.query(InputCache).filter(InputCache.name == self.name).\
filter(InputCache.hash == hash).first()
if db_cache and db_cache.entries:
log.error(
'There was an error during %s input (%s), using cache instead.'
% (self.name, e))
entries = [
Entry(e.entry) for e in db_cache.entries
]
log.verbose('Restored %s entries from db cache' %
len(entries))
# Store to in memory cache
self.cache[cache_name] = copy.deepcopy(entries)
return entries
# If there was nothing in the db cache, re-raise the error.
raise
if api_ver == 1:
response = task.entries
if not isinstance(response, list):
log.warning(
'Input %s did not return a list, cannot cache.' %
self.name)
return response
# store results to cache
log.debug('storing to cache %s %s entries' %
(cache_name, len(response)))
try:
self.cache[cache_name] = copy.deepcopy(response)
except TypeError:
# might be caused because of backlog restoring some idiotic stuff, so not neccessarily a bug
log.critical(
'Unable to save task content into cache, if problem persists longer than a day please report this as a bug'
)
if self.persist:
# Store to database
log.debug('Storing cache %s to database.' % cache_name)
db_cache = task.session.query(InputCache).filter(InputCache.name == self.name).\
filter(InputCache.hash == hash).first()
if not db_cache:
db_cache = InputCache(name=self.name, hash=hash)
db_cache.entries = [
InputCacheEntry(entry=e) for e in response
]
db_cache.added = datetime.now()
task.session.merge(db_cache)
return response
def create_entries(self, page_url, soup, config):
queue = []
duplicates = {}
duplicate_limit = 4
def title_exists(title):
"""Helper method. Return True if title is already added to entries"""
for entry in queue:
if entry['title'] == title:
return True
for link in soup.find_all('a'):
# not a valid link
if not link.has_attr('href'):
continue
# no content in the link
if not link.contents and not config.get('allow_empty_links',
False):
continue
url = link['href']
# fix broken urls
if url.startswith('//'):
url = 'http:' + url
elif not url.startswith('http://') or not url.startswith(
'https://'):
url = parse.urljoin(page_url, url)
log_link = url
log_link = log_link.replace('\n', '')
log_link = log_link.replace('\r', '')
# get only links matching regexp
regexps = config.get('links_re', None)
if regexps:
accept = False
for regexp in regexps:
if re.search(regexp, url):
accept = True
if not accept:
log.debug('url does not match any "links_re": %s' % url)
continue
title_from = config.get('title_from', 'auto')
if title_from == 'url':
title = self._title_from_url(url)
log.debug('title from url: %s' % title)
elif title_from == 'title':
if not link.has_attr('title'):
log.warning(
'Link `%s` doesn\'t have title attribute, ignored.' %
log_link)
continue
title = link['title']
log.debug('title from title: %s' % title)
elif title_from == 'auto':
title = self._title_from_link(link, log_link)
if title is None:
continue
# automatic mode, check if title is unique
# if there are too many duplicate titles, switch to title_from: url
if title_exists(title):
# ignore index links as a counter
if 'index' in title and len(title) < 10:
log.debug('ignored index title %s' % title)
continue
duplicates.setdefault(title, 0)
duplicates[title] += 1
if duplicates[title] > duplicate_limit:
# if from url seems to be bad choice use title
from_url = self._title_from_url(url)
switch_to = 'url'
for ext in ('.html', '.php'):
if from_url.endswith(ext):
switch_to = 'title'
log.info(
'Link names seem to be useless, auto-configuring \'title_from: %s\'. '
'This may not work well, you might need to configure it yourself.'
% switch_to)
config['title_from'] = switch_to
# start from the beginning ...
return self.create_entries(page_url, soup, config)
elif title_from == 'link' or title_from == 'contents':
# link from link name
title = self._title_from_link(link, log_link)
if title is None:
continue
log.debug('title from link: %s' % title)
else:
raise plugin.PluginError('Unknown title_from value %s' %
title_from)
if not title:
log.warning('title could not be determined for link %s' %
log_link)
continue
# strip unicode white spaces
title = title.replace(u'\u200B', u'').strip()
# in case the title contains xxxxxxx.torrent - foooo.torrent clean it a bit (get up to first .torrent)
# TODO: hack
if title.lower().find('.torrent') > 0:
title = title[:title.lower().find('.torrent')]
if title_exists(title):
# title link should be unique, add CRC32 to end if it's not
hash = zlib.crc32(url.encode("utf-8"))
crc32 = '%08X' % (hash & 0xFFFFFFFF)
title = '%s [%s]' % (title, crc32)
# truly duplicate, title + url crc already exists in queue
if title_exists(title):
continue
log.debug('uniqued title to %s' % title)
entry = Entry()
entry['url'] = url
entry['title'] = title
if 'username' in config and 'password' in config:
entry['download_auth'] = (config['username'],
config['password'])
queue.append(entry)
# add from queue to task
return queue
def search(self, task, entry, config=None):
"""
Search for name from iptorrents
"""
categories = config.get('category', 'All')
# Make sure categories is a list
if not isinstance(categories, list):
categories = [categories]
# If there are any text categories, turn them into their id number
categories = [
c if isinstance(c, int) else CATEGORIES[c] for c in categories
]
category_params = {str(c): '' for c in categories if str(c)}
entries = set()
for search_string in entry.get('search_strings', [entry['title']]):
search_params = {
key: value
for (key, value) in category_params.items()
}
query = normalize_unicode(search_string)
search_params.update({'q': query, 'qf': ''})
logger.debug('searching with params: {}', search_params)
if config.get('free'):
req = requests.get(FREE_SEARCH_URL,
params=search_params,
cookies={
'uid': str(config['uid']),
'pass': config['password']
})
else:
req = requests.get(SEARCH_URL,
params=search_params,
cookies={
'uid': str(config['uid']),
'pass': config['password']
})
logger.debug('full search URL: {}', req.url)
if '/u/' + str(config['uid']) not in req.text:
raise plugin.PluginError(
"Invalid cookies (user not logged in)...")
soup = get_soup(req.content, parser="html.parser")
torrents = soup.find('table', {'id': 'torrents'})
results = torrents.findAll('tr')
for torrent in results:
if torrent.th and 'ac' in torrent.th.get('class'):
# Header column
continue
if torrent.find('td', {'colspan': '99'}):
logger.debug('No results found for search {}',
search_string)
break
entry = Entry()
link = torrent.find('a', href=re.compile('download'))['href']
entry[
'url'] = f"{BASE_URL}{link}?torrent_pass={config.get('rss_key')}"
entry['title'] = torrent.find('a',
href=re.compile('details')).text
seeders = torrent.findNext('td', {
'class': 'ac t_seeders'
}).text
leechers = torrent.findNext('td', {
'class': 'ac t_leechers'
}).text
entry['torrent_seeds'] = int(seeders)
entry['torrent_leeches'] = int(leechers)
entry['torrent_availability'] = torrent_availability(
entry['torrent_seeds'], entry['torrent_leeches'])
size = torrent.findNext(
text=re.compile(r'^([\.\d]+) ([GMK]?)B$'))
size = re.search(r'^([\.\d]+) ([GMK]?)B$', size)
entry['content_size'] = parse_filesize(size.group(0))
logger.debug('Found entry {}', entry)
entries.add(entry)
return entries
def search(self, task, entry, config=None):
if not config:
return []
elif config == 'fail':
raise plugin.PluginError('search plugin failure')
return [Entry(entry)]
def inject_series(self, execute_task, release_name):
execute_task('inject_series',
options={
'inject': [Entry(title=release_name, url='')],
'disable_tracking': True
})
def on_task_input(self, task, config):
if config.get('account') and not config.get('username'):
config['username'] = '******'
session = get_session(account=config.get('account'))
endpoint = ['users', config['username']]
if config['list'] in ['collection', 'watchlist', 'watched']:
endpoint += (config['list'], config['type'])
else:
endpoint += ('lists', make_list_slug(config['list']), 'items')
log.verbose('Retrieving `%s` list `%s`' %
(config['type'], config['list']))
try:
result = session.get(get_api_url(endpoint))
try:
data = result.json()
except ValueError:
log.debug('Could not decode json from response: %s',
result.text)
raise plugin.PluginError('Error getting list from trakt.')
except RequestException as e:
raise plugin.PluginError(
'Could not retrieve list from trakt (%s)' % e.args[0])
if not data:
log.warning('No data returned from trakt for %s list %s.' %
(config['type'], config['list']))
return
entries = []
list_type = (config['type']).rstrip('s')
for item in data:
# Collection and watched lists don't return 'type' along with the items (right now)
if 'type' in item and item['type'] != list_type:
log.debug(
'Skipping %s because it is not a %s' %
(item[item['type']].get('title', 'unknown'), list_type))
continue
if not item[list_type]['title']:
# There seems to be some bad shows sometimes in lists with no titles. Skip them.
log.warning(
'Item in trakt list does not appear to have a title, skipping.'
)
continue
entry = Entry()
if list_type == 'episode':
entry[
'url'] = 'http://trakt.tv/shows/%s/seasons/%s/episodes/%s' % (
item['show']['ids']['slug'], item['episode']['season'],
item['episode']['number'])
else:
entry['url'] = 'http://trakt.tv/%s/%s' % (
list_type, item[list_type]['ids'].get('slug'))
entry.update_using_map(field_maps[list_type], item)
if entry.isvalid():
if config.get('strip_dates'):
# Remove year from end of name if present
entry['title'] = re.sub(r'\s+\(\d{4}\)$', '',
entry['title'])
entries.append(entry)
else:
log.debug('Invalid entry created? %s' % entry)
return entries
def on_task_input(self, task, config):
config = self.prepare_config(config)
if not config['enabled']:
return
if not self.client:
self.client = self.create_rpc_client(config)
entries = []
# Hack/Workaround for http://flexget.com/ticket/2002
# TODO: Proper fix
if 'username' in config and 'password' in config:
self.client.http_handler.set_authentication(
self.client.url, config['username'], config['password']
)
session = self.client.get_session()
for torrent in self.client.get_torrents():
seed_ratio_ok, idle_limit_ok = self.check_seed_limits(torrent, session)
if config['only_complete'] and not (
seed_ratio_ok and idle_limit_ok and torrent.progress == 100
):
continue
entry = Entry(
title=torrent.name,
url='',
torrent_info_hash=torrent.hashString,
content_size=torrent.totalSize / (1024 * 1024),
)
# Location of torrent is only valid if transmission is on same machine as flexget
if config['host'] in ('localhost', '127.0.0.1'):
entry['location'] = torrent.torrentFile
entry['url'] = 'file://' + torrent.torrentFile
for attr in [
'id',
'comment',
'downloadDir',
'isFinished',
'isPrivate',
'ratio',
'status',
'date_active',
'date_added',
'date_done',
'date_started',
'priority',
'progress',
'secondsDownloading',
'secondsSeeding',
]:
try:
entry['transmission_' + attr] = getattr(torrent, attr)
except Exception:
log.debug(
'error when requesting transmissionrpc attribute %s', attr, exc_info=True
)
entry['transmission_trackers'] = [t['announce'] for t in torrent.trackers]
entry['transmission_seed_ratio_ok'] = seed_ratio_ok
entry['transmission_idle_limit_ok'] = idle_limit_ok
st_error_to_desc = {
0: 'OK',
1: 'tracker_warning',
2: 'tracker_error',
3: 'local_error'
}
entry['transmission_error_state'] = st_error_to_desc[torrent.error]
# Built in done_date doesn't work when user adds an already completed file to transmission
if torrent.progress == 100:
entry['transmission_date_done'] = datetime.fromtimestamp(
max(torrent.addedDate, torrent.doneDate)
)
entries.append(entry)
return entries
def on_task_input(self, task, config):
subtitle_list = SubtitleList(config)
recursion_depth = config['recursion_depth']
# A hack to not output certain files without deleting them from the list
temp_discarded_items = set()
for item in subtitle_list:
if not config['force_file_existence'] and not os.path.exists(
item['location']):
log.error('File %s does not exist. Skipping.',
item['location'])
temp_discarded_items.add(item)
continue
if not os.path.exists(item['location']):
log.error('File %s does not exist. Removing from list.',
item['location'])
subtitle_list.discard(item)
continue
if self._expired(item, config):
log.info(
'File %s has been in the list for %s. Removing from list.',
item['location'],
item['remove_after'] or config['remove_after'],
)
subtitle_list.discard(item)
continue
languages = set(item['subtitle_languages']) or set(
config.get('languages', []))
num_potential_files = 0
num_added_files = 0
if os.path.isdir(item['location']):
# recursion depth 1 is no recursion
max_depth = (
len(normalize_path(item['location']).split(os.sep)) +
recursion_depth - 1)
for root_dir, _, files in os.walk(item['location']):
current_depth = len(root_dir.split(os.sep))
if current_depth > max_depth:
break
for file in files:
if os.path.splitext(file)[1] not in VIDEO_EXTENSIONS:
log.debug('File %s is not a video file. Skipping',
file)
continue
num_potential_files += 1
file_path = normalize_path(os.path.join(
root_dir, file))
if not config[
'check_subtitles'] or not self.all_subtitles_exist(
file_path, languages):
subtitle_list.config['languages'] = languages
subtitle_list.add(
Entry(
title=os.path.splitext(
os.path.basename(file_path))[0],
url='file://' + file_path,
location=file_path,
))
num_added_files += 1
# delete the original dir if it contains any video files
if num_added_files or num_potential_files:
log.debug(
'Added %s file(s) from %s to subtitle list %s',
num_added_files,
item['location'],
config['list'],
)
subtitle_list.discard(item)
else:
log.debug('No files found in %s. Skipping.',
item['location'])
temp_discarded_items.add(item)
elif config['check_subtitles'] and self.all_subtitles_exist(
item['location'], languages):
subtitle_list.discard(item)
return list(set(subtitle_list) - temp_discarded_items)
def to_entry(self):
entry = Entry()
entry['title'] = entry['regexp'] = self.regexp
entry['url'] = 'mock://localhost/regexp_list/%d' % self.id
return entry
def search(self, task, entry, config):
"""
Search for entries on RarBG
"""
categories = config.get('category', 'all')
# Ensure categories a list
if not isinstance(categories, list):
categories = [categories]
# Convert named category to its respective category id number
categories = [
c if isinstance(c, int) else CATEGORIES[c] for c in categories
]
category_url_fragment = ';'.join(str(c) for c in categories)
entries = set()
params = {
'mode': 'search',
'ranked': int(config['ranked']),
'min_seeders': config['min_seeders'],
'min_leechers': config['min_leechers'],
'sort': config['sorted_by'],
'category': category_url_fragment,
'format': 'json_extended',
'app_id': 'flexget',
}
for search_string in entry.get('search_strings', [entry['title']]):
params.pop('search_string', None)
params.pop('search_imdb', None)
params.pop('search_tvdb', None)
if entry.get('movie_name') and entry.get('imdb_id'):
params['search_imdb'] = entry.get('imdb_id')
else:
query = normalize_scene(search_string)
query_url_fragment = query.encode('utf8')
params['search_string'] = query_url_fragment
if config['use_tvdb']:
plugin.get('thetvdb_lookup',
self).lazy_series_lookup(entry, 'en')
params['search_tvdb'] = entry.get('tvdb_id')
logger.debug('Using tvdb id {}', entry.get('tvdb_id'))
response = self.get(params=params)
if not response:
continue
# error code 10 and 20 just mean no results were found
if response.get('error_code') in [10, 20]:
searched_string = (params.get('search_string')
or 'imdb={0}'.format(
params.get('search_imdb'))
or 'tvdb={0}'.format(params.get('tvdb_id')))
logger.debug(
'No results found for {}. Message from rarbg: {}',
searched_string,
response.get('error'),
)
continue
elif response.get('error'):
logger.error('Error code {}: {}', response.get('error_code'),
response.get('error'))
continue
else:
for result in response.get('torrent_results'):
e = Entry()
e['title'] = result.get('title')
e['url'] = result.get('download')
e['torrent_seeds'] = int(result.get('seeders'))
e['torrent_leeches'] = int(result.get('leechers'))
e['content_size'] = int(result.get('size')) / 1024 / 1024
episode_info = result.get('episode_info')
if episode_info:
e['imdb_id'] = episode_info.get('imdb')
e['tvdb_id'] = episode_info.get('tvdb')
e['tvrage_id'] = episode_info.get('tvrage')
entries.add(e)
return entries
def on_task_input(self, task, config):
config = self.prepare_config(config)
urlconfig = {}
urlappend = "?"
entries = []
if config['unwatched_only'] and config[
'section'] != 'recentlyViewedShows' and config[
'section'] != 'all':
urlconfig['unwatched'] = '1'
if config['username'] and config['password']:
accesstoken = self.plex_get_accesstoken(config)
log.debug("Got accesstoken: %s" % accesstoken)
urlconfig['X-Plex-Token'] = accesstoken
for key in urlconfig:
urlappend += '%s=%s&' % (key, urlconfig[key])
if not self.plex_section_is_int(config['section']):
try:
path = "/library/sections/"
r = requests.get(
"http://%s:%d%s%s" %
(config['plexserver'], config['port'], path, urlappend))
except requests.RequestException as e:
raise plugin.PluginError('Error retrieving source: %s' % e)
dom = parseString(r.text.encode("utf-8"))
for node in dom.getElementsByTagName('Directory'):
if node.getAttribute('title') == config['section']:
config['section'] = int(node.getAttribute('key'))
if not self.plex_section_is_int(config['section']):
raise plugin.PluginError('Could not find section \'%s\'' %
config['section'])
log.debug("Fetching http://%s:%d/library/sections/%s/%s%s" %
(config['server'], config['port'], config['section'],
config['selection'], urlappend))
try:
path = "/library/sections/%s/%s" % (config['section'],
config['selection'])
r = requests.get(
"http://%s:%d%s%s" %
(config['plexserver'], config['port'], path, urlappend))
except requests.RequestException as e:
raise plugin.PluginError(
'There is no section with number %d. (%s)' %
(config['section'], e))
dom = parseString(r.text.encode("utf-8"))
plexsectionname = dom.getElementsByTagName(
'MediaContainer')[0].getAttribute('title1')
viewgroup = dom.getElementsByTagName('MediaContainer')[0].getAttribute(
'viewGroup')
log.debug("Plex section \"%s\" is a \"%s\" section" %
(plexsectionname, viewgroup))
if viewgroup != "movie" and viewgroup != "show" and viewgroup != "episode":
raise plugin.PluginError(
"Section is neither a movie nor tv show section!")
domroot = "Directory"
titletag = "title"
if viewgroup == "episode":
domroot = "Video"
titletag = "grandparentTitle"
thumbtag = "thumb"
arttag = "art"
seasoncovertag = "parentThumb"
covertag = "grandparentThumb"
elif viewgroup == "movie":
domroot = "Video"
titletag = "title"
arttag = "art"
seasoncovertag = "thumb"
covertag = "thumb"
if config['fetch'] == "thumb":
raise plugin.PluginError(
"Movie sections does not have any thumbnails to download!")
for node in dom.getElementsByTagName(domroot):
e = Entry()
e['plex_server'] = config['plexserver']
e['plex_port'] = config['port']
e['plex_section'] = config['section']
e['plex_section_name'] = plexsectionname
e['plex_episode_thumb'] = ''
title = node.getAttribute(titletag)
if config['strip_year']:
title = re.sub(r'^(.*)\(\d{4}\)(.*)', r'\1\2', title)
if config['strip_parens']:
title = re.sub(r'\(.*?\)', r'', title)
title = title.strip()
if config['strip_non_alpha']:
title = re.sub(r'[\(\)]', r'', title)
title = re.sub(r'&', r'And', title)
title = re.sub(r'[^A-Za-z0-9- \']', r'', title)
if config['lowercase_title']:
title = title.lower()
if viewgroup == "show":
e['title'] = title
e['url'] = 'NULL'
entries.append(e)
# show ends here.
continue
e['plex_art'] = "http://%s:%d%s%s" % (
config['server'], config['port'], node.getAttribute(arttag),
urlappend)
e['plex_cover'] = "http://%s:%d%s%s" % (
config['server'], config['port'], node.getAttribute(covertag),
urlappend)
e['plex_season_cover'] = "http://%s:%d%s%s" % (
config['server'], config['port'],
node.getAttribute(seasoncovertag), urlappend)
if viewgroup == "episode":
e['plex_thumb'] = "http://%s:%d%s%s" % (
config['server'], config['port'],
node.getAttribute('thumb'), urlappend)
season = int(node.getAttribute('parentIndex'))
if node.getAttribute('parentIndex') == node.getAttribute(
'year'):
season = node.getAttribute('originallyAvailableAt')
filenamemap = "%s_%s%s_%s_%s_%s.%s"
episode = ""
elif node.getAttribute('index'):
episode = int(node.getAttribute('index'))
filenamemap = "%s_%02dx%02d_%s_%s_%s.%s"
else:
log.debug(
"Could not get episode number for '%s' (Hint, ratingKey: %s)"
% (title, node.getAttribute('ratingKey')))
break
elif viewgroup == "movie":
filenamemap = "%s_%s_%s_%s.%s"
e['plex_duration'] = node.getAttribute('duration')
e['plex_summary'] = node.getAttribute('summary')
count = node.getAttribute('viewCount')
offset = node.getAttribute('viewOffset')
if count:
e['plex_status'] = "seen"
elif offset:
e['plex_status'] = "inprogress"
else:
e['plex_status'] = "unwatched"
for media in node.getElementsByTagName('Media'):
vcodec = media.getAttribute('videoCodec')
acodec = media.getAttribute('audioCodec')
if config['fetch'] == "file" or not config['fetch']:
container = media.getAttribute('container')
else:
container = "jpg"
resolution = media.getAttribute('videoResolution') + "p"
for part in media.getElementsByTagName('Part'):
if config['fetch'] == "file" or not config['fetch']:
key = part.getAttribute('key')
elif config['fetch'] == "art":
key = node.getAttribute(arttag)
elif config['fetch'] == "cover":
key = node.getAttribute(arttag)
elif config['fetch'] == "season_cover":
key = node.getAttribute(seasoncovertag)
elif config['fetch'] == "thumb":
key = node.getAttribute(thumbtag)
# key = part.getAttribute('key')
duration = part.getAttribute('duration')
if viewgroup == "show":
e['plex_title'] = episodetitle
elif viewgroup == "movie":
e['plex_title'] = title
if config['original_filename']:
filename, fileext = os.path.splitext(
basename(part.getAttribute('file')))
if config['fetch'] != 'file':
filename += ".jpg"
else:
filename = "%s.%s" % (filename, fileext)
else:
if viewgroup == "episode":
filename = filenamemap % (title.replace(
" ", "."), season, episode, resolution, vcodec,
acodec, container)
title = filename
elif viewgroup == "movie":
filename = filenamemap % (title.replace(
" ",
"."), resolution, vcodec, acodec, container)
e['plex_url'] = "http://%s:%d%s%s" % (
config['server'], config['port'], key, urlappend)
e['plex_path'] = key
e['url'] = "http://%s:%d%s%s" % (
config['server'], config['port'], key, urlappend)
e['plex_duration'] = duration
e['filename'] = filename
e['title'] = title
if key == "":
log.debug("Could not find anything in PMS to download. Next!")
else:
entries.append(e)
return entries
def items(self):
if self._items is None:
log.debug('fetching items from IMDB')
try:
r = self.session.get(
'http://www.imdb.com/list/export?list_id=%s&author_id=%s' %
(self.list_id, self.user_id),
cookies=self.cookies)
except RequestException as e:
raise PluginError(e.args[0])
lines = r.iter_lines(decode_unicode=True)
# Throw away first line with headers
next(lines)
self._items = []
for row in csv_reader(lines):
log.debug('parsing line from csv: %s', ', '.join(row))
if not len(row) == 16:
log.debug('no movie row detected, skipping. %s',
', '.join(row))
continue
entry = Entry({
'title':
'%s (%s)' %
(row[5], row[11]) if row[11] != '????' else '%s' % row[5],
'url':
row[15],
'imdb_id':
row[1],
'imdb_url':
row[15],
'imdb_list_position':
int(row[0]),
'imdb_list_created':
datetime.strptime(row[2], '%a %b %d %H:%M:%S %Y')
if row[2] else None,
'imdb_list_modified':
datetime.strptime(row[3], '%a %b %d %H:%M:%S %Y')
if row[3] else None,
'imdb_list_description':
row[4],
'imdb_name':
row[5],
'imdb_year':
int(row[11]) if row[11] != '????' else None,
'imdb_score':
float(row[9]) if row[9] else None,
'imdb_user_score':
float(row[8]) if row[8] else None,
'imdb_votes':
int(row[13]) if row[13] else None,
'imdb_genres':
[genre.strip() for genre in row[12].split(',')]
})
item_type = row[6].lower()
name = row[5]
year = int(row[11]) if row[11] != '????' else None
if item_type in MOVIE_TYPES:
entry['movie_name'] = name
entry['movie_year'] = year
elif item_type in SERIES_TYPES:
entry['series_name'] = name
entry['series_year'] = year
elif item_type in OTHER_TYPES:
entry['title'] = name
else:
log.verbose(
'Unknown IMDB type entry received: %s. Skipping',
item_type)
continue
self._items.append(entry)
return self._items
def search(self, task, entry, config):
"""CPASBIEN search plugin
Config example:
tv_search_cpasbien:
discover:
what:
- trakt_list:
username: xxxxxxx
api_key: xxxxxxx
series: watchlist
from:
- cpasbien:
category: "series-vostfr"
interval: 1 day
ignore_estimations: yes
Category is ONE of:
all
films
series
musique
films-french
1080p
720p
series-francaise
films-dvdrip
films-vostfr
series-vostfr
ebook
"""
base_url = 'http://www.cpasbien.io'
entries = set()
for search_string in entry.get('search_strings', [entry['title']]):
search_string = search_string.replace(' ', '-').lower()
search_string = search_string.replace('(', '')
search_string = search_string.replace(')', '')
query = normalize_unicode(search_string)
query_url_fragment = urllib.quote_plus(query.encode('utf-8'))
# http://www.cpasbien.pe/recherche/ncis.html
if config['category'] == 'all':
str_url = (base_url, 'recherche', query_url_fragment)
url = '/'.join(str_url)
else:
category_url_fragment = '%s' % config['category']
str_url = (base_url, 'recherche', category_url_fragment,
query_url_fragment)
url = '/'.join(str_url)
log.debug('search url: %s' % url + '.html')
# GET URL
f = task.requests.get(url + '.html').content
soup = get_soup(f)
if soup.findAll(text=re.compile(' 0 torrents')):
log.debug('search returned no results')
else:
nextpage = 0
while (nextpage >= 0):
if (nextpage > 0):
newurl = url + '/page-' + str(nextpage)
log.debug('-----> NEXT PAGE : %s' % newurl)
f1 = task.requests.get(newurl).content
soup = get_soup(f1)
for result in soup.findAll(
'div', attrs={'class': re.compile('ligne')}):
entry = Entry()
link = result.find(
'a', attrs={'href': re.compile('dl-torrent')})
entry['title'] = link.contents[0]
# REWRITE URL
page_link = link.get('href')
link_rewrite = page_link.split('/')
# get last value in array remove .html and replace by .torrent
endlink = link_rewrite[-1]
str_url = (base_url, '/telechargement/', endlink[:-5],
'.torrent')
entry['url'] = ''.join(str_url)
log.debug('Title: %s | DL LINK: %s' %
(entry['title'], entry['url']))
entry['torrent_seeds'] = (int(
result.find('span',
attrs={
'class': re.compile('seed')
}).text))
entry['torrent_leeches'] = (int(
result.find('div',
attrs={
'class': re.compile('down')
}).text))
sizefull = (result.find('div',
attrs={
'class': re.compile('poid')
}).text)
size = sizefull[:-3]
unit = sizefull[-2:]
if unit == 'GB':
entry['content_size'] = int(float(size) * 1024)
elif unit == 'MB':
entry['content_size'] = int(float(size))
elif unit == 'KB':
entry['content_size'] = int(float(size) / 1024)
if (entry['torrent_seeds'] > 0):
entries.add(entry)
else:
log.debug('0 SEED, not adding entry')
if soup.find(text=re.compile('Suiv')):
nextpage += 1
else:
nextpage = -1
return entries
def process(self):
imdb_lookup = plugin.get_plugin_by_name('imdb_lookup').instance
self.changes.sort()
udata = load_uoccin_data(self.folder)
for line in self.changes:
tmp = line.split('|')
typ = tmp[1]
tid = tmp[2]
fld = tmp[3]
val = tmp[4]
self.log.verbose(
'processing: type=%s, target=%s, field=%s, value=%s' %
(typ, tid, fld, val))
if typ == 'movie':
# default
mov = udata['movies'].setdefault(
tid, {
'name': 'N/A',
'watchlist': False,
'collected': False,
'watched': False
})
# movie title is unknown at this time
fake = Entry()
fake['url'] = 'http://www.imdb.com/title/' + tid
fake['imdb_id'] = tid
try:
imdb_lookup.lookup(fake)
mov['name'] = fake.get('imdb_name')
except plugin.PluginError:
self.log.warning(
'Unable to lookup movie %s from imdb, using raw name.'
% tid)
# setting
if fld == 'watchlist':
mov['watchlist'] = val == 'true'
elif fld == 'collected':
mov['collected'] = val == 'true'
elif fld == 'watched':
mov['watched'] = val == 'true'
elif fld == 'tags':
mov['tags'] = re.split(',\s*', val)
elif fld == 'subtitles':
mov['subtitles'] = re.split(',\s*', val)
elif fld == 'rating':
mov['rating'] = int(val)
# cleaning
if not (mov['watchlist'] or mov['collected']
or mov['watched']):
self.log.verbose('deleting unused section: movies\%s' %
tid)
udata['movies'].pop(tid)
elif typ == 'series':
tmp = tid.split('.')
sid = tmp[0]
sno = tmp[1] if len(tmp) > 2 else None
eno = tmp[2] if len(tmp) > 2 else None
# default
ser = udata['series'].setdefault(
sid, {
'name': 'N/A',
'watchlist': False,
'collected': {},
'watched': {}
})
# series name is unknown at this time
try:
series = lookup_series(tvdb_id=sid)
ser['name'] = series.name
except LookupError:
self.log.warning(
'Unable to lookup series %s from tvdb, using raw name.'
% sid)
# setting
if fld == 'watchlist':
ser['watchlist'] = val == 'true'
elif fld == 'tags':
ser['tags'] = re.split(',\s*', val)
elif fld == 'rating':
ser['rating'] = int(val)
elif sno is None or eno is None:
self.log.warning(
'invalid line "%s": season and episode numbers are required'
% line)
elif fld == 'collected':
season = ser['collected'].setdefault(sno, {})
if val == 'true':
season.setdefault(eno, [])
else:
if eno in season:
season.pop(eno)
if not season:
self.log.verbose(
'deleting unused section: series\%s\collected\%s'
% (sid, sno))
ser['collected'].pop(sno)
elif fld == 'subtitles':
ser['collected'].setdefault(sno, {})[eno] = re.split(
',\s*', val)
elif fld == 'watched':
season = ser['watched'].setdefault(sno, [])
if val == 'true':
season = ser['watched'][sno] = list(
set(season) | set([int(eno)]))
elif int(eno) in season:
season.remove(int(eno))
season.sort()
if not season:
self.log.debug(
'deleting unused section: series\%s\watched\%s' %
(sid, sno))
ser['watched'].pop(sno)
# cleaning
if not (ser['watchlist'] or ser['collected']
or ser['watched']):
self.log.debug('deleting unused section: series\%s' % sid)
udata['series'].pop(sid)
else:
self.log.warning('invalid element type "%s"' % typ)
# save the updated uoccin.json
ufile = os.path.join(self.folder, 'uoccin.json')
try:
text = json.dumps(udata,
sort_keys=True,
indent=4,
separators=(',', ': '))
with open(ufile, 'w') as f:
f.write(text)
except Exception as err:
self.log.debug('error writing %s: %s' % (ufile, err))
raise plugin.PluginError('error writing %s: %s' % (ufile, err))
def search(self, task, entry, config=None):
config = self.prepare_config(config)
if not session.cookies:
log.debug('Logging in to %s...' % URL)
params = {
'username': config['username'],
'password': config['password'],
'keeplogged': '1',
'login': '******'
}
session.post(URL + 'login.php', data=params)
cat = ''.join([
'&' + ('filter_cat[%s]' % id) + '=1' for id in config['category']
])
rls = 'release_type=' + config['type']
url_params = rls + cat
multip = config['gravity_multiplier']
entries = set()
for search_string in entry.get('search_strings', [entry['title']]):
srch = normalize_unicode(clean_title(search_string))
srch = '&searchstr=' + quote(srch.encode('utf8'))
url = URL + 'torrents.php?' + url_params + srch
log.debug('Fetching URL for `%s`: %s' % (search_string, url))
page = session.get(url).content
soup = get_soup(page)
for result in soup.findAll('tr', attrs={'class': 'torrent'}):
entry = Entry()
entry['title'] = result.find('span',
attrs={
'class': 'torrent_name_link'
}).text
entry['url'] = URL + result.find(
'a', href=re.compile(
'torrents\.php\?action=download')).get('href')
entry['torrent_seeds'], entry['torrent_leeches'] = [
r.text for r in result.findAll('td')[-2:]
]
entry['search_sort'] = torrent_availability(
entry['torrent_seeds'], entry['torrent_leeches']) * multip
size = result.findAll('td')[-4].text
size = re.search('(\d+(?:[.,]\d+)*)\s?([KMG]B)', size)
if size:
if size.group(2) == 'GB':
entry['content_size'] = int(
float(size.group(1).replace(',', '')) * 1000**3 /
1024**2)
elif size.group(2) == 'MB':
entry['content_size'] = int(
float(size.group(1).replace(',', '')) * 1000**2 /
1024**2)
elif size.group(2) == 'KB':
entry['content_size'] = int(
float(size.group(1).replace(',', '')) * 1000 /
1024**2)
else:
entry['content_size'] = int(
float(size.group(1).replace(',', '')) / 1024**2)
entries.add(entry)
return entries
