def items(self):
if self._items is None:
if self.config['list'] in ['collection', 'watched'
] and self.config['type'] == 'auto':
raise plugin.PluginError(
'`type` cannot be `auto` for %s list.' %
self.config['list'])
endpoint = self.get_list_endpoint()
log.verbose('Retrieving `%s` list `%s`', self.config['type'],
self.config['list'])
try:
result = self.session.get(db.get_api_url(endpoint))
try:
data = result.json()
except ValueError:
log.debug('Could not decode json from response: %s',
result.text)
raise plugin.PluginError('Error getting list from trakt.')
current_page = int(result.headers.get('X-Pagination-Page', 1))
current_page_count = int(
result.headers.get('X-Pagination-Page-Count', 1))
if current_page < current_page_count:
# Pagination, gotta get it all, but we'll limit it to 1000 per page
# but we'll have to start over from 0
data = []
limit = 1000
pagination_item_count = int(
result.headers.get('X-Pagination-Item-Count', 0))
number_of_pages = math.ceil(pagination_item_count / limit)
log.debug(
'Response is paginated. Number of items: %s, number of pages: %s',
pagination_item_count,
number_of_pages,
)
page = int(result.headers.get('X-Pagination-Page'))
while page <= number_of_pages:
paginated_result = self.session.get(
db.get_api_url(endpoint),
params={
'limit': limit,
'page': page
})
page += 1
try:
data.extend(paginated_result.json())
except ValueError:
log.debug(
'Could not decode json from response: %s',
paginated_result.text)
raise plugin.PluginError(
'Error getting list from trakt.')
except RequestException as e:
raise plugin.PluginError(
'Could not retrieve list from trakt (%s)' % e)
if not data:
log.warning(
'No data returned from trakt for %s list %s.',
self.config['type'],
self.config['list'],
)
return []
entries = []
list_type = (self.config['type']).rstrip('s')
for item in data:
if self.config['type'] == 'auto':
list_type = item['type']
# Collection and watched lists don't return 'type' along with the items (right now)
if 'type' in item and item['type'] != list_type:
log.debug(
'Skipping %s because it is not a %s',
item[item['type']].get('title', 'unknown'),
list_type,
)
continue
if list_type != 'episode' and not item[list_type]['title']:
# Skip shows/movies with no title
log.warning(
'Item in trakt list does not appear to have a title, skipping.'
)
continue
entry = Entry()
if list_type == 'episode':
entry[
'url'] = 'https://trakt.tv/shows/%s/seasons/%s/episodes/%s' % (
item['show']['ids']['slug'],
item['episode']['season'],
item['episode']['number'],
)
else:
entry['url'] = 'https://trakt.tv/%ss/%s' % (
list_type,
item[list_type]['ids'].get('slug'),
)
entry.update_using_map(field_maps[list_type], item)
# get movie name translation
language = self.config.get('language')
if list_type == 'movie' and language:
endpoint = [
'movies', entry['trakt_movie_id'], 'translations',
language
]
try:
result = self.session.get(db.get_api_url(endpoint))
try:
translation = result.json()
except ValueError:
raise plugin.PluginError(
'Error decoding movie translation from trakt: %s.'
% result.text)
except RequestException as e:
raise plugin.PluginError(
'Could not retrieve movie translation from trakt: %s'
% str(e))
if not translation:
log.warning(
'No translation data returned from trakt for movie %s.',
entry['title'])
else:
log.verbose(
'Found `%s` translation for movie `%s`: %s',
language,
entry['movie_name'],
translation[0]['title'],
)
entry['title'] = translation[0]['title']
if entry.get('movie_year'):
entry['title'] += ' (' + str(
entry['movie_year']) + ')'
entry['movie_name'] = translation[0]['title']
# Override the title if strip_dates is on. TODO: a better way?
if self.config.get('strip_dates'):
if list_type in ['show', 'movie']:
entry['title'] = item[list_type]['title']
elif list_type == 'episode':
entry[
'title'] = '{show[title]} S{episode[season]:02}E{episode[number]:02}'.format(
**item)
if item['episode']['title']:
entry['title'] += ' {episode[title]}'.format(
**item)
if entry.isvalid():
if self.config.get('strip_dates'):
# Remove year from end of name if present
entry['title'] = split_title_year(entry['title'])[0]
entries.append(entry)
else:
log.debug('Invalid entry created? %s', entry)
self._items = entries
return self._items
def on_task_filter(self, task, config):
logger.debug('check for enforcing')
# parse config
if isinstance(config, bool):
# configured a boolean false, disable plugin
if not config:
return
# configured a boolean true, disable timeframe
timeframe = None
else:
# parse time window
logger.debug('interval: {}', config)
try:
timeframe = parse_timedelta(config)
except ValueError:
raise plugin.PluginError('Invalid time format', logger)
# throws DependencyError if not present aborting task
imdb_lookup = plugin.get_plugin_by_name('imdb_lookup').instance
for entry in task.entries:
parser = plugin.get('parsing', self).parse_movie(entry['title'])
# if we have imdb_id already evaluated
if entry.get('imdb_id', None, eval_lazy=False) is None:
try:
# TODO: fix imdb_id_lookup, cumbersome that it returns None and or throws exception
# Also it's crappy name!
imdb_id = imdb_lookup.imdb_id_lookup(
movie_title=parser.name, movie_year=parser.year, raw_title=entry['title']
)
if imdb_id is None:
continue
entry['imdb_id'] = imdb_id
except plugin.PluginError as pe:
log_once(pe.value)
continue
quality = parser.quality.name
logger.debug('quality: {}', quality)
logger.debug('imdb_id: {}', entry['imdb_id'])
logger.debug('current proper count: {}', parser.proper_count)
proper_movie = (
task.session.query(ProperMovie)
.filter(ProperMovie.imdb_id == entry['imdb_id'])
.filter(ProperMovie.quality == quality)
.order_by(desc(ProperMovie.proper_count))
.first()
)
if not proper_movie:
logger.debug('no previous download recorded for {}', entry['imdb_id'])
continue
highest_proper_count = proper_movie.proper_count
logger.debug('highest_proper_count: {}', highest_proper_count)
accept_proper = False
if parser.proper_count > highest_proper_count:
logger.debug('proper detected: {} ', proper_movie)
if timeframe is None:
accept_proper = True
else:
expires = proper_movie.added + timeframe
logger.debug('propers timeframe: {}', timeframe)
logger.debug('added: {}', proper_movie.added)
logger.debug('propers ignore after: {}', str(expires))
if datetime.now() < expires:
accept_proper = True
else:
logger.verbose("Proper `{}` has past it's expiration time", entry['title'])
if accept_proper:
logger.info(
'Accepting proper version previously downloaded movie `{}`', entry['title']
)
# TODO: does this need to be called?
# fire_event('forget', entry['imdb_url'])
fire_event('forget', entry['imdb_id'])
entry.accept('proper version of previously downloaded movie')
def on_task_input(self, task, config):
if config.get('account') and not config.get('username'):
config['username'] = '******'
session = get_session(account=config.get('account'))
listed_series = {}
if config.get('list'):
args = ('users', config['username'])
if config['list'] in ['collection', 'watchlist', 'watched']:
args += (config['list'], 'shows')
else:
args += ('lists', make_list_slug(config['list']), 'items')
try:
data = session.get(get_api_url(args)).json()
except RequestException as e:
raise plugin.PluginError('Unable to get trakt list `%s`: %s' %
(config['list'], e))
if not data:
log.warning('The list "%s" is empty.' % config['list'])
return
for item in data:
if item.get('show'):
if not item['show']['title']:
# Seems we can get entries with a blank show title sometimes
log.warning(
'Found trakt list show with no series name.')
continue
trakt_id = item['show']['ids']['trakt']
listed_series[trakt_id] = {
'series_name':
'%s (%s)' %
(item['show']['title'], item['show']['year']),
'trakt_id':
trakt_id,
'tvdb_id':
item['show']['ids']['tvdb'],
'trakt_list':
config.get('list')
}
context = config['context']
if context == 'collected':
context = 'collection'
entries = []
for trakt_id, fields in listed_series.items():
url = get_api_url('shows', trakt_id, 'progress', context)
try:
data = session.get(url).json()
except RequestException as e:
raise plugin.PluginError(
'An error has occured looking up: Trakt_id: %s Error: %s' %
(trakt_id, e))
if config['position'] == 'next' and data.get('next_episode'):
# If the next episode is already in the trakt database, we'll get it here
eps = data['next_episode']['season']
epn = data['next_episode']['number']
else:
# If we need last ep, or next_episode was not provided, search for last ep
for seas in reversed(data['seasons']):
# Find the first season with collected/watched episodes
if seas['completed'] > 0:
eps = seas['number']
# Pick the highest collected/watched episode
epn = max(item['number'] for item in seas['episodes']
if item['completed'])
# If we are in next episode mode, we have to increment this number
if config['position'] == 'next':
if seas['completed'] >= seas['aired']:
# TODO: next_episode doesn't count unaired episodes right now, this will skip to next
# season too early when there are episodes left to air this season.
eps += 1
epn = 1
else:
epn += 1
break
else:
if config['position'] == 'next':
eps = epn = 1
else:
# There were no watched/collected episodes, nothing to emit in 'last' mode
continue
if eps and epn:
if config.get('strip_dates'):
# remove year from end of series_name if present
fields['series_name'] = re.sub(r'\s+\(\d{4}\)$', '',
fields['series_name'])
entry = self.make_entry(fields, eps, epn)
entries.append(entry)
return entries
def items(self):
if self._items is None:
endpoint = self.get_list_endpoint()
log.verbose('Retrieving `%s` list `%s`' %
(self.config['type'], self.config['list']))
try:
result = self.session.get(get_api_url(endpoint))
try:
data = result.json()
except ValueError:
log.debug('Could not decode json from response: %s',
result.text)
raise plugin.PluginError('Error getting list from trakt.')
except RequestException as e:
raise plugin.PluginError(
'Could not retrieve list from trakt (%s)' % e.args[0])
if not data:
log.warning('No data returned from trakt for %s list %s.' %
(self.config['type'], self.config['list']))
return []
entries = []
list_type = (self.config['type']).rstrip('s')
for item in data:
if self.config['type'] == 'auto':
list_type = item['type']
# Collection and watched lists don't return 'type' along with the items (right now)
if 'type' in item and item['type'] != list_type:
log.debug('Skipping %s because it is not a %s' %
(item[item['type']].get('title',
'unknown'), list_type))
continue
if list_type != 'episode' and not item[list_type]['title']:
# Skip shows/movies with no title
log.warning(
'Item in trakt list does not appear to have a title, skipping.'
)
continue
entry = Entry()
if list_type == 'episode':
entry[
'url'] = 'http://trakt.tv/shows/%s/seasons/%s/episodes/%s' % (
item['show']['ids']['slug'],
item['episode']['season'],
item['episode']['number'])
else:
entry['url'] = 'http://trakt.tv/%s/%s' % (
list_type, item[list_type]['ids'].get('slug'))
entry.update_using_map(field_maps[list_type], item)
# Override the title if strip_dates is on. TODO: a better way?
if self.config.get('strip_dates'):
if list_type in ['show', 'movie']:
entry['title'] = item[list_type]['title']
elif list_type == 'episode':
entry[
'title'] = '{show[title]} S{episode[season]:02}E{episode[number]:02}'.format(
**item)
if item['episode']['title']:
entry['title'] += ' {episode[title]}'.format(
**item)
if entry.isvalid():
if self.config.get('strip_dates'):
# Remove year from end of name if present
entry['title'] = re.sub(r'\s+\(\d{4}\)$', '',
entry['title'])
entries.append(entry)
else:
log.debug('Invalid entry created? %s' % entry)
self._items = entries
return self._items
def on_task_exit(self, task, config):
"""Store finished / downloaded entries at exit"""
if not rss2gen:
raise plugin.PluginWarning(
'plugin make_rss requires PyRSS2Gen library.')
config = self.prepare_config(config)
# when history is disabled, remove everything from backlog on every run (a bit hackish, rarely useful)
if not config['history']:
log.debug('disabling history')
for item in task.session.query(RSSEntry).filter(
RSSEntry.file == config['file']).all():
task.session.delete(item)
# save entries into db for RSS generation
for entry in task.accepted:
rss = RSSEntry()
rss.title = entry.render(config['title'])
for field in config['link']:
if field in entry:
rss.link = entry[field]
break
try:
template = get_template(config['template'], 'rss')
except ValueError as e:
raise plugin.PluginError('Invalid template specified: %s' % e)
try:
rss.description = render_from_entry(template, entry)
except RenderError as e:
log.error(
'Error while rendering entry %s, falling back to plain title: %s'
% (entry, e))
rss.description = entry['title'] + ' - (Render Error)'
rss.file = config['file']
# TODO: check if this exists and suggest disabling history if it does since it shouldn't happen normally ...
log.debug('Saving %s into rss database' % entry['title'])
task.session.add(rss)
if not rss2gen:
return
# don't generate rss when learning
if task.options.learn:
return
db_items = task.session.query(RSSEntry).filter(RSSEntry.file == config['file']).\
order_by(RSSEntry.published.desc()).all()
# make items
rss_items = []
for db_item in db_items:
add = True
if config['items'] != -1:
if len(rss_items) > config['items']:
add = False
if config['days'] != -1:
if datetime.datetime.today() - datetime.timedelta(
days=config['days']) > db_item.published:
add = False
if add:
# add into generated feed
hasher = hashlib.sha1()
hasher.update(db_item.title.encode('utf8'))
hasher.update(db_item.description.encode('utf8'))
hasher.update(db_item.link.encode('utf8'))
guid = base64.urlsafe_b64encode(hasher.digest())
guid = PyRSS2Gen.Guid(guid, isPermaLink=False)
gen = {
'title': db_item.title,
'description': db_item.description,
'link': db_item.link,
'pubDate': db_item.published,
'guid': guid
}
log.trace('Adding %s into rss %s' %
(gen['title'], config['file']))
rss_items.append(PyRSS2Gen.RSSItem(**gen))
else:
# no longer needed
task.session.delete(db_item)
# make rss
rss = PyRSS2Gen.RSS2(title='FlexGet',
link=config.get('rsslink', 'http://flexget.com'),
description='FlexGet generated RSS feed',
lastBuildDate=datetime.datetime.utcnow(),
items=rss_items)
# don't run with --test
if task.options.test:
log.info('Would write rss file with %d entries.', len(rss_items))
return
# write rss
fn = os.path.expanduser(config['file'])
with open(fn, 'w') as file:
try:
log.verbose('Writing output rss to %s' % fn)
rss.write_xml(file, encoding=config['encoding'])
except LookupError:
log.critical('Unknown encoding %s' % config['encoding'])
return
except IOError:
# TODO: plugins cannot raise PluginWarnings in terminate event ..
log.critical('Unable to write %s' % fn)
return
def on_task_input(self, task, config):
config = self.prepare_config(config)
urlconfig = {}
urlappend = "?"
entries = []
if (config['unwatched_only']
and config['section'] != 'recentlyViewedShows'
and config['section'] != 'all'):
urlconfig['unwatched'] = '1'
if config.get('token'):
accesstoken = config['token']
log.debug("Using accesstoken: %s", accesstoken)
urlconfig['X-Plex-Token'] = accesstoken
elif config.get('username'):
accesstoken = self.plex_get_accesstoken(config)
log.debug("Got accesstoken: %s", accesstoken)
urlconfig['X-Plex-Token'] = accesstoken
for key in urlconfig:
urlappend += '%s=%s&' % (key, urlconfig[key])
if not self.plex_section_is_int(config['section']):
try:
path = "/library/sections/"
r = requests.get(
"http://%s:%d%s%s" %
(config['plexserver'], config['port'], path, urlappend))
except requests.RequestException as e:
raise plugin.PluginError('Error retrieving source: %s' % e)
dom = parseString(r.text.encode("utf-8"))
for node in dom.getElementsByTagName('Directory'):
if node.getAttribute('title') == config['section']:
config['section'] = int(node.getAttribute('key'))
if not self.plex_section_is_int(config['section']):
raise plugin.PluginError('Could not find section \'%s\'' %
config['section'])
log.debug(
"Fetching http://%s:%d/library/sections/%s/%s%s",
config['server'],
config['port'],
config['section'],
config['selection'],
urlappend,
)
try:
path = "/library/sections/%s/%s" % (config['section'],
config['selection'])
r = requests.get(
"http://%s:%d%s%s" %
(config['plexserver'], config['port'], path, urlappend))
except requests.RequestException as e:
raise plugin.PluginError(
'There is no section with number %d. (%s)' %
(config['section'], e))
dom = parseString(r.text.encode("utf-8"))
plexsectionname = dom.getElementsByTagName(
'MediaContainer')[0].getAttribute('title1')
viewgroup = dom.getElementsByTagName('MediaContainer')[0].getAttribute(
'viewGroup')
log.debug("Plex section \"%s\" is a \"%s\" section", plexsectionname,
viewgroup)
if viewgroup != "movie" and viewgroup != "show" and viewgroup != "episode":
raise plugin.PluginError(
"Section is neither a movie nor tv show section!")
domroot = "Directory"
titletag = "title"
if viewgroup == "episode":
domroot = "Video"
titletag = "grandparentTitle"
thumbtag = "thumb"
arttag = "art"
seasoncovertag = "parentThumb"
covertag = "grandparentThumb"
elif viewgroup == "movie":
domroot = "Video"
titletag = "title"
arttag = "art"
seasoncovertag = "thumb"
covertag = "thumb"
if config['fetch'] == "thumb":
raise plugin.PluginError(
"Movie sections does not have any thumbnails to download!")
for node in dom.getElementsByTagName(domroot):
e = Entry()
e['plex_server'] = config['plexserver']
e['plex_port'] = config['port']
e['plex_section'] = config['section']
e['plex_section_name'] = plexsectionname
e['plex_episode_thumb'] = ''
title = node.getAttribute(titletag)
if config['strip_year']:
title = re.sub(r'^(.*)\(\d{4}\)(.*)', r'\1\2', title)
if config['strip_parens']:
title = re.sub(r'\(.*?\)', r'', title)
title = title.strip()
if config['strip_non_alpha']:
title = re.sub(r'[\(\)]', r'', title)
title = re.sub(r'&', r'And', title)
title = re.sub(r'[^A-Za-z0-9- \']', r'', title)
if config['lowercase_title']:
title = title.lower()
if viewgroup == "show":
e['title'] = title
e['url'] = 'NULL'
entries.append(e)
# show ends here.
continue
e['plex_art'] = "http://%s:%d%s%s" % (
config['server'],
config['port'],
node.getAttribute(arttag),
urlappend,
)
e['plex_cover'] = "http://%s:%d%s%s" % (
config['server'],
config['port'],
node.getAttribute(covertag),
urlappend,
)
e['plex_season_cover'] = "http://%s:%d%s%s" % (
config['server'],
config['port'],
node.getAttribute(seasoncovertag),
urlappend,
)
if viewgroup == "episode":
e['plex_thumb'] = "http://%s:%d%s%s" % (
config['server'],
config['port'],
node.getAttribute('thumb'),
urlappend,
)
e['series_name'] = title
e['plex_ep_name'] = node.getAttribute('title')
season = int(node.getAttribute('parentIndex'))
if node.getAttribute('parentIndex') == node.getAttribute(
'year'):
season = node.getAttribute('originallyAvailableAt')
filenamemap = "%s_%s%s_%s_%s_%s.%s"
episode = ""
e['series_id_type'] = 'date'
e['series_date'] = season
elif node.getAttribute('index'):
episode = int(node.getAttribute('index'))
filenamemap = "%s_%02dx%02d_%s_%s_%s.%s"
e['series_season'] = season
e['series_episode'] = episode
e['series_id_type'] = 'ep'
e['series_id'] = 'S%02dE%02d' % (season, episode)
else:
log.debug(
"Could not get episode number for '%s' (Hint, ratingKey: %s)",
title,
node.getAttribute('ratingKey'),
)
break
elif viewgroup == "movie":
filenamemap = "%s_%s_%s_%s.%s"
e['plex_year'] = node.getAttribute('year')
e['plex_added'] = datetime.fromtimestamp(
int(node.getAttribute('addedAt')))
e['plex_duration'] = node.getAttribute('duration')
e['plex_summary'] = node.getAttribute('summary')
e['plex_userrating'] = node.getAttribute('userrating')
e['plex_key'] = node.getAttribute('ratingKey')
count = node.getAttribute('viewCount')
offset = node.getAttribute('viewOffset')
if count:
e['plex_status'] = "seen"
elif offset:
e['plex_status'] = "inprogress"
else:
e['plex_status'] = "unwatched"
for media in node.getElementsByTagName('Media'):
entry = Entry(e)
vcodec = media.getAttribute('videoCodec')
acodec = media.getAttribute('audioCodec')
if media.hasAttribute('title'):
entry['plex_media_title'] = media.getAttribute('title')
if media.hasAttribute('optimizedForStreaming'):
entry['plex_stream_optimized'] = media.getAttribute(
'optimizedForStreaming')
if config['fetch'] == "file" or not config['fetch']:
container = media.getAttribute('container')
else:
container = "jpg"
resolution = media.getAttribute('videoResolution') + "p"
for part in media.getElementsByTagName('Part'):
if config['fetch'] == "file" or not config['fetch']:
key = part.getAttribute('key')
elif config['fetch'] == "art":
key = node.getAttribute(arttag)
elif config['fetch'] == "cover":
key = node.getAttribute(arttag)
elif config['fetch'] == "season_cover":
key = node.getAttribute(seasoncovertag)
elif config['fetch'] == "thumb":
key = node.getAttribute(thumbtag)
# key = part.getAttribute('key')
duration = part.getAttribute('duration')
entry['plex_title'] = title
entry['title'] = title
if config['original_filename']:
filename, fileext = os.path.splitext(
basename(part.getAttribute('file')))
if config['fetch'] != 'file':
filename += ".jpg"
else:
filename = "%s%s" % (filename, fileext)
else:
if viewgroup == "episode":
filename = filenamemap % (
title.replace(" ", "."),
season,
episode,
resolution,
vcodec,
acodec,
container,
)
entry['title'] = filename
elif viewgroup == "movie":
filename = filenamemap % (
title.replace(" ", "."),
resolution,
vcodec,
acodec,
container,
)
entry['title'] = filename
entry['plex_url'] = "http://%s:%d%s%s" % (
config['server'],
config['port'],
key,
urlappend,
)
entry['plex_path'] = key
entry['url'] = "http://%s:%d%s%s" % (
config['server'],
config['port'],
key,
urlappend,
)
entry['plex_duration'] = duration
entry['filename'] = filename
if key == "":
log.debug(
"Could not find anything in PMS to download. Next!"
)
else:
entries.append(entry)
return entries
def on_task_output(self, task, config):
if 'aria_config' not in config:
config['aria_config'] = {}
if 'uri' not in config and config['do'] == 'add-new':
raise plugin.PluginError('uri (path to folder containing file(s) on server) is required when adding new '
'downloads.', log)
if 'dir' not in config['aria_config']:
if config['do'] == 'add-new':
raise plugin.PluginError('dir (destination directory) is required.', log)
else:
config['aria_config']['dir'] = ''
if config['keep_parent_folders'] and config['aria_config']['dir'].find('{{parent_folders}}') == -1:
raise plugin.PluginError('When using keep_parent_folders, you must specify {{parent_folders}} in the dir '
'option to show where it goes.', log)
if config['rename_content_files'] and not config['rename_template']:
raise plugin.PluginError('When using rename_content_files, you must specify a rename_template.', log)
if config['username'] and not config['password']:
raise plugin.PluginError('If you specify an aria2 username, you must specify a password.')
try:
userpass = ''
if config['username']:
userpass = '******' % (config['username'], config['password'])
baseurl = 'http://%s%s:%s/rpc' % (userpass, config['server'], config['port'])
log.debug('base url: %s' % baseurl)
s = xmlrpc.client.ServerProxy(baseurl)
log.info('Connected to daemon at ' + baseurl + '.')
except xmlrpc.client.ProtocolError as err:
raise plugin.PluginError('Could not connect to aria2 at %s. Protocol error %s: %s'
% (baseurl, err.errcode, err.errmsg), log)
except xmlrpc.client.Fault as err:
raise plugin.PluginError('XML-RPC fault: Unable to connect to aria2 daemon at %s: %s'
% (baseurl, err.faultString), log)
except socket_error as e:
(error, msg) = e.args
raise plugin.PluginError('Socket connection issue with aria2 daemon at %s: %s'
% (baseurl, msg), log)
except:
raise plugin.PluginError('Unidentified error during connection to aria2 daemon at %s' % baseurl, log)
# loop entries
for entry in task.accepted:
config['aria_dir'] = config['aria_config']['dir']
if 'aria_gid' in entry:
config['aria_config']['gid'] = entry['aria_gid']
elif 'torrent_info_hash' in entry:
config['aria_config']['gid'] = entry['torrent_info_hash'][0:16]
elif 'gid' in config['aria_config']:
del(config['aria_config']['gid'])
if 'content_files' not in entry:
if entry['url']:
entry['content_files'] = [entry['url']]
else:
entry['content_files'] = [entry['title']]
else:
if not isinstance(entry['content_files'], list):
entry['content_files'] = [entry['content_files']]
counter = 0
for cur_file in entry['content_files']:
entry['parent_folders'] = ''
# reset the 'dir' or it will only be rendered on the first loop
config['aria_config']['dir'] = config['aria_dir']
cur_filename = cur_file.split('/')[-1]
if cur_file.split('/')[0] != cur_filename and config['keep_parent_folders']:
lastSlash = cur_file.rfind('/')
cur_path = cur_file[:lastSlash]
if cur_path[0:1] == '/':
cur_path = cur_path[1:]
entry['parent_folders'] = cur_path
log.debug('parent folders: %s' % entry['parent_folders'])
file_dot = cur_filename.rfind(".")
file_ext = cur_filename[file_dot:]
if len(entry['content_files']) > 1 and 'gid' in config['aria_config']:
# if there is more than 1 file, need to give unique gids, this will work up to 999 files
counter += 1
strCounter = str(counter)
if len(entry['content_files']) > 99:
# sorry not sorry if you have more than 999 files
config['aria_config']['gid'] = ''.join([config['aria_config']['gid'][0:-3],
strCounter.rjust(3, str('0'))])
else:
config['aria_config']['gid'] = ''.join([config['aria_config']['gid'][0:-2],
strCounter.rjust(2, str('0'))])
if config['exclude_samples'] == True:
# remove sample files from download list
if cur_filename.lower().find('sample') > -1:
continue
if file_ext not in config['file_exts']:
if config['exclude_non_content'] == True:
# don't download non-content files, like nfos - definable in file_exts
continue
if config['parse_filename']:
if config['content_is_episodes']:
metainfo_series = plugin.get_plugin_by_name('metainfo_series')
guess_series = metainfo_series.instance.guess_series
if guess_series(cur_filename):
parser = guess_series(cur_filename)
entry['series_name'] = parser.name
# if the last four chars are numbers, REALLY good chance it's actually a year...
# fix it if so desired
log.verbose(entry['series_name'])
if re.search(r'\d{4}', entry['series_name'][-4:]) is not None and config['fix_year']:
entry['series_name'] = ''.join([entry['series_name'][0:-4], '(',
entry['series_name'][-4:], ')'])
log.verbose(entry['series_name'])
parser.data = cur_filename
parser.parse
log.debug(parser.id_type)
if parser.id_type == 'ep':
entry['series_id'] = ''.join(['S', str(parser.season).rjust(2, str('0')), 'E',
str(parser.episode).rjust(2, str('0'))])
elif parser.id_type == 'sequence':
entry['series_id'] = parser.episode
elif parser.id_type and parser.id:
entry['series_id'] = parser.id
else:
parser = get_plugin_by_name('parsing').instance.parse_movie(cur_filename)
parser.parse()
log.info(parser)
testname = parser.name
testyear = parser.year
parser.data = entry['title']
parser.parse()
log.info(parser)
if len(parser.name) > len(testname):
entry['name'] = parser.name
entry['movie_name'] = parser.name
else:
entry['name'] = testname
entry['movie_name'] = testname
if parser.year:
entry['year'] = parser.year
entry['movie_year'] = parser.year
else:
entry['year'] = testyear
entry['movie_year'] = testyear
if config['rename_content_files']:
if config['content_is_episodes']:
try:
config['aria_config']['out'] = entry.render(config['rename_template']) + file_ext
log.verbose(config['aria_config']['out'])
except RenderError as e:
log.error('Could not rename file %s: %s.' % (cur_filename, e))
continue
else:
try:
config['aria_config']['out'] = entry.render(config['rename_template']) + file_ext
log.verbose(config['aria_config']['out'])
except RenderError as e:
log.error('Could not rename file %s: %s. Try enabling imdb_lookup in this task'
' to assist.' % (cur_filename, e))
continue
elif 'torrent_info_hash' not in entry:
config['aria_config']['out'] = cur_filename
if config['do'] == 'add-new':
log.debug('Adding new file')
new_download = 0
if 'gid' in config['aria_config']:
try:
r = s.aria2.tellStatus(config['aria_config']['gid'], ['gid', 'status'])
log.info('Download status for %s (gid %s): %s' % (
config['aria_config'].get('out', config['uri']), r['gid'],
r['status']))
if r['status'] == 'paused':
try:
if not task.manager.options.test:
s.aria2.unpause(r['gid'])
log.info(' Unpaused download.')
except xmlrpc.client.Fault as err:
raise plugin.PluginError(
'aria2 response to unpause request: %s' % err.faultString, log)
else:
log.info(' Therefore, not re-adding.')
except xmlrpc.client.Fault as err:
if err.faultString[-12:] == 'is not found':
new_download = 1
else:
raise plugin.PluginError('aria2 response to download status request: %s'
% err.faultString, log)
except xmlrpc.client.ProtocolError as err:
raise plugin.PluginError('Could not connect to aria2 at %s. Protocol error %s: %s'
% (baseurl, err.errcode, err.errmsg), log)
except socket_error as e:
(error, msg) = e.args
raise plugin.PluginError('Socket connection issue with aria2 daemon at %s: %s'
% (baseurl, msg), log)
else:
new_download = 1
if new_download == 1:
try:
entry['filename'] = cur_file
cur_uri = entry.render(config['uri'])
log.verbose('uri: %s' % cur_uri)
except RenderError as e:
raise plugin.PluginError('Unable to render uri: %s' % e)
try:
for key, value in config['aria_config'].items():
log.trace('rendering %s: %s' % (key, value))
config['aria_config'][key] = entry.render(str(value))
log.debug('dir: %s' % config['aria_config']['dir'])
if not task.manager.options.test:
r = s.aria2.addUri([cur_uri], config['aria_config'])
else:
if 'gid' not in config['aria_config']:
r = '1234567890123456'
else:
r = config['aria_config']['gid']
log.info('%s successfully added to aria2 with gid %s.' % (
config['aria_config'].get('out', config['uri']),
r))
except xmlrpc.client.Fault as err:
raise plugin.PluginError('aria2 response to add URI request: %s' % err.faultString, log)
except socket_error as e:
(error, msg) = e.args
raise plugin.PluginError('Socket connection issue with aria2 daemon at %s: %s'
% (baseurl, msg), log)
except RenderError as e:
raise plugin.PluginError('Unable to render one of the fields being passed to aria2:'
'%s' % e)
elif config['do'] == 'remove-completed':
try:
r = s.aria2.tellStatus(config['aria_config']['gid'], ['gid', 'status'])
log.info('Status of download with gid %s: %s' % (r['gid'], r['status']))
if r['status'] in ['complete', 'removed']:
if not task.manager.options.test:
try:
a = s.aria2.removeDownloadResult(r['gid'])
if a == 'OK':
log.info('Download with gid %s removed from memory' % r['gid'])
except xmlrpc.client.Fault as err:
raise plugin.PluginError('aria2 response to remove request: %s'
% err.faultString, log)
except socket_error as e:
(error, msg) = e.args
raise plugin.PluginError('Socket connection issue with aria2 daemon at %s: %s'
% (baseurl, msg), log)
else:
log.info('Download with gid %s could not be removed because of its status: %s'
% (r['gid'], r['status']))
except xmlrpc.client.Fault as err:
if err.faultString[-12:] == 'is not found':
log.warning('Download with gid %s could not be removed because it was not found. It was '
'possibly previously removed or never added.' % config['aria_config']['gid'])
else:
raise plugin.PluginError('aria2 response to status request: %s' % err.faultString, log)
except socket_error as e:
(error, msg) = e.args
raise plugin.PluginError('Socket connection issue with aria2 daemon at %s: %s'
% (baseurl, msg), log)
def _get_info(task, link, cookie, adapter, user_agent):
headers = {'cookie': cookie, 'user-agent': user_agent}
detail_page = task.requests.get(link, headers=headers) # 详情
detail_page.encoding = 'utf-8'
if 'totheglory' in link:
peer_url = link
else:
peer_url = link.replace('details.php', 'viewpeerlist.php', 1)
try:
peer_page = task.requests.get(peer_url,
headers=headers).text # peer详情
except:
peer_page = ''
if 'login' in detail_page.url:
raise plugin.PluginError(
"Can't access the site. Your cookie may be wrong!")
if adapter:
convert = {value: key for key, value in adapter.items()}
discount_fn = NexusPHP.generate_discount_fn(convert)
return NexusPHP.info_from_page(detail_page, peer_page, discount_fn)
sites_discount = {
'chdbits': {
'pro_free.*?</h1>': 'free',
'pro_2up.*?</h1>': '2x',
'pro_free2up.*?</h1>': '2xfree',
'pro_30pctdown.*?</h1>': '30%',
'pro_50pctdown.*?</h1>': '50%',
'pro_50pctdown2up.*?</h1>': '2x50%'
},
'u2.dmhy': {
'<td.*?top.*?pro_free.*?优惠历史.*?</td>': 'free',
'<td.*?top.*?pro_2up.*?优惠历史.*?</td>': '2x',
'<td.*?top.*?pro_free2up.*?优惠历史.*?</td>': '2xfree',
'<td.*?top.*?pro_30pctdown.*?优惠历史.*?</td>': '30%',
'<td.*?top.*?pro_50pctdown.*?优惠历史.*?</td>': '50%',
'<td.*?top.*?pro_50pctdown2up.*?优惠历史.*?</td>': '2x50%',
'<td.*?top.*?pro_custom.*?优惠历史.*?</td>': '2xfree'
},
'yingk': {
'span_frees': 'free',
'span_twoupls': '2x',
'span_twoupfreels': '2xfree',
'span_thirtypercentls': '30%',
'span_halfdowns': '50%',
'span_twouphalfdownls': '2x50%'
},
'totheglory': {
'本种子限时不计流量': 'free',
'本种子的下载流量计为实际流量的30%': '30%',
'本种子的下载流量会减半': '50%',
},
'hdchina': {
'pro_free.*?</h2>': 'free',
'pro_2up.*?</h2>': '2x',
'pro_free2up.*?</h2>': '2xfree',
'pro_30pctdown.*?</h2>': '30%',
'pro_50pctdown.*?</h2>': '50%',
'pro_50pctdown2up.*?</h2>': '2x50%'
}
}
for site, convert in sites_discount.items():
if site in link:
discount_fn = NexusPHP.generate_discount_fn(convert)
return NexusPHP.info_from_page(detail_page, peer_page,
discount_fn)
discount_fn = NexusPHP.generate_discount_fn({
'class=\'free\'.*?免.*?</h1>':
'free',
'class=\'twoup\'.*?2X.*?</h1>':
'2x',
'class=\'twoupfree\'.*?2X免.*?</h1>':
'2xfree',
'class=\'thirtypercent\'.*?30%.*?</h1>':
'30%',
'class=\'halfdown\'.*?50%.*?</h1>':
'50%',
'class=\'twouphalfdown\'.*?2X 50%.*?</h1>':
'2x50%'
})
return NexusPHP.info_from_page(detail_page, peer_page, discount_fn)
def search(self, task, entry, config=None):
if not config:
return []
elif config == 'fail':
raise plugin.PluginError('search plugin failure')
return [Entry(entry)]
def process(self):
imdb_lookup = plugin.get_plugin_by_name('imdb_lookup').instance
self.changes.sort()
udata = load_uoccin_data(self.folder)
for line in self.changes:
tmp = line.split('|')
typ = tmp[1]
tid = tmp[2]
fld = tmp[3]
val = tmp[4]
self.log.verbose(
'processing: type=%s, target=%s, field=%s, value=%s' %
(typ, tid, fld, val))
if typ == 'movie':
# default
mov = udata['movies'].setdefault(
tid, {
'name': 'N/A',
'watchlist': False,
'collected': False,
'watched': False
})
# movie title is unknown at this time
fake = Entry()
fake['url'] = 'http://www.imdb.com/title/' + tid
fake['imdb_id'] = tid
try:
imdb_lookup.lookup(fake)
mov['name'] = fake.get('imdb_name')
except plugin.PluginError:
self.log.warning(
'Unable to lookup movie %s from imdb, using raw name.'
% tid)
# setting
if fld == 'watchlist':
mov['watchlist'] = val == 'true'
elif fld == 'collected':
mov['collected'] = val == 'true'
elif fld == 'watched':
mov['watched'] = val == 'true'
elif fld == 'tags':
mov['tags'] = re.split(',\s*', val)
elif fld == 'subtitles':
mov['subtitles'] = re.split(',\s*', val)
elif fld == 'rating':
mov['rating'] = int(val)
# cleaning
if not (mov['watchlist'] or mov['collected']
or mov['watched']):
self.log.verbose('deleting unused section: movies\%s' %
tid)
udata['movies'].pop(tid)
elif typ == 'series':
tmp = tid.split('.')
sid = tmp[0]
sno = tmp[1] if len(tmp) > 2 else None
eno = tmp[2] if len(tmp) > 2 else None
# default
ser = udata['series'].setdefault(
sid, {
'name': 'N/A',
'watchlist': False,
'collected': {},
'watched': {}
})
# series name is unknown at this time
try:
series = lookup_series(tvdb_id=sid)
ser['name'] = series.name
except LookupError:
self.log.warning(
'Unable to lookup series %s from tvdb, using raw name.'
% sid)
# setting
if fld == 'watchlist':
ser['watchlist'] = val == 'true'
elif fld == 'tags':
ser['tags'] = re.split(',\s*', val)
elif fld == 'rating':
ser['rating'] = int(val)
elif sno is None or eno is None:
self.log.warning(
'invalid line "%s": season and episode numbers are required'
% line)
elif fld == 'collected':
season = ser['collected'].setdefault(sno, {})
if val == 'true':
season.setdefault(eno, [])
else:
if eno in season:
season.pop(eno)
if not season:
self.log.verbose(
'deleting unused section: series\%s\collected\%s'
% (sid, sno))
ser['collected'].pop(sno)
elif fld == 'subtitles':
ser['collected'].setdefault(sno, {})[eno] = re.split(
',\s*', val)
elif fld == 'watched':
season = ser['watched'].setdefault(sno, [])
if val == 'true':
season = ser['watched'][sno] = list(
set(season) | set([int(eno)]))
elif int(eno) in season:
season.remove(int(eno))
season.sort()
if not season:
self.log.debug(
'deleting unused section: series\%s\watched\%s' %
(sid, sno))
ser['watched'].pop(sno)
# cleaning
if not (ser['watchlist'] or ser['collected']
or ser['watched']):
self.log.debug('deleting unused section: series\%s' % sid)
udata['series'].pop(sid)
else:
self.log.warning('invalid element type "%s"' % typ)
# save the updated uoccin.json
ufile = os.path.join(self.folder, 'uoccin.json')
try:
text = json.dumps(udata,
sort_keys=True,
indent=4,
separators=(',', ': '))
with open(ufile, 'w') as f:
f.write(text)
except Exception as err:
self.log.debug('error writing %s: %s' % (ufile, err))
raise plugin.PluginError('error writing %s: %s' % (ufile, err))
def on_task_filter(self, task, config):
config = self.build_config(config)
adapter = HTTPAdapter(max_retries=5)
task.requests.mount('http://', adapter)
task.requests.mount('https://', adapter)
# 先访问一次 预防异常
headers = {
'cookie': config['cookie'],
'user-agent': config['user-agent']
}
try:
task.requests.get(task.entries[0].get('link'), headers=headers)
except:
pass
def consider_entry(_entry, _link):
try:
discount, seeders, leechers, hr = NexusPHP._get_info(
task, _link, config['cookie'], config['adapter'],
config['user-agent'])
except plugin.PluginError as e:
raise e
except Exception as e:
log.info('NexusPHP._get_info: ' + str(e))
return
seeder_max = config['seeders']['max']
seeder_min = config['seeders']['min']
leecher_max = config['leechers']['max']
leecher_min = config['leechers']['min']
if config['discount']:
if discount not in config['discount']:
_entry.reject('%s does not match discount' %
discount) # 优惠信息不匹配
return
if config['hr'] is False and hr:
_entry.reject('it is HR') # 拒绝HR
if len(seeders) not in range(seeder_min, seeder_max + 1):
_entry.reject('%d is out of range of seeder' %
len(seeders)) # 做种人数不匹配
return
if len(leechers) not in range(leecher_min, leecher_max + 1):
_entry.reject('%d is out of range of leecher' %
len(leechers)) # 下载人数不匹配
return
if len(leechers) != 0:
max_complete = max(leechers,
key=lambda x: x['completed'])['completed']
else:
max_complete = 0
if max_complete > config['leechers']['max_complete']:
_entry.reject('%f is more than max_complete' %
max_complete) # 最大完成度不匹配
return
_entry.accept()
futures = [] # 线程任务
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
for entry in task.entries:
link = entry.get('link')
if not link:
raise plugin.PluginError(
"The rss plugin require 'other_fields' which contain 'link'. "
"For example: other_fields: - link")
futures.append(executor.submit(consider_entry, entry, link))
for f in concurrent.futures.as_completed(futures):
exception = f.exception()
if isinstance(exception, plugin.PluginError):
log.info(exception)
def on_task_input(self, task, config):
if isinstance(config, str):
config = {'username': config}
selected_list_status = config.get('status', ['current', 'planning'])
selected_release_status = config.get('release_status', ['all'])
selected_formats = config.get('format', ['all'])
selected_list_name = config.get('list', [])
if not isinstance(selected_list_status, list):
selected_list_status = [selected_list_status]
if not isinstance(selected_release_status, list):
selected_release_status = [selected_release_status]
if not isinstance(selected_formats, list):
selected_formats = [selected_formats]
if not isinstance(selected_list_name, list):
selected_list_name = [selected_list_name]
selected_list_name = [i.lower() for i in selected_list_name]
logger.debug('Selected List Status: {}', selected_list_status)
logger.debug('Selected Release Status: {}', selected_release_status)
logger.debug('Selected Formats: {}', selected_formats)
req_variables = {'user': config['username']}
req_chunk = 1
req_fields = (
'id, status, title{ romaji, english }, synonyms, siteUrl, idMal, format, episodes, '
'trailer{ site, id }, coverImage{ large }, bannerImage, genres, tags{ name }, '
'externalLinks{ site, url }, startDate{ year, month, day }, endDate{ year, month, day}'
)
while req_chunk:
req_query = (
f'query ($user: String){{ collection: MediaListCollection(userName: $user, '
f'type: ANIME, perChunk: 500, chunk: {req_chunk}, status_in: '
f'[{", ".join([s.upper() for s in selected_list_status])}]) {{ hasNextChunk, '
f'statuses: lists{{ status, name, list: entries{{ anime: media{{ {req_fields}'
f' }}}}}}}}}}'
)
try:
list_response = task.requests.post(
'https://graphql.anilist.co',
json={'query': req_query, 'variables': req_variables},
)
list_response = list_response.json()['data']
except RequestException as e:
raise plugin.PluginError(f'Error reading list - {e}')
except ValueError as e:
raise plugin.PluginError(f'Invalid JSON response {e}')
logger.debug('JSON output: {}', list_response)
for list_status in list_response.get('collection', {}).get('statuses', []):
if selected_list_name and (
list_status.get('name')
and list_status.get('name').lower() not in selected_list_name
):
continue
for anime in list_status['list']:
anime = anime.get('anime')
has_selected_release_status = (
anime.get('status')
and anime.get('status').lower() in selected_release_status
) or 'all' in selected_release_status
has_selected_type = (
anime.get('format') and anime.get('format').lower() in selected_formats
) or 'all' in selected_formats
if has_selected_type and has_selected_release_status:
try:
ids = task.requests.post(
'https://relations.yuna.moe/api/ids',
json={'anilist': anime.get('id')},
).json()
except RequestException as e:
ids = {}
raise plugin.PluginWarning(f'Couldn\'t fetch additional IDs - {e}')
entry = Entry()
entry['al_id'] = anime.get('id', ids.get('anilist'))
entry['anidb_id'] = ids.get('anidb')
entry['kitsu_id'] = ids.get('kitsu')
entry['mal_id'] = anime.get('idMal', ids.get('myanimelist'))
entry['al_banner'] = anime.get('bannerImage')
entry['al_cover'] = anime.get('coverImage', {}).get('large')
entry['al_date_end'] = (
datetime(
year=anime.get('endDate').get('year'),
month=anime.get('endDate').get('month', 1),
day=anime.get('endDate').get('day', 1),
)
if anime.get('endDate').get('year')
else None
)
entry['al_date_start'] = (
datetime(
year=anime.get('startDate').get('year'),
month=anime.get('startDate').get('month', 1),
day=anime.get('startDate').get('day', 1),
)
if anime.get('startDate').get('year')
else None
)
entry['al_episodes'] = anime.get('episodes')
entry['al_format'] = anime.get('format')
entry['al_genres'] = anime.get('genres')
entry['al_links'] = {
item['site']: item['url'] for item in anime.get('externalLinks')
}
entry['al_list'] = list_status.get('name')
entry['al_list_status'] = (
list_status['status'].capitalize()
if list_status.get('status')
else None
)
entry['al_release_status'] = (
anime['status'].capitalize() if anime.get('status') else None
)
entry['al_tags'] = [t.get('name') for t in anime.get('tags')]
entry['al_title'] = anime.get('title')
entry['al_trailer'] = (
TRAILER_SOURCE[anime.get('trailer', {}).get('site')]
+ anime.get('trailer', {}).get('id')
if anime.get('trailer')
and anime.get('trailer').get('site') in TRAILER_SOURCE
else None
)
entry['alternate_name'] = anime.get('synonyms', [])
eng_title = anime.get('title', {}).get('english')
if (
eng_title
and eng_title.lower() != anime.get('title', {}).get('romaji').lower()
and eng_title not in entry['alternate_name']
):
entry['alternate_name'].insert(0, eng_title)
entry['series_name'] = entry['al_title'].get('romaji') or entry[
'al_title'
].get('english')
entry['title'] = entry['series_name']
entry['url'] = anime.get('siteUrl')
if entry.isvalid():
yield entry
req_chunk = req_chunk + 1 if list_response['collection']['hasNextChunk'] else False
def on_connect_fail(self, result):
"""Pauses the reactor, returns PluginError. Gets called when connection to deluge daemon fails."""
log.debug('Connect to deluge daemon failed, result: %s' % result)
reactor.callLater(
0, reactor.pause,
plugin.PluginError('Could not connect to deluge daemon', log))
def create_entries(self, page_url, soup, config):
queue = []
duplicates = {}
duplicate_limit = 4
def title_exists(title):
"""Helper method. Return True if title is already added to entries"""
for entry in queue:
if entry['title'] == title:
return True
for link in soup.find_all('a'):
# not a valid link
if not link.has_attr('href'):
continue
# no content in the link
if not link.contents:
continue
url = link['href']
# fix broken urls
if url.startswith('//'):
url = 'http:' + url
elif not url.startswith('http://') or not url.startswith(
'https://'):
url = urlparse.urljoin(page_url, url)
log_link = url
log_link = log_link.replace('\n', '')
log_link = log_link.replace('\r', '')
# get only links matching regexp
regexps = config.get('links_re', None)
if regexps:
accept = False
for regexp in regexps:
if re.search(regexp, url):
accept = True
if not accept:
continue
title_from = config.get('title_from', 'auto')
if title_from == 'url':
title = self._title_from_url(url)
log.debug('title from url: %s' % title)
elif title_from == 'title':
if not link.has_attr('title'):
log.warning(
'Link `%s` doesn\'t have title attribute, ignored.' %
log_link)
continue
title = link['title']
log.debug('title from title: %s' % title)
elif title_from == 'auto':
title = self._title_from_link(link, log_link)
if title is None:
continue
# automatic mode, check if title is unique
# if there are too many duplicate titles, switch to title_from: url
if title_exists(title):
# ignore index links as a counter
if 'index' in title and len(title) < 10:
log.debug('ignored index title %s' % title)
continue
duplicates.setdefault(title, 0)
duplicates[title] += 1
if duplicates[title] > duplicate_limit:
# if from url seems to be bad choice use title
from_url = self._title_from_url(url)
switch_to = 'url'
for ext in ('.html', '.php'):
if from_url.endswith(ext):
switch_to = 'title'
log.info(
'Link names seem to be useless, auto-configuring \'title_from: %s\'. '
'This may not work well, you might need to configure it yourself.'
% switch_to)
config['title_from'] = switch_to
# start from the beginning ...
return self.create_entries(page_url, soup, config)
elif title_from == 'link' or title_from == 'contents':
# link from link name
title = self._title_from_link(link, log_link)
if title is None:
continue
log.debug('title from link: %s' % title)
else:
raise plugin.PluginError('Unknown title_from value %s' %
title_from)
if not title:
log.warning('title could not be determined for link %s' %
log_link)
continue
# strip unicode white spaces
title = title.replace(u'\u200B', u'').strip()
# in case the title contains xxxxxxx.torrent - foooo.torrent clean it a bit (get up to first .torrent)
# TODO: hack
if title.lower().find('.torrent') > 0:
title = title[:title.lower().find('.torrent')]
if title_exists(title):
# title link should be unique, add CRC32 to end if it's not
hash = zlib.crc32(url.encode("utf-8"))
crc32 = '%08X' % (hash & 0xFFFFFFFF)
title = '%s [%s]' % (title, crc32)
# truly duplicate, title + url crc already exists in queue
if title_exists(title):
continue
log.debug('uniqued title to %s' % title)
entry = Entry()
entry['url'] = url
entry['title'] = title
if 'username' in config and 'password' in config:
entry['download_auth'] = (config['username'],
config['password'])
queue.append(entry)
# add from queue to task
return queue
def parse_site(self, url, task):
"""Parse configured url and return releases array"""
try:
page = task.requests.get(url).content
except RequestException as e:
raise plugin.PluginError('Error getting input page: %e' % e)
soup = get_soup(page)
releases = []
for entry in soup.find_all('div', attrs={'class': 'entry'}):
release = {}
title = entry.find('h2')
if not title:
log.debug('No h2 entrytitle')
continue
release['title'] = title.a.contents[0].strip()
log.debug('Processing title %s' % (release['title']))
for link in entry.find_all('a'):
# no content in the link
if not link.contents:
continue
link_name = link.contents[0]
if link_name is None:
continue
if not isinstance(link_name, NavigableString):
continue
link_name = link_name.strip().lower()
if link.has_attr('href'):
link_href = link['href']
else:
continue
log.debug('found link %s -> %s' % (link_name, link_href))
# handle imdb link
if link_name.lower() == 'imdb':
log.debug('found imdb link %s' % link_href)
release['imdb_id'] = extract_id(link_href)
# test if entry with this url would be rewritable by known plugins (ie. downloadable)
temp = {}
temp['title'] = release['title']
temp['url'] = link_href
urlrewriting = plugin.get_plugin_by_name('urlrewriting')
if urlrewriting['instance'].url_rewritable(task, temp):
release['url'] = link_href
log.trace('--> accepting %s (resolvable)' % link_href)
else:
log.trace('<-- ignoring %s (non-resolvable)' % link_href)
# reject if no torrent link
if 'url' not in release:
from flexget.utils.log import log_once
log_once(
'%s skipped due to missing or unsupported (unresolvable) download link'
% (release['title']), log)
else:
releases.append(release)
return releases
def on_task_download(self, task, config):
config = self.prepare_config(config)
add_options = config.get('action').get('add')
if not add_options or not task.accepted:
return
if not self.client:
self.client = self.create_client(config)
if self.client:
logger.debug('Successfully connected to qBittorrent.')
else:
raise plugin.PluginError("Couldn't connect to qBittorrent.")
main_data_snapshot = self.client.get_main_data_snapshot(id(task))
server_state = main_data_snapshot.get('server_state')
reject_on = add_options.get('reject_on', {})
remember_reject = reject_on.get('remember', True)
bandwidth_limit = reject_on.get('bandwidth_limit')
reject_on_dl_speed = reject_on.get('dl_speed')
reject_on_dl_limit = reject_on.get('dl_limit')
up_bandwidth_limit = reject_on.get('up_bandwidth_limit')
reject_on_up_speed = reject_on.get('up_speed')
reject_on_all = reject_on.get('all')
reject_reason = ''
up_rate_limit = server_state.get('up_rate_limit')
dl_rate_limit = server_state.get('dl_rate_limit')
if reject_on_up_speed:
if isinstance(reject_on_up_speed, float):
up_rate_limit = up_rate_limit if up_rate_limit else up_bandwidth_limit
reject_on_up_speed = int(up_rate_limit * reject_on_up_speed)
up_info_speed = server_state.get('up_info_speed')
if up_info_speed and up_info_speed > reject_on_up_speed:
reject_reason = 'up_speed: {:.2F} MiB > reject_on_up_speed: {:.2F} MiB'.format(
up_info_speed / (1024 * 1024),
reject_on_up_speed / (1024 * 1024))
if reject_on_dl_limit and dl_rate_limit and dl_rate_limit < reject_on_dl_limit:
reject_reason = 'dl_limit: {:.2F} MiB < reject_on_dl_limit: {:.2F} MiB'.format(
dl_rate_limit / (1024 * 1024),
reject_on_dl_limit / (1024 * 1024))
elif reject_on_dl_speed:
if isinstance(reject_on_dl_speed, float):
dl_rate_limit = dl_rate_limit if dl_rate_limit else bandwidth_limit
reject_on_dl_speed = int(dl_rate_limit * reject_on_dl_speed)
dl_info_speed = server_state.get('dl_info_speed')
if dl_info_speed and dl_info_speed > reject_on_dl_speed:
reject_reason = 'dl_speed: {:.2F} MiB > reject_on_dl_speed: {:.2F} MiB'.format(
dl_info_speed / (1024 * 1024),
reject_on_dl_speed / (1024 * 1024))
if reject_on_all:
reject_reason = 'reject on all'
if 'download' not in task.config:
download = plugin.get('download', self)
headers = copy.deepcopy(task.requests.headers)
for entry in task.accepted:
if reject_reason:
entry.reject(reason=reject_reason, remember=remember_reject)
site_name = self._get_site_name(entry.get('url'))
logger.info('reject {}, because: {}, site: {}', entry['title'],
reject_reason, site_name)
continue
if entry.get('headers'):
task.requests.headers.update(entry['headers'])
else:
task.requests.headers.clear()
task.requests.headers = headers
if entry.get('cookie'):
task.requests.cookies.update(
NetUtils.cookie_str_to_dict(entry['cookie']))
else:
task.requests.cookies.clear()
download.get_temp_file(task,
entry,
handle_magnets=True,
fail_html=config['fail_html'])
def on_task_output(self, task, config):
config = self.prepare_config(config)
# don't add when learning
if task.options.learn:
return
if not config['enabled']:
return
# Do not run if there is nothing to do
if not task.accepted:
return
if self.client is None:
self.client = self.create_rpc_client(config)
if self.client:
logger.debug('Successfully connected to transmission.')
else:
raise plugin.PluginError("Couldn't connect to transmission.")
session_torrents = self.client.get_torrents()
for entry in task.accepted:
if task.options.test:
logger.info('Would {} {} in transmission.', config['action'],
entry['title'])
continue
# Compile user options into appropriate dict
options = self._make_torrent_options_dict(config, entry)
torrent_info = None
for t in session_torrents:
if t.hashString.lower() == entry.get(
'torrent_info_hash',
'').lower() or t.id == entry.get('transmission_id'):
torrent_info = t
logger.debug(
'Found {} already loaded in transmission as {}',
entry['title'],
torrent_info.name,
)
break
if not torrent_info:
if config['action'] != 'add':
logger.warning(
'Cannot {} {} because it is not loaded in transmission.',
config['action'],
entry['title'],
)
continue
downloaded = not entry['url'].startswith('magnet:')
# Check that file is downloaded
if downloaded and 'file' not in entry:
entry.fail('`file` field missing?')
continue
# Verify the temp file exists
if downloaded and not os.path.exists(entry['file']):
tmp_path = os.path.join(task.manager.config_base, 'temp')
logger.debug('entry: {}', entry)
logger.debug('temp: {}', ', '.join(os.listdir(tmp_path)))
entry.fail("Downloaded temp file '%s' doesn't exist!?" %
entry['file'])
continue
try:
if downloaded:
with open(entry['file'], 'rb') as f:
filedump = base64.b64encode(
f.read()).decode('utf-8')
torrent_info = self.client.add_torrent(
filedump, 30, **options['add'])
else:
if options['post'].get('magnetization_timeout', 0) > 0:
options['add']['paused'] = False
torrent_info = self.client.add_torrent(
entry['url'], timeout=30, **options['add'])
except TransmissionError as e:
logger.opt(exception=True).debug('TransmissionError')
logger.debug('Failed options dict: {}', options['add'])
msg = 'Error adding {} to transmission. TransmissionError: {}'.format(
entry['title'], e.message or 'N/A')
logger.error(msg)
entry.fail(msg)
continue
logger.info('"{}" torrent added to transmission',
entry['title'])
# The info returned by the add call is incomplete, refresh it
torrent_info = self.client.get_torrent(torrent_info.id)
else:
# Torrent already loaded in transmission
if options['add'].get('download_dir'):
logger.verbose('Moving {} to "{}"', torrent_info.name,
options['add']['download_dir'])
# Move data even if current reported torrent location matches new location
# as transmission may fail to automatically move completed file to final
# location but continue reporting final location instead of real location.
# In such case this will kick transmission to really move data.
# If data is already located at new location then transmission just ignore
# this command.
self.client.move_torrent_data(
torrent_info.id, options['add']['download_dir'], 120)
try:
total_size = torrent_info.totalSize
main_id = None
find_main_file = (options['post'].get('main_file_only')
or 'content_filename' in options['post'])
skip_files = options['post'].get('skip_files')
# We need to index the files if any of the following are defined
if find_main_file or skip_files:
file_list = self.client.get_files(
torrent_info.id)[torrent_info.id]
if options['post'].get('magnetization_timeout',
0) > 0 and not file_list:
logger.debug(
'Waiting {} seconds for "{}" to magnetize',
options['post']['magnetization_timeout'],
entry['title'],
)
for _ in range(
options['post']['magnetization_timeout']):
sleep(1)
file_list = self.client.get_files(
torrent_info.id)[torrent_info.id]
if file_list:
total_size = self.client.get_torrent(
torrent_info.id,
['id', 'totalSize']).totalSize
break
else:
logger.warning(
'"{}" did not magnetize before the timeout elapsed, file list unavailable for processing.',
entry['title'],
)
# Find files based on config
dl_list = []
skip_list = []
main_list = []
ext_list = ['*.srt', '*.sub', '*.idx', '*.ssa', '*.ass']
main_ratio = config['main_file_ratio']
if 'main_file_ratio' in options['post']:
main_ratio = options['post']['main_file_ratio']
for f in file_list:
# No need to set main_id if we're not going to need it
if find_main_file and file_list[f][
'size'] > total_size * main_ratio:
main_id = f
if 'include_files' in options['post']:
if any(
fnmatch(file_list[f]['name'], mask) for
mask in options['post']['include_files']):
dl_list.append(f)
elif options['post'].get('include_subs') and any(
fnmatch(file_list[f]['name'], mask)
for mask in ext_list):
dl_list.append(f)
if skip_files:
if any(
fnmatch(file_list[f]['name'], mask)
for mask in skip_files):
skip_list.append(f)
if main_id is not None:
# Look for files matching main ID title but with a different extension
if options['post'].get('rename_like_files'):
for f in file_list:
# if this filename matches main filename we want to rename it as well
fs = os.path.splitext(file_list[f]['name'])
if fs[0] == os.path.splitext(
file_list[main_id]['name'])[0]:
main_list.append(f)
else:
main_list = [main_id]
if main_id not in dl_list:
dl_list.append(main_id)
elif find_main_file:
logger.warning(
'No files in "{}" are > {:.0f}% of content size, no files renamed.',
entry['title'],
main_ratio * 100,
)
# If we have a main file and want to rename it and associated files
if 'content_filename' in options[
'post'] and main_id is not None:
if 'download_dir' not in options['add']:
download_dir = self.client.get_session(
).download_dir
else:
download_dir = options['add']['download_dir']
# Get new filename without ext
file_ext = os.path.splitext(
file_list[main_id]['name'])[1]
file_path = os.path.dirname(
os.path.join(download_dir,
file_list[main_id]['name']))
filename = options['post']['content_filename']
if config['host'] == 'localhost' or config[
'host'] == '127.0.0.1':
counter = 1
while os.path.exists(
os.path.join(file_path,
filename + file_ext)):
# Try appending a (#) suffix till a unique filename is found
filename = '%s(%s)' % (
options['post']['content_filename'],
counter,
)
counter += 1
else:
logger.debug(
'Cannot ensure content_filename is unique '
'when adding to a remote transmission daemon.')
for index in main_list:
file_ext = os.path.splitext(
file_list[index]['name'])[1]
logger.debug(
'File {} renamed to {}',
file_list[index]['name'],
filename + file_ext,
)
# change to below when set_files will allow setting name, more efficient to have one call
# fl[index]['name'] = os.path.basename(pathscrub(filename + file_ext).encode('utf-8'))
try:
self.client.rename_torrent_path(
torrent_info.id,
file_list[index]['name'],
os.path.basename(
str(pathscrub(filename + file_ext))),
)
except TransmissionError:
logger.error(
'content_filename only supported with transmission 2.8+'
)
if options['post'].get(
'main_file_only') and main_id is not None:
# Set Unwanted Files
options['change']['files_unwanted'] = [
x for x in file_list if x not in dl_list
]
options['change']['files_wanted'] = dl_list
logger.debug(
'Downloading {} of {} files in torrent.',
len(options['change']['files_wanted']),
len(file_list),
)
elif (not options['post'].get('main_file_only')
or main_id is None) and skip_files:
# If no main file and we want to skip files
if len(skip_list) >= len(file_list):
logger.debug(
'skip_files filter would cause no files to be downloaded; '
'including all files in torrent.')
else:
options['change']['files_unwanted'] = skip_list
options['change']['files_wanted'] = [
x for x in file_list if x not in skip_list
]
logger.debug(
'Downloading {} of {} files in torrent.',
len(options['change']['files_wanted']),
len(file_list),
)
# Set any changed file properties
if list(options['change'].keys()):
self.client.change_torrent(torrent_info.id, 30,
**options['change'])
if config['action'] == 'add':
# if add_paused was defined and set to False start the torrent;
# prevents downloading data before we set what files we want
start_paused = (
options['post']['paused']
if 'paused' in options['post'] else
not self.client.get_session().start_added_torrents)
if start_paused:
self.client.stop_torrent(torrent_info.id)
else:
self.client.start_torrent(torrent_info.id)
elif config['action'] in ('remove', 'purge'):
self.client.remove_torrent(
[torrent_info.id],
delete_data=config['action'] == 'purge')
logger.info('{}d {} from transmission', config['action'],
torrent_info.name)
elif config['action'] == 'pause':
self.client.stop_torrent([torrent_info.id])
logger.info('paused {} in transmission', torrent_info.name)
elif config['action'] == 'resume':
self.client.start_torrent([torrent_info.id])
logger.info('resumed {} in transmission',
torrent_info.name)
except TransmissionError as e:
logger.opt(exception=True).debug('TransmissionError')
logger.debug('Failed options dict: {}', options)
msg = 'Error trying to {} {}, TransmissionError: {}'.format(
config['action'], entry['title'], e.message or 'N/A')
logger.error(msg)
continue
def _fetch_passkey(results: ResultSet) -> str:
logger.debug('Trying to fetch hebits passkey from user profile')
for result in results:
if result.text == 'פאסקי':
return first(result.parent.select('td.prol')).text
raise plugin.PluginError('Could not fetch passkey, layout change?')
def on_task_output(self, task, config):
if not task.accepted:
log.debug('nothing accepted, aborting')
return
rooms = [s.encode('utf8').lower() for s in config.get('feeds', [])]
if task.options.test:
log.info('Test posting to feed(s): ' + ','.join(rooms))
else:
from flexget.plugins.local.friendfeed2 import FriendFeed, fetch_installed_app_access_token
consumer_token = {'key': config['app_key'], 'secret': config['app_secret']}
access_token = fetch_installed_app_access_token(consumer_token, config['username'], config['password'])
ff = FriendFeed(oauth_consumer_token=consumer_token, oauth_access_token=access_token)
if config['mode'] == 'posts':
for entry in task.accepted:
try:
fftext = entry.render(config['text'])
fflink = entry.render(config['link']) if 'link' in config else None
ffcomm = entry.render(config['comment']) if 'comment' in config else None
ffpict = entry.render(config['image']) if 'image' in config else None
except RenderError as e:
log.error('Error rendering data: %s' % e)
if task.options.test:
log.info('Test run for entry ' + entry['title'])
log.info('- Text would be: ' + fftext)
if fflink:
log.info('- Link would be: ' + fflink)
if ffpict:
log.info('- Image would be: ' + ffpict)
if ffcomm:
log.info('- Comment would be: ' + ffcomm)
else:
try:
res = ff.post_entry(fftext, link=fflink, comment=ffcomm,
to=','.join(rooms), image_url=ffpict)
log.info('Published id: %s' % res['id'])
except Exception as err:
log.info('post_entry() failed with %s' % str(err))
else:
if not config.get('comment'):
raise plugin.PluginError('"comment" option is required when "mode"=="comments".')
try:
fftext = render_from_task(config['text'], task)
fflink = render_from_task(config['link'], task) if 'link' in config else None
ffpict = render_from_task(config['image'], task) if 'image' in config else None
except RenderError as e:
log.error('Error rendering data: %s' % e)
if task.options.test:
log.info('Test run for task.')
log.info('- Text would be: ' + fftext)
if fflink:
log.info('- Link would be: ' + fflink)
if ffpict:
log.info('- Image would be: ' + ffpict)
else:
res = ff.post_entry(fftext, link=fflink, to=','.join(rooms), image_url=ffpict)
log.info('Published id: %s' % res['id'])
for entry in task.accepted:
try:
ffcomm = entry.render(config['comment'])
except RenderError as e:
log.error('Error rendering data: %s' % e)
if task.options.test:
log.info('- Comment would be: ' + ffcomm)
else:
try:
time.sleep(1)
rcm = ff.post_comment(res['id'], ffcomm)
log.verbose('Published comment id: %s' % rcm['id'])
except Exception as err:
log.info('post_comment() failed with %s' % str(err))
def user_profile(self) -> bytes:
logger.debug('Fetching user profile')
rsp = requests.get(self.profile_link)
if "returnto" in rsp.url:
raise plugin.PluginError('Could not fetch passkey from user profile, layout change?')
return rsp.content
def on_task_input(self, task, config):
config = self.prepare_config(config)
accesstoken = ""
urlconfig = {}
urlappend = "?"
if (config['unwatched_only']
and config['section'] != 'recentlyViewedShows'
and config['section'] != 'all'):
urlconfig['unwatched'] = '1'
plexserver = config['server']
if gethostbyname(config['server']) != config['server']:
config['server'] = gethostbyname(config['server'])
if config['username'] and config[
'password'] and config['server'] != '127.0.0.1':
header = {'X-Plex-Client-Identifier': 'flexget'}
log.debug("Trying to to connect to myplex.")
try:
r = requests.post('https://my.plexapp.com/users/sign_in.xml',
auth=(config['username'],
config['password']),
headers=header)
except requests.RequestException as e:
raise plugin.PluginError(
'Could not login to my.plexapp.com: %s. Username: %s Password: %s'
% (e, config['username'], config['password']))
log.debug("Connected to myplex.")
if 'Invalid email' in r.text:
raise plugin.PluginError(
'Could not login to my.plexapp.com: invalid username and/or password!'
)
log.debug("Managed to login to myplex.")
dom = parseString(r.text)
plextoken = dom.getElementsByTagName(
'authentication-token')[0].firstChild.nodeValue
log.debug("Got plextoken: %s" % plextoken)
try:
r = requests.get(
"https://my.plexapp.com/pms/servers?X-Plex-Token=%s" %
plextoken)
except requests.RequestException as e:
raise plugin.PluginError(
'Could not get servers from my.plexapp.com using authentication-token: %s.'
% plextoken)
dom = parseString(r.text)
for node in dom.getElementsByTagName('Server'):
if node.getAttribute('address') == config['server']:
accesstoken = node.getAttribute('accessToken')
log.debug("Got accesstoken: %s" % accesstoken)
urlconfig['X-Plex-Token'] = accesstoken
if accesstoken == "":
raise plugin.PluginError(
'Could not retrieve accesstoken for %s.' %
config['server'])
for key in urlconfig:
urlappend += '%s=%s&' % (key, urlconfig[key])
if not isinstance(config['section'], int):
try:
r = requests.get("http://%s:%d/library/sections/%s" %
(config['server'], config['port'], urlappend))
except requests.RequestException as e:
raise plugin.PluginError('Error retrieving source: %s' % e)
dom = parseString(r.text.encode("utf-8"))
for node in dom.getElementsByTagName('Directory'):
if node.getAttribute('title') == config['section']:
config['section'] = int(node.getAttribute('key'))
if not isinstance(config['section'], int):
raise plugin.PluginError('Could not find section \'%s\'' %
config['section'])
log.debug("Fetching http://%s:%d/library/sections/%s/%s%s" %
(config['server'], config['port'], config['section'],
config['selection'], urlappend))
try:
r = requests.get(
"http://%s:%d/library/sections/%s/%s%s" %
(config['server'], config['port'], config['section'],
config['selection'], urlappend))
except requests.RequestException as e:
raise plugin.PluginError('Error retrieving source: %s' % e)
dom = parseString(r.text.encode("utf-8"))
entries = []
plexsectionname = dom.getElementsByTagName(
'MediaContainer')[0].getAttribute('title1')
if dom.getElementsByTagName('MediaContainer')[0].getAttribute(
'viewGroup') == "show":
for node in dom.getElementsByTagName('Directory'):
title = node.getAttribute('title')
if config['strip_year']:
title = re.sub(r'^(.*)\(\d+\)$', r'\1', title)
title = re.sub(r'[\(\)]', r'', title)
title = re.sub(r'\&', r'And', title)
title = re.sub(r'[^A-Za-z0-9- ]', r'', title)
if config['lowercase_title']:
title = title.lower()
e = Entry()
e['title'] = title
e['url'] = "NULL"
e['plex_server'] = plexserver
e['plex_port'] = config['port']
e['plex_section'] = config['section']
e['plex_section_name'] = plexsectionname
entries.append(e)
elif dom.getElementsByTagName('MediaContainer')[0].getAttribute(
'viewGroup') == "episode":
for node in dom.getElementsByTagName('Video'):
title = node.getAttribute('grandparentTitle')
season = int(node.getAttribute('parentIndex'))
episodethumb = "http://%s:%d%s%s" % (
config['server'], config['port'],
node.getAttribute('thumb'), urlappend)
seriesart = "http://%s:%d%s%s" % (
config['server'], config['port'], node.getAttribute('art'),
urlappend)
seasoncover = "http://%s:%d%s%s" % (
config['server'], config['port'],
node.getAttribute('parentThumb'), urlappend)
seriescover = "http://%s:%d%s%s" % (
config['server'], config['port'],
node.getAttribute('grandparentThumb'), urlappend)
episodetitle = node.getAttribute('title')
episodesummary = node.getAttribute('summary')
if node.getAttribute('parentIndex') == node.getAttribute(
'year'):
season = node.getAttribute('originallyAvailableAt')
filenamemap = "%s_%s%s_%s_%s_%s.%s"
episode = ""
elif node.getAttribute('index'):
episode = int(node.getAttribute('index'))
filenamemap = "%s_%02dx%02d_%s_%s_%s.%s"
else:
log.debug(
"Could not get episode number for '%s' (Hint, ratingKey: %s)"
% (title, node.getAttribute('ratingKey')))
break
for media in node.getElementsByTagName('Media'):
vcodec = media.getAttribute('videoCodec')
acodec = media.getAttribute('audioCodec')
if config['fetch'] == "file" or not config['fetch']:
container = media.getAttribute('container')
else:
container = "jpg"
resolution = media.getAttribute('videoResolution') + "p"
for part in media.getElementsByTagName('Part'):
key = part.getAttribute('key')
e = Entry()
duration = part.getAttribute('duration')
if config['original_filename']:
filename, fileext = os.path.splitext(
basename(part.getAttribute('file')))
if config['fetch'] != 'file':
e['title'] = "%s.jpg" % filename
else:
e['title'] = "%s.%s" % (filename, fileext)
else:
if config['strip_year']:
title = re.sub(r'^(.*)\(\d+\)$', r'\1', title)
title = re.sub(r'[\(\)]', r'', title)
title = re.sub(r'\&', r'And', title).strip()
title = re.sub(r'[^A-Za-z0-9- ]', r'',
title).replace(" ", ".")
if config['lowercase_title']:
title = title.lower()
e['title'] = filenamemap % (title, season, episode,
resolution, vcodec,
acodec, container)
e['filename'] = e['title']
e['plex_url'] = "http://%s:%d%s%s" % (
config['server'], config['port'], key, urlappend)
e['url'] = "http://%s:%d%s%s" % (
config['server'], config['port'], key, urlappend)
e['plex_server'] = plexserver
e['plex_server_ip'] = config['server']
e['plex_port'] = config['port']
e['plex_section'] = config['section']
e['plex_section_name'] = plexsectionname
e['plex_path'] = key
e['plex_duration'] = duration
e['plex_episode_thumb'] = episodethumb
e['plex_series_art'] = seriesart
e['plex_season_cover'] = seasoncover
e['plex_episode_title'] = episodetitle
e['plex_episode_summary'] = episodesummary
if config['fetch'] == "file" or not config['fetch']:
e['url'] = e['plex_url']
elif config['fetch'] == "episode_thumb":
e['url'] = e['plex_episode_thumb']
elif config['fetch'] == "series_art":
e['url'] = e['plex_series_art']
elif config['fetch'] == "season_cover":
e['url'] = e['plex_season_cover']
log.debug("Setting url to %s since %s was selected." %
(e['url'], config['fetch']))
if find(e['url'], '/library/') == -1:
log.debug(
'Seems like the chosen item could not be found in the PMS. Oh, well. NEXT!'
)
break
entries.append(e)
else:
raise plugin.PluginError('Selected section is not a TV section.')
return entries
def search(self, task, entry, config):
"""Search for entries on HEBits"""
passkey = self.authenticate(config)
params = {}
if 'category' in config:
params['cata'] = HEBitsCategory[config['category']].value
entries = set()
params['sort'] = HEBitsSort[config['order_by']].value
params['type'] = 'desc' if config['order_desc'] else 'asc'
for value in ('free', 'double', 'triple', 'pack'):
if config.get(value):
params[value] = 'on'
for search_string in entry.get('search_strings', [entry['title']]):
params['search'] = search_string
logger.debug('Using search params: {}', params)
try:
page = requests.get(self.search_url, params=params)
page.raise_for_status()
except RequestException as e:
logger.error('HEBits request failed: {}', e)
continue
soup = get_soup(page.content)
table = first(soup.select("div.browse"), None)
if not table:
logger.debug(
'Could not find any results matching {} using the requested params {}',
search_string,
params,
)
continue
all_results = table.select("div.lineBrown, div.lineGray, div.lineBlue, div.lineGreen")
if not all_results:
raise plugin.PluginError(
'Result table found but not with any known items, layout change?'
)
for result in all_results:
torrent_id = first(result.select('a[href^=download]')).attrs['href'].split('=')[-1]
seeders = int(first(result.select("div.bUping")).text)
leechers = int(first(result.select("div.bDowning")).text)
size_strings = list(first(result.select("div.bSize")).strings)
size_text = f'{size_strings[1]}{size_strings[2]}'
size = parse_filesize(size_text)
title = first(result.select('a > b')).text.split("/")[-1].strip()
images = result.select("span > img")
freeleech, double_up, triple_up = self._fetch_bonus(images)
req = Request(
'GET', url=self.download_url, params={'passkey': passkey, 'id': torrent_id}
).prepare()
entry = Entry(
torrent_seeds=seeders,
torrent_leeches=leechers,
torrent_availability=torrent_availability(seeders, leechers),
content_size=size,
title=title,
freeleech=freeleech,
triple_up=triple_up,
double_up=double_up,
url=req.url,
)
entries.add(entry)
return entries
def on_task_input(self, task, config):
config = self.prepare_config(config)
passkeys = config.get('passkeys')
limit = config.get('limit')
show_detail = config.get('show_detail')
to = config.get('to')
result = []
from_client_method = None
to_client_method = None
for from_name, client_config in config['from'].items():
from_client = plugin.get_plugin_by_name(from_name)
start_method = from_client.phase_handlers['start']
input_method = from_client.phase_handlers['input']
if not to:
to = from_name[5:]
start_method(task, client_config)
result = input_method(task, client_config)
from_client_method = client_map[from_name]
to_client_method = client_map[to]
torrent_dict, torrents_hashes = self.get_torrents_data(
result, config, from_client_method)
if not torrent_dict:
return []
try:
data = {'sign': config['iyuu'], 'version': config['version']}
sites_response = task.requests.get(
'http://api.iyuu.cn/index.php?s=App.Api.Sites',
timeout=60,
params=data).json()
if sites_response.get('ret') != 200:
raise plugin.PluginError(
'http://api.iyuu.cn/index.php?s=App.Api.Sites: {}'.format(
sites_response))
sites_json = self.modify_sites(sites_response['data']['sites'])
reseed_response = task.requests.post(
'http://api.iyuu.cn/index.php?s=App.Api.Infohash',
json=torrents_hashes,
timeout=60).json()
if reseed_response.get('ret') != 200:
raise plugin.PluginError(
'http://api.iyuu.cn/index.php?s=App.Api.Infohash Error: {}'
.format(reseed_response))
reseed_json = reseed_response['data']
except (RequestException, JSONDecodeError) as e:
raise plugin.PluginError(
'Error when trying to send request to iyuu: {}'.format(e))
entries = []
site_limit = {}
if sites_json and reseed_json:
for info_hash, seeds_data in reseed_json.items():
client_torrent = torrent_dict[info_hash]
for torrent in seeds_data['torrent']:
site = sites_json.get(str(torrent['sid']))
if not site:
continue
if torrent['info_hash'] in torrent_dict.keys():
continue
site_name = self._get_site_name(site['base_url'])
passkey = passkeys.get(site_name)
if not passkey:
if show_detail:
logger.info(
'no passkey, skip site: {}, title: {}'.format(
site_name, client_torrent['title']))
continue
if not site_limit.get(site_name):
site_limit[site_name] = 1
else:
if site_limit[site_name] >= limit:
logger.info(
'site_limit:{} >= limit: {}, skip site: {}, title: {}'
.format(site_limit[site_name], limit,
site_name, client_torrent['title']))
continue
site_limit[site_name] = site_limit[site_name] + 1
torrent_id = str(torrent['torrent_id'])
entry = Entry(title=client_torrent['title'],
torrent_info_hash=torrent['info_hash'])
to_client_method(entry, client_torrent)
entry['class_name'] = site_name
Executor.build_reseed(entry, config, site, passkey,
torrent_id)
if show_detail:
logger.info(
f"accept site: {site_name}, title: {client_torrent['title']}, url: {entry.get('url', None)}"
)
if entry.get('url'):
entries.append(entry)
return entries
def on_task_input(self, task, config):
"""
This plugin returns ALL of the shows monitored by Sickbeard.
This includes both ongoing and ended.
Syntax:
sickbeard:
base_url=<value>
port=<value>
api_key=<value>
Options base_url and api_key are required.
Use with input plugin like discover and/or configure_series.
Example:
download-tv-task:
configure_series:
settings:
quality:
- 720p
from:
sickbeard:
base_url: http://localhost
port: 8531
api_key: MYAPIKEY1123
discover:
what:
- emit_series: yes
from:
torrentz: any
download:
/download/tv
Note that when using the configure_series plugin with Sickbeard
you are basically synced to it, so removing a show in Sickbeard will
remove it in flexget as well, which could be positive or negative,
depending on your usage.
"""
parsedurl = urlparse(config.get('base_url'))
url = '%s://%s:%s%s/api/%s/?cmd=shows' % (
parsedurl.scheme, parsedurl.netloc, config.get('port'),
parsedurl.path, config.get('api_key'))
try:
json = task.requests.get(url).json()
except RequestException as e:
raise plugin.PluginError(
'Unable to connect to Sickbeard at %s://%s:%s%s. Error: %s' %
(parsedurl.scheme, parsedurl.netloc, config.get('port'),
parsedurl.path, e))
entries = []
# Dictionary based on SB quality list.
qualities = {
'Any': '',
'HD': '720p-1080p',
'HD1080p': '1080p',
'HD720p': '720p',
'SD': '<hr'
}
for id, show in json['data'].items():
fg_quality = '' # Initializes the quality parameter
if show['paused'] and config.get('only_monitored'):
continue
if show['status'] == 'Ended' and not config.get('include_ended'):
continue
if config.get('include_data'):
show_url = '%s:%s/api/%s/?cmd=show&tvdbid=%s' % (
config['base_url'], config['port'], config['api_key'],
show['tvdbid'])
show_json = task.requests.get(show_url).json()
sb_quality = show_json['data']['quality']
fg_quality = qualities[sb_quality]
entry = Entry(
title=show['show_name'],
url='',
series_name=show['show_name'],
tvdb_id=show.get('tvdbid'),
tvrage_id=show.get('tvrage_id'),
# configure_series plugin requires that all settings will have the configure_series prefix
configure_series_quality=fg_quality)
if entry.isvalid():
entries.append(entry)
else:
log.error('Invalid entry created? %s' % entry)
# Test mode logging
if task.options.test:
log.info("Test mode. Entry includes:")
log.info(" Title: %s" % entry["title"])
log.info(" URL: %s" % entry["url"])
log.info(" Show name: %s" % entry["series_name"])
log.info(" TVDB ID: %s" % entry["tvdb_id"])
log.info(" TVRAGE ID: %s" % entry["tvrage_id"])
log.info(" Quality: %s" % entry["configure_series_quality"])
return entries
def on_task_output(self, task, config):
"""
Configuration::
subliminal:
languages: List of languages (as IETF codes) in order of preference. At least one is required.
alternatives: List of second-choice languages; subs will be downloaded but entries rejected.
exact_match: Use file hash only to search for subs, otherwise Subliminal will try to guess by filename.
providers: List of providers from where to download subtitles.
single: Download subtitles in single mode (no language code added to subtitle filename).
directory: Path to directory where to save the subtitles, default is next to the video.
hearing_impaired: Prefer subtitles for the hearing impaired when available
authentication: >
Dictionary of configuration options for different providers.
Keys correspond to provider names, and values are dictionaries, usually specifying `username` and
`password`.
"""
if not task.accepted:
log.debug('nothing accepted, aborting')
return
from babelfish import Language
from dogpile.cache.exception import RegionAlreadyConfigured
import subliminal
from subliminal.cli import MutexLock
from subliminal.score import episode_scores, movie_scores
try:
cachefile = os.path.join(os.path.expanduser('~'), 'AppData',
'Local', 'subliminal', 'subliminal',
'Cache', 'subliminal.dbm')
subliminal.region.configure('dogpile.cache.dbm',
arguments={
'filename': cachefile,
'lock_factory': MutexLock,
})
except RegionAlreadyConfigured:
pass
# Let subliminal be more verbose if our logger is set to DEBUG
if log.isEnabledFor(logging.DEBUG):
logging.getLogger("subliminal").setLevel(logging.INFO)
else:
logging.getLogger("subliminal").setLevel(logging.CRITICAL)
logging.getLogger("dogpile").setLevel(logging.CRITICAL)
logging.getLogger("enzyme").setLevel(logging.WARNING)
try:
languages = set(
[Language.fromietf(s) for s in config.get('languages', [])])
alternative_languages = set(
[Language.fromietf(s) for s in config.get('alternatives', [])])
except ValueError as e:
raise plugin.PluginError(e)
# keep all downloaded subtitles and save to disk when done (no need to write every time)
downloaded_subtitles = collections.defaultdict(list)
providers_list = config.get('providers', None)
provider_configs = config.get('authentication', None)
# test if only one language was provided, if so we will download in single mode
# (aka no language code added to subtitle filename)
# unless we are forced not to by configuration
# if we pass 'yes' for single in configuration but choose more than one language
# we ignore the configuration and add the language code to the
# potentially downloaded files
single_mode = config.get(
'single', '') and len(languages | alternative_languages) <= 1
hearing_impaired = config.get('hearing_impaired', False)
with subliminal.core.ProviderPool(
providers=providers_list,
provider_configs=provider_configs) as provider_pool:
for entry in task.accepted:
if 'location' not in entry:
log.warning(
'Cannot act on entries that do not represent a local file.'
)
continue
if not os.path.exists(entry['location']):
entry.fail('file not found: %s' % entry['location'])
continue
if '$RECYCLE.BIN' in entry[
'location']: # ignore deleted files in Windows shares
continue
try:
video = subliminal.scan_video(entry['location'])
# use metadata refiner to get mkv metadata
subliminal.core.refine(video,
episode_refiners=('metadata', ),
movie_refiners=('metadata', ))
video.subtitle_languages |= set(
subliminal.core.search_external_subtitles(
entry['location']).values())
primary_languages = set(entry.get('subtitle_languages',
[])) or languages
if primary_languages.issubset(
video.subtitle_languages) or (
single_mode and video.subtitle_languages):
log.debug(
'All preferred languages already exist for "%s"',
entry['title'])
continue # subs for preferred language(s) already exists
if isinstance(video, subliminal.Episode):
title = video.series
hash_scores = episode_scores['hash']
else:
title = video.title
hash_scores = movie_scores['hash']
log.debug('Name computed for %s was %s', entry['location'],
title)
msc = hash_scores if config['exact_match'] else 0
####################################################################################################
all_languages = primary_languages | alternative_languages
subtitles_list = provider_pool.list_subtitles(
video, all_languages - video.subtitle_languages)
subtitles = provider_pool.download_best_subtitles(
subtitles_list,
video,
all_languages,
min_score=msc,
hearing_impaired=hearing_impaired)
if subtitles:
downloaded_subtitles[video].extend(subtitles)
downloaded_languages = set([
Language.fromietf(str(l.language))
for l in subtitles
])
if len(downloaded_languages & primary_languages):
log.info('Subtitles found for %s',
entry['location'])
else:
log.info(
'subtitles found for a second-choice language.'
)
video.subtitle_languages |= downloaded_languages
entry['subtitles'] = [
l.alpha3 for l in video.subtitle_languages
]
else:
log.verbose('cannot find any subtitles for now.')
'''
subtitles_list = provider_pool.list_subtitles(video, primary_languages - video.subtitle_languages)
subtitles = provider_pool.download_best_subtitles(subtitles_list, video, primary_languages,
min_score=msc, hearing_impaired=hearing_impaired)
if subtitles:
downloaded_subtitles[video].extend(subtitles)
log.info('Subtitles found for %s', entry['location'])
else:
# only try to download for alternatives that aren't already downloaded
subtitles_list = provider_pool.list_subtitles(video, alternative_languages - video.subtitle_languages)
subtitles = provider_pool.download_best_subtitles(subtitles_list, video,
alternative_languages, min_score=msc,
hearing_impaired=hearing_impaired)
if subtitles:
downloaded_subtitles[video].extend(subtitles)
log.info('subtitles found for a second-choice language.')
else:
log.verbose('cannot find any subtitles for now.')
if subtitles:
downloaded_languages = set([Language.fromietf(str(l.language)) for l in subtitles])
entry['subtitles'] = [l.alpha3 for l in video.subtitle_languages]
for l in downloaded_subtitles[video]:
code = Language.fromietf(unicode(l.language)).alpha3
if not code in entry['subtitles']:
entry['subtitles'].append(code)
'''
####################################################################################################
except ValueError as e:
log.error('subliminal error: %s', e)
entry.fail()
if downloaded_subtitles:
if task.options.test:
log.verbose('Test mode. Found subtitles:')
# save subtitles to disk
for video, subtitle in downloaded_subtitles.items():
if subtitle:
_directory = config.get('directory')
if _directory:
_directory = os.path.expanduser(_directory)
if task.options.test:
log.verbose(' FOUND LANGUAGES %s for %s',
[str(l.language) for l in subtitle],
video.name)
continue
subliminal.save_subtitles(video,
subtitle,
single=single_mode,
directory=_directory)
def on_task_input(self, task, config):
user_payload = {'filter[name]': config['username']}
try:
user_response = task.requests.get(
'https://kitsu.io/api/edge/users', params=user_payload
)
except RequestException as e:
error_message = 'Error finding User url: {url}'.format(url=e.request.url)
if hasattr(e, 'response'):
error_message += ' status: {status}'.format(status=e.response.status_code)
log.debug(error_message, exc_info=True)
raise plugin.PluginError(error_message)
user = user_response.json()
if not len(user['data']):
raise plugin.PluginError(
'no such username found "{name}"'.format(name=config['username'])
)
next_url = 'https://kitsu.io/api/edge/users/{id}/library-entries'.format(
id=user['data'][0]['id']
)
payload = {
'filter[status]': ','.join(config['lists']),
'filter[media_type]': 'Anime',
'include': 'media',
'page[limit]': 20,
}
try:
response = task.requests.get(next_url, params=payload)
except RequestException as e:
error_message = 'Error getting list from {url}'.format(url=e.request.url)
if hasattr(e, 'response'):
error_message += ' status: {status}'.format(status=e.response.status_code)
log.debug(error_message, exc_info=True)
raise plugin.PluginError(error_message)
json_data = response.json()
while json_data:
for item, anime in zip(json_data['data'], json_data['included']):
if item['relationships']['media']['data']['id'] != anime['id']:
raise ValueError(
'Anime IDs {id1} and {id2} do not match'.format(
id1=item['relationships']['media']['data']['id'], id2=anime['id']
)
)
status = config.get('status')
if status is not None:
if status == 'airing' and anime['attributes']['endDate'] is not None:
continue
if status == 'finished' and anime['attributes']['endDate'] is None:
continue
types = config.get('type')
if types is not None:
subType = anime['attributes']['subtype']
if subType is None or not subType.lower() in types:
continue
entry = Entry()
entry['title'] = anime['attributes']['canonicalTitle']
titles_en = anime['attributes']['titles'].get('en')
if titles_en:
entry['kitsu_title_en'] = titles_en
titles_en_jp = anime['attributes']['titles'].get('en_jp')
if titles_en_jp:
entry['kitsu_title_en_jp'] = titles_en_jp
titles_ja_jp = anime['attributes']['titles'].get('ja_jp')
if titles_ja_jp:
entry['kitsu_title_ja_jp'] = titles_ja_jp
entry['url'] = anime['links']['self']
if entry.isvalid():
if config.get('latest'):
entry['series_episode'] = item['progress']
entry['series_id_type'] = 'sequence'
entry['title'] += ' ' + str(entry['progress'])
yield entry
next_url = json_data['links'].get('next')
if next_url:
try:
response = task.requests.get(next_url)
except RequestException as e:
error_message = 'Error getting list from next page url: {url}'.format(
url=e.request.url
)
if hasattr(e, 'response'):
error_message += ' status: {status}'.format(status=e.response.status_code)
log.debug(error_message, exc_info=True)
raise plugin.PluginError(error_message)
json_data = response.json()
else:
break
def on_task_prepare(self, task, config):
if config is False: # handles 'template: no' form to turn off template on this task
return
# implements --template NAME
if task.options.template:
if not config or task.options.template not in config:
task.abort('does not use `%s` template' %
task.options.template,
silent=True)
config = self.prepare_config(config)
# add global in except when disabled with no_global
if 'no_global' in config:
config.remove('no_global')
if 'global' in config:
config.remove('global')
elif 'global' not in config:
config.append('global')
toplevel_templates = task.manager.config.get('templates', {})
# apply templates
for template in config:
if template not in toplevel_templates:
if template == 'global':
continue
raise plugin.PluginError(
'Unable to find template %s for task %s' %
(template, task.name), log)
if toplevel_templates[template] is None:
log.warning('Template `%s` is empty. Nothing to merge.' %
template)
continue
log.debug('Merging template %s into task %s' %
(template, task.name))
# We make a copy here because we need to remove
template_config = toplevel_templates[template]
# When there are templates within templates we remove the template
# key from the config and append it's items to our own
if 'template' in template_config:
nested_templates = self.prepare_config(
template_config['template'])
for nested_template in nested_templates:
if nested_template not in config:
config.append(nested_template)
else:
log.warning('Templates contain each other in a loop.')
# Replace template_config with a copy without the template key, to avoid merging errors
template_config = dict(template_config)
del template_config['template']
# Merge
try:
task.merge_config(template_config)
except MergeException as exc:
raise plugin.PluginError(
'Failed to merge template %s to task %s. Error: %s' %
(template, task.name, exc.value))
log.trace('templates: %s', config)
def parse(self, imdb_id, soup=None):
self.imdb_id = extract_id(imdb_id)
url = make_url(self.imdb_id)
self.url = url
if not soup:
page = requests.get(url)
soup = get_soup(page.text)
title_wrapper = soup.find('div', attrs={'class': 'title_wrapper'})
data = json.loads(
soup.find('script', {
'type': 'application/ld+json'
}).text)
if not data:
raise plugin.PluginError(
'IMDB parser needs updating, imdb format changed. Please report on Github.'
)
# Parse stuff from the title-overview section
name_elem = data['name']
if name_elem:
self.name = name_elem.strip()
else:
logger.error(
'Possible IMDB parser needs updating, Please report on Github.'
)
raise plugin.PluginError('Unable to set imdb_name for %s from %s' %
(self.imdb_id, self.url))
year = soup.find('span', attrs={'id': 'titleYear'})
if year:
m = re.search(r'([0-9]{4})', year.text)
if m:
self.year = int(m.group(1))
if not self.year:
logger.debug('No year found for {}', self.imdb_id)
mpaa_rating_elem = data.get('contentRating')
if mpaa_rating_elem:
self.mpaa_rating = mpaa_rating_elem
else:
logger.debug('No rating found for {}', self.imdb_id)
photo_elem = data.get('image')
if photo_elem:
self.photo = photo_elem
else:
logger.debug('No photo found for {}', self.imdb_id)
original_name_elem = title_wrapper.find('div',
{'class': 'originalTitle'})
if original_name_elem:
self.name = title_wrapper.find('h1').contents[0].strip()
self.original_name = original_name_elem.contents[0].strip().strip(
'"')
else:
logger.debug('No original title found for {}', self.imdb_id)
votes_elem = data.get('aggregateRating', {}).get('ratingCount')
if votes_elem:
self.votes = str_to_int(votes_elem) if not isinstance(
votes_elem, int) else votes_elem
else:
logger.debug('No votes found for {}', self.imdb_id)
score_elem = data.get('aggregateRating', {}).get('ratingValue')
if score_elem:
self.score = float(score_elem)
else:
logger.debug('No score found for {}', self.imdb_id)
meta_score_elem = soup.find(attrs={'class': 'metacriticScore'})
if meta_score_elem:
self.meta_score = str_to_int(meta_score_elem.text)
else:
logger.debug('No Metacritic score found for {}', self.imdb_id)
# get director(s)
directors = data.get('director', [])
if not isinstance(directors, list):
directors = [directors]
for director in directors:
if director['@type'] != 'Person':
continue
director_id = extract_id(director['url'])
director_name = director['name']
self.directors[director_id] = director_name
# get writer(s)
writers = data.get('creator', [])
if not isinstance(writers, list):
writers = [writers]
for writer in writers:
if writer['@type'] != 'Person':
continue
writer_id = extract_id(writer['url'])
writer_name = writer['name']
self.writers[writer_id] = writer_name
# Details section
title_details = soup.find('div', attrs={'id': 'titleDetails'})
if title_details:
# get languages
for link in title_details.find_all(
'a',
href=re.compile(r'^/search/title\?title_type=feature'
'&primary_language=')):
lang = link.text.strip().lower()
if lang not in self.languages:
self.languages.append(lang.strip())
# Storyline section
storyline = soup.find('div', attrs={'id': 'titleStoryLine'})
if storyline:
plot_elem = storyline.find('p')
if plot_elem:
# Remove the "Written By" part.
if plot_elem.em:
plot_elem.em.replace_with('')
self.plot_outline = plot_elem.text.strip()
else:
logger.debug('No storyline found for {}', self.imdb_id)
keyword_elem = storyline.find('h4').parent
if keyword_elem:
# The last "a" tag is a link to the full list
self.plot_keywords = [
keyword_elem.text.strip()
for keyword_elem in keyword_elem.find_all("a")[:-1]
]
genres = data.get('genre', [])
if not isinstance(genres, list):
genres = [genres]
self.genres = [g.strip().lower() for g in genres]
# Cast section
cast = soup.find('table', attrs={'class': 'cast_list'})
if cast:
for actor in cast.select('tr > td:nth-of-type(2) > a'):
actor_id = extract_id(actor['href'])
actor_name = actor.text.strip()
# tag instead of name
if isinstance(actor_name, Tag):
actor_name = None
self.actors[actor_id] = actor_name
def on_task_input(self, task, config):
config = self.build_config(config)
log.debug('Requesting task `%s` url `%s`', task.name, config['url'])
# Used to identify which etag/modified to use
url_hash = str(hash(config['url']))
# set etag and last modified headers if config has not changed since
# last run and if caching wasn't disabled with --no-cache argument.
all_entries = (config['all_entries'] or task.config_modified
or task.options.nocache or task.options.retry)
headers = {}
if not all_entries:
etag = task.simple_persistence.get('%s_etag' % url_hash, None)
if etag:
log.debug('Sending etag %s for task %s', etag, task.name)
headers['If-None-Match'] = etag
modified = task.simple_persistence.get('%s_modified' % url_hash,
None)
if modified:
if not isinstance(modified, basestring):
log.debug(
'Invalid date was stored for last modified time.')
else:
headers['If-Modified-Since'] = modified
log.debug('Sending last-modified %s for task %s',
headers['If-Modified-Since'], task.name)
# Get the feed content
if config['url'].startswith(('http', 'https', 'ftp', 'file')):
# Get feed using requests library
auth = None
if 'username' in config and 'password' in config:
auth = (config['username'], config['password'])
try:
# Use the raw response so feedparser can read the headers and status values
response = task.requests.get(config['url'],
timeout=60,
headers=headers,
raise_status=False,
auth=auth)
content = response.content
except RequestException as e:
raise plugin.PluginError(
'Unable to download the RSS for task %s (%s): %s' %
(task.name, config['url'], e))
if config.get('ascii'):
# convert content to ascii (cleanup), can also help with parsing problems on malformed feeds
content = response.text.encode('ascii', 'ignore')
# status checks
status = response.status_code
if status == 304:
log.verbose(
'%s hasn\'t changed since last run. Not creating entries.',
config['url'])
# Let details plugin know that it is ok if this feed doesn't produce any entries
task.no_entries_ok = True
return []
elif status == 401:
raise plugin.PluginError(
'Authentication needed for task %s (%s): %s' %
(task.name, config['url'],
response.headers['www-authenticate']), log)
elif status == 404:
raise plugin.PluginError(
'RSS Feed %s (%s) not found' % (task.name, config['url']),
log)
elif status == 500:
raise plugin.PluginError(
'Internal server exception on task %s (%s)' %
(task.name, config['url']), log)
elif status != 200:
raise plugin.PluginError(
'HTTP error %s received from %s' % (status, config['url']),
log)
# update etag and last modified
if not config['all_entries']:
etag = response.headers.get('etag')
if etag:
task.simple_persistence['%s_etag' % url_hash] = etag
log.debug('etag %s saved for task %s', etag, task.name)
if response.headers.get('last-modified'):
modified = response.headers['last-modified']
task.simple_persistence['%s_modified' %
url_hash] = modified
log.debug('last modified %s saved for task %s', modified,
task.name)
else:
# This is a file, open it
with open(config['url'], 'rb') as f:
content = f.read()
if config.get('ascii'):
# Just assuming utf-8 file in this case
content = content.decode('utf-8',
'ignore').encode('ascii', 'ignore')
if not content:
log.error('No data recieved for rss feed.')
return
try:
rss = feedparser.parse(content)
except LookupError as e:
raise plugin.PluginError('Unable to parse the RSS (from %s): %s' %
(config['url'], e))
# check for bozo
ex = rss.get('bozo_exception', False)
if ex or rss.get('bozo'):
if rss.entries:
msg = 'Bozo error %s while parsing feed, but entries were produced, ignoring the error.' % type(
ex)
if config.get('silent', False):
log.debug(msg)
else:
log.verbose(msg)
else:
if isinstance(ex, feedparser.NonXMLContentType):
# see: http://www.feedparser.org/docs/character-encoding.html#advanced.encoding.nonxml
log.debug('ignoring feedparser.NonXMLContentType')
elif isinstance(ex, feedparser.CharacterEncodingOverride):
# see: ticket 88
log.debug('ignoring feedparser.CharacterEncodingOverride')
elif isinstance(ex, UnicodeEncodeError):
raise plugin.PluginError(
'Feed has UnicodeEncodeError while parsing...')
elif isinstance(ex, (xml.sax._exceptions.SAXParseException,
xml.sax._exceptions.SAXException)):
# save invalid data for review, this is a bit ugly but users seem to really confused when
# html pages (login pages) are received
self.process_invalid_content(task, content, config['url'])
if task.options.debug:
log.exception(ex)
raise plugin.PluginError(
'Received invalid RSS content from task %s (%s)' %
(task.name, config['url']))
elif isinstance(ex, httplib.BadStatusLine) or isinstance(
ex, IOError):
raise ex # let the @internet decorator handle
else:
# all other bozo errors
self.process_invalid_content(task, content, config['url'])
raise plugin.PluginError(
'Unhandled bozo_exception. Type: %s (task: %s)' %
(ex.__class__.__name__, task.name), log)
log.debug('encoding %s', rss.encoding)
last_entry_id = ''
if not all_entries:
# Test to make sure entries are in descending order
if rss.entries and rss.entries[0].get(
'published_parsed') and rss.entries[-1].get(
'published_parsed'):
if rss.entries[0]['published_parsed'] < rss.entries[-1][
'published_parsed']:
# Sort them if they are not
rss.entries.sort(key=lambda x: x['published_parsed'],
reverse=True)
last_entry_id = task.simple_persistence.get('%s_last_entry' %
url_hash)
# new entries to be created
entries = []
# field name for url can be configured by setting link.
# default value is auto but for example guid is used in some feeds
ignored = 0
for entry in rss.entries:
# Check if title field is overridden in config
title_field = config.get('title', 'title')
# ignore entries without title
if not entry.get(title_field):
log.debug('skipping entry without title')
ignored += 1
continue
# Set the title from the source field
entry.title = entry[title_field]
# Check we haven't already processed this entry in a previous run
if last_entry_id == entry.title + entry.get('guid', ''):
log.verbose('Not processing entries from last run.')
# Let details plugin know that it is ok if this task doesn't produce any entries
task.no_entries_ok = True
break
# remove annoying zero width spaces
entry.title = entry.title.replace(u'\u200B', u'')
# Dict with fields to grab mapping from rss field name to FlexGet field name
fields = {
'guid': 'guid',
'author': 'author',
'description': 'description',
'infohash': 'torrent_info_hash'
}
# extend the dict of fields to grab with other_fields list in config
for field_map in config.get('other_fields', []):
fields.update(field_map)
# helper
# TODO: confusing? refactor into class member ...
def add_entry(ea):
ea['title'] = entry.title
for rss_field, flexget_field in fields.iteritems():
if rss_field in entry:
if not isinstance(getattr(entry, rss_field),
basestring):
# Error if this field is not a string
log.error(
'Cannot grab non text field `%s` from rss.',
rss_field)
# Remove field from list of fields to avoid repeated error
config['other_fields'].remove(rss_field)
continue
if not getattr(entry, rss_field):
log.debug(
'Not grabbing blank field %s from rss for %s.',
rss_field, ea['title'])
continue
try:
ea[flexget_field] = decode_html(entry[rss_field])
if rss_field in config.get('other_fields', []):
# Print a debug message for custom added fields
log.debug('Field `%s` set to `%s` for `%s`',
rss_field, ea[rss_field],
ea['title'])
except UnicodeDecodeError:
log.warning(
'Failed to decode entry `%s` field `%s`',
ea['title'], rss_field)
# Also grab pubdate if available
if hasattr(entry,
'published_parsed') and entry.published_parsed:
ea['rss_pubdate'] = datetime(*entry.published_parsed[:6])
# store basic auth info
if 'username' in config and 'password' in config:
ea['download_auth'] = (config['username'],
config['password'])
entries.append(ea)
# create from enclosures if present
enclosures = entry.get('enclosures', [])
if len(enclosures) > 1 and not config.get('group_links'):
# There is more than 1 enclosure, create an Entry for each of them
log.debug('adding %i entries from enclosures', len(enclosures))
for enclosure in enclosures:
if not 'href' in enclosure:
log.debug('RSS-entry `%s` enclosure does not have URL',
entry.title)
continue
# There is a valid url for this enclosure, create an Entry for it
ee = Entry()
self.add_enclosure_info(ee, enclosure,
config.get('filename', True), True)
add_entry(ee)
# If we created entries for enclosures, we should not create an Entry for the main rss item
continue
# create flexget entry
e = Entry()
if not isinstance(config.get('link'), list):
# If the link field is not a list, search for first valid url
if config['link'] == 'auto':
# Auto mode, check for a single enclosure url first
if len(entry.get(
'enclosures',
[])) == 1 and entry['enclosures'][0].get('href'):
self.add_enclosure_info(e, entry['enclosures'][0],
config.get('filename', True))
else:
# If there is no enclosure url, check link, then guid field for urls
for field in ['link', 'guid']:
if entry.get(field):
e['url'] = entry[field]
break
else:
if entry.get(config['link']):
e['url'] = entry[config['link']]
else:
# If link was passed as a list, we create a list of urls
for field in config['link']:
if entry.get(field):
e.setdefault('url', entry[field])
if entry[field] not in e.setdefault('urls', []):
e['urls'].append(entry[field])
if config.get('group_links'):
# Append a list of urls from enclosures to the urls field if group_links is enabled
e.setdefault('urls', [e['url']]).extend([
enc.href for enc in entry.get('enclosures', [])
if enc.get('href') not in e['urls']
])
if not e.get('url'):
log.debug('%s does not have link (%s) or enclosure',
entry.title, config['link'])
ignored += 1
continue
add_entry(e)
# Save last spot in rss
if rss.entries:
log.debug('Saving location in rss feed.')
task.simple_persistence[
'%s_last_entry' %
url_hash] = rss.entries[0].title + rss.entries[0].get(
'guid', '')
if ignored:
if not config.get('silent'):
log.warning(
'Skipped %s RSS-entries without required information (title, link or enclosures)',
ignored)
return entries
def lookup(self, entry, search_allowed=True, session=None):
"""
Perform imdb lookup for entry.
:param entry: Entry instance
:param search_allowed: Allow fallback to search
:raises PluginError: Failure reason
"""
from flexget.manager import manager
if entry.get('imdb_id', eval_lazy=False):
logger.debug('No title passed. Lookup for {}', entry['imdb_id'])
elif entry.get('imdb_url', eval_lazy=False):
logger.debug('No title passed. Lookup for {}', entry['imdb_url'])
elif entry.get('title', eval_lazy=False):
logger.debug('lookup for {}', entry['title'])
else:
raise plugin.PluginError(
'looking up IMDB for entry failed, no title, imdb_url or imdb_id passed.'
)
# if imdb_id is included, build the url.
if entry.get('imdb_id', eval_lazy=False) and not entry.get('imdb_url', eval_lazy=False):
entry['imdb_url'] = make_url(entry['imdb_id'])
# make sure imdb url is valid
if entry.get('imdb_url', eval_lazy=False):
imdb_id = extract_id(entry['imdb_url'])
if imdb_id:
entry['imdb_url'] = make_url(imdb_id)
else:
logger.debug('imdb url {} is invalid, removing it', entry['imdb_url'])
entry['imdb_url'] = ''
# no imdb_url, check if there is cached result for it or if the
# search is known to fail
if not entry.get('imdb_url', eval_lazy=False):
result = (
session.query(db.SearchResult)
.filter(db.SearchResult.title == entry['title'])
.first()
)
if result:
# TODO: 1.2 this should really be checking task.options.retry
if result.fails and not manager.options.execute.retry:
# this movie cannot be found, not worth trying again ...
logger.debug('{} will fail lookup', entry['title'])
raise plugin.PluginError('IMDB lookup failed for %s' % entry['title'])
else:
if result.url:
logger.trace('Setting imdb url for {} from db', entry['title'])
entry['imdb_id'] = result.imdb_id
entry['imdb_url'] = result.url
# no imdb url, but information required, try searching
if not entry.get('imdb_url', eval_lazy=False) and search_allowed:
logger.verbose('Searching from imdb `{}`', entry['title'])
search = ImdbSearch()
search_name = entry.get('movie_name', entry['title'], eval_lazy=False)
search_result = search.smart_match(search_name)
if search_result:
entry['imdb_url'] = search_result['url']
# store url for this movie, so we don't have to search on every run
result = db.SearchResult(entry['title'], entry['imdb_url'])
session.add(result)
session.commit()
logger.verbose('Found {}', entry['imdb_url'])
else:
log_once(
'IMDB lookup failed for %s' % entry['title'],
logger,
'WARNING',
session=session,
)
# store FAIL for this title
result = db.SearchResult(entry['title'])
result.fails = True
session.add(result)
session.commit()
raise plugin.PluginError('Title `%s` lookup failed' % entry['title'])
# check if this imdb page has been parsed & cached
movie = session.query(db.Movie).filter(db.Movie.url == entry['imdb_url']).first()
# If we have a movie from cache, we are done
if movie and not movie.expired:
entry.update_using_map(self.field_map, movie)
return
# Movie was not found in cache, or was expired
if movie is not None:
if movie.expired:
logger.verbose('Movie `{}` details expired, refreshing ...', movie.title)
# Remove the old movie, we'll store another one later.
session.query(db.MovieLanguage).filter(db.MovieLanguage.movie_id == movie.id).delete()
session.query(db.Movie).filter(db.Movie.url == entry['imdb_url']).delete()
session.commit()
# search and store to cache
if 'title' in entry:
logger.verbose('Parsing imdb for `{}`', entry['title'])
else:
logger.verbose('Parsing imdb for `{}`', entry['imdb_id'])
try:
movie = self._parse_new_movie(entry['imdb_url'], session)
except UnicodeDecodeError:
logger.error(
'Unable to determine encoding for {}. Installing chardet library may help.',
entry['imdb_url'],
)
# store cache so this will not be tried again
movie = db.Movie()
movie.url = entry['imdb_url']
session.add(movie)
session.commit()
raise plugin.PluginError('UnicodeDecodeError')
except ValueError as e:
# TODO: might be a little too broad catch, what was this for anyway? ;P
if manager.options.debug:
logger.exception(e)
raise plugin.PluginError('Invalid parameter: %s' % entry['imdb_url'], logger)
for att in [
'title',
'score',
'votes',
'meta_score',
'year',
'genres',
'languages',
'actors',
'directors',
'writers',
'mpaa_rating',
]:
logger.trace('movie.{}: {}', att, getattr(movie, att))
# Update the entry fields
entry.update_using_map(self.field_map, movie)
