def on_task_output(self, task, config):
if not config:
return
if not isinstance(config, dict):
config = {'action': config}
config.setdefault('queue_name', 'default')
for entry in task.accepted:
# Tell tmdb_lookup to add lazy lookup fields if not already present
try:
plugin.get_plugin_by_name('tmdb_lookup').instance.lookup(entry)
except plugin.DependencyError:
log.debug(
'tmdb_lookup is not available, queue will not work if movie ids are not populated'
)
# Find one or both movie id's for this entry. See if an id is already populated before incurring lazy lookup
kwargs = {}
for lazy in [False, True]:
if entry.get('imdb_id', eval_lazy=lazy):
kwargs['imdb_id'] = entry['imdb_id']
if entry.get('tmdb_id', eval_lazy=lazy):
kwargs['tmdb_id'] = entry['tmdb_id']
if kwargs:
break
if not kwargs:
log.warning(
'Could not determine a movie id for %s, it will not be added to queue.'
% entry['title'])
continue
# Provide movie title if it is already available, to avoid movie_queue doing a lookup
kwargs['title'] = (entry.get('imdb_name', eval_lazy=False)
or entry.get('tmdb_name', eval_lazy=False)
or entry.get('movie_name', eval_lazy=False))
log.debug('movie_queue kwargs: %s' % kwargs)
kwargs['queue_name'] = config.get('queue_name')
try:
action = config.get('action')
if action == 'add':
# since entries usually have unknown quality we need to ignore that ..
if entry.get('quality_req'):
kwargs['quality'] = qualities.Requirements(
entry['quality_req'])
elif entry.get('quality'):
kwargs['quality'] = qualities.Requirements(
entry['quality'].name)
else:
kwargs['quality'] = qualities.Requirements(
config.get('quality', 'any'))
queue_add(**kwargs)
elif action == 'remove':
queue_del(**kwargs)
elif action == 'forget':
queue_forget(**kwargs)
except QueueError as e:
# Ignore already in queue errors
if e.errno != 1:
entry.fail('Error adding movie to queue: %s' % e.message)
def validate(self, data):
try:
qualities.Requirements(data)
except ValueError, e:
self.errors.add('`%s` is not a valid quality requirement: %s' %
(data, e.message))
return False
def on_task_start(self, task, config):
if isinstance(config, basestring):
config = {'any': config}
assume = namedtuple('assume', ['target', 'quality'])
self.assumptions = []
for target, quality in list(config.items()):
log.verbose('New assumption: %s is %s' % (target, quality))
try:
target = qualities.Requirements(target)
except ValueError:
raise plugin.PluginError(
'%s is not a valid quality. Forgetting assumption.' %
target)
try:
quality = qualities.get(quality)
except ValueError:
raise plugin.PluginError(
'%s is not a valid quality. Forgetting assumption.' %
quality)
self.assumptions.append(assume(target, quality))
self.assumptions.sort(
key=lambda assumption: self.precision(assumption.target),
reverse=True)
for assumption in self.assumptions:
log.debug('Target %s - Priority %s' %
(assumption.target, self.precision(assumption.target)))
def queue_add(title=None,
imdb_id=None,
tmdb_id=None,
quality=None,
session=None,
queue_name='default'):
"""
Add an item to the queue with the specified quality requirements.
One or more of `title` `imdb_id` or `tmdb_id` must be specified when calling this function.
:param title: Title of the movie. (optional)
:param imdb_id: IMDB id for the movie. (optional)
:param tmdb_id: TMDB id for the movie. (optional)
:param quality: A QualityRequirements object defining acceptable qualities.
:param queue_name: Name of movie queue to get items from
:param session: Optional session to use for database updates
"""
quality = quality or qualities.Requirements('any')
if not title or not (imdb_id or tmdb_id):
# We don't have all the info we need to add movie, do a lookup for more info
result = parse_what(imdb_id or title or tmdb_id, session=session)
title = result['title']
if not title:
raise QueueError(
'Could not parse movie info for given parameters: title=%s, imdb_id=%s, tmdb_id=%s'
% (title, imdb_id, tmdb_id))
imdb_id = result['imdb_id']
tmdb_id = result['tmdb_id']
# check if the item is already queued
item = session.query(QueuedMovie).filter(
and_((func.lower(QueuedMovie.queue_name) == queue_name.lower()),
or_(
and_(QueuedMovie.imdb_id != None,
QueuedMovie.imdb_id == imdb_id),
and_(QueuedMovie.tmdb_id != None,
QueuedMovie.tmdb_id == tmdb_id)))).first()
if not item:
item = QueuedMovie(title=title,
imdb_id=imdb_id,
tmdb_id=tmdb_id,
quality=quality.text,
queue_name=queue_name)
session.add(item)
session.commit()
log.info('Adding %s to movie queue %s with quality=%s.', title,
queue_name, quality)
return item.to_dict()
else:
if item.downloaded:
raise QueueError(
'ERROR: %s has already been queued and downloaded' % title,
errno=1)
else:
raise QueueError('ERROR: %s is already in the queue %s' %
(title, queue_name),
errno=1)
def filter_entries(self, entries, existing, target, action_on_lower):
target_requirement = qualities.Requirements(target) if target else None
filtered = []
for entry in entries:
# Filter out entries within target
if target:
if not target_requirement.allows(entry['quality']):
log.debug(
'Skipping %s as does not meet upgrade quality requirements', entry['title']
)
if action_on_lower:
action_on_lower(entry, 'does not meet upgrade quality requirements')
continue
if entry['quality'] < existing.quality:
log.debug('Skipping %s as lower quality then existing', entry['title'])
if action_on_lower:
action_on_lower(entry, 'lower quality then existing')
continue
if (
entry['quality'] == existing.quality
and entry.get('proper_count', 0) <= existing.proper_count
):
log.debug('Skipping %s as same quality but lower proper', entry['title'])
if action_on_lower:
action_on_lower(entry, 'lower proper then existing')
continue
filtered.append(entry)
return filtered
def __call__(self, parser, namespace, values, option_string=None):
options = namespace.movie_queue = {}
# Assume 'list' if no action was given
if not values:
values = ['list']
if values[0].lower() not in ACTIONS:
raise ArgumentError(self, '`%s` is not a valid action.\nUsage: ' % values[0] + USAGE)
options['action'] = values[0].lower()
if len(values) == 1:
if options['action'] not in ('list', 'downloaded', 'clear'):
raise ArgumentError(self, 'You must specify the movie.\nUsage: ' + USAGE)
# 2, args is the minimum allowed (operation + item) for actions other than list
if len(values) >= 2:
options['what'] = values[1]
# 3, quality
if len(values) >= 3:
try:
options['quality'] = qualities.Requirements(values[2])
except ValueError, e:
raise ArgumentError(self, '`%s` is an invalid quality requirement string: %s' %
(values[2], e.message))
def on_task_output(self, task, config):
if not config:
return
if not isinstance(config, dict):
config = {}
for entry in task.accepted:
# Tell tmdb_lookup to add lazy lookup fields if not already present
try:
get_plugin_by_name('tmdb_lookup').instance.lookup(entry)
except DependencyError:
log.debug(
'tmdb_lookup is not available, queue will not work if movie ids are not populated'
)
# Find one or both movie id's for this entry. See if an id is already populated before incurring lazy lookup
kwargs = {}
for lazy in [False, True]:
if entry.get('imdb_id', eval_lazy=lazy):
kwargs['imdb_id'] = entry['imdb_id']
if entry.get('tmdb_id', eval_lazy=lazy):
kwargs['tmdb_id'] = entry['tmdb_id']
if kwargs:
break
if not kwargs:
log.warning(
'Could not determine a movie id for %s, it will not be added to queue.'
% entry['title'])
continue
# since entries usually have unknown quality we need to ignore that ..
if entry.get('quality'):
quality = qualities.Requirements(entry['quality'].name)
else:
quality = qualities.Requirements(config.get('quality', 'any'))
kwargs['quality'] = quality
force = entry.get('force', config.get('force'))
if force is not None:
kwargs['force'] = force
# Provide movie title if it is already available, to avoid movie_queue doing a lookup
kwargs['title'] = entry.get('imdb_name') or entry.get(
'tmdb_name') or entry.get('movie_name')
log.debug('queueing kwargs: %s' % kwargs)
try:
queue_add(**kwargs)
except QueueError, e:
task.fail(entry, 'Error adding movie to queue: %s' % e.message)
def __call__(self, parser, namespace, values, option_string=None):
options = namespace.movie_queue = {}
# Assume 'list' if no action was given
if not values:
values = ['list']
if values[0].lower() not in ACTIONS:
raise ArgumentError(
self,
'`%s` is not a valid action.\nUsage: ' % values[0] + USAGE)
options['action'] = values[0].lower()
if len(values) == 1:
if options['action'] not in ('list', 'downloaded', 'clear'):
raise ArgumentError(
self, 'You must specify the movie.\nUsage: ' + USAGE)
# 2, args is the minimum allowed (operation + item) for actions other than list
if len(values) >= 2:
options['what'] = values[1]
# 3, quality
if len(values) >= 3:
try:
options['quality'] = qualities.Requirements(values[2])
except ValueError as e:
raise ArgumentError(
self, '`%s` is an invalid quality requirement string: %s' %
(values[2], e.message))
else:
options['quality'] = qualities.Requirements('any')
# TODO: Get default from config somehow?
# why not use the quality user has queued most, ie option called 'auto' ?
# and if none is queued default to something good like '720p bluray'
# 4, force download
if len(values) >= 4:
options['force'] = str_to_boolean(values[3])
else:
options['force'] = True
if len(values) > 4:
raise ArgumentError(self,
'Too many arguments passed.\nUsage: ' + USAGE)
def add_to_queue():
what = request.values.get('what')
imdb_id = request.values.get('imdb_id')
# TODO: This is a rather limited selection of quality considering the new quality system. Improve it.
quality = qualities.Requirements(request.values.get('quality', 'ANY'))
force = request.values.get('force') == 'on'
try:
title = queue_add(title=what, imdb_id=imdb_id, quality=quality, force=force)['title']
except QueueError, e:
flash(e.message, 'error')
def on_task_filter(self, task, config):
if not isinstance(config, list):
config = [config]
reqs = [quals.Requirements(req) for req in config]
for entry in task.entries:
if not entry.get('quality'):
entry.reject('Entry doesn\'t have a quality')
continue
if not any(req.allows(entry['quality']) for req in reqs):
entry.reject('%s does not match quality requirement %s' % (entry['quality'], reqs))
def queue_add(title=None,
imdb_id=None,
tmdb_id=None,
quality=None,
force=True,
session=None):
"""
Add an item to the queue with the specified quality requirements.
One or more of `title` `imdb_id` or `tmdb_id` must be specified when calling this function.
:param title: Title of the movie. (optional)
:param imdb_id: IMDB id for the movie. (optional)
:param tmdb_id: TMDB id for the movie. (optional)
:param quality: A QualityRequirements object defining acceptable qualities.
:param force: If this is true, accepted movie will be marked as immortal.
:param session: Optional session to use for database updates
"""
quality = quality or qualities.Requirements('any')
if not title or not (imdb_id or tmdb_id):
# We don't have all the info we need to add movie, do a lookup for more info
result = parse_what(imdb_id or title, session=session)
title = result['title']
imdb_id = result['imdb_id']
tmdb_id = result['tmdb_id']
# check if the item is already queued
item = session.query(QueuedMovie).filter(or_(and_(QueuedMovie.imdb_id != None, QueuedMovie.imdb_id == imdb_id),
and_(QueuedMovie.tmdb_id != None, QueuedMovie.tmdb_id == tmdb_id))).\
first()
if not item:
item = QueuedMovie(title=title,
imdb_id=imdb_id,
tmdb_id=tmdb_id,
quality=quality.text,
immortal=force)
session.add(item)
log.info('Adding %s to movie queue with quality=%s and force=%s.' %
(title, quality, force))
return {
'title': title,
'imdb_id': imdb_id,
'tmdb_id': tmdb_id,
'quality': quality,
'force': force
}
else:
if item.downloaded:
raise QueueError(
'ERROR: %s has already been queued and downloaded' % title)
else:
raise QueueError('ERROR: %s is already in the queue' % title)
def on_task_filter(self, task, config):
if not isinstance(config, list):
config = [config]
reqs = [qualities.Requirements(req) for req in config]
for entry in task.entries:
if entry.get('quality') is None:
entry.reject('Entry doesn\'t have a quality')
continue
if not any(req.allows(entry['quality']) for req in reqs):
text_reqs = ', '.join(f'`{req}`' for req in reqs)
entry.reject(
f'`{entry["quality"]}` does not match any of quality requirements: {text_reqs}'
)
def add_to_queue():
what = request.values.get('what')
imdb_id = request.values.get('imdb_id')
# TODO: This is a rather limited selection of quality considering the new quality system. Improve it.
quality = qualities.Requirements(request.values.get('quality', 'ANY'))
force = request.values.get('force') == 'on'
try:
title = queue_add(title=what, imdb_id=imdb_id, quality=quality, force=force)['title']
except QueueError as e:
flash(e.message, 'error')
else:
flash('%s successfully added to queue.' % title, 'success')
return redirect(url_for('.index'))
def post(self, session=None):
""" Add movies to movie queue """
kwargs = request.json
kwargs['quality'] = qualities.Requirements(kwargs.get('quality'))
kwargs['session'] = session
try:
movie = mq.queue_add(**kwargs)
except mq.QueueError as e:
reply = {'status': 'error', 'message': e.message}
return reply, 500
reply = jsonify(movie)
reply.status_code = 201
return reply
def getter(self):
return qualities.Requirements(getattr(self, text_attr))
def on_task_filter(self, task, config):
if not config:
return
identified_by = '{{ id }}' if config[
'identified_by'] == 'auto' else config['identified_by']
grouped_entries = group_entries(task.accepted + task.undecided,
identified_by)
if not grouped_entries:
return
action_on_waiting = entry_actions[config[
'on_waiting']] if config['on_waiting'] != 'do_nothing' else None
action_on_reached = entry_actions[config[
'on_reached']] if config['on_reached'] != 'do_nothing' else None
with Session() as session:
# Prefetch Data
existing_ids = session.query(EntryTimeFrame).filter(
EntryTimeFrame.id.in_(grouped_entries.keys())).all()
existing_ids = {e.id: e for e in existing_ids}
for identifier, entries in grouped_entries.items():
if not entries:
continue
id_timeframe = existing_ids.get(identifier)
if not id_timeframe:
id_timeframe = EntryTimeFrame()
id_timeframe.id = identifier
id_timeframe.status = 'waiting'
id_timeframe.first_seen = datetime.now()
session.add(id_timeframe)
if id_timeframe.status == 'accepted':
log.debug('Previously accepted %s with %s skipping',
identifier, id_timeframe.title)
continue
# Sort entities in order of quality and best proper
entries.sort(key=lambda e:
(e['quality'], e.get('proper_count', 0)),
reverse=True)
best_entry = entries[0]
log.debug('Current best for identifier %s is %s', identifier,
best_entry['title'])
id_timeframe.title = best_entry['title']
id_timeframe.quality = best_entry['quality']
id_timeframe.proper_count = best_entry.get('proper_count', 0)
# Check we hit target or better
target_requirement = qualities.Requirements(config['target'])
target_quality = qualities.Quality(config['target'])
if target_requirement.allows(
best_entry['quality']
) or best_entry['quality'] >= target_quality:
log.debug(
'timeframe reach target quality %s or higher for %s' %
(target_quality, identifier))
if action_on_reached:
action_on_reached(
best_entry,
'timeframe reached target quality or higher')
continue
# Check if passed wait time
expires = id_timeframe.first_seen + parse_timedelta(
config['wait'])
if expires <= datetime.now():
log.debug(
'timeframe expired, releasing quality restriction for %s'
% identifier)
if action_on_reached:
action_on_reached(best_entry, 'timeframe wait expired')
continue
# Verbose waiting, add to backlog
if action_on_waiting:
for entry in entries:
action_on_waiting(entry, 'timeframe waiting')
diff = expires - datetime.now()
hours, remainder = divmod(diff.seconds, 3600)
hours += diff.days * 24
minutes, _ = divmod(remainder, 60)
log.info(
'`%s`: timeframe waiting for %02dh:%02dmin. Currently best is `%s`.',
identifier, hours, minutes, best_entry['title'])
# add best entry to backlog (backlog is able to handle duplicate adds)
if self.backlog:
self.backlog.instance.add_backlog(task,
best_entry,
session=session)
def is_quality_req(instance):
if not isinstance(instance, str_types):
return True
return qualities.Requirements(instance)
if options['action'] not in ('list', 'downloaded', 'clear'):
raise ArgumentError(self, 'You must specify the movie.\nUsage: ' + USAGE)
# 2, args is the minimum allowed (operation + item) for actions other than list
if len(values) >= 2:
options['what'] = values[1]
# 3, quality
if len(values) >= 3:
try:
options['quality'] = qualities.Requirements(values[2])
except ValueError, e:
raise ArgumentError(self, '`%s` is an invalid quality requirement string: %s' %
(values[2], e.message))
else:
options['quality'] = qualities.Requirements('any')
# TODO: Get default from config somehow?
# why not use the quality user has queued most, ie option called 'auto' ?
# and if none is queued default to something good like '720p bluray'
# 4, force download
if len(values) >= 4:
options['force'] = str_to_boolean(values[3])
else:
options['force'] = True
if len(values) > 4:
raise ArgumentError(self, 'Too many arguments passed.\nUsage: ' + USAGE)
register_plugin(MovieQueueManager, 'movie_queue_manager', builtin=True)
def do_cli(manager, options):
"""Handle movie-queue subcommand"""
if options.queue_action == 'list':
queue_list(options)
return
# If the action affects make sure all entries are processed again next run.
manager.config_changed()
if options.queue_action == 'clear':
clear()
return
if options.queue_action == 'del':
try:
what = parse_what(options.movie_name, lookup=False)
title = queue_del(**what)
except QueueError as e:
console('ERROR: %s' % e.message)
else:
console('Removed %s from queue' % title)
return
if options.queue_action == 'forget':
try:
what = parse_what(options.movie_name, lookup=False)
title = queue_forget(**what)
except QueueError as e:
console('ERROR: %s' % e.message)
else:
console(
'Forgot that %s was downloaded. Movie will be downloaded again.'
% title)
return
if options.queue_action == 'add':
try:
quality = qualities.Requirements(options.quality)
except ValueError as e:
console('`%s` is an invalid quality requirement string: %s' %
(options.quality, e.message))
return
# Adding to queue requires a lookup for missing information
what = {}
try:
what = parse_what(options.movie_name)
except QueueError as e:
console('ERROR: %s' % e.message)
if not what.get('title') or not (what.get('imdb_id')
or what.get('tmdb_id')):
console('could not determine movie') # TODO: Rethink errors
return
try:
queue_add(quality=quality, **what)
except QueueError as e:
console(e.message)
if e.errno == 1:
# This is an invalid quality error, display some more info
# TODO: Fix this error?
# console('Recognized qualities are %s' % ', '.join([qual.name for qual in qualities.all()]))
console(
'ANY is the default and can also be used explicitly to specify that quality should be ignored.'
)
except OperationalError:
console('OperationalError')
return
