def on_task_output(self, task, config): if not config: return if not isinstance(config, dict): config = {'action': config} config.setdefault('queue_name', 'default') for entry in task.accepted: # Tell tmdb_lookup to add lazy lookup fields if not already present try: plugin.get_plugin_by_name('tmdb_lookup').instance.lookup(entry) except plugin.DependencyError: log.debug( 'tmdb_lookup is not available, queue will not work if movie ids are not populated' ) # Find one or both movie id's for this entry. See if an id is already populated before incurring lazy lookup kwargs = {} for lazy in [False, True]: if entry.get('imdb_id', eval_lazy=lazy): kwargs['imdb_id'] = entry['imdb_id'] if entry.get('tmdb_id', eval_lazy=lazy): kwargs['tmdb_id'] = entry['tmdb_id'] if kwargs: break if not kwargs: log.warning( 'Could not determine a movie id for %s, it will not be added to queue.' % entry['title']) continue # Provide movie title if it is already available, to avoid movie_queue doing a lookup kwargs['title'] = (entry.get('imdb_name', eval_lazy=False) or entry.get('tmdb_name', eval_lazy=False) or entry.get('movie_name', eval_lazy=False)) log.debug('movie_queue kwargs: %s' % kwargs) kwargs['queue_name'] = config.get('queue_name') try: action = config.get('action') if action == 'add': # since entries usually have unknown quality we need to ignore that .. if entry.get('quality_req'): kwargs['quality'] = qualities.Requirements( entry['quality_req']) elif entry.get('quality'): kwargs['quality'] = qualities.Requirements( entry['quality'].name) else: kwargs['quality'] = qualities.Requirements( config.get('quality', 'any')) queue_add(**kwargs) elif action == 'remove': queue_del(**kwargs) elif action == 'forget': queue_forget(**kwargs) except QueueError as e: # Ignore already in queue errors if e.errno != 1: entry.fail('Error adding movie to queue: %s' % e.message)
def validate(self, data): try: qualities.Requirements(data) except ValueError, e: self.errors.add('`%s` is not a valid quality requirement: %s' % (data, e.message)) return False
def on_task_start(self, task, config): if isinstance(config, basestring): config = {'any': config} assume = namedtuple('assume', ['target', 'quality']) self.assumptions = [] for target, quality in list(config.items()): log.verbose('New assumption: %s is %s' % (target, quality)) try: target = qualities.Requirements(target) except ValueError: raise plugin.PluginError( '%s is not a valid quality. Forgetting assumption.' % target) try: quality = qualities.get(quality) except ValueError: raise plugin.PluginError( '%s is not a valid quality. Forgetting assumption.' % quality) self.assumptions.append(assume(target, quality)) self.assumptions.sort( key=lambda assumption: self.precision(assumption.target), reverse=True) for assumption in self.assumptions: log.debug('Target %s - Priority %s' % (assumption.target, self.precision(assumption.target)))
def queue_add(title=None, imdb_id=None, tmdb_id=None, quality=None, session=None, queue_name='default'): """ Add an item to the queue with the specified quality requirements. One or more of `title` `imdb_id` or `tmdb_id` must be specified when calling this function. :param title: Title of the movie. (optional) :param imdb_id: IMDB id for the movie. (optional) :param tmdb_id: TMDB id for the movie. (optional) :param quality: A QualityRequirements object defining acceptable qualities. :param queue_name: Name of movie queue to get items from :param session: Optional session to use for database updates """ quality = quality or qualities.Requirements('any') if not title or not (imdb_id or tmdb_id): # We don't have all the info we need to add movie, do a lookup for more info result = parse_what(imdb_id or title or tmdb_id, session=session) title = result['title'] if not title: raise QueueError( 'Could not parse movie info for given parameters: title=%s, imdb_id=%s, tmdb_id=%s' % (title, imdb_id, tmdb_id)) imdb_id = result['imdb_id'] tmdb_id = result['tmdb_id'] # check if the item is already queued item = session.query(QueuedMovie).filter( and_((func.lower(QueuedMovie.queue_name) == queue_name.lower()), or_( and_(QueuedMovie.imdb_id != None, QueuedMovie.imdb_id == imdb_id), and_(QueuedMovie.tmdb_id != None, QueuedMovie.tmdb_id == tmdb_id)))).first() if not item: item = QueuedMovie(title=title, imdb_id=imdb_id, tmdb_id=tmdb_id, quality=quality.text, queue_name=queue_name) session.add(item) session.commit() log.info('Adding %s to movie queue %s with quality=%s.', title, queue_name, quality) return item.to_dict() else: if item.downloaded: raise QueueError( 'ERROR: %s has already been queued and downloaded' % title, errno=1) else: raise QueueError('ERROR: %s is already in the queue %s' % (title, queue_name), errno=1)
def filter_entries(self, entries, existing, target, action_on_lower): target_requirement = qualities.Requirements(target) if target else None filtered = [] for entry in entries: # Filter out entries within target if target: if not target_requirement.allows(entry['quality']): log.debug( 'Skipping %s as does not meet upgrade quality requirements', entry['title'] ) if action_on_lower: action_on_lower(entry, 'does not meet upgrade quality requirements') continue if entry['quality'] < existing.quality: log.debug('Skipping %s as lower quality then existing', entry['title']) if action_on_lower: action_on_lower(entry, 'lower quality then existing') continue if ( entry['quality'] == existing.quality and entry.get('proper_count', 0) <= existing.proper_count ): log.debug('Skipping %s as same quality but lower proper', entry['title']) if action_on_lower: action_on_lower(entry, 'lower proper then existing') continue filtered.append(entry) return filtered
def __call__(self, parser, namespace, values, option_string=None): options = namespace.movie_queue = {} # Assume 'list' if no action was given if not values: values = ['list'] if values[0].lower() not in ACTIONS: raise ArgumentError(self, '`%s` is not a valid action.\nUsage: ' % values[0] + USAGE) options['action'] = values[0].lower() if len(values) == 1: if options['action'] not in ('list', 'downloaded', 'clear'): raise ArgumentError(self, 'You must specify the movie.\nUsage: ' + USAGE) # 2, args is the minimum allowed (operation + item) for actions other than list if len(values) >= 2: options['what'] = values[1] # 3, quality if len(values) >= 3: try: options['quality'] = qualities.Requirements(values[2]) except ValueError, e: raise ArgumentError(self, '`%s` is an invalid quality requirement string: %s' % (values[2], e.message))
def on_task_output(self, task, config): if not config: return if not isinstance(config, dict): config = {} for entry in task.accepted: # Tell tmdb_lookup to add lazy lookup fields if not already present try: get_plugin_by_name('tmdb_lookup').instance.lookup(entry) except DependencyError: log.debug( 'tmdb_lookup is not available, queue will not work if movie ids are not populated' ) # Find one or both movie id's for this entry. See if an id is already populated before incurring lazy lookup kwargs = {} for lazy in [False, True]: if entry.get('imdb_id', eval_lazy=lazy): kwargs['imdb_id'] = entry['imdb_id'] if entry.get('tmdb_id', eval_lazy=lazy): kwargs['tmdb_id'] = entry['tmdb_id'] if kwargs: break if not kwargs: log.warning( 'Could not determine a movie id for %s, it will not be added to queue.' % entry['title']) continue # since entries usually have unknown quality we need to ignore that .. if entry.get('quality'): quality = qualities.Requirements(entry['quality'].name) else: quality = qualities.Requirements(config.get('quality', 'any')) kwargs['quality'] = quality force = entry.get('force', config.get('force')) if force is not None: kwargs['force'] = force # Provide movie title if it is already available, to avoid movie_queue doing a lookup kwargs['title'] = entry.get('imdb_name') or entry.get( 'tmdb_name') or entry.get('movie_name') log.debug('queueing kwargs: %s' % kwargs) try: queue_add(**kwargs) except QueueError, e: task.fail(entry, 'Error adding movie to queue: %s' % e.message)
def __call__(self, parser, namespace, values, option_string=None): options = namespace.movie_queue = {} # Assume 'list' if no action was given if not values: values = ['list'] if values[0].lower() not in ACTIONS: raise ArgumentError( self, '`%s` is not a valid action.\nUsage: ' % values[0] + USAGE) options['action'] = values[0].lower() if len(values) == 1: if options['action'] not in ('list', 'downloaded', 'clear'): raise ArgumentError( self, 'You must specify the movie.\nUsage: ' + USAGE) # 2, args is the minimum allowed (operation + item) for actions other than list if len(values) >= 2: options['what'] = values[1] # 3, quality if len(values) >= 3: try: options['quality'] = qualities.Requirements(values[2]) except ValueError as e: raise ArgumentError( self, '`%s` is an invalid quality requirement string: %s' % (values[2], e.message)) else: options['quality'] = qualities.Requirements('any') # TODO: Get default from config somehow? # why not use the quality user has queued most, ie option called 'auto' ? # and if none is queued default to something good like '720p bluray' # 4, force download if len(values) >= 4: options['force'] = str_to_boolean(values[3]) else: options['force'] = True if len(values) > 4: raise ArgumentError(self, 'Too many arguments passed.\nUsage: ' + USAGE)
def add_to_queue(): what = request.values.get('what') imdb_id = request.values.get('imdb_id') # TODO: This is a rather limited selection of quality considering the new quality system. Improve it. quality = qualities.Requirements(request.values.get('quality', 'ANY')) force = request.values.get('force') == 'on' try: title = queue_add(title=what, imdb_id=imdb_id, quality=quality, force=force)['title'] except QueueError, e: flash(e.message, 'error')
def on_task_filter(self, task, config): if not isinstance(config, list): config = [config] reqs = [quals.Requirements(req) for req in config] for entry in task.entries: if not entry.get('quality'): entry.reject('Entry doesn\'t have a quality') continue if not any(req.allows(entry['quality']) for req in reqs): entry.reject('%s does not match quality requirement %s' % (entry['quality'], reqs))
def queue_add(title=None, imdb_id=None, tmdb_id=None, quality=None, force=True, session=None): """ Add an item to the queue with the specified quality requirements. One or more of `title` `imdb_id` or `tmdb_id` must be specified when calling this function. :param title: Title of the movie. (optional) :param imdb_id: IMDB id for the movie. (optional) :param tmdb_id: TMDB id for the movie. (optional) :param quality: A QualityRequirements object defining acceptable qualities. :param force: If this is true, accepted movie will be marked as immortal. :param session: Optional session to use for database updates """ quality = quality or qualities.Requirements('any') if not title or not (imdb_id or tmdb_id): # We don't have all the info we need to add movie, do a lookup for more info result = parse_what(imdb_id or title, session=session) title = result['title'] imdb_id = result['imdb_id'] tmdb_id = result['tmdb_id'] # check if the item is already queued item = session.query(QueuedMovie).filter(or_(and_(QueuedMovie.imdb_id != None, QueuedMovie.imdb_id == imdb_id), and_(QueuedMovie.tmdb_id != None, QueuedMovie.tmdb_id == tmdb_id))).\ first() if not item: item = QueuedMovie(title=title, imdb_id=imdb_id, tmdb_id=tmdb_id, quality=quality.text, immortal=force) session.add(item) log.info('Adding %s to movie queue with quality=%s and force=%s.' % (title, quality, force)) return { 'title': title, 'imdb_id': imdb_id, 'tmdb_id': tmdb_id, 'quality': quality, 'force': force } else: if item.downloaded: raise QueueError( 'ERROR: %s has already been queued and downloaded' % title) else: raise QueueError('ERROR: %s is already in the queue' % title)
def on_task_filter(self, task, config): if not isinstance(config, list): config = [config] reqs = [qualities.Requirements(req) for req in config] for entry in task.entries: if entry.get('quality') is None: entry.reject('Entry doesn\'t have a quality') continue if not any(req.allows(entry['quality']) for req in reqs): text_reqs = ', '.join(f'`{req}`' for req in reqs) entry.reject( f'`{entry["quality"]}` does not match any of quality requirements: {text_reqs}' )
def add_to_queue(): what = request.values.get('what') imdb_id = request.values.get('imdb_id') # TODO: This is a rather limited selection of quality considering the new quality system. Improve it. quality = qualities.Requirements(request.values.get('quality', 'ANY')) force = request.values.get('force') == 'on' try: title = queue_add(title=what, imdb_id=imdb_id, quality=quality, force=force)['title'] except QueueError as e: flash(e.message, 'error') else: flash('%s successfully added to queue.' % title, 'success') return redirect(url_for('.index'))
def post(self, session=None): """ Add movies to movie queue """ kwargs = request.json kwargs['quality'] = qualities.Requirements(kwargs.get('quality')) kwargs['session'] = session try: movie = mq.queue_add(**kwargs) except mq.QueueError as e: reply = {'status': 'error', 'message': e.message} return reply, 500 reply = jsonify(movie) reply.status_code = 201 return reply
def getter(self): return qualities.Requirements(getattr(self, text_attr))
def on_task_filter(self, task, config): if not config: return identified_by = '{{ id }}' if config[ 'identified_by'] == 'auto' else config['identified_by'] grouped_entries = group_entries(task.accepted + task.undecided, identified_by) if not grouped_entries: return action_on_waiting = entry_actions[config[ 'on_waiting']] if config['on_waiting'] != 'do_nothing' else None action_on_reached = entry_actions[config[ 'on_reached']] if config['on_reached'] != 'do_nothing' else None with Session() as session: # Prefetch Data existing_ids = session.query(EntryTimeFrame).filter( EntryTimeFrame.id.in_(grouped_entries.keys())).all() existing_ids = {e.id: e for e in existing_ids} for identifier, entries in grouped_entries.items(): if not entries: continue id_timeframe = existing_ids.get(identifier) if not id_timeframe: id_timeframe = EntryTimeFrame() id_timeframe.id = identifier id_timeframe.status = 'waiting' id_timeframe.first_seen = datetime.now() session.add(id_timeframe) if id_timeframe.status == 'accepted': log.debug('Previously accepted %s with %s skipping', identifier, id_timeframe.title) continue # Sort entities in order of quality and best proper entries.sort(key=lambda e: (e['quality'], e.get('proper_count', 0)), reverse=True) best_entry = entries[0] log.debug('Current best for identifier %s is %s', identifier, best_entry['title']) id_timeframe.title = best_entry['title'] id_timeframe.quality = best_entry['quality'] id_timeframe.proper_count = best_entry.get('proper_count', 0) # Check we hit target or better target_requirement = qualities.Requirements(config['target']) target_quality = qualities.Quality(config['target']) if target_requirement.allows( best_entry['quality'] ) or best_entry['quality'] >= target_quality: log.debug( 'timeframe reach target quality %s or higher for %s' % (target_quality, identifier)) if action_on_reached: action_on_reached( best_entry, 'timeframe reached target quality or higher') continue # Check if passed wait time expires = id_timeframe.first_seen + parse_timedelta( config['wait']) if expires <= datetime.now(): log.debug( 'timeframe expired, releasing quality restriction for %s' % identifier) if action_on_reached: action_on_reached(best_entry, 'timeframe wait expired') continue # Verbose waiting, add to backlog if action_on_waiting: for entry in entries: action_on_waiting(entry, 'timeframe waiting') diff = expires - datetime.now() hours, remainder = divmod(diff.seconds, 3600) hours += diff.days * 24 minutes, _ = divmod(remainder, 60) log.info( '`%s`: timeframe waiting for %02dh:%02dmin. Currently best is `%s`.', identifier, hours, minutes, best_entry['title']) # add best entry to backlog (backlog is able to handle duplicate adds) if self.backlog: self.backlog.instance.add_backlog(task, best_entry, session=session)
def is_quality_req(instance): if not isinstance(instance, str_types): return True return qualities.Requirements(instance)
if options['action'] not in ('list', 'downloaded', 'clear'): raise ArgumentError(self, 'You must specify the movie.\nUsage: ' + USAGE) # 2, args is the minimum allowed (operation + item) for actions other than list if len(values) >= 2: options['what'] = values[1] # 3, quality if len(values) >= 3: try: options['quality'] = qualities.Requirements(values[2]) except ValueError, e: raise ArgumentError(self, '`%s` is an invalid quality requirement string: %s' % (values[2], e.message)) else: options['quality'] = qualities.Requirements('any') # TODO: Get default from config somehow? # why not use the quality user has queued most, ie option called 'auto' ? # and if none is queued default to something good like '720p bluray' # 4, force download if len(values) >= 4: options['force'] = str_to_boolean(values[3]) else: options['force'] = True if len(values) > 4: raise ArgumentError(self, 'Too many arguments passed.\nUsage: ' + USAGE) register_plugin(MovieQueueManager, 'movie_queue_manager', builtin=True)
def do_cli(manager, options): """Handle movie-queue subcommand""" if options.queue_action == 'list': queue_list(options) return # If the action affects make sure all entries are processed again next run. manager.config_changed() if options.queue_action == 'clear': clear() return if options.queue_action == 'del': try: what = parse_what(options.movie_name, lookup=False) title = queue_del(**what) except QueueError as e: console('ERROR: %s' % e.message) else: console('Removed %s from queue' % title) return if options.queue_action == 'forget': try: what = parse_what(options.movie_name, lookup=False) title = queue_forget(**what) except QueueError as e: console('ERROR: %s' % e.message) else: console( 'Forgot that %s was downloaded. Movie will be downloaded again.' % title) return if options.queue_action == 'add': try: quality = qualities.Requirements(options.quality) except ValueError as e: console('`%s` is an invalid quality requirement string: %s' % (options.quality, e.message)) return # Adding to queue requires a lookup for missing information what = {} try: what = parse_what(options.movie_name) except QueueError as e: console('ERROR: %s' % e.message) if not what.get('title') or not (what.get('imdb_id') or what.get('tmdb_id')): console('could not determine movie') # TODO: Rethink errors return try: queue_add(quality=quality, **what) except QueueError as e: console(e.message) if e.errno == 1: # This is an invalid quality error, display some more info # TODO: Fix this error? # console('Recognized qualities are %s' % ', '.join([qual.name for qual in qualities.all()])) console( 'ANY is the default and can also be used explicitly to specify that quality should be ignored.' ) except OperationalError: console('OperationalError') return