def __init__(self, config): self.config = config with Session() as session: if not self._db_list(session): session.add(PendingListList(name=self.config))
def discard(self, entry): with Session() as session: db_movie = self._find_entry(entry, session=session) if db_movie: logger.debug('deleting movie {}', db_movie) session.delete(db_movie)
def _execute(self): """Executes the task without rerunning.""" if not self.enabled: log.debug('Not running disabled task %s' % self.name) return log.debug('executing %s' % self.name) # Handle keyword args if self.options.learn: log.info('Disabling download and output phases because of --learn') self.disable_phase('download') self.disable_phase('output') if self.options.disable_phases: map(self.disable_phase, self.options.disable_phases) if self.options.inject: # If entries are passed for this execution (eg. rerun), disable the input phase self.disable_phase('input') self.all_entries.extend(self.options.inject) # Save current config hash and set config_modidied flag with Session() as session: config_hash = hashlib.md5(str(sorted(self.config.items()))).hexdigest() last_hash = session.query(TaskConfigHash).filter(TaskConfigHash.task == self.name).first() if self.is_rerun: # Restore the config to state right after start phase if self.prepared_config: self.config = copy.deepcopy(self.prepared_config) else: log.error('BUG: No prepared_config on rerun, please report.') self.config_modified = False elif not last_hash: self.config_modified = True last_hash = TaskConfigHash(task=self.name, hash=config_hash) session.add(last_hash) elif last_hash.hash != config_hash: self.config_modified = True last_hash.hash = config_hash else: self.config_modified = False # run phases try: for phase in task_phases: if phase in self.disabled_phases: # log keywords not executed for plugin in self.plugins(phase): if plugin.name in self.config: log.info('Plugin %s is not executed because %s phase is disabled (e.g. --test)' % (plugin.name, phase)) continue if phase == 'start' and self.is_rerun: log.debug('skipping task_start during rerun') elif phase == 'exit' and self._rerun and self._rerun_count < self.max_reruns: log.debug('not running task_exit yet because task will rerun') else: # run all plugins with this phase self.__run_task_phase(phase) if phase == 'start': # Store a copy of the config state after start phase to restore for reruns self.prepared_config = copy.deepcopy(self.config) except TaskAbort: try: self.__run_task_phase('abort') except TaskAbort as e: log.exception('abort handlers aborted: %s' % e) raise else: for entry in self.all_entries: entry.complete()
def learn_backlog(self, task, amount=''): """Learn current entries into backlog. All task inputs must have been executed.""" with Session() as session: for entry in task.entries: self.add_backlog(task, entry, amount, session=session)
def __iter__(self): with Session() as session: return iter([ movie.to_entry(self.strip_year) for movie in self._db_list(session).movies ])
def get_access_token(account, token=None, refresh=False, re_auth=False, called_from_cli=False): """ Gets authorization info from a pin or refresh token. :param account: Arbitrary account name to attach authorization to. :param unicode token: The pin or refresh token, as supplied by the trakt website. :param bool refresh: If True, refresh the access token using refresh_token from db. :param bool re_auth: If True, account is re-authorized even if it already exists in db. :raises RequestException: If there is a network error while authorizing. """ data = { 'client_id': CLIENT_ID, 'client_secret': CLIENT_SECRET, 'redirect_uri': 'urn:ietf:wg:oauth:2.0:oob', } with Session() as session: acc = session.query(TraktUserAuth).filter( TraktUserAuth.account == account).first() if acc and datetime.now( ) < acc.expires and not refresh and not re_auth: return acc.access_token else: if (acc and (refresh or datetime.now() >= acc.expires - timedelta(days=5)) and not re_auth): logger.debug('Using refresh token to re-authorize account {}.', account) data['refresh_token'] = acc.refresh_token data['grant_type'] = 'refresh_token' token_dict = token_oauth(data) elif token: # We are only in here if a pin was specified, so it's safe to use console instead of logging console( 'Warning: PIN authorization has been deprecated. Use Device Authorization instead.' ) data['code'] = token data['grant_type'] = 'authorization_code' token_dict = token_oauth(data) elif called_from_cli: logger.debug( 'No pin specified for an unknown account {}. Attempting to authorize device.', account, ) token_dict = device_auth() else: raise plugin.PluginError( 'Account %s has not been authorized. See `flexget trakt auth -h` on how to.' % account) try: new_acc = TraktUserAuth( account, token_dict['access_token'], token_dict['refresh_token'], token_dict.get('created_at', time.time()), token_dict['expires_in'], ) session.merge(new_acc) return new_acc.access_token except requests.RequestException as e: raise plugin.PluginError( 'Token exchange with trakt failed: {0}'.format(e))
def authenticate(self): """Authenticates a session with IMDB, and grabs any IDs needed for getting/modifying list.""" cached_credentials = False with Session() as session: user = (session.query(IMDBListUser).filter( IMDBListUser.user_name == self.config.get( 'login')).one_or_none()) if user and user.cookies and user.user_id: log.debug('login credentials found in cache, testing') self.user_id = user.user_id if not self.get_user_id_and_hidden_value(cookies=user.cookies): log.debug('cache credentials expired') user.cookies = None self._session.cookies.clear() else: self.cookies = user.cookies cached_credentials = True if not cached_credentials: log.debug( 'user credentials not found in cache or outdated, fetching from IMDB' ) url_credentials = ( 'https://www.imdb.com/ap/signin?openid.return_to=https%3A%2F%2Fwww.imdb.com%2Fap-signin-' 'handler&openid.identity=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&' 'openid.assoc_handle=imdb_mobile_us&openid.mode=checkid_setup&openid.claimed_id=http%3A%' '2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&openid.ns=http%3A%2F%2Fspecs.ope' 'nid.net%2Fauth%2F2.0') try: # we need to get some cookies first self._session.get('https://www.imdb.com') r = self._session.get(url_credentials) except RequestException as e: raise PluginError(e.args[0]) soup = get_soup(r.content) form = soup.find('form', attrs={'name': 'signIn'}) inputs = form.select('input') data = dict((i['name'], i.get('value')) for i in inputs if i.get('name')) data['email'] = self.config['login'] data['password'] = self.config['password'] action = form.get('action') log.debug('email=%s, password=%s', data['email'], data['password']) self._session.headers.update({'Referer': url_credentials}) self._session.post(action, data=data) self._session.headers.update( {'Referer': 'https://www.imdb.com/'}) self.user_id = self.get_user_id_and_hidden_value() if not self.user_id: raise plugin.PluginError( 'Login to IMDB failed. Check your credentials.') self.cookies = self._session.cookies.get_dict( domain='.imdb.com') # Get list ID if user: for list in user.lists: if self.config['list'] == list.list_name: log.debug( 'found list ID %s matching list name %s in cache', list.list_id, list.list_name, ) self.list_id = list.list_id if not self.list_id: log.debug( 'could not find list ID in cache, fetching from IMDB') if self.config['list'] == 'watchlist': data = { 'consts[]': 'tt0133093', 'tracking_tag': 'watchlistRibbon' } wl_data = self._session.post( 'https://www.imdb.com/list/_ajax/watchlist_has', data=data, cookies=self.cookies, ).json() try: self.list_id = wl_data['list_id'] except KeyError: raise PluginError( 'No list ID could be received. Please initialize list by ' 'manually adding an item to it and try again') elif self.config['list'] in IMMUTABLE_LISTS or self.config[ 'list'].startswith('ls'): self.list_id = self.config['list'] else: data = {'tconst': 'tt0133093'} list_data = self._session.post( 'https://www.imdb.com/list/_ajax/wlb_dropdown', data=data, cookies=self.cookies, ).json() for li in list_data['items']: if li['wlb_text'] == self.config['list']: self.list_id = li['data_list_id'] break else: raise plugin.PluginError('Could not find list %s' % self.config['list']) user = IMDBListUser(self.config['login'], self.user_id, self.cookies) list = IMDBListList(self.list_id, self.config['list'], self.user_id) user.lists.append(list) session.merge(user) self._authenticated = True
def get_flexget_db_version(): with Session() as session: version = session.query(FlexgetVersion).first() if version: return version.version
def delete_account(account): with Session() as session: acc = session.query(TraktUserAuth).filter(TraktUserAuth.account == account).first() if not acc: raise plugin.PluginError('Account %s not found.' % account) session.delete(acc)
def load(cls, task=None): """Load all key/values from `task` into memory from database.""" with Session() as session: for skv in session.query(SimpleKeyValue).filter(SimpleKeyValue.task == task).all(): cls.class_store[task][skv.plugin][skv.key] = skv.value
def search(*args, **kwargs): if 'count' in kwargs: return 0 else: with Session() as session: return session.query(series.Series)
def on_task_input(self, task, config): # Let details plugin know that it is ok if this task doesn't produce any entries task.no_entries_ok = True filename = os.path.expanduser(config['file']) encoding = config.get('encoding', 'utf-8') with Session() as session: db_pos = (session.query(TailPosition).filter( TailPosition.task == task.name).filter( TailPosition.filename == filename).first()) if db_pos: last_pos = db_pos.position else: last_pos = 0 with open(filename, 'r', encoding=encoding, errors='replace') as file: if task.options.tail_reset == filename or task.options.tail_reset == task.name: if last_pos == 0: logger.info('Task {} tail position is already zero', task.name) else: logger.info('Task {} tail position ({}) reset to zero', task.name, last_pos) last_pos = 0 if os.path.getsize(filename) < last_pos: logger.info( 'File size is smaller than in previous execution, resetting to beginning of the file' ) last_pos = 0 file.seek(last_pos) logger.debug('continuing from last position {}', last_pos) entry_config = config.get('entry') format_config = config.get('format', {}) # keep track what fields have been found used = {} entries = [] entry = Entry() # now parse text for line in file: if not line: break for field, regexp in entry_config.items(): # log.debug('search field: %s regexp: %s' % (field, regexp)) match = re.search(regexp, line) if match: # check if used field detected, in such case start with new entry if field in used: if entry.isvalid(): logger.info( 'Found field {} again before entry was completed. Adding current incomplete, but valid entry and moving to next.', field, ) self.format_entry(entry, format_config) entries.append(entry) else: logger.info( 'Invalid data, entry field {} is already found once. Ignoring entry.', field, ) # start new entry entry = Entry() used = {} # add field to entry entry[field] = match.group(1) used[field] = True logger.debug('found field: {} value: {}', field, entry[field]) # if all fields have been found if len(used) == len(entry_config): # check that entry has at least title and url if not entry.isvalid(): logger.info( 'Invalid data, constructed entry is missing mandatory fields (title or url)' ) else: self.format_entry(entry, format_config) entries.append(entry) logger.debug('Added entry {}', entry) # start new entry entry = Entry() used = {} last_pos = file.tell() if db_pos: db_pos.position = last_pos else: session.add( TailPosition(task=task.name, filename=filename, position=last_pos)) return entries
def test_seen_sorting(self, api_client): seen_entry_1 = dict(title='test_title_1', reason='test_reason_c', task='test_task_2', local=True) field_1 = dict(field='test_field_1', value='test_value_1') field_2 = dict(field='test_field_2', value='test_value_2') seen_entry_2 = dict(title='test_title_2', reason='test_reason_b', task='test_task_3', local=True) field_3 = dict(field='test_field_3', value='test_value_3') field_4 = dict(field='test_field_4', value='test_value_4') seen_entry_3 = dict(title='test_title_3', reason='test_reason_a', task='test_task_1', local=False) field_5 = dict(field='test_field_3', value='test_value_3') field_6 = dict(field='test_field_4', value='test_value_4') with Session() as session: seen_db_1 = SeenEntry(**seen_entry_1) session.add(seen_db_1) seen_db_1.fields = [SeenField(**field_1), SeenField(**field_2)] seen_db_2 = SeenEntry(**seen_entry_2) session.add(seen_db_2) seen_db_2.fields = [SeenField(**field_3), SeenField(**field_4)] seen_db_3 = SeenEntry(**seen_entry_3) session.add(seen_db_3) seen_db_3.fields = [SeenField(**field_5), SeenField(**field_6)] # Sort by title rsp = api_client.get('/seen/?sort_by=title') assert rsp.status_code == 200, 'Response code is %s' % rsp.status_code data = json.loads(rsp.get_data(as_text=True)) assert data[0]['title'] == 'test_title_3' rsp = api_client.get('/seen/?sort_by=title&order=asc') assert rsp.status_code == 200, 'Response code is %s' % rsp.status_code data = json.loads(rsp.get_data(as_text=True)) assert data[0]['title'] == 'test_title_1' # Sort by task rsp = api_client.get('/seen/?sort_by=task') assert rsp.status_code == 200, 'Response code is %s' % rsp.status_code data = json.loads(rsp.get_data(as_text=True)) assert data[0]['task'] == 'test_task_3' rsp = api_client.get('/seen/?sort_by=task&order=asc') assert rsp.status_code == 200, 'Response code is %s' % rsp.status_code data = json.loads(rsp.get_data(as_text=True)) assert data[0]['task'] == 'test_task_1' # Sort by reason rsp = api_client.get('/seen/?sort_by=reason') assert rsp.status_code == 200, 'Response code is %s' % rsp.status_code data = json.loads(rsp.get_data(as_text=True)) assert data[0]['reason'] == 'test_reason_c' rsp = api_client.get('/seen/?sort_by=reason&order=asc') assert rsp.status_code == 200, 'Response code is %s' % rsp.status_code data = json.loads(rsp.get_data(as_text=True)) assert data[0]['reason'] == 'test_reason_a' # Sort by local rsp = api_client.get('/seen/?sort_by=local') assert rsp.status_code == 200, 'Response code is %s' % rsp.status_code data = json.loads(rsp.get_data(as_text=True)) assert data[0]['local'] == True rsp = api_client.get('/seen/?sort_by=local&order=asc') assert rsp.status_code == 200, 'Response code is %s' % rsp.status_code data = json.loads(rsp.get_data(as_text=True)) assert data[0]['local'] == False # Combine sorting and pagination rsp = api_client.get('/seen/?sort_by=reason&per_page=2&page=2') assert rsp.status_code == 200, 'Response code is %s' % rsp.status_code data = json.loads(rsp.get_data(as_text=True)) assert data[0]['reason'] == 'test_reason_a'
def test_seen_pagination(self, api_client, link_headers): base_seen_entry = dict(title='test_title_', task='test_task_', reason='test_reason_') base_seen_field = dict(field='test_field_', value='test_value_') number_of_entries = 200 with Session() as session: for i in range(number_of_entries): entry = copy.deepcopy(base_seen_entry) field = copy.deepcopy(base_seen_field) for key, value in entry.items(): entry[key] = value + str(i) for key, value in field.items(): field[key] = value + str(i) seen_entry = SeenEntry(**entry) session.add(seen_entry) seen_entry.fields = [SeenField(**field)] # Default values rsp = api_client.get('/seen/') assert rsp.status_code == 200, 'Response code is %s' % rsp.status_code data = json.loads(rsp.get_data(as_text=True)) assert len(data) == 50 assert int(rsp.headers['total-count']) == 200 assert int(rsp.headers['count']) == 50 links = link_headers(rsp) assert links['last']['page'] == 4 assert links['next']['page'] == 2 # Change page size rsp = api_client.get('/seen/?per_page=100') assert rsp.status_code == 200 data = json.loads(rsp.get_data(as_text=True)) assert len(data) == 100 assert int(rsp.headers['total-count']) == 200 assert int(rsp.headers['count']) == 100 links = link_headers(rsp) assert links['last']['page'] == 2 assert links['next']['page'] == 2 # Get different page rsp = api_client.get('/seen/?page=2') assert rsp.status_code == 200 data = json.loads(rsp.get_data(as_text=True)) assert len(data) == 50 assert int(rsp.headers['total-count']) == 200 assert int(rsp.headers['count']) == 50 links = link_headers(rsp) assert links['last']['page'] == 4 assert links['next']['page'] == 3 assert links['prev']['page'] == 1
def discard(self, entry): with Session() as session: db_file = self._find_entry(entry, session=session) if db_file: log.debug('deleting file %s', db_file) session.delete(db_file)
def estimate(self, entry): if not all(field in entry for field in ['series_name', 'series_season']): return series_name = entry['series_name'] season = entry['series_season'] episode_number = entry.get('series_episode') title, year_match = split_title_year(series_name) # This value should be added to input plugins to trigger a season lookup season_pack = entry.get('season_pack_lookup') kwargs = { 'title': title, 'year': entry.get('trakt_series_year') or entry.get('year') or entry.get('imdb_year') or year_match, 'trakt_slug': entry.get('trakt_slug'), 'tmdb_id': entry.get('tmdb_id'), 'tvdb_id': entry.get('tvdb_id') or entry.get('trakt_series_tvdb_id'), 'imdb_id': entry.get('imdb_id'), 'tvrage_id': entry.get('tvrage_id') or entry.get('trakt_series_tvrage_id'), } api_trakt = plugin.get_plugin_by_name('api_trakt').instance log.debug('Searching api_trakt for series') for k, v in list(kwargs.items()): if v: log.debug('%s: %s', k, v) with Session(expire_on_commit=False) as session: try: trakt_series = api_trakt.lookup_series(session=session, **kwargs) if trakt_series is None: return trakt_season = trakt_series.get_season(season, session) if trakt_season is None: log.debug('%s doesn\'t have a season %s in trakt' % (series_name, season)) return datetime.max if season_pack: entity = trakt_season else: entity = trakt_series.get_episode(season, episode_number, session) if entity is None: log.debug( '%s doesn\'t have a season %s episode %s in trakt' % (series_name, season, episode_number)) return datetime.max except LookupError as e: log.debug(str(e)) return if entity and entity.first_aired: log.debug('received first-aired: %s', entity.first_aired) return entity.first_aired return
def lookup(self, entry, search_allowed=True): """ Perform imdb lookup for entry. :param entry: Entry instance :param search_allowed: Allow fallback to search :raises PluginError: Failure reason """ from flexget.manager import manager if entry.get('imdb_id', eval_lazy=False): log.debug('No title passed. Lookup for %s' % entry['imdb_id']) elif entry.get('imdb_url', eval_lazy=False): log.debug('No title passed. Lookup for %s' % entry['imdb_url']) elif entry.get('title', eval_lazy=False): log.debug('lookup for %s' % entry['title']) else: raise plugin.PluginError('looking up IMDB for entry failed, no title, imdb_url or imdb_id passed.') session = Session() try: # entry sanity checks for field in ['imdb_votes', 'imdb_score']: if entry.get(field, eval_lazy=False): value = entry[field] if not isinstance(value, (int, float)): raise plugin.PluginError('Entry field %s should be a number!' % field) # if imdb_id is included, build the url. if entry.get('imdb_id', eval_lazy=False) and not entry.get('imdb_url', eval_lazy=False): entry['imdb_url'] = make_url(entry['imdb_id']) # make sure imdb url is valid if entry.get('imdb_url', eval_lazy=False): imdb_id = extract_id(entry['imdb_url']) if imdb_id: entry['imdb_url'] = make_url(imdb_id) else: log.debug('imdb url %s is invalid, removing it' % entry['imdb_url']) del(entry['imdb_url']) # no imdb_url, check if there is cached result for it or if the # search is known to fail if not entry.get('imdb_url', eval_lazy=False): result = session.query(SearchResult).\ filter(SearchResult.title == entry['title']).first() if result: # TODO: 1.2 this should really be checking task.options.retry if result.fails and not manager.options.execute.retry: # this movie cannot be found, not worth trying again ... log.debug('%s will fail lookup' % entry['title']) raise plugin.PluginError('IMDB lookup failed for %s' % entry['title']) else: if result.url: log.trace('Setting imdb url for %s from db' % entry['title']) entry['imdb_url'] = result.url # no imdb url, but information required, try searching if not entry.get('imdb_url', eval_lazy=False) and search_allowed: log.verbose('Searching from imdb `%s`' % entry['title']) search = ImdbSearch() search_name = entry.get('movie_name', entry['title'], eval_lazy=False) search_result = search.smart_match(search_name) if search_result: entry['imdb_url'] = search_result['url'] # store url for this movie, so we don't have to search on # every run result = SearchResult(entry['title'], entry['imdb_url']) session.add(result) log.verbose('Found %s' % (entry['imdb_url'])) else: log_once('IMDB lookup failed for %s' % entry['title'], log, logging.WARN) # store FAIL for this title result = SearchResult(entry['title']) result.fails = True session.add(result) raise plugin.PluginError('Title `%s` lookup failed' % entry['title']) # check if this imdb page has been parsed & cached movie = session.query(Movie).filter(Movie.url == entry['imdb_url']).first() # determine whether or not movie details needs to be parsed req_parse = False if not movie: req_parse = True elif movie.expired: req_parse = True if req_parse: if movie is not None: if movie.expired: log.verbose('Movie `%s` details expired, refreshing ...' % movie.title) # Remove the old movie, we'll store another one later. session.query(MovieLanguage).filter(MovieLanguage.movie_id == movie.id).delete() session.query(Movie).filter(Movie.url == entry['imdb_url']).delete() # search and store to cache if 'title' in entry: log.verbose('Parsing imdb for `%s`' % entry['title']) else: log.verbose('Parsing imdb for `%s`' % entry['imdb_id']) try: movie = self._parse_new_movie(entry['imdb_url'], session) except UnicodeDecodeError: log.error('Unable to determine encoding for %s. Installing chardet library may help.' % entry['imdb_url']) # store cache so this will not be tried again movie = Movie() movie.url = entry['imdb_url'] session.add(movie) raise plugin.PluginError('UnicodeDecodeError') except ValueError as e: # TODO: might be a little too broad catch, what was this for anyway? ;P if manager.options.debug: log.exception(e) raise plugin.PluginError('Invalid parameter: %s' % entry['imdb_url'], log) for att in ['title', 'score', 'votes', 'year', 'genres', 'languages', 'actors', 'directors', 'mpaa_rating']: log.trace('movie.%s: %s' % (att, getattr(movie, att))) # store to entry entry.update_using_map(self.field_map, movie) finally: log.trace('committing session') session.commit()
def lookup_movie(title=None, year=None, rottentomatoes_id=None, imdb_id=None, smart_match=None, only_cached=False, session=None): """ Do a lookup from Rotten Tomatoes for the movie matching the passed arguments. Any combination of criteria can be passed, the most specific criteria specified will be used. :param rottentomatoes_id: rottentomatoes_id of desired movie :param imdb_id: imdb_id of desired movie :param title: title of desired movie :param year: release year of desired movie :param smart_match: attempt to clean and parse title and year from a string :param only_cached: if this is specified, an online lookup will not occur if the movie is not in the cache :param session: optionally specify a session to use, if specified, returned Movie will be live in that session :returns: The Movie object populated with data from Rotten Tomatoes :raises: PluginError if a match cannot be found or there are other problems with the lookup """ if smart_match: # If smart_match was specified, and we don't have more specific criteria, parse it into a title and year title_parser = MovieParser() title_parser.parse(smart_match) title = title_parser.name year = title_parser.year if title == '' and not (rottentomatoes_id or imdb_id or title): raise PluginError('Failed to parse name from %s' % smart_match) if title: search_string = title.lower() if year: search_string = '%s %s' % (search_string, year) elif not (rottentomatoes_id or imdb_id): raise PluginError('No criteria specified for rotten tomatoes lookup') def id_str(): return '<title=%s,year=%s,rottentomatoes_id=%s,imdb_id=%s>' % (title, year, rottentomatoes_id, imdb_id) if not session: session = Session() log.debug('Looking up rotten tomatoes information for %s' % id_str()) movie = None # Try to lookup from cache if rottentomatoes_id: movie = session.query(RottenTomatoesMovie).\ filter(RottenTomatoesMovie.id == rottentomatoes_id).first() if not movie and imdb_id: alt_id = session.query(RottenTomatoesAlternateId).\ filter(RottenTomatoesAlternateId.name.in_(['imdb', 'flexget_imdb'])).\ filter(RottenTomatoesAlternateId.id == imdb_id.lstrip('t')).first() if alt_id: movie = session.query(RottenTomatoesMovie).filter(RottenTomatoesMovie.id == alt_id.movie_id).first() if not movie and title: movie_filter = session.query(RottenTomatoesMovie).filter(func.lower(RottenTomatoesMovie.title) == title.lower()) if year: movie_filter = movie_filter.filter(RottenTomatoesMovie.year == year) movie = movie_filter.first() if not movie: log.debug('No matches in movie cache found, checking search cache.') found = session.query(RottenTomatoesSearchResult).\ filter(func.lower(RottenTomatoesSearchResult.search) == search_string).first() if found and found.movie: log.debug('Movie found in search cache.') movie = found.movie if movie: # Movie found in cache, check if cache has expired. if movie.expired and not only_cached: log.debug('Cache has expired for %s, attempting to refresh from Rotten Tomatoes.' % id_str()) try: imdb_alt_id = movie.alternate_ids and filter( lambda alt_id: alt_id.name in ['imdb', 'flexget_imdb'], movie.alternate_ids)[0].id if imdb_alt_id: result = movies_alias(imdb_alt_id, 'imdb') else: result = movies_info(movie.id) movie = _set_movie_details(movie, session, result) session.merge(movie) except URLError: log.error('Error refreshing movie details from Rotten Tomatoes, cached info being used.') else: log.debug('Movie %s information restored from cache.' % id_str()) else: if only_cached: raise PluginError('Movie %s not found from cache' % id_str()) # There was no movie found in the cache, do a lookup from Rotten Tomatoes log.debug('Movie %s not found in cache, looking up from rotten tomatoes.' % id_str()) try: # Lookups using imdb_id # TODO: extract to method if imdb_id: log.debug('Using IMDB alias %s.' % imdb_id) result = movies_alias(imdb_id, 'imdb') if result: mismatch = [] min_match = difflib.SequenceMatcher(lambda x: x == ' ', re.sub('\s+\(.*\)$', '', result['title'].lower()), title.lower()).ratio() < MIN_MATCH if title and min_match: mismatch.append('the title (%s <-?-> %s)' % (title, result['title'])) result['year'] = int(result['year']) if year and fabs(result['year'] - year) > 1: mismatch.append('the year (%s <-?-> %s)' % (year, result['year'])) release_year = None if result.get('release_dates', {}).get('theater'): log.debug('Checking year against theater release date') release_year = time.strptime(result['release_dates'].get('theater'), '%Y-%m-%d').tm_year if fabs(release_year - year) > 1: mismatch.append('the theater release (%s)' % release_year) elif result.get('release_dates', {}).get('dvd'): log.debug('Checking year against dvd release date') release_year = time.strptime(result['release_dates'].get('dvd'), '%Y-%m-%d').tm_year if fabs(release_year - year) > 1: mismatch.append('the DVD release (%s)' % release_year) if mismatch: log.warning('Rotten Tomatoes had an imdb alias for %s but it didn\'t match %s.' % (imdb_id, ', or '.join(mismatch))) else: log.debug('imdb_id %s maps to rt_id %s, checking db for info.' % (imdb_id, result['id'])) movie = session.query(RottenTomatoesMovie).\ filter(RottenTomatoesMovie.id == result.get('id')).first() if movie: log.debug('Movie %s was in database, but did not have the imdb_id stored, ' 'forcing an update' % movie) movie = _set_movie_details(movie, session, result) session.merge(movie) else: log.debug('%s was not in database, setting info.' % result['title']) movie = RottenTomatoesMovie() movie = _set_movie_details(movie, session, result) if not movie: raise PluginError('set_movie_details returned %s' % movie) session.add(movie) else: log.debug('IMDB alias %s returned no results.' % imdb_id) if not movie and rottentomatoes_id: result = movies_info(rottentomatoes_id) if result: movie = RottenTomatoesMovie() movie = _set_movie_details(movie, session, result) session.add(movie) if not movie and title: # TODO: Extract to method log.verbose('Searching from rt `%s`' % search_string) results = movies_search(search_string) if results: results = results.get('movies') if results: for movie_res in results: seq = difflib.SequenceMatcher( lambda x: x == ' ', movie_res['title'].lower(), title.lower()) movie_res['match'] = seq.ratio() results.sort(key=lambda x: x['match'], reverse=True) # Remove all movies below MIN_MATCH, and different year for movie_res in results[:]: if year and movie_res.get('year'): movie_res['year'] = int(movie_res['year']) if movie_res['year'] != year: release_year = False if movie_res.get('release_dates', {}).get('theater'): log.debug('Checking year against theater release date') release_year = time.strptime(movie_res['release_dates'].get('theater'), '%Y-%m-%d').tm_year elif movie_res.get('release_dates', {}).get('dvd'): log.debug('Checking year against dvd release date') release_year = time.strptime(movie_res['release_dates'].get('dvd'), '%Y-%m-%d').tm_year if not (release_year and release_year == year): log.debug('removing %s - %s (wrong year: %s)' % (movie_res['title'], movie_res['id'], str(release_year or movie_res['year']))) results.remove(movie_res) continue if movie_res['match'] < MIN_MATCH: log.debug('removing %s (min_match)' % movie_res['title']) results.remove(movie_res) continue if not results: raise PluginError('no appropiate results') if len(results) == 1: log.debug('SUCCESS: only one movie remains') else: # Check min difference between best two hits diff = results[0]['match'] - results[1]['match'] if diff < MIN_DIFF: log.debug('unable to determine correct movie, min_diff too small' '(`%s (%d) - %s` <-?-> `%s (%d) - %s`)' % (results[0]['title'], results[0]['year'], results[0]['id'], results[1]['title'], results[1]['year'], results[1]['id'])) for r in results: log.debug('remain: %s (match: %s) %s' % (r['title'], r['match'], r['id'])) raise PluginError('min_diff') imdb_alt_id = results[0].get('alternate_ids', {}).get('imdb') if imdb_alt_id: result = movies_alias(imdb_alt_id) else: result = movies_info(results[0].get('id')) if not result: result = results[0] movie = RottenTomatoesMovie() movie = _set_movie_details(movie, session, result) if imdb_id and not filter( lambda alt_id: alt_id.name == 'imdb' and alt_id.id == imdb_id.lstrip('t'), movie.alternate_ids): # TODO: get rid of these confusing lambdas log.warning('Adding flexget_imdb alternate id %s for movie %s' % (imdb_id, movie)) movie.alternate_ids.append(RottenTomatoesAlternateId('flexget_imdb', imdb_id.lstrip('t'))) session.add(movie) session.commit() if title.lower() != movie.title.lower(): log.debug('Saving search result for \'%s\'' % search_string) session.add(RottenTomatoesSearchResult(search=search_string, movie=movie)) except URLError: raise PluginError('Error looking up movie from RottenTomatoes') if not movie: raise PluginError('No results found from rotten tomatoes for %s' % id_str()) else: # Access attributes to force the relationships to eager load before we detach from session for attr in ['alternate_ids', 'cast', 'directors', 'genres', 'links', 'posters', 'release_dates']: getattr(movie, attr) session.commit() return movie
def consolidate(): """ Converts previous archive data model to new one. """ session = Session() try: log.verbose('Checking archive size ...') count = session.query(ArchiveEntry).count() log.verbose( 'Found %i items to migrate, this can be aborted with CTRL-C safely.' % count) # consolidate old data from progressbar import ProgressBar, Percentage, Bar, ETA widgets = [ 'Process - ', ETA(), ' ', Percentage(), ' ', Bar(left='[', right=']') ] bar = ProgressBar(widgets=widgets, maxval=count).start() # id's for duplicates duplicates = [] for index, orig in enumerate(session.query(ArchiveEntry).yield_per(5)): bar.update(index) # item already processed if orig.id in duplicates: continue # item already migrated if orig.sources: log.info( 'Database looks like it has already been consolidated, ' 'item %s has already sources ...' % orig.title) session.rollback() return # add legacy task to the sources list orig.sources.append(get_source(orig.task, session)) # remove task, deprecated .. well, let's still keep it .. # orig.task = None for dupe in session.query(ArchiveEntry).\ filter(ArchiveEntry.id != orig.id).\ filter(ArchiveEntry.title == orig.title).\ filter(ArchiveEntry.url == orig.url).all(): orig.sources.append(get_source(dupe.task, session)) duplicates.append(dupe.id) if duplicates: log.info('Consolidated %i items, removing duplicates ...' % len(duplicates)) for id in duplicates: session.query(ArchiveEntry).filter( ArchiveEntry.id == id).delete() session.commit() log.info('Completed! This does NOT need to be ran again.') except KeyboardInterrupt: session.rollback() log.critical('Aborted, no changes saved') finally: session.close()
def display_details(name): """Display detailed series information, ie. series show NAME""" from flexget.manager import Session with Session() as session: name = normalize_series_name(name) # Sort by length of name, so that partial matches always show shortest matching title matches = (session.query(Series).filter( Series._name_normalized.contains(name)).order_by( func.char_length(Series.name)).all()) if not matches: console('ERROR: Unknown series `%s`' % name) return # Pick the best matching series series = matches[0] console('Showing results for `%s`.' % series.name) if len(matches) > 1: console('WARNING: Multiple series match to `%s`.' % name) console('Be more specific to see the results of other matches:') for s in matches[1:]: console(' - %s' % s.name) console(' %-63s%-15s' % ('Identifier, Title', 'Quality')) console('-' * 79) # Query episodes in sane order instead of iterating from series.episodes episodes = session.query(Episode).filter( Episode.series_id == series.id) if series.identified_by == 'sequence': episodes = episodes.order_by(Episode.number).all() elif series.identified_by == 'ep': episodes = episodes.order_by(Episode.season, Episode.number).all() else: episodes = episodes.order_by(Episode.identifier).all() for episode in episodes: if episode.identifier is None: console(' None <--- Broken!') else: console(' %s (%s) - %s' % (episode.identifier, episode.identified_by or 'N/A', episode.age)) for release in episode.releases: status = release.quality.name title = release.title if len(title) > 55: title = title[:55] + '...' if release.proper_count > 0: status += '-proper' if release.proper_count > 1: status += str(release.proper_count) if release.downloaded: console(' * %-60s%-15s' % (title, status)) else: console(' %-60s%-15s' % (title, status)) console('-' * 79) console(' * = downloaded') if not series.identified_by: console('') console( ' Series plugin is still learning which episode numbering mode is ' ) console(' correct for this series (identified_by: auto).') console( ' Few duplicate downloads can happen with different numbering schemes' ) console(' during this time.') else: console( ' Series uses `%s` mode to identify episode numbering (identified_by).' % series.identified_by) console(' See option `identified_by` for more information.') if series.begin: console(' Begin episode for this series set to `%s`.' % series.begin.identifier)
def __run_task_phase(self, phase): """Executes task phase, ie. call all enabled plugins on the task. Fires events: * task.execute.before_plugin * task.execute.after_plugin :param string phase: Name of the phase """ if phase not in phase_methods: raise Exception('%s is not a valid task phase' % phase) # warn if no inputs, filters or outputs in the task if phase in ['input', 'filter', 'output']: if not self.manager.unit_test: # Check that there is at least one manually configured plugin for these phases for p in self.plugins(phase): if not p.builtin: break else: if phase not in self.suppress_warnings: if phase == 'filter': log.warning( 'Task does not have any filter plugins to accept entries. ' 'You need at least one to accept the entries you want.' ) else: log.warning( 'Task doesn\'t have any %s plugins, you should add (at least) one!' % phase) for plugin in self.plugins(phase): # Abort this phase if one of the plugins disables it if phase in self.disabled_phases: return # store execute info, except during entry events self.current_phase = phase self.current_plugin = plugin.name if plugin.api_ver == 1: # backwards compatibility # pass method only task (old behaviour) args = (self, ) else: # pass method task, copy of config (so plugin cannot modify it) args = (self, copy.copy(self.config.get(plugin.name))) # Hack to make task.session only active for a single plugin with Session() as session: self.session = session try: fire_event('task.execute.before_plugin', self, plugin.name) response = self.__run_plugin(plugin, phase, args) if phase == 'input' and response: # add entries returned by input to self.all_entries for e in response: e.task = self self.all_entries.extend(response) finally: fire_event('task.execute.after_plugin', self, plugin.name) self.session = None # check config hash for changes at the end of 'prepare' phase if phase == 'prepare': self.check_config_hash()
def display_summary(options): """ Display series summary. :param options: argparse options from the CLI """ session = Session() try: query = (session.query(Series).outerjoin(Series.episodes).outerjoin( Episode.releases).outerjoin(Series.in_tasks).group_by(Series.id)) if options.configured == 'configured': query = query.having(func.count(SeriesTask.id) >= 1) elif options.configured == 'unconfigured': query = query.having(func.count(SeriesTask.id) < 1) if options.premieres: query = (query.having(func.max(Episode.season) <= 1).having( func.max(Episode.number) <= 2).having( func.count(SeriesTask.id) < 1)).filter( Release.downloaded == True) if options.new: query = query.having( func.max(Episode.first_seen) > datetime.now() - timedelta(days=options.new)) if options.stale: query = query.having( func.max(Episode.first_seen) < datetime.now() - timedelta(days=options.stale)) if options.porcelain: formatting = '%-30s %s %-10s %s %-10s %s %-20s' console(formatting % ('Name', '|', 'Latest', '|', 'Age', '|', 'Downloaded')) else: formatting = ' %-30s %-10s %-10s %-20s' console('-' * 79) console(formatting % ('Name', 'Latest', 'Age', 'Downloaded')) console('-' * 79) for series in query.order_by(Series.name).yield_per(10): series_name = series.name if len(series_name) > 30: series_name = series_name[:27] + '...' new_ep = ' ' behind = 0 status = 'N/A' age = 'N/A' episode_id = 'N/A' latest = get_latest_release(series) if latest: if latest.first_seen > datetime.now() - timedelta(days=2): if options.porcelain: pass else: new_ep = '>' behind = new_eps_after(latest) status = get_latest_status(latest) age = latest.age episode_id = latest.identifier if behind: episode_id += ' +%s' % behind if options.porcelain: console(formatting % (series_name, '|', episode_id, '|', age, '|', status)) else: console(new_ep + formatting[1:] % (series_name, episode_id, age, status)) if behind >= 3: console( ' ! Latest download is %d episodes behind, this may require ' 'manual intervention' % behind) if options.porcelain: pass else: console('-' * 79) console(' > = new episode ') console( ' Use `flexget series show NAME` to get detailed information') finally: session.close()
def on_task_input(self, task, config): if not config: return config = self.prepare_config(config) entries = [] queue_name = config.get('queue_name') with Session() as session: for queue_item in queue_get(session=session, downloaded=False, queue_name=queue_name): entry = Entry() # make sure the entry has IMDB fields filled entry['url'] = '' if queue_item.imdb_id: entry['imdb_id'] = queue_item.imdb_id entry['imdb_url'] = make_imdb_url(queue_item.imdb_id) if queue_item.tmdb_id: entry['tmdb_id'] = queue_item.tmdb_id # check if title is a imdb url (leftovers from old database?) # TODO: maybe this should be fixed at the queue_get ... if 'http://' in queue_item.title: plugin.get_plugin_by_name('tmdb_lookup').instance.lookup( entry) log.debug('queue contains url instead of title') if entry.get('movie_name'): entry['title'] = entry['movie_name'] else: log.error( 'Found imdb url in imdb queue, but lookup failed: %s' % entry['title']) continue else: # normal title entry['title'] = queue_item.title # Add the year and quality if configured to (make sure not to double it up) if config.get('year') and entry.get('movie_year') \ and str(entry['movie_year']) not in entry['title']: plugin.get_plugin_by_name('tmdb_lookup').instance.lookup( entry) entry['title'] += ' %s' % entry['movie_year'] # TODO: qualities can now be ranges.. how should we handle this? if config.get('quality') and queue_item.quality != 'ANY': log.info( 'quality option of emit_movie_queue is disabled while we figure out how to handle ranges' ) # entry['title'] += ' %s' % queue_item.quality entries.append(entry) if entry.get('imdb_id'): log.debug('Added title and IMDB id to new entry: %s - %s', entry['title'], entry['imdb_id']) elif entry.get('tmdb_id'): log.debug('Added title and TMDB id to new entry: %s - %s', entry['title'], entry['tmdb_id']) else: # should this ever happen though? log.debug('Added title to new entry: %s', entry['title']) return entries
def get_login_cookies(self, username, password): url_auth = 'http://www.t411.in/users/login' db_session = Session() account = db_session.query(torrent411Account).filter( torrent411Account.username == username).first() if account: if account.expiry_time < datetime.now(): db_session.delete(account) db_session.commit() log.debug("Cookies found in db!") return account.auth else: log.debug("Getting login cookies from : %s " % url_auth) params = {'login': username, 'password': password, 'remember': '1'} cj = cookielib.CookieJar() # WE NEED A COOKIE HOOK HERE TO AVOID REDIRECT COOKIES opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) # NEED TO BE SAME USER_AGENT THAN DOWNLOAD LINK opener.addheaders = [('User-agent', self.USER_AGENT)] login_output = None try: login_output = opener.open(url_auth, urllib.urlencode(params)).read() except Exception as e: raise UrlRewritingError("Connection Error for %s : %s" % (url_auth, e)) if b'confirmer le captcha' in login_output: log.warn("Captcha requested for login.") login_output = self._solveCaptcha(login_output, url_auth, params, opener) if b'logout' in login_output: authKey = None uid = None password = None for cookie in cj: if cookie.name == "authKey": authKey = cookie.value if cookie.name == "uid": uid = cookie.value if cookie.name == "pass": password = cookie.value if authKey is not None and \ uid is not None and \ password is not None: authCookie = { 'uid': uid, 'password': password, 'authKey': authKey } db_session.add( torrent411Account(username=username, auth=authCookie, expiry_time=datetime.now() + timedelta(days=1))) db_session.commit() return authCookie else: log.error( "Login failed (Torrent411). Check your login and password." ) return {}
def __len__(self): with Session() as session: return len(self._db_list(session).movies)
def __iter__(self): with Session() as session: return iter( [file.to_entry() for file in self._db_list(session).files])
def execute(self): """ Executes the the task. If :attr:`.enabled` is False task is not executed. Certain :attr:`.options` affect how execution is handled. - :attr:`.options.disable_phases` is a list of phases that are not enabled for this execution. - :attr:`.options.inject` is a list of :class:`Entry` instances used instead of running input phase. """ if not self.enabled: log.debug('Not running disabled task %s' % self.name) if self.options.cron: self.manager.db_cleanup() self._reset() log.debug('executing %s' % self.name) if not self.enabled: log.debug('task %s disabled during preparation, not running' % self.name) return # Handle keyword args if self.options.learn: log.info('Disabling download and output phases because of --learn') self.disable_phase('download') self.disable_phase('output') if self.options.disable_phases: map(self.disable_phase, self.options.disable_phases) if self.options.inject: # If entries are passed for this execution (eg. rerun), disable the input phase self.disable_phase('input') self.all_entries.extend(self.options.inject) log.debug('starting session') self.session = Session() # Save current config hash and set config_modidied flag config_hash = hashlib.md5(str(sorted(self.config.items()))).hexdigest() last_hash = self.session.query(TaskConfigHash).filter( TaskConfigHash.task == self.name).first() if self.is_rerun: # Restore the config to state right after start phase if self.prepared_config: self.config = copy.deepcopy(self.prepared_config) else: log.error('BUG: No prepared_config on rerun, please report.') self.config_modified = False elif not last_hash: self.config_modified = True last_hash = TaskConfigHash(task=self.name, hash=config_hash) self.session.add(last_hash) elif last_hash.hash != config_hash: self.config_modified = True last_hash.hash = config_hash else: self.config_modified = False # run phases try: for phase in task_phases: if phase in self.disabled_phases: # log keywords not executed for plugin in self.plugins(phase): if plugin.name in self.config: log.info( 'Plugin %s is not executed because %s phase is disabled (e.g. --test)' % (plugin.name, phase)) continue if phase == 'start' and self.is_rerun: log.debug('skipping task_start during rerun') elif phase == 'exit' and self._rerun: log.debug( 'not running task_exit yet because task will rerun') else: # run all plugins with this phase self.__run_task_phase(phase) if phase == 'start': # Store a copy of the config state after start phase to restore for reruns self.prepared_config = copy.deepcopy(self.config) except TaskAbort: # Roll back the session before calling abort handlers self.session.rollback() try: self.__run_task_phase('abort') # Commit just the abort handler changes if no exceptions are raised there self.session.commit() except TaskAbort as e: log.exception('abort handlers aborted: %s' % e) raise else: for entry in self.all_entries: entry.complete() log.debug('committing session') self.session.commit() fire_event('task.execute.completed', self) finally: # this will cause database rollback on exception self.session.close() # rerun task if self._rerun: log.info( 'Rerunning the task in case better resolution can be achieved.' ) self._rerun_count += 1 # TODO: Potential optimization is to take snapshots (maybe make the ones backlog uses built in instead of # taking another one) after input and just inject the same entries for the rerun self.execute()
def __len__(self): with Session() as session: return self._db_list(session).files.count()
def test_failed_sorting(self, api_client): failed_entry_dict_1 = dict(title='Failed title_1', url='http://jhb.com', reason='Test reason_3') failed_entry_dict_2 = dict(title='Failed title_2', url='http://def.com', reason='Test reason_1') failed_entry_dict_3 = dict(title='Failed title_3', url='http://abc.com', reason='Test reason_2') with Session() as session: failed_entry1 = FailedEntry(**failed_entry_dict_1) failed_entry2 = FailedEntry(**failed_entry_dict_2) failed_entry3 = FailedEntry(**failed_entry_dict_3) session.bulk_save_objects( [failed_entry1, failed_entry2, failed_entry3]) # Sort by title rsp = api_client.get('/failed/?sort_by=title') assert rsp.status_code == 200 data = json.loads(rsp.get_data(as_text=True)) assert data[0]['title'] == 'Failed title_3' rsp = api_client.get('/failed/?sort_by=title&order=asc') assert rsp.status_code == 200 data = json.loads(rsp.get_data(as_text=True)) assert data[0]['title'] == 'Failed title_1' # Sort by url rsp = api_client.get('/failed/?sort_by=url') assert rsp.status_code == 200 data = json.loads(rsp.get_data(as_text=True)) assert data[0]['url'] == 'http://jhb.com' rsp = api_client.get('/failed/?sort_by=url&order=asc') assert rsp.status_code == 200 data = json.loads(rsp.get_data(as_text=True)) assert data[0]['url'] == 'http://abc.com' # Sort by reason rsp = api_client.get('/failed/?sort_by=reason') assert rsp.status_code == 200 data = json.loads(rsp.get_data(as_text=True)) assert data[0]['reason'] == 'Test reason_3' rsp = api_client.get('/failed/?sort_by=reason&order=asc') assert rsp.status_code == 200 data = json.loads(rsp.get_data(as_text=True)) assert data[0]['reason'] == 'Test reason_1' # Combine sorting and pagination rsp = api_client.get('/failed/?sort_by=reason&per_page=2&page=2') assert rsp.status_code == 200 data = json.loads(rsp.get_data(as_text=True)) assert data[0]['reason'] == 'Test reason_1'
def get(self, entry): with Session() as session: match = self._entry_query(session=session, entry=entry, approved=True) return Entry(match.entry) if match else None