def cli_search(options): search_term = ' '.join(options.keywords) tags = options.tags sources = options.sources def print_ae(ae): diff = datetime.now() - ae.added console('ID: %-6s | Title: %s\nAdded: %s (%d days ago)\nURL: %s' % (ae.id, ae.title, ae.added, diff.days, ae.url)) source_names = ', '.join([s.name for s in ae.sources]) tag_names = ', '.join([t.name for t in ae.tags]) console('Source(s): %s | Tag(s): %s' % (source_names or 'N/A', tag_names or 'N/A')) if ae.description: console('Description: %s' % strip_html(ae.description)) console('---') session = Session() try: console('Searching: %s' % search_term) if tags: console('Tags: %s' % ', '.join(tags)) if sources: console('Sources: %s' % ', '.join(sources)) console('Please wait...') console('') results = False for ae in search(session, search_term, tags=tags, sources=sources): print_ae(ae) results = True if not results: console('No results found.') finally: session.close()
def estimate(self, entry): if all(field in entry for field in ['series_name', 'series_season', 'series_episode']): # Try to get airdate from tvrage first if api_tvrage: season = entry['series_season'] if entry.get('series_id_type') == 'sequence': # Tvrage has absolute numbered shows under season 1 season = 1 log.debug("Querying release estimation for %s S%02dE%02d ..." % (entry['series_name'], season, entry['series_episode'])) try: series_info = lookup_series(name=entry['series_name']) except LookupError as e: log.debug('tvrage lookup error: %s' % e) else: if series_info: try: episode_info = series_info.find_episode(season, entry['series_episode']) if episode_info: return episode_info.airdate else: # If episode does not exist in tvrage database, we always return a future date log.verbose('%s S%02dE%02d does not exist in tvrage database, assuming unreleased', series_info.name, season, entry['series_episode']) return datetime.now() + timedelta(weeks=4) except Exception as e: log.exception(e) else: log.debug('No series info obtained from TVRage to %s' % entry['series_name']) log.debug('No episode info obtained from TVRage for %s season %s episode %s' % (entry['series_name'], entry['series_season'], entry['series_episode'])) # If no results from tvrage, estimate a date based on series history session = Session() series = session.query(Series).filter(Series.name == entry['series_name']).first() if not series: return episodes = (session.query(Episode).join(Episode.series). filter(Episode.season != None). filter(Series.id == series.id). filter(Episode.season == func.max(Episode.season).select()). order_by(desc(Episode.number)).limit(2).all()) if len(episodes) < 2: return # If last two eps were not contiguous, don't guess if episodes[0].number != episodes[1].number + 1: return last_diff = episodes[0].first_seen - episodes[1].first_seen # If last eps were grabbed close together, we might be catching up, don't guess # Or, if last eps were too far apart, don't guess # TODO: What range? if last_diff < timedelta(days=2) or last_diff > timedelta(days=10): return # Estimate next season somewhat more than a normal episode break if entry['series_season'] > episodes[0].season: # TODO: How big should this be? return episodes[0].first_seen + multiply_timedelta(last_diff, 2) # Estimate next episode comes out about same length as last ep span, with a little leeway return episodes[0].first_seen + multiply_timedelta(last_diff, 0.9)
def do_cli(manager, options): session = Session() try: console('-- History: ' + '-' * 67) query = session.query(History) if options.search: search_term = options.search.replace(' ', '%').replace('.', '%') query = query.filter(History.title.like('%' + search_term + '%')) if options.task: query = query.filter(History.task.like('%' + options.task + '%')) query = query.order_by(desc(History.time)).limit(options.limit) for item in reversed(query.all()): if options.short: console(' %-25s %s' % (item.time.strftime("%c"), item.title)) else: console(' Task : %s' % item.task) console(' Title : %s' % item.title) console(' Url : %s' % item.url) if item.filename: console(' Stored : %s' % item.filename) console(' Time : %s' % item.time.strftime("%c")) console(' Details : %s' % item.details) console('-' * 79) finally: session.close()
def get_series_summary(self): """ :return: Dictionary where key is series name and value is dictionary of summary details. """ result = {} session = Session() try: seriestasks = session.query(SeriesTask).all() if seriestasks: all_series = set(st.series for st in seriestasks) else: all_series = session.query(Series).all() for series in all_series: name = series.name # capitalize if user hasn't, better look and sorting ... if name.islower(): name = capwords(name) result[name] = {'identified_by': series.identified_by} result[name]['in_tasks'] = [task.name for task in series.in_tasks] episode = self.get_latest_download(series) if episode: latest = {'first_seen': episode.first_seen, 'episode_instance': episode, 'episode_id': episode.identifier, 'age': episode.age, 'status': self.get_latest_status(episode), 'behind': self.new_eps_after(episode)} result[name]['latest'] = latest finally: session.close() return result
def get_file(self, only_cached=False): """Makes sure the poster is downloaded to the local cache (in userstatic folder) and returns the path split into a list of directory and file components""" from flexget.manager import manager base_dir = os.path.join(manager.config_base, 'userstatic') if self.file and os.path.isfile(os.path.join(base_dir, self.file)): return self.file.split(os.sep) elif only_cached: return # If we don't already have a local copy, download one. log.debug('Downloading poster %s' % self.url) dirname = os.path.join('tmdb', 'posters', str(self.movie_id)) # Create folders if they don't exist fullpath = os.path.join(base_dir, dirname) if not os.path.isdir(fullpath): os.makedirs(fullpath) filename = os.path.join(dirname, posixpath.basename(self.url)) thefile = file(os.path.join(base_dir, filename), 'wb') thefile.write(requests.get(self.url).content) self.file = filename # If we are detached from a session, update the db if not Session.object_session(self): session = Session() try: poster = session.query(TMDBPoster).filter(TMDBPoster.db_id == self.db_id).first() if poster: poster.file = filename finally: session.close() return filename.split(os.sep)
def on_process_start(self, task): if not task.manager.options.seen_search: return task.manager.disable_tasks() session = Session() shown = [] for field in session.query(SeenField).\ filter(SeenField.value.like(unicode('%' + task.manager.options.seen_search + '%'))).\ order_by(asc(SeenField.added)).all(): se = session.query(SeenEntry).filter(SeenEntry.id == field.seen_entry_id).first() if not se: print 'ERROR: <SeenEntry(id=%s)> missing' % field.seen_entry_id continue # don't show duplicates if se.id in shown: continue shown.append(se.id) print 'ID: %s Name: %s Task: %s Added: %s' % (se.id, se.title, se.task, se.added.strftime('%c')) for sf in se.fields: print ' %s: %s' % (sf.field, sf.value) print '' if not shown: print 'No results' session.close()
def age_series(**kwargs): from flexget.plugins.filter.series import Release from flexget.manager import Session import datetime session = Session() session.query(Release).update({'first_seen': datetime.datetime.now() - datetime.timedelta(**kwargs)}) session.commit()
def cli_search(options): search_term = " ".join(options.keywords) tags = options.tags sources = options.sources def print_ae(ae): diff = datetime.now() - ae.added console( "ID: %-6s | Title: %s\nAdded: %s (%d days ago)\nURL: %s" % (ae.id, ae.title, ae.added, diff.days, ae.url) ) source_names = ", ".join([s.name for s in ae.sources]) tag_names = ", ".join([t.name for t in ae.tags]) console("Source(s): %s | Tag(s): %s" % (source_names or "N/A", tag_names or "N/A")) if ae.description: console("Description: %s" % strip_html(ae.description)) console("---") session = Session() try: console("Searching: %s" % search_term) if tags: console("Tags: %s" % ", ".join(tags)) if sources: console("Sources: %s" % ", ".join(sources)) console("Please wait...") console("") results = False for ae in search(session, search_term, tags=tags, sources=sources): print_ae(ae) results = True if not results: console("No results found.") finally: session.close()
def on_process_start(self, task): # migrate seen to seen_entry session = Session() from flexget.utils.sqlalchemy_utils import table_exists if table_exists('seen', session): self.migrate2() session.close()
def assert_series_count_in_db(expected_count): from flexget.plugins.filter.series import Series from flexget.manager import Session session = Session() actual_series_count = session.query(Series).count() assert expected_count == actual_series_count, "expecting %s series stored in db, got %s instead" % \ (expected_count, actual_series_count)
def search(self, search_term, tags=None): def print_ae(ae): diff = datetime.now() - ae.added console('ID: %-6s | Title: %s\nAdded: %s (%d days ago)\nURL: %s' % (ae.id, ae.title, ae.added, diff.days, ae.url)) source_names = ', '.join([s.name for s in ae.sources]) tag_names = ', '.join([t.name for t in ae.tags]) console('Source(s): %s | Tag(s): %s' % (source_names or 'N/A', tag_names or 'N/A')) if ae.description: console('Description: %s' % strip_html(ae.description)) console('---') session = Session() try: console('Searching: %s' % search_term) if tags: console('Tags: %s' % ', '.join(tags)) console('Please wait ...') console('') for ae in search(session, search_term, tags): print_ae(ae) finally: session.close()
def on_process_start(self, feed): if not feed.manager.options.repair_seen_movies: return feed.manager.disable_feeds() from progressbar import ProgressBar, Percentage, Bar, ETA from flexget.manager import Session from seen import SeenField from flexget.utils.imdb import extract_id session = Session() index = 0 count = 0 total = session.query(SeenField).filter(SeenField.field == u'imdb_url').count() widgets = ['Repairing: ', ETA(), ' ', Percentage(), ' ', Bar(left='[', right=']')] bar = ProgressBar(widgets=widgets, maxval=total).start() for seen in session.query(SeenField).filter(SeenField.field == u'imdb_url').all(): index += 1 if index % 5 == 0: bar.update(index) value = u'http://www.imdb.com/title/%s/' % extract_id(seen.value) if value != seen.value: count += 1 seen.value = value seen.field = unicode('imdb_url') bar.finish() session.commit() print 'Fixed %s/%s URLs' % (count, total)
def test_seen_get(self, mock_seen_search): session = Session() entry_list = session.query(SeenEntry).join(SeenField).all() mock_seen_search.return_value = entry_list # No params rsp = self.get('/seen/') assert rsp.status_code == 200, 'Response code is %s' % rsp.status_code # Default params rsp = self.get('/seen/?page=1&max=100&local_seen=true&sort_by=added&order=desc') assert rsp.status_code == 200, 'Response code is %s' % rsp.status_code # Changed params rsp = self.get('/seen/?max=1000&local_seen=false&sort_by=title&order=asc') assert rsp.status_code == 200, 'Response code is %s' % rsp.status_code # Negative test, invalid parameter rsp = self.get('/seen/?max=1000&local_seen=BLA&sort_by=title &order=asc') assert rsp.status_code == 400, 'Response code is %s' % rsp.status_code # With value rsp = self.get('/seen/?value=bla') assert rsp.status_code == 200, 'Response code is %s' % rsp.status_code assert mock_seen_search.call_count == 4, 'Should have 4 calls, is actually %s' % mock_seen_search.call_count
def get_series_summary(self): """ :return: Dictionary where key is series name and value is dictionary of summary details. """ result = {} session = Session() try: for series in session.query(Series).all(): name = series.name # capitalize if user hasn't, better look and sorting ... if name.islower(): name = capwords(name) result[name] = {"identified_by": series.identified_by} episode = self.get_latest_download(series) if episode: latest = { "first_seen": episode.first_seen, "episode_instance": episode, "episode_id": episode.identifier, "age": episode.age, "status": self.get_latest_status(episode), "behind": self.new_eps_after(episode), } result[name]["latest"] = latest finally: session.close() return result
def test_seen_delete_all(self, mock_seen_search, api_client): session = Session() entry_list = session.query(SeenEntry).join(SeenField) mock_seen_search.return_value = entry_list # No params rsp = api_client.delete('/seen/') assert rsp.status_code == 404, 'Response code is %s' % rsp.status_code fields = { 'url': 'http://test.com/file.torrent', 'title': 'Test.Title', 'torrent_hash_id': 'dsfgsdfg34tq34tq34t' } entry = { 'local': False, 'reason': 'test_reason', 'task': 'test_task', 'title': 'Test.Title', 'fields': fields } rsp = api_client.json_post('/seen/', data=json.dumps(entry)) assert rsp.status_code == 200, 'Response code is %s' % rsp.status_code # With value rsp = api_client.delete('/seen/?value=Test.Title') assert rsp.status_code == 200, 'Response code is %s' % rsp.status_code assert mock_seen_search.call_count == 2, 'Should have 2 calls, is actually %s' % mock_seen_search.call_count
def clear_backlog(manager): if not manager.options.clear_backlog: return manager.disable_tasks() session = Session() num = session.query(BacklogEntry).delete() session.close() console('%s entries cleared from backlog.' % num)
def test_withsession(self): session = Session() persist = SimplePersistence('testplugin', session=session) persist['aoeu'] = 'test' assert persist['aoeu'] == 'test' # Make sure it didn't commit or close our session session.rollback() assert 'aoeu' not in persist
def assert_series_count_in_db(expected_count): from flexget.components.series.db import Series from flexget.manager import Session session = Session() actual_series_count = session.query(Series).count() assert ( expected_count == actual_series_count ), "expecting %s series stored in db, got %s instead" % (expected_count, actual_series_count)
def get_login_cookies(self, username, password): url_auth = 'http://www.t411.li/users/login' db_session = Session() account = db_session.query(torrent411Account).filter( torrent411Account.username == username).first() if account: if account.expiry_time < datetime.now(): db_session.delete(account) db_session.commit() log.debug("Cookies found in db!") return account.auth else: log.debug("Getting login cookies from : %s " % url_auth) params = {'login': username, 'password': password, 'remember': '1'} cj = http.cookiejar.CookieJar() # WE NEED A COOKIE HOOK HERE TO AVOID REDIRECT COOKIES opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj)) # NEED TO BE SAME USER_AGENT THAN DOWNLOAD LINK opener.addheaders = [('User-agent', self.USER_AGENT)] login_output = None try: login_output = opener.open(url_auth, urllib.parse.urlencode(params)).read() except Exception as e: raise UrlRewritingError("Connection Error for %s : %s" % (url_auth, e)) if b'confirmer le captcha' in login_output: log.warning("Captcha requested for login.") login_output = self._solveCaptcha(login_output, url_auth, params, opener) if b'logout' in login_output: authKey = None uid = None password = None for cookie in cj: if cookie.name == "authKey": authKey = cookie.value if cookie.name == "uid": uid = cookie.value if cookie.name == "pass": password = cookie.value if authKey is not None and \ uid is not None and \ password is not None: authCookie = {'uid': uid, 'password': password, 'authKey': authKey } db_session.add(torrent411Account(username=username, auth=authCookie, expiry_time=datetime.now() + timedelta(days=1))) db_session.commit() return authCookie else: log.error("Login failed (Torrent411). Check your login and password.") return {}
def _get_db_last_run(self): session = Session() try: db_trigger = session.query(DBTrigger).get(self.uid) if db_trigger: self.last_run = db_trigger.last_run log.debug("loaded last_run from the database") finally: session.close()
def display_summary(options): """ Display series summary. :param options: argparse options from the CLI """ formatting = ' %-30s %-10s %-10s %-20s' console(formatting % ('Name', 'Latest', 'Age', 'Downloaded')) console('-' * 79) session = Session() try: query = (session.query(Series).outerjoin(Series.episodes).outerjoin(Episode.releases). outerjoin(Series.in_tasks).group_by(Series.id)) if options.configured == 'configured': query = query.having(func.count(SeriesTask.id) >= 1) elif options.configured == 'unconfigured': query = query.having(func.count(SeriesTask.id) < 1) if options.premieres: query = (query.having(func.max(Episode.season) <= 1).having(func.max(Episode.number) <= 2). having(func.count(SeriesTask.id) < 1)).filter(Release.downloaded == True) if options.new: query = query.having(func.max(Episode.first_seen) > datetime.now() - timedelta(days=options.new)) if options.stale: query = query.having(func.max(Episode.first_seen) < datetime.now() - timedelta(days=options.stale)) for series in query.order_by(Series.name).yield_per(10): series_name = series.name if len(series_name) > 30: series_name = series_name[:27] + '...' new_ep = ' ' behind = 0 status = 'N/A' age = 'N/A' episode_id = 'N/A' latest = get_latest_release(series) if latest: if latest.first_seen > datetime.now() - timedelta(days=2): new_ep = '>' behind = new_eps_after(latest) status = get_latest_status(latest) age = latest.age episode_id = latest.identifier if behind: episode_id += ' +%s' % behind console(new_ep + formatting[1:] % (series_name, episode_id, age, status)) if behind >= 3: console(' ! Latest download is %d episodes behind, this may require ' 'manual intervention' % behind) console('-' * 79) console(' > = new episode ') console(' Use `flexget series show NAME` to get detailed information') finally: session.close()
def display_details(self, name): """Display detailed series information, ie. --series NAME""" from flexget.manager import Session session = Session() name = unicode(name.lower()) series = session.query(Series).filter(Series.name == name).first() if not series: console("Unknown series `%s`" % name) return console(" %-63s%-15s" % ("Identifier, Title", "Quality")) console("-" * 79) # Query episodes in sane order instead of iterating from series.episodes episodes = session.query(Episode).filter(Episode.series_id == series.id) if series.identified_by == "sequence": episodes = episodes.order_by(Episode.number).all() else: episodes = episodes.order_by(Episode.identifier).all() for episode in episodes: if episode.identifier is None: console(" None <--- Broken!") else: console(" %s (%s) - %s" % (episode.identifier, episode.identified_by or "N/A", episode.age)) for release in episode.releases: status = release.quality.name title = release.title if len(title) > 55: title = title[:55] + "..." if release.proper_count > 0: status += "-proper" if release.proper_count > 1: status += str(release.proper_count) if release.downloaded: console(" * %-60s%-15s" % (title, status)) else: console(" %-60s%-15s" % (title, status)) console("-" * 79) console(" * = downloaded") if not series.identified_by: console("") console(" Series plugin is still learning which episode numbering mode is ") console(" correct for this series (identified_by: auto).") console(" Few duplicate downloads can happen with different numbering schemes") console(" during this time.") else: console(" Series uses `%s` mode to identify episode numbering (identified_by)." % series.identified_by) console(" See option `identified_by` for more information.") session.close()
def cli_perf_test(manager, options): if options.test_name not in TESTS: console('Unknown performance test %s' % options.test_name) return session = Session() try: if options.test_name == 'imdb_query': imdb_query(session) finally: session.close()
def upgrade_required(): """Returns true if an upgrade of the database is required.""" session = Session() try: for old_schema in session.query(PluginSchema).all(): if old_schema.plugin in plugin_schemas and old_schema.version < plugin_schemas[old_schema.plugin]['version']: return True return False finally: session.close()
def get_login_cookies(self, username, password): url_auth = 'http://www.t411.me/users/login' db_session = Session() account = db_session.query(torrent411Account).filter( torrent411Account.username == username).first() if account: if account.expiry_time < datetime.now(): db_session.delete(account) db_session.commit() log.debug("Cookies found in db!") return account.auth else: log.debug("Getting login cookies from : %s " % url_auth) params = urllib.urlencode({'login': username, 'password': password, 'remember': '1'}) cj = cookielib.CookieJar() # WE NEED A COOKIE HOOK HERE TO AVOID REDIRECT COOKIES opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) # NEED TO BE SAME USER_AGENT THAN DOWNLOAD LINK opener.addheaders = [('User-agent', self.USER_AGENT)] try: opener.open(url_auth, params) except Exception as e: raise UrlRewritingError("Connection Error for %s : %s" % (url_auth, e)) authKey = None uid = None password = None for cookie in cj: if cookie.name == "authKey": authKey = cookie.value if cookie.name == "uid": uid = cookie.value if cookie.name == "pass": password = cookie.value if authKey is not None and \ uid is not None and \ password is not None: authCookie = {'uid': uid, 'password': password, 'authKey': authKey } db_session.add(torrent411Account(username=username, auth=authCookie, expiry_time=datetime.now() + timedelta(days=1))) db_session.commit() return authCookie return {"uid": "", "password": "", "authKey": "" }
def display_details(self, name): """Display detailed series information, ie. --series NAME""" from flexget.manager import Session session = Session() name = unicode(name.lower()) series = session.query(Series).filter(Series.name == name).first() if not series: print 'Unknown series `%s`' % name return print ' %-63s%-15s' % ('Identifier, Title', 'Quality') print '-' * 79 # Query episodes in sane order instead of iterating from series.episodes episodes = session.query(Episode).filter(Episode.series_id == series.id) if series.identified_by == 'sequence': episodes = episodes.order_by(Episode.number).all() else: episodes = episodes.order_by(Episode.identifier).all() for episode in episodes: if episode.identifier is None: print ' None <--- Broken!' else: print ' %s (%s) - %s' % (episode.identifier, episode.identified_by or 'N/A', episode.age) for release in episode.releases: status = release.quality.name title = release.title if len(title) > 55: title = title[:55] + '...' if release.proper_count > 0: status += '-proper' if release.proper_count > 1: status += str(release.proper_count) if release.downloaded: print ' * %-60s%-15s' % (title, status) else: print ' %-60s%-15s' % (title, status) print '-' * 79 print ' * = downloaded' if not series.identified_by: print '' print ' Series plugin is still learning which episode numbering mode is ' print ' correct for this series (identified_by: auto).' print ' Few duplicate downloads can happen with different numbering schemes' print ' during this time.' else: print ' Series uses `%s` mode to identify episode numbering (identified_by).' % series.identified_by print ' See option `identified_by` for more information.' session.close()
def list_failed(): session = Session() try: results = session.query(FailedEntry).all() if not results: console('No failed entries recorded') for entry in results: console('%16s - %s - %s times - %s' % (entry.tof.strftime('%Y-%m-%d %H:%M'), entry.title, entry.count, entry.reason)) finally: session.close()
def get_version(plugin): session = Session() try: schema = session.query(PluginSchema).filter(PluginSchema.plugin == plugin).first() if not schema: log.debug('No schema version stored for %s' % plugin) return None else: return schema.version finally: session.close()
def on_process_start(self, feed): # migrate shelve -> sqlalchemy if feed.manager.shelve_session: self.migrate(feed) # migrate seen to seen_entry session = Session() from flexget.utils.sqlalchemy_utils import table_exists if table_exists('seen', session): self.migrate2() session.close()
def list_rejected(): session = Session() try: results = session.query(RememberEntry).all() if not results: console('No rejected entries recorded by remember_rejected') else: console('Rejections remembered by remember_rejected:') for entry in results: console('%s from %s by %s because %s' % (entry.title, entry.task.name, entry.rejected_by, entry.reason)) finally: session.close()
def on_task_input(self, task, config): if not config: return if isinstance(config, bool): config = {} if task.is_rerun: # Just return calculated next eps on reruns entries = self.rerun_entries self.rerun_entries = [] return entries else: self.rerun_entries = [] entries = [] with Session() as session: for seriestask in session.query(SeriesTask).filter( SeriesTask.name == task.name).all(): series = seriestask.series if not series: # TODO: How can this happen? log.debug( 'Found SeriesTask item without series specified. Cleaning up.' ) session.delete(seriestask) continue if series.identified_by not in ['ep', 'sequence']: log.verbose('Can only emit ep or sequence based series. ' '`%s` is identified_by %s' % (series.name, series.identified_by or 'auto')) continue low_season = 0 if series.identified_by == 'ep' else -1 check_downloaded = not config.get('backfill') latest_season = get_latest_release(series, downloaded=check_downloaded) if latest_season: latest_season = latest_season.season else: latest_season = low_season + 1 for season in range(latest_season, low_season, -1): log.debug('Adding episodes for season %d' % season) latest = get_latest_release(series, season=season, downloaded=check_downloaded) if series.begin and (not latest or latest < series.begin): entries.append( self.search_entry(series, series.begin.season, series.begin.number, task)) elif latest and not config.get('backfill'): entries.append( self.search_entry(series, latest.season, latest.number + 1, task)) elif latest: start_at_ep = 1 episodes_this_season = (session.query(Episode).filter( Episode.series_id == series.id).filter( Episode.season == season)) if series.identified_by == 'sequence': # Don't look for missing too far back with sequence shows start_at_ep = max(latest.number - 10, 1) episodes_this_season = episodes_this_season.filter( Episode.number >= start_at_ep) latest_ep_this_season = episodes_this_season.order_by( desc(Episode.number)).first() downloaded_this_season = (episodes_this_season.join( Episode.releases).filter( Release.downloaded == True).all()) # Calculate the episodes we still need to get from this season if series.begin and series.begin.season == season: start_at_ep = max(start_at_ep, series.begin.number) eps_to_get = list( range(start_at_ep, latest_ep_this_season.number + 1)) for ep in downloaded_this_season: try: eps_to_get.remove(ep.number) except ValueError: pass entries.extend( self.search_entry( series, season, x, task, rerun=False) for x in eps_to_get) # If we have already downloaded the latest known episode, try the next episode if latest_ep_this_season.releases: entries.append( self.search_entry( series, season, latest_ep_this_season.number + 1, task)) else: if config.get('from_start') or config.get('backfill'): entries.append( self.search_entry(series, season, 1, task)) else: log.verbose( 'Series `%s` has no history. Set begin option, ' 'or use CLI `series begin` ' 'subcommand to set first episode to emit' % series.name) break # Skip older seasons if we are not in backfill mode if not config.get('backfill'): break # Don't look for seasons older than begin ep if series.begin and series.begin.season >= season: break return entries
def learn_backlog(self, task, amount=''): """Learn current entries into backlog. All task inputs must have been executed.""" with Session() as session: for entry in task.entries: self.add_backlog(task, entry, amount, session=session)
def build(self, task): if not (plt and pd): logger.warning('Dependency does not exist: [matplotlib, pandas]') return if not task.accepted and not task.failed: return session = Session() columns = [ 'site', 'uploaded', 'downloaded', 'share_ratio', 'points', 'seeding', 'leeching', 'hr', ] data = {'sort_column': [], 'default_order': []} total_details = {} total_changed = {} user_classes_dict = {} for column in columns: data[column] = [] total_details[column] = 0 total_changed[column] = 0 order = len(task.all_entries) for entry in task.all_entries: user_classes_dict[entry['site_name']] = entry.get('user_classes') if entry.get('do_not_draw'): continue data['default_order'].append(order) order = order - 1 user_details_db = self._get_user_details(session, entry['site_name']) if user_details_db is None: user_details_db = UserDetailsEntry( site=entry['site_name'], uploaded=0, downloaded=0, share_ratio=0, join_date=datetime.now().date(), points=0, seeding=0, leeching=0, hr=0) session.add(user_details_db) session.commit() user_details_db = self._get_user_details( session, entry['site_name']) # failed if not entry.get('details'): for column in columns: value = getattr(user_details_db, column) if entry.failed: data[column].append( self.buid_data_text(column, value) + '*') else: data[column].append(self.buid_data_text(column, value)) if not entry.get('do_not_count') and column not in [ 'site' ]: self.count(total_details, column, value) data['sort_column'].append(0) continue # now details_now = {} for key, value in entry['details'].items(): details_now[key] = self.transfer_data(key, value) # changed details_changed = {} for key, value_now in details_now.items(): if value_now != '*' and key not in ['join_date']: details_changed[key] = value_now - getattr( user_details_db, key) else: details_changed[key] = '*' if details_changed['uploaded'] == '*': data['sort_column'].append(0) else: data['sort_column'].append(details_changed['uploaded']) # append to data data['site'].append(entry['site_name']) for column in columns: if column == 'site': continue data[column].append('{}{}'.format( self.buid_data_text(column, getattr(user_details_db, column)), self.buid_data_text(column, details_changed[column], append=True))) if total_details.get(column) is None: total_details[column] = 0 if total_changed.get(column) is None: total_changed[column] = 0 if not entry.get('do_not_count') and column not in [ 'share_ratio', 'points' ]: total_details[column] = total_details[column] + getattr( user_details_db, column) if details_changed[column] != '*': total_changed[column] = total_changed[ column] + details_changed[column] # update db for key, value in details_now.items(): if key == 'join_date': value = parse(value) if value != '*': setattr(user_details_db, key, value) session.commit() data['site'].append('total') for column in columns: if column == 'site': continue data[column].append('{}{}'.format( self.buid_data_text(column, total_details[column]), self.buid_data_text(column, total_changed[column], append=True))) data['sort_column'].append(float('inf')) data['default_order'].append(float('inf')) df = pd.DataFrame(data) df.sort_values(by=['sort_column', 'default_order'], ascending=False, inplace=True) df.drop(columns=['sort_column', 'default_order'], inplace=True) line_count = len(data['site']) fig = plt.figure(figsize=(8, line_count / 1.8)) plt.axis('off') colors = [] for x in df.values: cc = [] for y in x: if '\n-' in y: cc.append('#f38181') elif '+' in y: cc.append('#95e1d3') elif '*' in y: cc.append('#eff48e') else: cc.append('white') colors.append(cc) col_widths = [0.15, 0.16, 0.16, 0.13, 0.14, 0.1, 0.1, 0.06] table = plt.table(cellText=df.values, cellColours=colors, bbox=[0, 0, 1, 1], colLabels=df.columns, colWidths=col_widths, loc='best') table.auto_set_font_size(False) table.set_fontsize(10) fig.tight_layout() plt.title(datetime.now().replace(microsecond=0)) plt.savefig('details_report.png', bbox_inches='tight', dpi=300) self.draw_user_classes(user_classes_dict, session, df)
def authenticate(self): """Authenticates a session with IMDB, and grabs any IDs needed for getting/modifying list.""" cached_credentials = False with Session() as session: if self.cookies: if not self.user_id: self.user_id = self.get_user_id_and_hidden_value( self.cookies) if not self.user_id: raise PluginError( 'Not possible to retrive userid, please check cookie information' ) user = IMDBListUser(self.config['login'], self.user_id, self.cookies) self.cookies = user.cookies cached_credentials = True else: user = (session.query(IMDBListUser).filter( IMDBListUser.user_name == self.config.get( 'login')).one_or_none()) if user and user.cookies and user.user_id: logger.debug('login credentials found in cache, testing') self.user_id = user.user_id if not self.get_user_id_and_hidden_value( cookies=user.cookies): logger.debug('cache credentials expired') user.cookies = None self._session.cookies.clear() else: self.cookies = user.cookies cached_credentials = True if not cached_credentials: logger.debug( 'user credentials not found in cache or outdated, fetching from IMDB' ) url_credentials = ( 'https://www.imdb.com/ap/signin?openid.return_to=https%3A%2F%2Fwww.imdb.com%2Fap-signin-' 'handler&openid.identity=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&' 'openid.assoc_handle=imdb_mobile_us&openid.mode=checkid_setup&openid.claimed_id=http%3A%' '2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&openid.ns=http%3A%2F%2Fspecs.ope' 'nid.net%2Fauth%2F2.0') try: # we need to get some cookies first self._session.get('https://www.imdb.com') r = self._session.get(url_credentials) except RequestException as e: raise PluginError(e.args[0]) soup = get_soup(r.content) form = soup.find('form', attrs={'name': 'signIn'}) inputs = form.select('input') data = dict((i['name'], i.get('value')) for i in inputs if i.get('name')) data['email'] = self.config['login'] data['password'] = self.config['password'] action = form.get('action') logger.debug('email={}, password={}', data['email'], data['password']) self._session.headers.update({'Referer': url_credentials}) self._session.post(action, data=data) self._session.headers.update( {'Referer': 'https://www.imdb.com/'}) self.user_id = self.get_user_id_and_hidden_value() if not self.user_id: raise plugin.PluginError( 'Login to IMDB failed. Check your credentials.') self.cookies = self._session.cookies.get_dict( domain='.imdb.com') # Get list ID if user: for list in user.lists: if self.config['list'] == list.list_name: logger.debug( 'found list ID {} matching list name {} in cache', list.list_id, list.list_name, ) self.list_id = list.list_id if not self.list_id: logger.debug( 'could not find list ID in cache, fetching from IMDB') if self.config['list'] == 'watchlist': data = { 'consts[]': 'tt0133093', 'tracking_tag': 'watchlistRibbon' } wl_data = self._session.post( 'https://www.imdb.com/list/_ajax/watchlist_has', data=data, cookies=self.cookies, ).json() try: self.list_id = wl_data['list_id'] except KeyError: raise PluginError( 'No list ID could be received. Please initialize list by ' 'manually adding an item to it and try again') elif self.config['list'] in IMMUTABLE_LISTS or self.config[ 'list'].startswith('ls'): self.list_id = self.config['list'] else: data = {'tconst': 'tt0133093'} list_data = self._session.post( 'https://www.imdb.com/list/_ajax/wlb_dropdown', data=data, cookies=self.cookies, ).json() for li in list_data['items']: if li['wlb_text'] == self.config['list']: self.list_id = li['data_list_id'] break else: raise plugin.PluginError('Could not find list %s' % self.config['list']) user = IMDBListUser(self.config['login'], self.user_id, self.cookies) list = IMDBListList(self.list_id, self.config['list'], self.user_id) user.lists.append(list) session.merge(user) self._authenticated = True
def add_failed(self, entry, reason=None): """Adds entry to internal failed list, displayed with --failed""" reason = unicode(reason or 'Unknown') failed = Session() try: # query item's existence item = failed.query(FailedEntry).filter(FailedEntry.title == entry['title']).\ filter(FailedEntry.url == entry['original_url']).first() if not item: item = FailedEntry(entry['title'], entry['original_url'], reason) else: item.count += 1 item.tof = datetime.now() item.reason = reason failed.merge(item) log.debug('Marking %s in failed list. Has failed %s times.' % (item.title, item.count)) # limit item number to 25 for row in failed.query(FailedEntry).order_by( FailedEntry.tof.desc())[25:]: failed.delete(row) failed.commit() finally: failed.close()
def __len__(self): with Session() as session: return self._db_list(session).regexps.count()
def discard(self, entry): with Session() as session: db_regexp = self._find_entry(entry, session=session) if db_regexp: log.debug('deleting file %s', db_regexp) session.delete(db_regexp)
def _execute(self): """Executes the task without rerunning.""" if not self.enabled: log.debug('Not running disabled task %s' % self.name) return log.debug('executing %s' % self.name) # Handle keyword args if self.options.learn: log.info('Disabling download and output phases because of --learn') self.disable_phase('download') self.disable_phase('output') if self.options.disable_phases: map(self.disable_phase, self.options.disable_phases) if self.options.inject: # If entries are passed for this execution (eg. rerun), disable the input phase self.disable_phase('input') self.all_entries.extend(self.options.inject) # Save current config hash and set config_modidied flag with Session() as session: config_hash = hashlib.md5(str(sorted(self.config.items()))).hexdigest() last_hash = session.query(TaskConfigHash).filter(TaskConfigHash.task == self.name).first() if self.is_rerun: # Restore the config to state right after start phase if self.prepared_config: self.config = copy.deepcopy(self.prepared_config) else: log.error('BUG: No prepared_config on rerun, please report.') self.config_modified = False elif not last_hash: self.config_modified = True last_hash = TaskConfigHash(task=self.name, hash=config_hash) session.add(last_hash) elif last_hash.hash != config_hash: self.config_modified = True last_hash.hash = config_hash else: self.config_modified = False # run phases try: for phase in task_phases: if phase in self.disabled_phases: # log keywords not executed for plugin in self.plugins(phase): if plugin.name in self.config: log.info('Plugin %s is not executed because %s phase is disabled (e.g. --test)' % (plugin.name, phase)) continue if phase == 'start' and self.is_rerun: log.debug('skipping task_start during rerun') elif phase == 'exit' and self._rerun: log.debug('not running task_exit yet because task will rerun') else: # run all plugins with this phase self.__run_task_phase(phase) if phase == 'start': # Store a copy of the config state after start phase to restore for reruns self.prepared_config = copy.deepcopy(self.config) except TaskAbort: try: self.__run_task_phase('abort') except TaskAbort as e: log.exception('abort handlers aborted: %s' % e) raise else: for entry in self.all_entries: entry.complete()
def __iter__(self): with Session() as session: return iter([ movie.to_entry(self.strip_year) for movie in self._db_list(session).movies ])
def display_details(options): """Display detailed series information, ie. series show NAME""" name = options.series_name with Session() as session: name = normalize_series_name(name) # Sort by length of name, so that partial matches always show shortest matching title matches = shows_by_name(name, session=session) if not matches: console(colorize(ERROR_COLOR, 'ERROR: Unknown series `%s`' % name)) return # Pick the best matching series series = matches[0] table_title = colorize('white', series.name) if len(matches) > 1: warning = ( colorize('red', ' WARNING: ') + 'Multiple series match to `{}`.\n ' 'Be more specific to see the results of other matches:\n\n' ' {}'.format(name, ', '.join(s.name for s in matches[1:]))) if not options.table_type == 'porcelain': console(warning) header = [ 'Episode ID', 'Latest age', 'Release titles', 'Release Quality', 'Proper' ] table_data = [header] episodes = show_episodes(series, session=session) for episode in episodes: if episode.identifier is None: identifier = colorize(ERROR_COLOR, 'MISSING') age = '' else: identifier = episode.identifier age = episode.age ep_data = [identifier, age] release_titles = [] release_qualities = [] release_propers = [] for release in episode.releases: title = release.title quality = release.quality.name if not release.downloaded: title = colorize(UNDOWNLOADED_RELEASE_COLOR, title) quality = quality else: title = colorize(DOWNLOADED_RELEASE_COLOR, title) quality = quality release_titles.append(title) release_qualities.append(quality) release_propers.append( 'Yes' if release.proper_count > 0 else '') ep_data.append('\n'.join(release_titles)) ep_data.append('\n'.join(release_qualities)) ep_data.append('\n'.join(release_propers)) table_data.append(ep_data) footer = (' %s %s\n' % (colorize(DOWNLOADED_RELEASE_COLOR, 'Downloaded'), colorize(UNDOWNLOADED_RELEASE_COLOR, 'Un-downloaded'))) if not series.identified_by: footer += ( '\n' ' Series plugin is still learning which episode numbering mode is \n' ' correct for this series (identified_by: auto).\n' ' Few duplicate downloads can happen with different numbering schemes\n' ' during this time.') else: footer += ' \n Series uses `%s` mode to identify episode numbering (identified_by).' % series.identified_by footer += ' \n See option `identified_by` for more information.\n' if series.begin: footer += ' Begin episode for this series set to `%s`.' % series.begin.identifier table = TerminalTable(options.table_type, table_data, table_title, drop_columns=[4, 3, 1]) try: console(table.output) except TerminalTableError as e: console('ERROR: %s' % str(e)) return if not options.table_type == 'porcelain': console(footer)
def discard(self, entry): with Session() as session: db_movie = self._find_entry(entry, session=session) if db_movie: log.debug('deleting movie %s', db_movie) session.delete(db_movie)
def display_summary(options): """ Display series summary. :param options: argparse options from the CLI """ porcelain = options.table_type == 'porcelain' with Session() as session: kwargs = { 'configured': options.configured, 'premieres': options.premieres, 'session': session, 'sort_by': options.sort_by, 'descending': options.order } if options.new: kwargs['status'] = 'new' kwargs['days'] = options.new elif options.stale: kwargs['status'] = 'stale' kwargs['days'] = options.stale if options.sort_by == 'name': kwargs['sort_by'] = 'show_name' else: kwargs['sort_by'] = 'last_download_date' query = get_series_summary(**kwargs) header = ['Name', 'Latest', 'Age', 'Downloaded', 'Identified By'] for index, value in enumerate(header): if value.lower() == options.sort_by: header[index] = colorize(SORT_COLUMN_COLOR, value) footer = 'Use `flexget series show NAME` to get detailed information' table_data = [header] for series in query: name_column = series.name new_ep = False behind = 0 latest_release = '-' age_col = '-' episode_id = '-' latest = get_latest_release(series) identifier_type = series.identified_by if identifier_type == 'auto': identifier_type = colorize('yellow', 'auto') if latest: behind = new_eps_after(latest) latest_release = get_latest_status(latest) # colorize age age_col = latest.age if latest.age_timedelta is not None: if latest.age_timedelta < timedelta(days=1): age_col = colorize(NEW_EP_COLOR, latest.age) elif latest.age_timedelta < timedelta(days=3): age_col = colorize(FRESH_EP_COLOR, latest.age) elif latest.age_timedelta > timedelta(days=400): age_col = colorize(OLD_EP_COLOR, latest.age) episode_id = latest.identifier if not porcelain: if behind > 0: name_column += colorize(BEHIND_EP_COLOR, ' {} behind'.format(behind)) table_data.append([ name_column, episode_id, age_col, latest_release, identifier_type ]) table = TerminalTable(options.table_type, table_data, wrap_columns=[3], drop_columns=[4, 3, 2]) try: console(table.output) except TerminalTableError as e: console('ERROR: %s' % str(e)) return if not porcelain: console(footer)
def on_process_end(self, task): """Write RSS file at application terminate.""" if not rss2gen: return # don't generate rss when learning if task.manager.options.learn: return config = self.get_config(task) if config['file'] in self.written: log.trace('skipping already written file %s' % config['file']) return # in terminate phase there is no open session in task, so open new one from flexget.manager import Session session = Session() db_items = session.query(RSSEntry).filter(RSSEntry.file == config['file']).\ order_by(RSSEntry.published.desc()).all() # make items rss_items = [] for db_item in db_items: add = True if config['items'] != -1: if len(rss_items) > config['items']: add = False if config['days'] != -1: if datetime.datetime.today() - datetime.timedelta( days=config['days']) > db_item.published: add = False if add: # add into generated feed gen = { 'title': db_item.title, 'description': db_item.description, 'link': db_item.link, 'pubDate': db_item.published } log.trace('Adding %s into rss %s' % (gen['title'], config['file'])) rss_items.append(PyRSS2Gen.RSSItem(**gen)) else: # no longer needed session.delete(db_item) session.commit() session.close() # make rss rss = PyRSS2Gen.RSS2(title='FlexGet', link=config.get('rsslink', 'http://flexget.com'), description='FlexGet generated RSS feed', lastBuildDate=datetime.datetime.utcnow(), items=rss_items) # write rss fn = os.path.expanduser(config['file']) with open(fn, 'w') as file: try: log.verbose('Writing output rss to %s' % fn) rss.write_xml(file, encoding=config['encoding']) except LookupError: log.critical('Unknown encoding %s' % config['encoding']) return except IOError: # TODO: plugins cannot raise PluginWarnings in terminate event .. log.critical('Unable to write %s' % fn) return self.written[config['file']] = True
def search(*args, **kwargs): if 'count' in kwargs: return 0 else: with Session() as session: return session.query(SeenEntry).join(SeenField)
def authenticate(self): """Authenticates a session with IMDB, and grabs any IDs needed for getting/modifying list.""" cached_credentials = False with Session() as session: user = session.query(IMDBListUser).filter( IMDBListUser.user_name == self.config.get( 'login')).one_or_none() if user and user.cookies and user.user_id: log.debug('login credentials found in cache, testing') self.cookies = user.cookies self.user_id = user.user_id r = self._session.head('http://www.imdb.com/profile', allow_redirects=False, cookies=self.cookies) if not r.headers.get( 'location') or 'login' in r.headers['location']: log.debug('cache credentials expired') else: cached_credentials = True if not cached_credentials: log.debug( 'user credentials not found in cache or outdated, fetching from IMDB' ) try: r = self._session.get( 'https://www.imdb.com/ap/signin?openid.return_to=https%3A%2F%2Fwww.imdb.com%2Fap-signin-' 'handler&openid.identity=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&' 'openid.assoc_handle=imdb_mobile_us&openid.mode=checkid_setup&openid.claimed_id=http%3A%' '2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&openid.ns=http%3A%2F%2Fspecs.ope' 'nid.net%2Fauth%2F2.0') except RequestException as e: raise PluginError(e.args[0]) soup = get_soup(r.content) inputs = soup.select('form#ap_signin_form input') data = dict((i['name'], i.get('value')) for i in inputs if i.get('name')) data['email'] = self.config['login'] data['password'] = self.config['password'] log.debug('email=%s, password=%s', data['email'], data['password']) d = self._session.post('https://www.imdb.com/ap/signin', data=data) # Get user id by extracting from redirect url r = self._session.head('http://www.imdb.com/profile', allow_redirects=False) if not r.headers.get( 'location') or 'login' in r.headers['location']: raise plugin.PluginError( 'Login to IMDB failed. Check your credentials.') self.user_id = re.search('ur\d+(?!\d)', r.headers['location']).group() self.cookies = dict(d.cookies) # Get list ID if user: for list in user.lists: if self.config['list'] == list.list_name: log.debug( 'found list ID %s matching list name %s in cache', list.list_id, list.list_name) self.list_id = list.list_id if not self.list_id: log.debug( 'could not find list ID in cache, fetching from IMDB') if self.config['list'] == 'watchlist': data = { 'consts[]': 'tt0133093', 'tracking_tag': 'watchlistRibbon' } wl_data = self._session.post( 'http://www.imdb.com/list/_ajax/watchlist_has', data=data, cookies=self.cookies).json() try: self.list_id = wl_data['list_id'] except KeyError: raise PluginError( 'No list ID could be received. Please initialize list by ' 'manually adding an item to it and try again') elif self.config['list'] in IMMUTABLE_LISTS or self.config[ 'list'].startswith('ls'): self.list_id = self.config['list'] else: data = {'tconst': 'tt0133093'} list_data = self._session.post( 'http://www.imdb.com/list/_ajax/wlb_dropdown', data=data, cookies=self.cookies).json() for li in list_data['items']: if li['wlb_text'] == self.config['list']: self.list_id = li['data_list_id'] break else: raise plugin.PluginError('Could not find list %s' % self.config['list']) user = IMDBListUser(self.config['login'], self.user_id, self.cookies) list = IMDBListList(self.list_id, self.config['list'], self.user_id) user.lists.append(list) session.merge(user) self._authenticated = True
def load(cls, task=None): """Load all key/values from `task` into memory from database.""" with Session() as session: for skv in session.query(SimpleKeyValue).filter(SimpleKeyValue.task == task).all(): cls.class_store[task][skv.plugin][skv.key] = skv.value
def display_details(self, name): """Display detailed series information, ie. --series NAME""" from flexget.manager import Session session = Session() name = unicode(name.lower()) series = session.query(Series).filter(Series.name == name).first() if not series: console('Unknown series `%s`' % name) return console(' %-63s%-15s' % ('Identifier, Title', 'Quality')) console('-' * 79) # Query episodes in sane order instead of iterating from series.episodes episodes = session.query(Episode).filter( Episode.series_id == series.id) if series.identified_by == 'sequence': episodes = episodes.order_by(Episode.number).all() else: episodes = episodes.order_by(Episode.identifier).all() for episode in episodes: if episode.identifier is None: console(' None <--- Broken!') else: console(' %s (%s) - %s' % (episode.identifier, episode.identified_by or 'N/A', episode.age)) for release in episode.releases: status = release.quality.name title = release.title if len(title) > 55: title = title[:55] + '...' if release.proper_count > 0: status += '-proper' if release.proper_count > 1: status += str(release.proper_count) if release.downloaded: console(' * %-60s%-15s' % (title, status)) else: console(' %-60s%-15s' % (title, status)) console('-' * 79) console(' * = downloaded') if not series.identified_by: console('') console( ' Series plugin is still learning which episode numbering mode is ' ) console(' correct for this series (identified_by: auto).') console( ' Few duplicate downloads can happen with different numbering schemes' ) console(' during this time.') else: console( ' Series uses `%s` mode to identify episode numbering (identified_by).' % series.identified_by) console(' See option `identified_by` for more information.') session.close()
def age_timeframe(**kwargs): with Session() as session: session.query(EntryTimeFrame).update({ 'first_seen': datetime.datetime.now() - datetime.timedelta(**kwargs) })
def __len__(self): with Session() as session: return len(self._db_list(session).movies)
def __run_task_phase(self, phase): """Executes task phase, ie. call all enabled plugins on the task. Fires events: * task.execute.before_plugin * task.execute.after_plugin :param string phase: Name of the phase """ if phase not in phase_methods: raise Exception('%s is not a valid task phase' % phase) # warn if no inputs, filters or outputs in the task if phase in ['input', 'filter', 'output']: if not self.manager.unit_test: # Check that there is at least one manually configured plugin for these phases for p in self.plugins(phase): if not p.builtin: break else: if phase not in self.suppress_warnings: if phase == 'filter': logger.warning( 'Task does not have any filter plugins to accept entries. ' 'You need at least one to accept the entries you want.' ) else: logger.warning( 'Task doesn\'t have any {} plugins, you should add (at least) one!', phase, ) for plugin in self.plugins(phase): # Abort this phase if one of the plugins disables it if phase in self.disabled_phases: return if plugin.name in self.disabled_plugins: continue # store execute info, except during entry events self.current_phase = phase self.current_plugin = plugin.name if plugin.api_ver == 1: # backwards compatibility # pass method only task (old behaviour) args = (self, ) else: # pass method task, copy of config (so plugin cannot modify it) args = (self, copy.copy(self.config.get(plugin.name))) # Hack to make task.session only active for a single plugin with Session() as session: self.session = session try: fire_event('task.execute.before_plugin', self, plugin.name) response = self.__run_plugin(plugin, phase, args) if phase == 'input' and response: # add entries returned by input to self.all_entries for e in response: e.task = self self.all_entries.append(e) finally: fire_event('task.execute.after_plugin', self, plugin.name) self.session = None # check config hash for changes at the end of 'prepare' phase if phase == 'prepare': self.check_config_hash()
def forget(value): """ See module docstring :param string value: Can be task name, entry title or field value :return: count, field_count where count is number of entries removed and field_count number of fields """ log.debug('forget called with %s' % value) session = Session() try: count = 0 field_count = 0 for se in session.query(SeenEntry).filter( or_(SeenEntry.title == value, SeenEntry.task == value)).all(): field_count += len(se.fields) count += 1 log.debug('forgetting %s' % se) session.delete(se) for sf in session.query(SeenField).filter( SeenField.value == value).all(): se = session.query(SeenEntry).filter( SeenEntry.id == sf.seen_entry_id).first() field_count += len(se.fields) count += 1 log.debug('forgetting %s' % se) session.delete(se) return count, field_count finally: session.commit() session.close()
def test_history_pagination(self, api_client, schema_match, link_headers): history_entry = dict( task='test_task_', title='test_title_', url='test_url_', filename='test_filename_', details='test_details_', ) num_of_entries = 200 with Session() as session: for i in range(num_of_entries): item = History() for key, value in history_entry.items(): setattr(item, key, value + str(i)) session.add(item) rsp = api_client.get('/history/') assert rsp.status_code == 200 data = json.loads(rsp.get_data(as_text=True)) errors = schema_match(OC.history_list_object, data) assert not errors assert len(data) == 50 # Default page size assert int(rsp.headers['total-count']) == 200 assert int(rsp.headers['count']) == 50 links = link_headers(rsp) assert links['last']['page'] == 4 assert links['next']['page'] == 2 rsp = api_client.get('/history/?per_page=100') assert rsp.status_code == 200 data = json.loads(rsp.get_data(as_text=True)) errors = schema_match(OC.history_list_object, data) assert not errors assert len(data) == 100 assert int(rsp.headers['count']) == 100 links = link_headers(rsp) assert links['last']['page'] == 2 assert links['next']['page'] == 2 # Per page is limited to 100 rsp = api_client.get('/history/?per_page=200') assert rsp.status_code == 200 data = json.loads(rsp.get_data(as_text=True)) errors = schema_match(OC.history_list_object, data) assert not errors assert len(data) == 100 rsp = api_client.get('/history/?page=2&sort_by=id&order=asc') assert rsp.status_code == 200 data = json.loads(rsp.get_data(as_text=True)) errors = schema_match(OC.history_list_object, data) assert not errors assert data[0]['task'] == 'test_task_50' links = link_headers(rsp) assert links['last']['page'] == 4 assert links['next']['page'] == 3 assert links['prev']['page'] == 1 # Non existent page rsp = api_client.get('/history/?page=5') assert rsp.status_code == 404 data = json.loads(rsp.get_data(as_text=True)) errors = schema_match(base_message, data) assert not errors
def __iter__(self): with Session() as session: return iter([regexp.to_entry() for regexp in self._db_list(session).regexps])
def test_history_sorting(self, api_client, schema_match, link_headers): history_entry1 = dict( task='test_task_1', title='test_title_a', url='test_url_1', filename='test_filename_a', details='test_details_1', ) history_entry2 = dict( task='test_task_2', title='test_title_b', url='test_url_2', filename='test_filename_b', details='test_details_2', ) history_entry3 = dict( task='test_task_3', title='test_title_c', url='test_url_3', filename='test_filename_c', details='test_details_3', ) entries = [history_entry1, history_entry2, history_entry3] with Session() as session: for entry in entries: item = History() for key, value in entry.items(): setattr(item, key, value) session.add(item) rsp = api_client.get('/history/?sort_by=id') assert rsp.status_code == 200 data = json.loads(rsp.get_data(as_text=True)) errors = schema_match(OC.history_list_object, data) assert not errors assert data[0]['id'] == 3 assert int(rsp.headers['total-count']) == 3 assert int(rsp.headers['count']) == 3 rsp = api_client.get('/history/?sort_by=task&order=asc') assert rsp.status_code == 200 data = json.loads(rsp.get_data(as_text=True)) errors = schema_match(OC.history_list_object, data) assert not errors assert data[0]['task'] == 'test_task_1' rsp = api_client.get('/history/?sort_by=details') assert rsp.status_code == 200 data = json.loads(rsp.get_data(as_text=True)) errors = schema_match(OC.history_list_object, data) assert not errors assert data[0]['details'] == 'test_details_3' rsp = api_client.get('/history/?per_page=2') assert rsp.status_code == 200 data = json.loads(rsp.get_data(as_text=True)) errors = schema_match(OC.history_list_object, data) assert not errors assert len(data) == 2 assert int(rsp.headers['total-count']) == 3 assert int(rsp.headers['count']) == 2 rsp = api_client.get('/history/?per_page=2&page=2') assert rsp.status_code == 200 data = json.loads(rsp.get_data(as_text=True)) errors = schema_match(OC.history_list_object, data) assert not errors assert len(data) == 1 assert int(rsp.headers['total-count']) == 3 assert int(rsp.headers['count']) == 1
def wrapped_func(*args, **kwargs): # get task from method parameters task = args[1] # detect api version api_ver = 1 if len(args) == 3: api_ver = 2 if api_ver == 1: # get name for a cache from tasks configuration if self.name not in task.config: raise Exception( '@cache config name %s is not configured in task %s' % (self.name, task.name)) hash = config_hash(task.config[self.name]) else: hash = config_hash(args[2]) log.trace('self.name: %s' % self.name) log.trace('hash: %s' % hash) cache_name = self.name + '_' + hash log.debug('cache name: %s (has: %s)' % (cache_name, ', '.join(list(self.cache.keys())))) cache_value = self.cache.get(cache_name, None) if not task.options.nocache and cache_value: # return from the cache log.trace('cache hit') entries = [] for entry in cache_value: fresh = copy.deepcopy(entry) entries.append(fresh) if entries: log.verbose('Restored %s entries from cache' % len(entries)) return entries else: if self.persist and not task.options.nocache: # Check database cache with Session() as session: db_cache = session.query(InputCache).filter(InputCache.name == self.name).\ filter(InputCache.hash == hash).\ filter(InputCache.added > datetime.now() - self.persist).\ first() if db_cache: entries = [e.entry for e in db_cache.entries] log.verbose('Restored %s entries from db cache' % len(entries)) # Store to in memory cache self.cache[cache_name] = copy.deepcopy(entries) return entries # Nothing was restored from db or memory cache, run the function log.trace('cache miss') # call input event try: response = func(*args, **kwargs) except PluginError as e: # If there was an error producing entries, but we have valid entries in the db cache, return those. if self.persist and not task.options.nocache: with Session() as session: db_cache = session.query(InputCache).filter(InputCache.name == self.name).\ filter(InputCache.hash == hash).first() if db_cache and db_cache.entries: log.error( 'There was an error during %s input (%s), using cache instead.' % (self.name, e)) entries = [e.entry for e in db_cache.entries] log.verbose( 'Restored %s entries from db cache' % len(entries)) # Store to in memory cache self.cache[cache_name] = copy.deepcopy(entries) return entries # If there was nothing in the db cache, re-raise the error. raise if api_ver == 1: response = task.entries if not isinstance(response, list): log.warning( 'Input %s did not return a list, cannot cache.' % self.name) return response # store results to cache log.debug('storing to cache %s %s entries' % (cache_name, len(response))) try: self.cache[cache_name] = copy.deepcopy(response) except TypeError: # might be caused because of backlog restoring some idiotic stuff, so not neccessarily a bug log.critical( 'Unable to save task content into cache, ' 'if problem persists longer than a day please report this as a bug' ) if self.persist: # Store to database log.debug('Storing cache %s to database.' % cache_name) with Session() as session: db_cache = session.query(InputCache).filter(InputCache.name == self.name).\ filter(InputCache.hash == hash).first() if not db_cache: db_cache = InputCache(name=self.name, hash=hash) db_cache.entries = [ InputCacheEntry(entry=e) for e in response ] db_cache.added = datetime.now() session.merge(db_cache) return response
def __init__(self, config): self.config = config with Session() as session: if not self._db_list(session): session.add(EntryListList(name=self.config))
def __iter__(self): with Session() as session: return iter( [file.to_entry() for file in self._db_list(session).files])
def get_login_cookies(self, username, password): url_auth = 'http://www.t411.ch/users/login' db_session = Session() account = db_session.query(torrent411Account).filter( torrent411Account.username == username).first() if account: if account.expiry_time < datetime.now(): db_session.delete(account) db_session.commit() log.debug("Cookies found in db!") return account.auth else: log.debug("Getting login cookies from : %s " % url_auth) params = {'login': username, 'password': password, 'remember': '1'} cj = http.cookiejar.CookieJar() # WE NEED A COOKIE HOOK HERE TO AVOID REDIRECT COOKIES opener = urllib.request.build_opener( urllib.request.HTTPCookieProcessor(cj)) # NEED TO BE SAME USER_AGENT THAN DOWNLOAD LINK opener.addheaders = [('User-agent', self.USER_AGENT)] login_output = None try: login_output = opener.open( url_auth, urllib.parse.urlencode(params)).read() except Exception as e: raise UrlRewritingError("Connection Error for %s : %s" % (url_auth, e)) if b'confirmer le captcha' in login_output: log.warning("Captcha requested for login.") login_output = self._solveCaptcha(login_output, url_auth, params, opener) if b'logout' in login_output: authKey = None uid = None password = None for cookie in cj: if cookie.name == "authKey": authKey = cookie.value if cookie.name == "uid": uid = cookie.value if cookie.name == "pass": password = cookie.value if authKey is not None and \ uid is not None and \ password is not None: authCookie = { 'uid': uid, 'password': password, 'authKey': authKey } db_session.add( torrent411Account(username=username, auth=authCookie, expiry_time=datetime.now() + timedelta(days=1))) db_session.commit() return authCookie else: log.error( "Login failed (Torrent411). Check your login and password." ) return {}
def on_task_start(self, task, config): if config is False: # handles 'template: no' form to turn off template on this task return # implements --template NAME if task.options.template: if not config or task.options.template not in config: task.abort('does not use `%s` template' % task.options.template, silent=True) config = self.prepare_config(config) # add global in except when disabled with no_global if 'no_global' in config: config.remove('no_global') if 'global' in config: config.remove('global') elif 'global' not in config: config.append('global') toplevel_templates = task.manager.config.get('templates', {}) # apply templates for template in config: if template not in toplevel_templates: if template == 'global': continue raise plugin.PluginError( 'Unable to find template %s for task %s' % (template, task.name), log) if toplevel_templates[template] is None: log.warning('Template `%s` is empty. Nothing to merge.' % template) continue log.debug('Merging template %s into task %s' % (template, task.name)) # We make a copy here because we need to remove template_config = toplevel_templates[template] # When there are templates within templates we remove the template # key from the config and append it's items to our own if 'template' in template_config: nested_templates = self.prepare_config( template_config['template']) for nested_template in nested_templates: if nested_template not in config: config.append(nested_template) else: log.warning('Templates contain each other in a loop.') # Replace template_config with a copy without the template key, to avoid merging errors template_config = dict(template_config) del template_config['template'] # Merge try: merge_dict_from_to(template_config, task.config) except MergeException as exc: raise plugin.PluginError( 'Failed to merge template %s to task %s. Error: %s' % (template, task.name, exc.value)) # TODO: Better handling of config_modified flag for templates??? with Session() as session: last_hash = session.query(TemplateConfigHash).filter( TemplateConfigHash.task == task.name).first() task.config_modified, config_hash = task.is_config_modified( last_hash) if task.config_modified: session.add( TemplateConfigHash(task=task.name, hash=config_hash)) log.trace('templates: %s' % config)
def get_login_cookie(self, username, password, force=False): """ Retrieves login cookie :param str username: :param str password: :param bool force: if True, then retrieve a fresh cookie instead of looking in the DB :return: """ if not force: with Session() as session: saved_cookie = (session.query(FileListCookie).filter( FileListCookie.username == username.lower()).first()) if (saved_cookie and saved_cookie.expires and saved_cookie.expires >= datetime.datetime.now()): logger.debug('Found valid login cookie') return saved_cookie.cookie url = BASE_URL + 'takelogin.php' try: # get validator token response = requests.get(BASE_URL + 'login.php') soup = get_soup(response.content) login_validator = soup.find("input", {"name": "validator"}) if not login_validator: raise plugin.PluginError( 'FileList.ro could not get login validator') logger.debug('Login Validator: {}'.format( login_validator.get('value'))) logger.debug('Attempting to retrieve FileList.ro cookie') response = requests.post( url, data={ 'username': username, 'password': password, 'validator': login_validator.get('value'), 'login': '******', 'unlock': '1', }, timeout=30, ) except RequestException as e: raise plugin.PluginError('FileList.ro login failed: %s' % e) if 'https://filelist.ro/my.php' != response.url: raise plugin.PluginError( 'FileList.ro login failed: Your username or password was incorrect.' ) with Session() as session: expires = None for c in requests.cookies: if c.name == 'pass': expires = c.expires if expires: expires = datetime.datetime.fromtimestamp(expires) logger.debug('Saving or updating FileList.ro cookie in db') cookie = FileListCookie(username=username.lower(), cookie=dict(requests.cookies), expires=expires) session.merge(cookie) return cookie.cookie