def cli_search(options):
search_term = ' '.join(options.keywords)
tags = options.tags
sources = options.sources
def print_ae(ae):
diff = datetime.now() - ae.added
console('ID: %-6s | Title: %s\nAdded: %s (%d days ago)\nURL: %s' %
(ae.id, ae.title, ae.added, diff.days, ae.url))
source_names = ', '.join([s.name for s in ae.sources])
tag_names = ', '.join([t.name for t in ae.tags])
console('Source(s): %s | Tag(s): %s' % (source_names or 'N/A', tag_names or 'N/A'))
if ae.description:
console('Description: %s' % strip_html(ae.description))
console('---')
session = Session()
try:
console('Searching: %s' % search_term)
if tags:
console('Tags: %s' % ', '.join(tags))
if sources:
console('Sources: %s' % ', '.join(sources))
console('Please wait...')
console('')
results = False
for ae in search(session, search_term, tags=tags, sources=sources):
print_ae(ae)
results = True
if not results:
console('No results found.')
finally:
session.close()
def estimate(self, entry):
if all(field in entry for field in ['series_name', 'series_season', 'series_episode']):
# Try to get airdate from tvrage first
if api_tvrage:
season = entry['series_season']
if entry.get('series_id_type') == 'sequence':
# Tvrage has absolute numbered shows under season 1
season = 1
log.debug("Querying release estimation for %s S%02dE%02d ..." %
(entry['series_name'], season, entry['series_episode']))
try:
series_info = lookup_series(name=entry['series_name'])
except LookupError as e:
log.debug('tvrage lookup error: %s' % e)
else:
if series_info:
try:
episode_info = series_info.find_episode(season, entry['series_episode'])
if episode_info:
return episode_info.airdate
else:
# If episode does not exist in tvrage database, we always return a future date
log.verbose('%s S%02dE%02d does not exist in tvrage database, assuming unreleased',
series_info.name, season, entry['series_episode'])
return datetime.now() + timedelta(weeks=4)
except Exception as e:
log.exception(e)
else:
log.debug('No series info obtained from TVRage to %s' % entry['series_name'])
log.debug('No episode info obtained from TVRage for %s season %s episode %s' %
(entry['series_name'], entry['series_season'], entry['series_episode']))
# If no results from tvrage, estimate a date based on series history
session = Session()
series = session.query(Series).filter(Series.name == entry['series_name']).first()
if not series:
return
episodes = (session.query(Episode).join(Episode.series).
filter(Episode.season != None).
filter(Series.id == series.id).
filter(Episode.season == func.max(Episode.season).select()).
order_by(desc(Episode.number)).limit(2).all())
if len(episodes) < 2:
return
# If last two eps were not contiguous, don't guess
if episodes[0].number != episodes[1].number + 1:
return
last_diff = episodes[0].first_seen - episodes[1].first_seen
# If last eps were grabbed close together, we might be catching up, don't guess
# Or, if last eps were too far apart, don't guess
# TODO: What range?
if last_diff < timedelta(days=2) or last_diff > timedelta(days=10):
return
# Estimate next season somewhat more than a normal episode break
if entry['series_season'] > episodes[0].season:
# TODO: How big should this be?
return episodes[0].first_seen + multiply_timedelta(last_diff, 2)
# Estimate next episode comes out about same length as last ep span, with a little leeway
return episodes[0].first_seen + multiply_timedelta(last_diff, 0.9)
def do_cli(manager, options):
session = Session()
try:
console('-- History: ' + '-' * 67)
query = session.query(History)
if options.search:
search_term = options.search.replace(' ', '%').replace('.', '%')
query = query.filter(History.title.like('%' + search_term + '%'))
if options.task:
query = query.filter(History.task.like('%' + options.task + '%'))
query = query.order_by(desc(History.time)).limit(options.limit)
for item in reversed(query.all()):
if options.short:
console(' %-25s %s' % (item.time.strftime("%c"), item.title))
else:
console(' Task : %s' % item.task)
console(' Title : %s' % item.title)
console(' Url : %s' % item.url)
if item.filename:
console(' Stored : %s' % item.filename)
console(' Time : %s' % item.time.strftime("%c"))
console(' Details : %s' % item.details)
console('-' * 79)
finally:
session.close()
def get_series_summary(self):
"""
:return: Dictionary where key is series name and value is dictionary of summary details.
"""
result = {}
session = Session()
try:
seriestasks = session.query(SeriesTask).all()
if seriestasks:
all_series = set(st.series for st in seriestasks)
else:
all_series = session.query(Series).all()
for series in all_series:
name = series.name
# capitalize if user hasn't, better look and sorting ...
if name.islower():
name = capwords(name)
result[name] = {'identified_by': series.identified_by}
result[name]['in_tasks'] = [task.name for task in series.in_tasks]
episode = self.get_latest_download(series)
if episode:
latest = {'first_seen': episode.first_seen,
'episode_instance': episode,
'episode_id': episode.identifier,
'age': episode.age,
'status': self.get_latest_status(episode),
'behind': self.new_eps_after(episode)}
result[name]['latest'] = latest
finally:
session.close()
return result
def get_file(self, only_cached=False):
"""Makes sure the poster is downloaded to the local cache (in userstatic folder) and
returns the path split into a list of directory and file components"""
from flexget.manager import manager
base_dir = os.path.join(manager.config_base, 'userstatic')
if self.file and os.path.isfile(os.path.join(base_dir, self.file)):
return self.file.split(os.sep)
elif only_cached:
return
# If we don't already have a local copy, download one.
log.debug('Downloading poster %s' % self.url)
dirname = os.path.join('tmdb', 'posters', str(self.movie_id))
# Create folders if they don't exist
fullpath = os.path.join(base_dir, dirname)
if not os.path.isdir(fullpath):
os.makedirs(fullpath)
filename = os.path.join(dirname, posixpath.basename(self.url))
thefile = file(os.path.join(base_dir, filename), 'wb')
thefile.write(requests.get(self.url).content)
self.file = filename
# If we are detached from a session, update the db
if not Session.object_session(self):
session = Session()
try:
poster = session.query(TMDBPoster).filter(TMDBPoster.db_id == self.db_id).first()
if poster:
poster.file = filename
finally:
session.close()
return filename.split(os.sep)
def on_process_start(self, task):
if not task.manager.options.seen_search:
return
task.manager.disable_tasks()
session = Session()
shown = []
for field in session.query(SeenField).\
filter(SeenField.value.like(unicode('%' + task.manager.options.seen_search + '%'))).\
order_by(asc(SeenField.added)).all():
se = session.query(SeenEntry).filter(SeenEntry.id == field.seen_entry_id).first()
if not se:
print 'ERROR: <SeenEntry(id=%s)> missing' % field.seen_entry_id
continue
# don't show duplicates
if se.id in shown:
continue
shown.append(se.id)
print 'ID: %s Name: %s Task: %s Added: %s' % (se.id, se.title, se.task, se.added.strftime('%c'))
for sf in se.fields:
print ' %s: %s' % (sf.field, sf.value)
print ''
if not shown:
print 'No results'
session.close()
def age_series(**kwargs):
from flexget.plugins.filter.series import Release
from flexget.manager import Session
import datetime
session = Session()
session.query(Release).update({'first_seen': datetime.datetime.now() - datetime.timedelta(**kwargs)})
session.commit()
def cli_search(options):
search_term = " ".join(options.keywords)
tags = options.tags
sources = options.sources
def print_ae(ae):
diff = datetime.now() - ae.added
console(
"ID: %-6s | Title: %s\nAdded: %s (%d days ago)\nURL: %s" % (ae.id, ae.title, ae.added, diff.days, ae.url)
)
source_names = ", ".join([s.name for s in ae.sources])
tag_names = ", ".join([t.name for t in ae.tags])
console("Source(s): %s | Tag(s): %s" % (source_names or "N/A", tag_names or "N/A"))
if ae.description:
console("Description: %s" % strip_html(ae.description))
console("---")
session = Session()
try:
console("Searching: %s" % search_term)
if tags:
console("Tags: %s" % ", ".join(tags))
if sources:
console("Sources: %s" % ", ".join(sources))
console("Please wait...")
console("")
results = False
for ae in search(session, search_term, tags=tags, sources=sources):
print_ae(ae)
results = True
if not results:
console("No results found.")
finally:
session.close()
def on_process_start(self, task):
# migrate seen to seen_entry
session = Session()
from flexget.utils.sqlalchemy_utils import table_exists
if table_exists('seen', session):
self.migrate2()
session.close()
def assert_series_count_in_db(expected_count):
from flexget.plugins.filter.series import Series
from flexget.manager import Session
session = Session()
actual_series_count = session.query(Series).count()
assert expected_count == actual_series_count, "expecting %s series stored in db, got %s instead" % \
(expected_count, actual_series_count)
def search(self, search_term, tags=None):
def print_ae(ae):
diff = datetime.now() - ae.added
console('ID: %-6s | Title: %s\nAdded: %s (%d days ago)\nURL: %s' %
(ae.id, ae.title, ae.added, diff.days, ae.url))
source_names = ', '.join([s.name for s in ae.sources])
tag_names = ', '.join([t.name for t in ae.tags])
console('Source(s): %s | Tag(s): %s' % (source_names or 'N/A', tag_names or 'N/A'))
if ae.description:
console('Description: %s' % strip_html(ae.description))
console('---')
session = Session()
try:
console('Searching: %s' % search_term)
if tags:
console('Tags: %s' % ', '.join(tags))
console('Please wait ...')
console('')
for ae in search(session, search_term, tags):
print_ae(ae)
finally:
session.close()
def on_process_start(self, feed):
if not feed.manager.options.repair_seen_movies:
return
feed.manager.disable_feeds()
from progressbar import ProgressBar, Percentage, Bar, ETA
from flexget.manager import Session
from seen import SeenField
from flexget.utils.imdb import extract_id
session = Session()
index = 0
count = 0
total = session.query(SeenField).filter(SeenField.field == u'imdb_url').count()
widgets = ['Repairing: ', ETA(), ' ', Percentage(), ' ', Bar(left='[', right=']')]
bar = ProgressBar(widgets=widgets, maxval=total).start()
for seen in session.query(SeenField).filter(SeenField.field == u'imdb_url').all():
index += 1
if index % 5 == 0:
bar.update(index)
value = u'http://www.imdb.com/title/%s/' % extract_id(seen.value)
if value != seen.value:
count += 1
seen.value = value
seen.field = unicode('imdb_url')
bar.finish()
session.commit()
print 'Fixed %s/%s URLs' % (count, total)
def test_seen_get(self, mock_seen_search):
session = Session()
entry_list = session.query(SeenEntry).join(SeenField).all()
mock_seen_search.return_value = entry_list
# No params
rsp = self.get('/seen/')
assert rsp.status_code == 200, 'Response code is %s' % rsp.status_code
# Default params
rsp = self.get('/seen/?page=1&max=100&local_seen=true&sort_by=added&order=desc')
assert rsp.status_code == 200, 'Response code is %s' % rsp.status_code
# Changed params
rsp = self.get('/seen/?max=1000&local_seen=false&sort_by=title&order=asc')
assert rsp.status_code == 200, 'Response code is %s' % rsp.status_code
# Negative test, invalid parameter
rsp = self.get('/seen/?max=1000&local_seen=BLA&sort_by=title &order=asc')
assert rsp.status_code == 400, 'Response code is %s' % rsp.status_code
# With value
rsp = self.get('/seen/?value=bla')
assert rsp.status_code == 200, 'Response code is %s' % rsp.status_code
assert mock_seen_search.call_count == 4, 'Should have 4 calls, is actually %s' % mock_seen_search.call_count
def get_series_summary(self):
"""
:return: Dictionary where key is series name and value is dictionary of summary details.
"""
result = {}
session = Session()
try:
for series in session.query(Series).all():
name = series.name
# capitalize if user hasn't, better look and sorting ...
if name.islower():
name = capwords(name)
result[name] = {"identified_by": series.identified_by}
episode = self.get_latest_download(series)
if episode:
latest = {
"first_seen": episode.first_seen,
"episode_instance": episode,
"episode_id": episode.identifier,
"age": episode.age,
"status": self.get_latest_status(episode),
"behind": self.new_eps_after(episode),
}
result[name]["latest"] = latest
finally:
session.close()
return result
def test_seen_delete_all(self, mock_seen_search, api_client):
session = Session()
entry_list = session.query(SeenEntry).join(SeenField)
mock_seen_search.return_value = entry_list
# No params
rsp = api_client.delete('/seen/')
assert rsp.status_code == 404, 'Response code is %s' % rsp.status_code
fields = {
'url': 'http://test.com/file.torrent',
'title': 'Test.Title',
'torrent_hash_id': 'dsfgsdfg34tq34tq34t'
}
entry = {
'local': False,
'reason': 'test_reason',
'task': 'test_task',
'title': 'Test.Title',
'fields': fields
}
rsp = api_client.json_post('/seen/', data=json.dumps(entry))
assert rsp.status_code == 200, 'Response code is %s' % rsp.status_code
# With value
rsp = api_client.delete('/seen/?value=Test.Title')
assert rsp.status_code == 200, 'Response code is %s' % rsp.status_code
assert mock_seen_search.call_count == 2, 'Should have 2 calls, is actually %s' % mock_seen_search.call_count
def clear_backlog(manager):
if not manager.options.clear_backlog:
return
manager.disable_tasks()
session = Session()
num = session.query(BacklogEntry).delete()
session.close()
console('%s entries cleared from backlog.' % num)
def test_withsession(self):
session = Session()
persist = SimplePersistence('testplugin', session=session)
persist['aoeu'] = 'test'
assert persist['aoeu'] == 'test'
# Make sure it didn't commit or close our session
session.rollback()
assert 'aoeu' not in persist
def assert_series_count_in_db(expected_count):
from flexget.components.series.db import Series
from flexget.manager import Session
session = Session()
actual_series_count = session.query(Series).count()
assert (
expected_count == actual_series_count
), "expecting %s series stored in db, got %s instead" % (expected_count, actual_series_count)
def get_login_cookies(self, username, password):
url_auth = 'http://www.t411.li/users/login'
db_session = Session()
account = db_session.query(torrent411Account).filter(
torrent411Account.username == username).first()
if account:
if account.expiry_time < datetime.now():
db_session.delete(account)
db_session.commit()
log.debug("Cookies found in db!")
return account.auth
else:
log.debug("Getting login cookies from : %s " % url_auth)
params = {'login': username, 'password': password, 'remember': '1'}
cj = http.cookiejar.CookieJar()
# WE NEED A COOKIE HOOK HERE TO AVOID REDIRECT COOKIES
opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj))
# NEED TO BE SAME USER_AGENT THAN DOWNLOAD LINK
opener.addheaders = [('User-agent', self.USER_AGENT)]
login_output = None
try:
login_output = opener.open(url_auth, urllib.parse.urlencode(params)).read()
except Exception as e:
raise UrlRewritingError("Connection Error for %s : %s" % (url_auth, e))
if b'confirmer le captcha' in login_output:
log.warning("Captcha requested for login.")
login_output = self._solveCaptcha(login_output, url_auth, params, opener)
if b'logout' in login_output:
authKey = None
uid = None
password = None
for cookie in cj:
if cookie.name == "authKey":
authKey = cookie.value
if cookie.name == "uid":
uid = cookie.value
if cookie.name == "pass":
password = cookie.value
if authKey is not None and \
uid is not None and \
password is not None:
authCookie = {'uid': uid,
'password': password,
'authKey': authKey
}
db_session.add(torrent411Account(username=username,
auth=authCookie,
expiry_time=datetime.now() + timedelta(days=1)))
db_session.commit()
return authCookie
else:
log.error("Login failed (Torrent411). Check your login and password.")
return {}
def _get_db_last_run(self):
session = Session()
try:
db_trigger = session.query(DBTrigger).get(self.uid)
if db_trigger:
self.last_run = db_trigger.last_run
log.debug("loaded last_run from the database")
finally:
session.close()
def display_summary(options):
"""
Display series summary.
:param options: argparse options from the CLI
"""
formatting = ' %-30s %-10s %-10s %-20s'
console(formatting % ('Name', 'Latest', 'Age', 'Downloaded'))
console('-' * 79)
session = Session()
try:
query = (session.query(Series).outerjoin(Series.episodes).outerjoin(Episode.releases).
outerjoin(Series.in_tasks).group_by(Series.id))
if options.configured == 'configured':
query = query.having(func.count(SeriesTask.id) >= 1)
elif options.configured == 'unconfigured':
query = query.having(func.count(SeriesTask.id) < 1)
if options.premieres:
query = (query.having(func.max(Episode.season) <= 1).having(func.max(Episode.number) <= 2).
having(func.count(SeriesTask.id) < 1)).filter(Release.downloaded == True)
if options.new:
query = query.having(func.max(Episode.first_seen) > datetime.now() - timedelta(days=options.new))
if options.stale:
query = query.having(func.max(Episode.first_seen) < datetime.now() - timedelta(days=options.stale))
for series in query.order_by(Series.name).yield_per(10):
series_name = series.name
if len(series_name) > 30:
series_name = series_name[:27] + '...'
new_ep = ' '
behind = 0
status = 'N/A'
age = 'N/A'
episode_id = 'N/A'
latest = get_latest_release(series)
if latest:
if latest.first_seen > datetime.now() - timedelta(days=2):
new_ep = '>'
behind = new_eps_after(latest)
status = get_latest_status(latest)
age = latest.age
episode_id = latest.identifier
if behind:
episode_id += ' +%s' % behind
console(new_ep + formatting[1:] % (series_name, episode_id, age, status))
if behind >= 3:
console(' ! Latest download is %d episodes behind, this may require '
'manual intervention' % behind)
console('-' * 79)
console(' > = new episode ')
console(' Use `flexget series show NAME` to get detailed information')
finally:
session.close()
def display_details(self, name):
"""Display detailed series information, ie. --series NAME"""
from flexget.manager import Session
session = Session()
name = unicode(name.lower())
series = session.query(Series).filter(Series.name == name).first()
if not series:
console("Unknown series `%s`" % name)
return
console(" %-63s%-15s" % ("Identifier, Title", "Quality"))
console("-" * 79)
# Query episodes in sane order instead of iterating from series.episodes
episodes = session.query(Episode).filter(Episode.series_id == series.id)
if series.identified_by == "sequence":
episodes = episodes.order_by(Episode.number).all()
else:
episodes = episodes.order_by(Episode.identifier).all()
for episode in episodes:
if episode.identifier is None:
console(" None <--- Broken!")
else:
console(" %s (%s) - %s" % (episode.identifier, episode.identified_by or "N/A", episode.age))
for release in episode.releases:
status = release.quality.name
title = release.title
if len(title) > 55:
title = title[:55] + "..."
if release.proper_count > 0:
status += "-proper"
if release.proper_count > 1:
status += str(release.proper_count)
if release.downloaded:
console(" * %-60s%-15s" % (title, status))
else:
console(" %-60s%-15s" % (title, status))
console("-" * 79)
console(" * = downloaded")
if not series.identified_by:
console("")
console(" Series plugin is still learning which episode numbering mode is ")
console(" correct for this series (identified_by: auto).")
console(" Few duplicate downloads can happen with different numbering schemes")
console(" during this time.")
else:
console(" Series uses `%s` mode to identify episode numbering (identified_by)." % series.identified_by)
console(" See option `identified_by` for more information.")
session.close()
def cli_perf_test(manager, options):
if options.test_name not in TESTS:
console('Unknown performance test %s' % options.test_name)
return
session = Session()
try:
if options.test_name == 'imdb_query':
imdb_query(session)
finally:
session.close()
def upgrade_required():
"""Returns true if an upgrade of the database is required."""
session = Session()
try:
for old_schema in session.query(PluginSchema).all():
if old_schema.plugin in plugin_schemas and old_schema.version < plugin_schemas[old_schema.plugin]['version']:
return True
return False
finally:
session.close()
def get_login_cookies(self, username, password):
url_auth = 'http://www.t411.me/users/login'
db_session = Session()
account = db_session.query(torrent411Account).filter(
torrent411Account.username == username).first()
if account:
if account.expiry_time < datetime.now():
db_session.delete(account)
db_session.commit()
log.debug("Cookies found in db!")
return account.auth
else:
log.debug("Getting login cookies from : %s " % url_auth)
params = urllib.urlencode({'login': username,
'password': password,
'remember': '1'})
cj = cookielib.CookieJar()
# WE NEED A COOKIE HOOK HERE TO AVOID REDIRECT COOKIES
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
# NEED TO BE SAME USER_AGENT THAN DOWNLOAD LINK
opener.addheaders = [('User-agent', self.USER_AGENT)]
try:
opener.open(url_auth, params)
except Exception as e:
raise UrlRewritingError("Connection Error for %s : %s" % (url_auth, e))
authKey = None
uid = None
password = None
for cookie in cj:
if cookie.name == "authKey":
authKey = cookie.value
if cookie.name == "uid":
uid = cookie.value
if cookie.name == "pass":
password = cookie.value
if authKey is not None and \
uid is not None and \
password is not None:
authCookie = {'uid': uid,
'password': password,
'authKey': authKey
}
db_session.add(torrent411Account(username=username,
auth=authCookie,
expiry_time=datetime.now() + timedelta(days=1)))
db_session.commit()
return authCookie
return {"uid": "",
"password": "",
"authKey": ""
}
def display_details(self, name):
"""Display detailed series information, ie. --series NAME"""
from flexget.manager import Session
session = Session()
name = unicode(name.lower())
series = session.query(Series).filter(Series.name == name).first()
if not series:
print 'Unknown series `%s`' % name
return
print ' %-63s%-15s' % ('Identifier, Title', 'Quality')
print '-' * 79
# Query episodes in sane order instead of iterating from series.episodes
episodes = session.query(Episode).filter(Episode.series_id == series.id)
if series.identified_by == 'sequence':
episodes = episodes.order_by(Episode.number).all()
else:
episodes = episodes.order_by(Episode.identifier).all()
for episode in episodes:
if episode.identifier is None:
print ' None <--- Broken!'
else:
print ' %s (%s) - %s' % (episode.identifier, episode.identified_by or 'N/A', episode.age)
for release in episode.releases:
status = release.quality.name
title = release.title
if len(title) > 55:
title = title[:55] + '...'
if release.proper_count > 0:
status += '-proper'
if release.proper_count > 1:
status += str(release.proper_count)
if release.downloaded:
print ' * %-60s%-15s' % (title, status)
else:
print ' %-60s%-15s' % (title, status)
print '-' * 79
print ' * = downloaded'
if not series.identified_by:
print ''
print ' Series plugin is still learning which episode numbering mode is '
print ' correct for this series (identified_by: auto).'
print ' Few duplicate downloads can happen with different numbering schemes'
print ' during this time.'
else:
print ' Series uses `%s` mode to identify episode numbering (identified_by).' % series.identified_by
print ' See option `identified_by` for more information.'
session.close()
def list_failed():
session = Session()
try:
results = session.query(FailedEntry).all()
if not results:
console('No failed entries recorded')
for entry in results:
console('%16s - %s - %s times - %s' %
(entry.tof.strftime('%Y-%m-%d %H:%M'), entry.title, entry.count, entry.reason))
finally:
session.close()
def get_version(plugin):
session = Session()
try:
schema = session.query(PluginSchema).filter(PluginSchema.plugin == plugin).first()
if not schema:
log.debug('No schema version stored for %s' % plugin)
return None
else:
return schema.version
finally:
session.close()
def on_process_start(self, feed):
# migrate shelve -> sqlalchemy
if feed.manager.shelve_session:
self.migrate(feed)
# migrate seen to seen_entry
session = Session()
from flexget.utils.sqlalchemy_utils import table_exists
if table_exists('seen', session):
self.migrate2()
session.close()
def list_rejected():
session = Session()
try:
results = session.query(RememberEntry).all()
if not results:
console('No rejected entries recorded by remember_rejected')
else:
console('Rejections remembered by remember_rejected:')
for entry in results:
console('%s from %s by %s because %s' % (entry.title, entry.task.name, entry.rejected_by, entry.reason))
finally:
session.close()
def on_task_input(self, task, config):
if not config:
return
if isinstance(config, bool):
config = {}
if task.is_rerun:
# Just return calculated next eps on reruns
entries = self.rerun_entries
self.rerun_entries = []
return entries
else:
self.rerun_entries = []
entries = []
with Session() as session:
for seriestask in session.query(SeriesTask).filter(
SeriesTask.name == task.name).all():
series = seriestask.series
if not series:
# TODO: How can this happen?
log.debug(
'Found SeriesTask item without series specified. Cleaning up.'
)
session.delete(seriestask)
continue
if series.identified_by not in ['ep', 'sequence']:
log.verbose('Can only emit ep or sequence based series. '
'`%s` is identified_by %s' %
(series.name, series.identified_by or 'auto'))
continue
low_season = 0 if series.identified_by == 'ep' else -1
check_downloaded = not config.get('backfill')
latest_season = get_latest_release(series,
downloaded=check_downloaded)
if latest_season:
latest_season = latest_season.season
else:
latest_season = low_season + 1
for season in range(latest_season, low_season, -1):
log.debug('Adding episodes for season %d' % season)
latest = get_latest_release(series,
season=season,
downloaded=check_downloaded)
if series.begin and (not latest or latest < series.begin):
entries.append(
self.search_entry(series, series.begin.season,
series.begin.number, task))
elif latest and not config.get('backfill'):
entries.append(
self.search_entry(series, latest.season,
latest.number + 1, task))
elif latest:
start_at_ep = 1
episodes_this_season = (session.query(Episode).filter(
Episode.series_id == series.id).filter(
Episode.season == season))
if series.identified_by == 'sequence':
# Don't look for missing too far back with sequence shows
start_at_ep = max(latest.number - 10, 1)
episodes_this_season = episodes_this_season.filter(
Episode.number >= start_at_ep)
latest_ep_this_season = episodes_this_season.order_by(
desc(Episode.number)).first()
downloaded_this_season = (episodes_this_season.join(
Episode.releases).filter(
Release.downloaded == True).all())
# Calculate the episodes we still need to get from this season
if series.begin and series.begin.season == season:
start_at_ep = max(start_at_ep, series.begin.number)
eps_to_get = list(
range(start_at_ep,
latest_ep_this_season.number + 1))
for ep in downloaded_this_season:
try:
eps_to_get.remove(ep.number)
except ValueError:
pass
entries.extend(
self.search_entry(
series, season, x, task, rerun=False)
for x in eps_to_get)
# If we have already downloaded the latest known episode, try the next episode
if latest_ep_this_season.releases:
entries.append(
self.search_entry(
series, season,
latest_ep_this_season.number + 1, task))
else:
if config.get('from_start') or config.get('backfill'):
entries.append(
self.search_entry(series, season, 1, task))
else:
log.verbose(
'Series `%s` has no history. Set begin option, '
'or use CLI `series begin` '
'subcommand to set first episode to emit' %
series.name)
break
# Skip older seasons if we are not in backfill mode
if not config.get('backfill'):
break
# Don't look for seasons older than begin ep
if series.begin and series.begin.season >= season:
break
return entries
def learn_backlog(self, task, amount=''):
"""Learn current entries into backlog. All task inputs must have been executed."""
with Session() as session:
for entry in task.entries:
self.add_backlog(task, entry, amount, session=session)
def build(self, task):
if not (plt and pd):
logger.warning('Dependency does not exist: [matplotlib, pandas]')
return
if not task.accepted and not task.failed:
return
session = Session()
columns = [
'site',
'uploaded',
'downloaded',
'share_ratio',
'points',
'seeding',
'leeching',
'hr',
]
data = {'sort_column': [], 'default_order': []}
total_details = {}
total_changed = {}
user_classes_dict = {}
for column in columns:
data[column] = []
total_details[column] = 0
total_changed[column] = 0
order = len(task.all_entries)
for entry in task.all_entries:
user_classes_dict[entry['site_name']] = entry.get('user_classes')
if entry.get('do_not_draw'):
continue
data['default_order'].append(order)
order = order - 1
user_details_db = self._get_user_details(session,
entry['site_name'])
if user_details_db is None:
user_details_db = UserDetailsEntry(
site=entry['site_name'],
uploaded=0,
downloaded=0,
share_ratio=0,
join_date=datetime.now().date(),
points=0,
seeding=0,
leeching=0,
hr=0)
session.add(user_details_db)
session.commit()
user_details_db = self._get_user_details(
session, entry['site_name'])
# failed
if not entry.get('details'):
for column in columns:
value = getattr(user_details_db, column)
if entry.failed:
data[column].append(
self.buid_data_text(column, value) + '*')
else:
data[column].append(self.buid_data_text(column, value))
if not entry.get('do_not_count') and column not in [
'site'
]:
self.count(total_details, column, value)
data['sort_column'].append(0)
continue
# now
details_now = {}
for key, value in entry['details'].items():
details_now[key] = self.transfer_data(key, value)
# changed
details_changed = {}
for key, value_now in details_now.items():
if value_now != '*' and key not in ['join_date']:
details_changed[key] = value_now - getattr(
user_details_db, key)
else:
details_changed[key] = '*'
if details_changed['uploaded'] == '*':
data['sort_column'].append(0)
else:
data['sort_column'].append(details_changed['uploaded'])
# append to data
data['site'].append(entry['site_name'])
for column in columns:
if column == 'site':
continue
data[column].append('{}{}'.format(
self.buid_data_text(column,
getattr(user_details_db, column)),
self.buid_data_text(column,
details_changed[column],
append=True)))
if total_details.get(column) is None:
total_details[column] = 0
if total_changed.get(column) is None:
total_changed[column] = 0
if not entry.get('do_not_count') and column not in [
'share_ratio', 'points'
]:
total_details[column] = total_details[column] + getattr(
user_details_db, column)
if details_changed[column] != '*':
total_changed[column] = total_changed[
column] + details_changed[column]
# update db
for key, value in details_now.items():
if key == 'join_date':
value = parse(value)
if value != '*':
setattr(user_details_db, key, value)
session.commit()
data['site'].append('total')
for column in columns:
if column == 'site':
continue
data[column].append('{}{}'.format(
self.buid_data_text(column, total_details[column]),
self.buid_data_text(column, total_changed[column],
append=True)))
data['sort_column'].append(float('inf'))
data['default_order'].append(float('inf'))
df = pd.DataFrame(data)
df.sort_values(by=['sort_column', 'default_order'],
ascending=False,
inplace=True)
df.drop(columns=['sort_column', 'default_order'], inplace=True)
line_count = len(data['site'])
fig = plt.figure(figsize=(8, line_count / 1.8))
plt.axis('off')
colors = []
for x in df.values:
cc = []
for y in x:
if '\n-' in y:
cc.append('#f38181')
elif '+' in y:
cc.append('#95e1d3')
elif '*' in y:
cc.append('#eff48e')
else:
cc.append('white')
colors.append(cc)
col_widths = [0.15, 0.16, 0.16, 0.13, 0.14, 0.1, 0.1, 0.06]
table = plt.table(cellText=df.values,
cellColours=colors,
bbox=[0, 0, 1, 1],
colLabels=df.columns,
colWidths=col_widths,
loc='best')
table.auto_set_font_size(False)
table.set_fontsize(10)
fig.tight_layout()
plt.title(datetime.now().replace(microsecond=0))
plt.savefig('details_report.png', bbox_inches='tight', dpi=300)
self.draw_user_classes(user_classes_dict, session, df)
def authenticate(self):
"""Authenticates a session with IMDB, and grabs any IDs needed for getting/modifying list."""
cached_credentials = False
with Session() as session:
if self.cookies:
if not self.user_id:
self.user_id = self.get_user_id_and_hidden_value(
self.cookies)
if not self.user_id:
raise PluginError(
'Not possible to retrive userid, please check cookie information'
)
user = IMDBListUser(self.config['login'], self.user_id,
self.cookies)
self.cookies = user.cookies
cached_credentials = True
else:
user = (session.query(IMDBListUser).filter(
IMDBListUser.user_name == self.config.get(
'login')).one_or_none())
if user and user.cookies and user.user_id:
logger.debug('login credentials found in cache, testing')
self.user_id = user.user_id
if not self.get_user_id_and_hidden_value(
cookies=user.cookies):
logger.debug('cache credentials expired')
user.cookies = None
self._session.cookies.clear()
else:
self.cookies = user.cookies
cached_credentials = True
if not cached_credentials:
logger.debug(
'user credentials not found in cache or outdated, fetching from IMDB'
)
url_credentials = (
'https://www.imdb.com/ap/signin?openid.return_to=https%3A%2F%2Fwww.imdb.com%2Fap-signin-'
'handler&openid.identity=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&'
'openid.assoc_handle=imdb_mobile_us&openid.mode=checkid_setup&openid.claimed_id=http%3A%'
'2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&openid.ns=http%3A%2F%2Fspecs.ope'
'nid.net%2Fauth%2F2.0')
try:
# we need to get some cookies first
self._session.get('https://www.imdb.com')
r = self._session.get(url_credentials)
except RequestException as e:
raise PluginError(e.args[0])
soup = get_soup(r.content)
form = soup.find('form', attrs={'name': 'signIn'})
inputs = form.select('input')
data = dict((i['name'], i.get('value')) for i in inputs
if i.get('name'))
data['email'] = self.config['login']
data['password'] = self.config['password']
action = form.get('action')
logger.debug('email={}, password={}', data['email'],
data['password'])
self._session.headers.update({'Referer': url_credentials})
self._session.post(action, data=data)
self._session.headers.update(
{'Referer': 'https://www.imdb.com/'})
self.user_id = self.get_user_id_and_hidden_value()
if not self.user_id:
raise plugin.PluginError(
'Login to IMDB failed. Check your credentials.')
self.cookies = self._session.cookies.get_dict(
domain='.imdb.com')
# Get list ID
if user:
for list in user.lists:
if self.config['list'] == list.list_name:
logger.debug(
'found list ID {} matching list name {} in cache',
list.list_id,
list.list_name,
)
self.list_id = list.list_id
if not self.list_id:
logger.debug(
'could not find list ID in cache, fetching from IMDB')
if self.config['list'] == 'watchlist':
data = {
'consts[]': 'tt0133093',
'tracking_tag': 'watchlistRibbon'
}
wl_data = self._session.post(
'https://www.imdb.com/list/_ajax/watchlist_has',
data=data,
cookies=self.cookies,
).json()
try:
self.list_id = wl_data['list_id']
except KeyError:
raise PluginError(
'No list ID could be received. Please initialize list by '
'manually adding an item to it and try again')
elif self.config['list'] in IMMUTABLE_LISTS or self.config[
'list'].startswith('ls'):
self.list_id = self.config['list']
else:
data = {'tconst': 'tt0133093'}
list_data = self._session.post(
'https://www.imdb.com/list/_ajax/wlb_dropdown',
data=data,
cookies=self.cookies,
).json()
for li in list_data['items']:
if li['wlb_text'] == self.config['list']:
self.list_id = li['data_list_id']
break
else:
raise plugin.PluginError('Could not find list %s' %
self.config['list'])
user = IMDBListUser(self.config['login'], self.user_id,
self.cookies)
list = IMDBListList(self.list_id, self.config['list'],
self.user_id)
user.lists.append(list)
session.merge(user)
self._authenticated = True
def add_failed(self, entry, reason=None):
"""Adds entry to internal failed list, displayed with --failed"""
reason = unicode(reason or 'Unknown')
failed = Session()
try:
# query item's existence
item = failed.query(FailedEntry).filter(FailedEntry.title == entry['title']).\
filter(FailedEntry.url == entry['original_url']).first()
if not item:
item = FailedEntry(entry['title'], entry['original_url'],
reason)
else:
item.count += 1
item.tof = datetime.now()
item.reason = reason
failed.merge(item)
log.debug('Marking %s in failed list. Has failed %s times.' %
(item.title, item.count))
# limit item number to 25
for row in failed.query(FailedEntry).order_by(
FailedEntry.tof.desc())[25:]:
failed.delete(row)
failed.commit()
finally:
failed.close()
def __len__(self):
with Session() as session:
return self._db_list(session).regexps.count()
def discard(self, entry):
with Session() as session:
db_regexp = self._find_entry(entry, session=session)
if db_regexp:
log.debug('deleting file %s', db_regexp)
session.delete(db_regexp)
def _execute(self):
"""Executes the task without rerunning."""
if not self.enabled:
log.debug('Not running disabled task %s' % self.name)
return
log.debug('executing %s' % self.name)
# Handle keyword args
if self.options.learn:
log.info('Disabling download and output phases because of --learn')
self.disable_phase('download')
self.disable_phase('output')
if self.options.disable_phases:
map(self.disable_phase, self.options.disable_phases)
if self.options.inject:
# If entries are passed for this execution (eg. rerun), disable the input phase
self.disable_phase('input')
self.all_entries.extend(self.options.inject)
# Save current config hash and set config_modidied flag
with Session() as session:
config_hash = hashlib.md5(str(sorted(self.config.items()))).hexdigest()
last_hash = session.query(TaskConfigHash).filter(TaskConfigHash.task == self.name).first()
if self.is_rerun:
# Restore the config to state right after start phase
if self.prepared_config:
self.config = copy.deepcopy(self.prepared_config)
else:
log.error('BUG: No prepared_config on rerun, please report.')
self.config_modified = False
elif not last_hash:
self.config_modified = True
last_hash = TaskConfigHash(task=self.name, hash=config_hash)
session.add(last_hash)
elif last_hash.hash != config_hash:
self.config_modified = True
last_hash.hash = config_hash
else:
self.config_modified = False
# run phases
try:
for phase in task_phases:
if phase in self.disabled_phases:
# log keywords not executed
for plugin in self.plugins(phase):
if plugin.name in self.config:
log.info('Plugin %s is not executed because %s phase is disabled (e.g. --test)' %
(plugin.name, phase))
continue
if phase == 'start' and self.is_rerun:
log.debug('skipping task_start during rerun')
elif phase == 'exit' and self._rerun:
log.debug('not running task_exit yet because task will rerun')
else:
# run all plugins with this phase
self.__run_task_phase(phase)
if phase == 'start':
# Store a copy of the config state after start phase to restore for reruns
self.prepared_config = copy.deepcopy(self.config)
except TaskAbort:
try:
self.__run_task_phase('abort')
except TaskAbort as e:
log.exception('abort handlers aborted: %s' % e)
raise
else:
for entry in self.all_entries:
entry.complete()
def __iter__(self):
with Session() as session:
return iter([
movie.to_entry(self.strip_year)
for movie in self._db_list(session).movies
])
def display_details(options):
"""Display detailed series information, ie. series show NAME"""
name = options.series_name
with Session() as session:
name = normalize_series_name(name)
# Sort by length of name, so that partial matches always show shortest matching title
matches = shows_by_name(name, session=session)
if not matches:
console(colorize(ERROR_COLOR, 'ERROR: Unknown series `%s`' % name))
return
# Pick the best matching series
series = matches[0]
table_title = colorize('white', series.name)
if len(matches) > 1:
warning = (
colorize('red', ' WARNING: ') +
'Multiple series match to `{}`.\n '
'Be more specific to see the results of other matches:\n\n'
' {}'.format(name, ', '.join(s.name for s in matches[1:])))
if not options.table_type == 'porcelain':
console(warning)
header = [
'Episode ID', 'Latest age', 'Release titles', 'Release Quality',
'Proper'
]
table_data = [header]
episodes = show_episodes(series, session=session)
for episode in episodes:
if episode.identifier is None:
identifier = colorize(ERROR_COLOR, 'MISSING')
age = ''
else:
identifier = episode.identifier
age = episode.age
ep_data = [identifier, age]
release_titles = []
release_qualities = []
release_propers = []
for release in episode.releases:
title = release.title
quality = release.quality.name
if not release.downloaded:
title = colorize(UNDOWNLOADED_RELEASE_COLOR, title)
quality = quality
else:
title = colorize(DOWNLOADED_RELEASE_COLOR, title)
quality = quality
release_titles.append(title)
release_qualities.append(quality)
release_propers.append(
'Yes' if release.proper_count > 0 else '')
ep_data.append('\n'.join(release_titles))
ep_data.append('\n'.join(release_qualities))
ep_data.append('\n'.join(release_propers))
table_data.append(ep_data)
footer = (' %s %s\n' %
(colorize(DOWNLOADED_RELEASE_COLOR, 'Downloaded'),
colorize(UNDOWNLOADED_RELEASE_COLOR, 'Un-downloaded')))
if not series.identified_by:
footer += (
'\n'
' Series plugin is still learning which episode numbering mode is \n'
' correct for this series (identified_by: auto).\n'
' Few duplicate downloads can happen with different numbering schemes\n'
' during this time.')
else:
footer += ' \n Series uses `%s` mode to identify episode numbering (identified_by).' % series.identified_by
footer += ' \n See option `identified_by` for more information.\n'
if series.begin:
footer += ' Begin episode for this series set to `%s`.' % series.begin.identifier
table = TerminalTable(options.table_type,
table_data,
table_title,
drop_columns=[4, 3, 1])
try:
console(table.output)
except TerminalTableError as e:
console('ERROR: %s' % str(e))
return
if not options.table_type == 'porcelain':
console(footer)
def discard(self, entry):
with Session() as session:
db_movie = self._find_entry(entry, session=session)
if db_movie:
log.debug('deleting movie %s', db_movie)
session.delete(db_movie)
def display_summary(options):
"""
Display series summary.
:param options: argparse options from the CLI
"""
porcelain = options.table_type == 'porcelain'
with Session() as session:
kwargs = {
'configured': options.configured,
'premieres': options.premieres,
'session': session,
'sort_by': options.sort_by,
'descending': options.order
}
if options.new:
kwargs['status'] = 'new'
kwargs['days'] = options.new
elif options.stale:
kwargs['status'] = 'stale'
kwargs['days'] = options.stale
if options.sort_by == 'name':
kwargs['sort_by'] = 'show_name'
else:
kwargs['sort_by'] = 'last_download_date'
query = get_series_summary(**kwargs)
header = ['Name', 'Latest', 'Age', 'Downloaded', 'Identified By']
for index, value in enumerate(header):
if value.lower() == options.sort_by:
header[index] = colorize(SORT_COLUMN_COLOR, value)
footer = 'Use `flexget series show NAME` to get detailed information'
table_data = [header]
for series in query:
name_column = series.name
new_ep = False
behind = 0
latest_release = '-'
age_col = '-'
episode_id = '-'
latest = get_latest_release(series)
identifier_type = series.identified_by
if identifier_type == 'auto':
identifier_type = colorize('yellow', 'auto')
if latest:
behind = new_eps_after(latest)
latest_release = get_latest_status(latest)
# colorize age
age_col = latest.age
if latest.age_timedelta is not None:
if latest.age_timedelta < timedelta(days=1):
age_col = colorize(NEW_EP_COLOR, latest.age)
elif latest.age_timedelta < timedelta(days=3):
age_col = colorize(FRESH_EP_COLOR, latest.age)
elif latest.age_timedelta > timedelta(days=400):
age_col = colorize(OLD_EP_COLOR, latest.age)
episode_id = latest.identifier
if not porcelain:
if behind > 0:
name_column += colorize(BEHIND_EP_COLOR,
' {} behind'.format(behind))
table_data.append([
name_column, episode_id, age_col, latest_release,
identifier_type
])
table = TerminalTable(options.table_type,
table_data,
wrap_columns=[3],
drop_columns=[4, 3, 2])
try:
console(table.output)
except TerminalTableError as e:
console('ERROR: %s' % str(e))
return
if not porcelain:
console(footer)
def on_process_end(self, task):
"""Write RSS file at application terminate."""
if not rss2gen:
return
# don't generate rss when learning
if task.manager.options.learn:
return
config = self.get_config(task)
if config['file'] in self.written:
log.trace('skipping already written file %s' % config['file'])
return
# in terminate phase there is no open session in task, so open new one
from flexget.manager import Session
session = Session()
db_items = session.query(RSSEntry).filter(RSSEntry.file == config['file']).\
order_by(RSSEntry.published.desc()).all()
# make items
rss_items = []
for db_item in db_items:
add = True
if config['items'] != -1:
if len(rss_items) > config['items']:
add = False
if config['days'] != -1:
if datetime.datetime.today() - datetime.timedelta(
days=config['days']) > db_item.published:
add = False
if add:
# add into generated feed
gen = {
'title': db_item.title,
'description': db_item.description,
'link': db_item.link,
'pubDate': db_item.published
}
log.trace('Adding %s into rss %s' %
(gen['title'], config['file']))
rss_items.append(PyRSS2Gen.RSSItem(**gen))
else:
# no longer needed
session.delete(db_item)
session.commit()
session.close()
# make rss
rss = PyRSS2Gen.RSS2(title='FlexGet',
link=config.get('rsslink', 'http://flexget.com'),
description='FlexGet generated RSS feed',
lastBuildDate=datetime.datetime.utcnow(),
items=rss_items)
# write rss
fn = os.path.expanduser(config['file'])
with open(fn, 'w') as file:
try:
log.verbose('Writing output rss to %s' % fn)
rss.write_xml(file, encoding=config['encoding'])
except LookupError:
log.critical('Unknown encoding %s' % config['encoding'])
return
except IOError:
# TODO: plugins cannot raise PluginWarnings in terminate event ..
log.critical('Unable to write %s' % fn)
return
self.written[config['file']] = True
def search(*args, **kwargs):
if 'count' in kwargs:
return 0
else:
with Session() as session:
return session.query(SeenEntry).join(SeenField)
def authenticate(self):
"""Authenticates a session with IMDB, and grabs any IDs needed for getting/modifying list."""
cached_credentials = False
with Session() as session:
user = session.query(IMDBListUser).filter(
IMDBListUser.user_name == self.config.get(
'login')).one_or_none()
if user and user.cookies and user.user_id:
log.debug('login credentials found in cache, testing')
self.cookies = user.cookies
self.user_id = user.user_id
r = self._session.head('http://www.imdb.com/profile',
allow_redirects=False,
cookies=self.cookies)
if not r.headers.get(
'location') or 'login' in r.headers['location']:
log.debug('cache credentials expired')
else:
cached_credentials = True
if not cached_credentials:
log.debug(
'user credentials not found in cache or outdated, fetching from IMDB'
)
try:
r = self._session.get(
'https://www.imdb.com/ap/signin?openid.return_to=https%3A%2F%2Fwww.imdb.com%2Fap-signin-'
'handler&openid.identity=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&'
'openid.assoc_handle=imdb_mobile_us&openid.mode=checkid_setup&openid.claimed_id=http%3A%'
'2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&openid.ns=http%3A%2F%2Fspecs.ope'
'nid.net%2Fauth%2F2.0')
except RequestException as e:
raise PluginError(e.args[0])
soup = get_soup(r.content)
inputs = soup.select('form#ap_signin_form input')
data = dict((i['name'], i.get('value')) for i in inputs
if i.get('name'))
data['email'] = self.config['login']
data['password'] = self.config['password']
log.debug('email=%s, password=%s', data['email'],
data['password'])
d = self._session.post('https://www.imdb.com/ap/signin',
data=data)
# Get user id by extracting from redirect url
r = self._session.head('http://www.imdb.com/profile',
allow_redirects=False)
if not r.headers.get(
'location') or 'login' in r.headers['location']:
raise plugin.PluginError(
'Login to IMDB failed. Check your credentials.')
self.user_id = re.search('ur\d+(?!\d)',
r.headers['location']).group()
self.cookies = dict(d.cookies)
# Get list ID
if user:
for list in user.lists:
if self.config['list'] == list.list_name:
log.debug(
'found list ID %s matching list name %s in cache',
list.list_id, list.list_name)
self.list_id = list.list_id
if not self.list_id:
log.debug(
'could not find list ID in cache, fetching from IMDB')
if self.config['list'] == 'watchlist':
data = {
'consts[]': 'tt0133093',
'tracking_tag': 'watchlistRibbon'
}
wl_data = self._session.post(
'http://www.imdb.com/list/_ajax/watchlist_has',
data=data,
cookies=self.cookies).json()
try:
self.list_id = wl_data['list_id']
except KeyError:
raise PluginError(
'No list ID could be received. Please initialize list by '
'manually adding an item to it and try again')
elif self.config['list'] in IMMUTABLE_LISTS or self.config[
'list'].startswith('ls'):
self.list_id = self.config['list']
else:
data = {'tconst': 'tt0133093'}
list_data = self._session.post(
'http://www.imdb.com/list/_ajax/wlb_dropdown',
data=data,
cookies=self.cookies).json()
for li in list_data['items']:
if li['wlb_text'] == self.config['list']:
self.list_id = li['data_list_id']
break
else:
raise plugin.PluginError('Could not find list %s' %
self.config['list'])
user = IMDBListUser(self.config['login'], self.user_id,
self.cookies)
list = IMDBListList(self.list_id, self.config['list'],
self.user_id)
user.lists.append(list)
session.merge(user)
self._authenticated = True
def load(cls, task=None):
"""Load all key/values from `task` into memory from database."""
with Session() as session:
for skv in session.query(SimpleKeyValue).filter(SimpleKeyValue.task == task).all():
cls.class_store[task][skv.plugin][skv.key] = skv.value
def display_details(self, name):
"""Display detailed series information, ie. --series NAME"""
from flexget.manager import Session
session = Session()
name = unicode(name.lower())
series = session.query(Series).filter(Series.name == name).first()
if not series:
console('Unknown series `%s`' % name)
return
console(' %-63s%-15s' % ('Identifier, Title', 'Quality'))
console('-' * 79)
# Query episodes in sane order instead of iterating from series.episodes
episodes = session.query(Episode).filter(
Episode.series_id == series.id)
if series.identified_by == 'sequence':
episodes = episodes.order_by(Episode.number).all()
else:
episodes = episodes.order_by(Episode.identifier).all()
for episode in episodes:
if episode.identifier is None:
console(' None <--- Broken!')
else:
console(' %s (%s) - %s' %
(episode.identifier, episode.identified_by
or 'N/A', episode.age))
for release in episode.releases:
status = release.quality.name
title = release.title
if len(title) > 55:
title = title[:55] + '...'
if release.proper_count > 0:
status += '-proper'
if release.proper_count > 1:
status += str(release.proper_count)
if release.downloaded:
console(' * %-60s%-15s' % (title, status))
else:
console(' %-60s%-15s' % (title, status))
console('-' * 79)
console(' * = downloaded')
if not series.identified_by:
console('')
console(
' Series plugin is still learning which episode numbering mode is '
)
console(' correct for this series (identified_by: auto).')
console(
' Few duplicate downloads can happen with different numbering schemes'
)
console(' during this time.')
else:
console(
' Series uses `%s` mode to identify episode numbering (identified_by).'
% series.identified_by)
console(' See option `identified_by` for more information.')
session.close()
def age_timeframe(**kwargs):
with Session() as session:
session.query(EntryTimeFrame).update({
'first_seen':
datetime.datetime.now() - datetime.timedelta(**kwargs)
})
def __len__(self):
with Session() as session:
return len(self._db_list(session).movies)
def __run_task_phase(self, phase):
"""Executes task phase, ie. call all enabled plugins on the task.
Fires events:
* task.execute.before_plugin
* task.execute.after_plugin
:param string phase: Name of the phase
"""
if phase not in phase_methods:
raise Exception('%s is not a valid task phase' % phase)
# warn if no inputs, filters or outputs in the task
if phase in ['input', 'filter', 'output']:
if not self.manager.unit_test:
# Check that there is at least one manually configured plugin for these phases
for p in self.plugins(phase):
if not p.builtin:
break
else:
if phase not in self.suppress_warnings:
if phase == 'filter':
logger.warning(
'Task does not have any filter plugins to accept entries. '
'You need at least one to accept the entries you want.'
)
else:
logger.warning(
'Task doesn\'t have any {} plugins, you should add (at least) one!',
phase,
)
for plugin in self.plugins(phase):
# Abort this phase if one of the plugins disables it
if phase in self.disabled_phases:
return
if plugin.name in self.disabled_plugins:
continue
# store execute info, except during entry events
self.current_phase = phase
self.current_plugin = plugin.name
if plugin.api_ver == 1:
# backwards compatibility
# pass method only task (old behaviour)
args = (self, )
else:
# pass method task, copy of config (so plugin cannot modify it)
args = (self, copy.copy(self.config.get(plugin.name)))
# Hack to make task.session only active for a single plugin
with Session() as session:
self.session = session
try:
fire_event('task.execute.before_plugin', self, plugin.name)
response = self.__run_plugin(plugin, phase, args)
if phase == 'input' and response:
# add entries returned by input to self.all_entries
for e in response:
e.task = self
self.all_entries.append(e)
finally:
fire_event('task.execute.after_plugin', self, plugin.name)
self.session = None
# check config hash for changes at the end of 'prepare' phase
if phase == 'prepare':
self.check_config_hash()
def forget(value):
"""
See module docstring
:param string value: Can be task name, entry title or field value
:return: count, field_count where count is number of entries removed and field_count number of fields
"""
log.debug('forget called with %s' % value)
session = Session()
try:
count = 0
field_count = 0
for se in session.query(SeenEntry).filter(
or_(SeenEntry.title == value, SeenEntry.task == value)).all():
field_count += len(se.fields)
count += 1
log.debug('forgetting %s' % se)
session.delete(se)
for sf in session.query(SeenField).filter(
SeenField.value == value).all():
se = session.query(SeenEntry).filter(
SeenEntry.id == sf.seen_entry_id).first()
field_count += len(se.fields)
count += 1
log.debug('forgetting %s' % se)
session.delete(se)
return count, field_count
finally:
session.commit()
session.close()
def test_history_pagination(self, api_client, schema_match, link_headers):
history_entry = dict(
task='test_task_',
title='test_title_',
url='test_url_',
filename='test_filename_',
details='test_details_',
)
num_of_entries = 200
with Session() as session:
for i in range(num_of_entries):
item = History()
for key, value in history_entry.items():
setattr(item, key, value + str(i))
session.add(item)
rsp = api_client.get('/history/')
assert rsp.status_code == 200
data = json.loads(rsp.get_data(as_text=True))
errors = schema_match(OC.history_list_object, data)
assert not errors
assert len(data) == 50 # Default page size
assert int(rsp.headers['total-count']) == 200
assert int(rsp.headers['count']) == 50
links = link_headers(rsp)
assert links['last']['page'] == 4
assert links['next']['page'] == 2
rsp = api_client.get('/history/?per_page=100')
assert rsp.status_code == 200
data = json.loads(rsp.get_data(as_text=True))
errors = schema_match(OC.history_list_object, data)
assert not errors
assert len(data) == 100
assert int(rsp.headers['count']) == 100
links = link_headers(rsp)
assert links['last']['page'] == 2
assert links['next']['page'] == 2
# Per page is limited to 100
rsp = api_client.get('/history/?per_page=200')
assert rsp.status_code == 200
data = json.loads(rsp.get_data(as_text=True))
errors = schema_match(OC.history_list_object, data)
assert not errors
assert len(data) == 100
rsp = api_client.get('/history/?page=2&sort_by=id&order=asc')
assert rsp.status_code == 200
data = json.loads(rsp.get_data(as_text=True))
errors = schema_match(OC.history_list_object, data)
assert not errors
assert data[0]['task'] == 'test_task_50'
links = link_headers(rsp)
assert links['last']['page'] == 4
assert links['next']['page'] == 3
assert links['prev']['page'] == 1
# Non existent page
rsp = api_client.get('/history/?page=5')
assert rsp.status_code == 404
data = json.loads(rsp.get_data(as_text=True))
errors = schema_match(base_message, data)
assert not errors
def __iter__(self):
with Session() as session:
return iter([regexp.to_entry() for regexp in self._db_list(session).regexps])
def test_history_sorting(self, api_client, schema_match, link_headers):
history_entry1 = dict(
task='test_task_1',
title='test_title_a',
url='test_url_1',
filename='test_filename_a',
details='test_details_1',
)
history_entry2 = dict(
task='test_task_2',
title='test_title_b',
url='test_url_2',
filename='test_filename_b',
details='test_details_2',
)
history_entry3 = dict(
task='test_task_3',
title='test_title_c',
url='test_url_3',
filename='test_filename_c',
details='test_details_3',
)
entries = [history_entry1, history_entry2, history_entry3]
with Session() as session:
for entry in entries:
item = History()
for key, value in entry.items():
setattr(item, key, value)
session.add(item)
rsp = api_client.get('/history/?sort_by=id')
assert rsp.status_code == 200
data = json.loads(rsp.get_data(as_text=True))
errors = schema_match(OC.history_list_object, data)
assert not errors
assert data[0]['id'] == 3
assert int(rsp.headers['total-count']) == 3
assert int(rsp.headers['count']) == 3
rsp = api_client.get('/history/?sort_by=task&order=asc')
assert rsp.status_code == 200
data = json.loads(rsp.get_data(as_text=True))
errors = schema_match(OC.history_list_object, data)
assert not errors
assert data[0]['task'] == 'test_task_1'
rsp = api_client.get('/history/?sort_by=details')
assert rsp.status_code == 200
data = json.loads(rsp.get_data(as_text=True))
errors = schema_match(OC.history_list_object, data)
assert not errors
assert data[0]['details'] == 'test_details_3'
rsp = api_client.get('/history/?per_page=2')
assert rsp.status_code == 200
data = json.loads(rsp.get_data(as_text=True))
errors = schema_match(OC.history_list_object, data)
assert not errors
assert len(data) == 2
assert int(rsp.headers['total-count']) == 3
assert int(rsp.headers['count']) == 2
rsp = api_client.get('/history/?per_page=2&page=2')
assert rsp.status_code == 200
data = json.loads(rsp.get_data(as_text=True))
errors = schema_match(OC.history_list_object, data)
assert not errors
assert len(data) == 1
assert int(rsp.headers['total-count']) == 3
assert int(rsp.headers['count']) == 1
def wrapped_func(*args, **kwargs):
# get task from method parameters
task = args[1]
# detect api version
api_ver = 1
if len(args) == 3:
api_ver = 2
if api_ver == 1:
# get name for a cache from tasks configuration
if self.name not in task.config:
raise Exception(
'@cache config name %s is not configured in task %s' %
(self.name, task.name))
hash = config_hash(task.config[self.name])
else:
hash = config_hash(args[2])
log.trace('self.name: %s' % self.name)
log.trace('hash: %s' % hash)
cache_name = self.name + '_' + hash
log.debug('cache name: %s (has: %s)' %
(cache_name, ', '.join(list(self.cache.keys()))))
cache_value = self.cache.get(cache_name, None)
if not task.options.nocache and cache_value:
# return from the cache
log.trace('cache hit')
entries = []
for entry in cache_value:
fresh = copy.deepcopy(entry)
entries.append(fresh)
if entries:
log.verbose('Restored %s entries from cache' %
len(entries))
return entries
else:
if self.persist and not task.options.nocache:
# Check database cache
with Session() as session:
db_cache = session.query(InputCache).filter(InputCache.name == self.name).\
filter(InputCache.hash == hash).\
filter(InputCache.added > datetime.now() - self.persist).\
first()
if db_cache:
entries = [e.entry for e in db_cache.entries]
log.verbose('Restored %s entries from db cache' %
len(entries))
# Store to in memory cache
self.cache[cache_name] = copy.deepcopy(entries)
return entries
# Nothing was restored from db or memory cache, run the function
log.trace('cache miss')
# call input event
try:
response = func(*args, **kwargs)
except PluginError as e:
# If there was an error producing entries, but we have valid entries in the db cache, return those.
if self.persist and not task.options.nocache:
with Session() as session:
db_cache = session.query(InputCache).filter(InputCache.name == self.name).\
filter(InputCache.hash == hash).first()
if db_cache and db_cache.entries:
log.error(
'There was an error during %s input (%s), using cache instead.'
% (self.name, e))
entries = [e.entry for e in db_cache.entries]
log.verbose(
'Restored %s entries from db cache' %
len(entries))
# Store to in memory cache
self.cache[cache_name] = copy.deepcopy(entries)
return entries
# If there was nothing in the db cache, re-raise the error.
raise
if api_ver == 1:
response = task.entries
if not isinstance(response, list):
log.warning(
'Input %s did not return a list, cannot cache.' %
self.name)
return response
# store results to cache
log.debug('storing to cache %s %s entries' %
(cache_name, len(response)))
try:
self.cache[cache_name] = copy.deepcopy(response)
except TypeError:
# might be caused because of backlog restoring some idiotic stuff, so not neccessarily a bug
log.critical(
'Unable to save task content into cache, '
'if problem persists longer than a day please report this as a bug'
)
if self.persist:
# Store to database
log.debug('Storing cache %s to database.' % cache_name)
with Session() as session:
db_cache = session.query(InputCache).filter(InputCache.name == self.name).\
filter(InputCache.hash == hash).first()
if not db_cache:
db_cache = InputCache(name=self.name, hash=hash)
db_cache.entries = [
InputCacheEntry(entry=e) for e in response
]
db_cache.added = datetime.now()
session.merge(db_cache)
return response
def __init__(self, config):
self.config = config
with Session() as session:
if not self._db_list(session):
session.add(EntryListList(name=self.config))
def __iter__(self):
with Session() as session:
return iter(
[file.to_entry() for file in self._db_list(session).files])
def get_login_cookies(self, username, password):
url_auth = 'http://www.t411.ch/users/login'
db_session = Session()
account = db_session.query(torrent411Account).filter(
torrent411Account.username == username).first()
if account:
if account.expiry_time < datetime.now():
db_session.delete(account)
db_session.commit()
log.debug("Cookies found in db!")
return account.auth
else:
log.debug("Getting login cookies from : %s " % url_auth)
params = {'login': username, 'password': password, 'remember': '1'}
cj = http.cookiejar.CookieJar()
# WE NEED A COOKIE HOOK HERE TO AVOID REDIRECT COOKIES
opener = urllib.request.build_opener(
urllib.request.HTTPCookieProcessor(cj))
# NEED TO BE SAME USER_AGENT THAN DOWNLOAD LINK
opener.addheaders = [('User-agent', self.USER_AGENT)]
login_output = None
try:
login_output = opener.open(
url_auth, urllib.parse.urlencode(params)).read()
except Exception as e:
raise UrlRewritingError("Connection Error for %s : %s" %
(url_auth, e))
if b'confirmer le captcha' in login_output:
log.warning("Captcha requested for login.")
login_output = self._solveCaptcha(login_output, url_auth,
params, opener)
if b'logout' in login_output:
authKey = None
uid = None
password = None
for cookie in cj:
if cookie.name == "authKey":
authKey = cookie.value
if cookie.name == "uid":
uid = cookie.value
if cookie.name == "pass":
password = cookie.value
if authKey is not None and \
uid is not None and \
password is not None:
authCookie = {
'uid': uid,
'password': password,
'authKey': authKey
}
db_session.add(
torrent411Account(username=username,
auth=authCookie,
expiry_time=datetime.now() +
timedelta(days=1)))
db_session.commit()
return authCookie
else:
log.error(
"Login failed (Torrent411). Check your login and password."
)
return {}
def on_task_start(self, task, config):
if config is False: # handles 'template: no' form to turn off template on this task
return
# implements --template NAME
if task.options.template:
if not config or task.options.template not in config:
task.abort('does not use `%s` template' %
task.options.template,
silent=True)
config = self.prepare_config(config)
# add global in except when disabled with no_global
if 'no_global' in config:
config.remove('no_global')
if 'global' in config:
config.remove('global')
elif 'global' not in config:
config.append('global')
toplevel_templates = task.manager.config.get('templates', {})
# apply templates
for template in config:
if template not in toplevel_templates:
if template == 'global':
continue
raise plugin.PluginError(
'Unable to find template %s for task %s' %
(template, task.name), log)
if toplevel_templates[template] is None:
log.warning('Template `%s` is empty. Nothing to merge.' %
template)
continue
log.debug('Merging template %s into task %s' %
(template, task.name))
# We make a copy here because we need to remove
template_config = toplevel_templates[template]
# When there are templates within templates we remove the template
# key from the config and append it's items to our own
if 'template' in template_config:
nested_templates = self.prepare_config(
template_config['template'])
for nested_template in nested_templates:
if nested_template not in config:
config.append(nested_template)
else:
log.warning('Templates contain each other in a loop.')
# Replace template_config with a copy without the template key, to avoid merging errors
template_config = dict(template_config)
del template_config['template']
# Merge
try:
merge_dict_from_to(template_config, task.config)
except MergeException as exc:
raise plugin.PluginError(
'Failed to merge template %s to task %s. Error: %s' %
(template, task.name, exc.value))
# TODO: Better handling of config_modified flag for templates???
with Session() as session:
last_hash = session.query(TemplateConfigHash).filter(
TemplateConfigHash.task == task.name).first()
task.config_modified, config_hash = task.is_config_modified(
last_hash)
if task.config_modified:
session.add(
TemplateConfigHash(task=task.name, hash=config_hash))
log.trace('templates: %s' % config)
def get_login_cookie(self, username, password, force=False):
"""
Retrieves login cookie
:param str username:
:param str password:
:param bool force: if True, then retrieve a fresh cookie instead of looking in the DB
:return:
"""
if not force:
with Session() as session:
saved_cookie = (session.query(FileListCookie).filter(
FileListCookie.username == username.lower()).first())
if (saved_cookie and saved_cookie.expires
and saved_cookie.expires >= datetime.datetime.now()):
logger.debug('Found valid login cookie')
return saved_cookie.cookie
url = BASE_URL + 'takelogin.php'
try:
# get validator token
response = requests.get(BASE_URL + 'login.php')
soup = get_soup(response.content)
login_validator = soup.find("input", {"name": "validator"})
if not login_validator:
raise plugin.PluginError(
'FileList.ro could not get login validator')
logger.debug('Login Validator: {}'.format(
login_validator.get('value')))
logger.debug('Attempting to retrieve FileList.ro cookie')
response = requests.post(
url,
data={
'username': username,
'password': password,
'validator': login_validator.get('value'),
'login': '******',
'unlock': '1',
},
timeout=30,
)
except RequestException as e:
raise plugin.PluginError('FileList.ro login failed: %s' % e)
if 'https://filelist.ro/my.php' != response.url:
raise plugin.PluginError(
'FileList.ro login failed: Your username or password was incorrect.'
)
with Session() as session:
expires = None
for c in requests.cookies:
if c.name == 'pass':
expires = c.expires
if expires:
expires = datetime.datetime.fromtimestamp(expires)
logger.debug('Saving or updating FileList.ro cookie in db')
cookie = FileListCookie(username=username.lower(),
cookie=dict(requests.cookies),
expires=expires)
session.merge(cookie)
return cookie.cookie
