def get(self):
shows = ShowDetailsDb().get_all_shows()
total_shows = len(shows)
failed_shows = FailedShowsDb().get_failed_shows()
total_episodes = 0
total_subtitles_wanted = 0
total_subtitles_available = 0
total_subtitles_missing = 0
show_settings_db = ShowSettingsDb()
show_details_db = ShowEpisodeDetailsDb()
for show in shows:
show_settings = show_settings_db.get_show_settings(show.tvdb_id)
show_episodes = show_details_db.get_show_episodes(
show.tvdb_id, available_only=True, subtitles=False)
wanted_languages = show_settings.wanted_languages
for show_episode in show_episodes:
total_episodes += 1
total_subtitles_wanted += len(wanted_languages)
total_subtitles_missing += len(show_episode.missing_languages)
total_subtitles_available += len(wanted_languages) - len(
show_episode.missing_languages)
return {
'total_shows': total_shows,
'total_episodes': total_episodes,
'total_subtitles_wanted': total_subtitles_wanted,
'total_subtitles_missing': total_subtitles_missing,
'total_subtitles_available': total_subtitles_available,
'failed_shows': failed_shows
}
def put(self, tvdb_id):
"""Save the settings for a show."""
input_json = cherrypy.request.json
if all(k in input_json for k in ('wanted_languages', 'refine',
'hearing_impaired', 'utf8_encoding')):
wanted_languages = input_json['wanted_languages']
refine = get_boolean(input_json['refine'])
hearing_impaired = get_boolean(input_json['hearing_impaired'])
utf8_encoding = get_boolean(input_json['utf8_encoding'])
# Update settings
db = ShowSettingsDb()
show_settings = db.get_show_settings(tvdb_id)
show_settings.wanted_languages = wanted_languages
show_settings.refine = refine
show_settings.hearing_impaired = hearing_impaired
show_settings.utf8_encoding = utf8_encoding
db.update_show_settings(show_settings)
# Delete wanted items for the show so the new settings will be used in the next disk scan
WantedItemsDb().delete_wanted_items_for_show(tvdb_id)
# Send notification
send_websocket_notification(
'Settings will be applied on next disk scan.')
return self._no_content()
return self._bad_request('Missing data')
def put(self, tvdb_id):
"""Save the settings for a show."""
input_json = cherrypy.request.json
if all(k in input_json for k in ('wanted_languages', 'refine', 'hearing_impaired', 'utf8_encoding')):
wanted_languages = input_json['wanted_languages']
refine = get_boolean(input_json['refine'])
hearing_impaired = get_boolean(input_json['hearing_impaired'])
utf8_encoding = get_boolean(input_json['utf8_encoding'])
# Update settings
db = ShowSettingsDb()
show_settings = db.get_show_settings(tvdb_id)
show_settings.wanted_languages = wanted_languages
show_settings.refine = refine
show_settings.hearing_impaired = hearing_impaired
show_settings.utf8_encoding = utf8_encoding
db.update_show_settings(show_settings)
# Delete wanted items for the show so the new settings will be used in the next disk scan
WantedItemsDb().delete_wanted_items_for_show(tvdb_id)
# Send notification
send_websocket_notification('Settings will be applied on next disk scan.')
return self._no_content()
return self._bad_request('Missing data')
def get(self):
shows = ShowDetailsDb().get_all_shows()
total_shows = len(shows)
failed_shows = FailedShowsDb().get_failed_shows()
total_episodes = 0
total_subtitles_wanted = 0
total_subtitles_available = 0
total_subtitles_missing = 0
show_settings_db = ShowSettingsDb()
show_details_db = ShowEpisodeDetailsDb()
for show in shows:
show_settings = show_settings_db.get_show_settings(show.tvdb_id)
show_episodes = show_details_db.get_show_episodes(show.tvdb_id, available_only=True, subtitles=False)
wanted_languages = show_settings.wanted_languages
for show_episode in show_episodes:
total_episodes += 1
total_subtitles_wanted += len(wanted_languages)
total_subtitles_missing += len(show_episode.missing_languages)
total_subtitles_available += len(wanted_languages) - len(show_episode.missing_languages)
return {
'total_shows': total_shows,
'total_episodes': total_episodes,
'total_subtitles_wanted': total_subtitles_wanted,
'total_subtitles_missing': total_subtitles_missing,
'total_subtitles_available': total_subtitles_available,
'failed_shows': failed_shows
}
def get(self, tvdb_id):
"""Get the settings for a show"""
show_settings_db = ShowSettingsDb()
show_settings = show_settings_db.get_show_settings(tvdb_id)
# If no settings are defined yet, use the default settings
if not show_settings:
show_settings = ShowSettings.default_settings(tvdb_id)
show_settings_db.set_show_settings(show_settings)
return show_settings.to_json()
def get(self, tvdb_id):
"""Get the settings for a show"""
show_settings_db = ShowSettingsDb()
show_settings = show_settings_db.get_show_settings(tvdb_id)
# If no settings are defined yet, use the default settings
if not show_settings:
show_settings = ShowSettings.default_settings(tvdb_id)
show_settings_db.set_show_settings(show_settings)
return show_settings.to_json()
def get(self, tvdb_id=None):
"""Get the list of shows or the details of a single show."""
if tvdb_id:
db_show = ShowDetailsDb().get_show(tvdb_id)
db_show_settings = ShowSettingsDb().get_show_settings(tvdb_id)
# Return NotFound if movie does not longer exists on disk
if not os.path.exists(db_show.path):
raise NotFound()
# Return show details
return self._to_show_json(db_show, db_show_settings, details=True)
else:
shows = []
show_settings_db = ShowSettingsDb()
db_shows = ShowDetailsDb().get_all_shows()
for db_show in db_shows:
db_show_settings = show_settings_db.get_show_settings(db_show.tvdb_id)
shows.append(self._to_show_json(db_show, db_show_settings))
return shows
def get(self, tvdb_id=None):
"""Get the list of shows or the details of a single show."""
if tvdb_id:
db_show = ShowDetailsDb().get_show(tvdb_id)
db_show_settings = ShowSettingsDb().get_show_settings(tvdb_id)
# Return NotFound if movie does not longer exists on disk
if not os.path.exists(db_show.path):
raise NotFound()
# Return show details
return self._to_show_json(db_show, db_show_settings, details=True)
else:
shows = []
show_settings_db = ShowSettingsDb()
db_shows = ShowDetailsDb().get_all_shows()
for db_show in db_shows:
db_show_settings = show_settings_db.get_show_settings(
db_show.tvdb_id)
shows.append(self._to_show_json(db_show, db_show_settings))
return shows
class DiskScanner(ScheduledProcess):
"""Disk scanner.
Scan the specified path for episodes and movies with missing subtitles.
If found, episodes and movies are added to the WANTEDQUEUE.
"""
def __init__(self):
super(DiskScanner, self).__init__()
self.wanted_db = WantedItemsDb()
self.show_settings_db = ShowSettingsDb()
self.movie_settings_db = MovieSettingsDb()
@release_wanted_queue_lock_on_exception
def run(self, force_run):
paths = autosubliminal.VIDEOPATHS
log.info('Starting round of local disk checking at %r', paths)
# Show info message (only when run was forced manually)
if force_run:
send_websocket_notification('Scanning disk...')
# Check if a path exists to scan
if not one_path_exists(paths):
log.error('None of the configured video paths (%r) exists, aborting...', paths)
return
# Walk through paths to search for wanted items
new_wanted_items = []
old_wanted_items = self.wanted_db.get_wanted_items()
for path in paths:
try:
new_wanted_items.extend(self._scan_path(path))
except Exception:
log.exception('Could not scan the video path (%s), skipping it', path)
# Cleanup wanted items that have been removed from disk manually but are still stored in the db
log.debug('Checking for non existing wanted items in wanted_items database')
for item in old_wanted_items:
if item not in new_wanted_items:
self.wanted_db.delete_wanted_item(item)
log.debug('Deleted non existing wanted item: %s', item.videopath)
# Populate WANTEDQUEUE with all items from wanted_items database
log.info('Listing videos with missing subtitles:')
autosubliminal.WANTEDQUEUE = []
for item in self.wanted_db.get_wanted_items():
log.info('%s %s', item.videopath, item.languages)
autosubliminal.WANTEDQUEUE.append(item)
# Send home page reload event
send_websocket_event(PAGE_RELOAD, data={'name': 'home'})
log.info('Finished round of local disk checking')
def _scan_path(self, path):
log.info('Scanning video path: %s', path)
wanted_items = []
# Check all folders and files
for dirname, dirnames, filenames in os.walk(os.path.join(path)):
log.debug('Directory: %s', dirname)
# Ignore skipped dirs
if is_skipped_dir(dirname):
continue
# Check files
for filename in filenames:
# Only scan valid video files
if is_valid_video_file(filename):
log.debug('Video file found: %s', filename)
wanted_item = None
try:
wanted_item = self._scan_file(dirname, filename)
except Exception:
log.exception('Error while scanning video file: %s', os.path.join(dirname, filename))
# Add it to list of wanted items
if wanted_item:
log.debug('Video added to list of wanted items')
wanted_items.append(wanted_item)
return wanted_items
def _scan_file(self, dirname, filename):
# Check if video file has already been processed before, so we don't need to process it again
wanted_item = self.wanted_db.get_wanted_item(os.path.join(dirname, filename))
if wanted_item:
log.debug('Video found in wanted_items database, no need to scan it again')
else:
log.debug('Video not found in wanted_items database, start scanning it')
# Process file
wanted_item = fileprocessor.process_file(dirname, filename)
if wanted_item:
# Determine wanted languages
wanted_languages = []
if wanted_item.is_episode and wanted_item.tvdbid:
settings = self.show_settings_db.get_show_settings(wanted_item.tvdbid)
if settings and settings.wanted_languages:
wanted_languages = settings.wanted_languages
elif wanted_item.is_movie and wanted_item.imdbid:
settings = self.movie_settings_db.get_movie_settings(wanted_item.imdbid)
if settings and settings.wanted_languages:
wanted_languages = settings.wanted_languages
# Check for missing subtitles (scan embedded and detect invalid if configured to do so)
languages = check_missing_subtitle_languages(dirname, filename,
scan_embedded=autosubliminal.SCANEMBEDDEDSUBS,
scan_hardcoded=autosubliminal.SCANHARDCODEDSUBS,
detect_invalid=autosubliminal.DETECTINVALIDSUBLANGUAGE,
wanted_languages=wanted_languages)
# Process the video file if there are missing subtitles
if len(languages) > 0:
# Add missing languages and store in wanted_items database
wanted_item.languages = languages
self.wanted_db.set_wanted_item(wanted_item)
else:
log.debug('Video has no missing subtitles')
return None
else:
return None
# Check if we need to skip it and delete it from the database
log.debug('Checking if the video needs to be skipped')
if wanted_item:
# Skip episode check
if wanted_item.is_episode:
title = wanted_item.title
season = wanted_item.season
if skip_show(title, season):
self.wanted_db.delete_wanted_item(wanted_item)
log.info('Skipping %s - Season %s', title, season)
return None
# Skip movie check
if wanted_item.is_movie:
title = wanted_item.title
year = wanted_item.year
if skip_movie(title, year):
self.wanted_db.delete_wanted_item(wanted_item)
log.info('Skipping %s (%s)', title, year)
return None
return wanted_item
class LibraryPathScanner(object):
def __init__(self):
self.show_db = ShowDetailsDb()
self.failed_shows_db = FailedShowsDb()
self.show_episodes_db = ShowEpisodeDetailsDb()
self.show_settings_db = ShowSettingsDb()
self.failed_movies_db = FailedMoviesDb()
self.movie_db = MovieDetailsDb()
self.movie_settings_db = MovieSettingsDb()
self.show_indexer = ShowIndexer()
self.movie_indexer = MovieIndexer()
def scan_path(self, path):
log.info('Scanning path: %s', path)
# Check all folders and files
for dirname, dirnames, filenames in os.walk(os.path.join(path)):
log.debug('Directory: %s', dirname)
# Ignore skipped dirs
if is_skipped_dir(dirname):
continue
# Check files
for filename in filenames:
# Only scan valid video files
if is_valid_video_file(filename, video_extensions=EXTENDED_VIDEO_EXTENSIONS):
log.debug('Video file found: %s', filename)
try:
self._scan_file(dirname, filename)
except Exception:
log.exception('Error while scanning video file: %s', os.path.join(dirname, filename))
def _scan_file(self, dirname, filename):
wanted_item = process_file(dirname, filename)
if wanted_item:
if wanted_item.is_episode:
# Do a force search if no tvdb id found
if not wanted_item.tvdbid:
wanted_item.tvdbid = self.show_indexer.get_tvdb_id(wanted_item.title, year=wanted_item.year,
force_search=True)
# Skip if no tvdb id is found
if not wanted_item.tvdbid:
log.warning('Skipping show episode file with unknown tvdb id: %s', os.path.join(dirname, filename))
show_path = self._get_show_path(dirname)
if not self.failed_shows_db.get_failed_show(show_path):
self.failed_shows_db.set_failed_show(show_path)
return
# Store default show settings if not yet available
if not self.show_settings_db.get_show_settings(wanted_item.tvdbid):
self.show_settings_db.set_show_settings(ShowSettings.default_settings(wanted_item.tvdbid))
show_settings = self.show_settings_db.get_show_settings(wanted_item.tvdbid)
# Get show details
show_details = self.show_db.get_show(wanted_item.tvdbid)
# Add show and episodes to db if not yet in db
if not show_details:
show_details = self.show_indexer.get_show_details(wanted_item.tvdbid)
if show_details:
show_details.path = self._get_show_path(dirname)
self.show_db.set_show(show_details)
episodes = self.show_indexer.get_show_episodes(wanted_item.tvdbid)
if episodes:
for episode in episodes:
self.show_episodes_db.set_show_episode(episode)
# Cache artwork (fullsize and thumbnail) if not yet cached
if show_details:
# Poster
if show_details.poster:
if not is_artwork_cached(self.show_indexer.name, show_details.tvdb_id, 'poster'):
cache_artwork(self.show_indexer.name, show_details.tvdb_id, 'poster',
get_artwork_url(show_details.poster))
if not is_artwork_cached(self.show_indexer.name, show_details.tvdb_id, 'poster',
thumbnail=True):
cache_artwork(self.show_indexer.name, show_details.tvdb_id, 'poster',
get_artwork_url(show_details.poster, thumbnail=True), thumbnail=True)
# Banner
if show_details.banner:
if not is_artwork_cached(self.show_indexer.name, show_details.tvdb_id, 'banner'):
cache_artwork(self.show_indexer.name, show_details.tvdb_id, 'banner',
get_artwork_url(show_details.banner))
if not is_artwork_cached(self.show_indexer.name, show_details.tvdb_id, 'banner',
thumbnail=True):
cache_artwork(self.show_indexer.name, show_details.tvdb_id, 'banner',
get_artwork_url(show_details.banner, thumbnail=True), thumbnail=True)
# Check episode details
if isinstance(wanted_item.episode, list):
for episode in wanted_item.episode:
self._update_episode_details(show_settings, dirname, filename, wanted_item.tvdbid,
wanted_item.season, episode)
else:
self._update_episode_details(show_settings, dirname, filename, wanted_item.tvdbid,
wanted_item.season, wanted_item.episode)
if wanted_item.is_movie:
# Do a force search if no imdb id found
if not wanted_item.imdbid:
wanted_item.imdbid, _ = self.movie_indexer.get_imdb_id_and_year(wanted_item.title,
year=wanted_item.year,
force_search=True)
# Skip if no imdb id is found
if not wanted_item.imdbid:
log.warning('Skipping movie file with unknown imdb id: %s', os.path.join(dirname, filename))
if not self.failed_movies_db.get_failed_movie(dirname):
self.failed_movies_db.set_failed_movie(dirname)
return
# Store default movie settings if not yet available
if not self.movie_settings_db.get_movie_settings(wanted_item.imdbid):
self.movie_settings_db.set_movie_settings(MovieSettings.default_settings(wanted_item.imdbid))
movie_settings = self.movie_settings_db.get_movie_settings(wanted_item.imdbid)
# Get movie details
movie_details = self.movie_db.get_movie(wanted_item.imdbid)
# Add movie to db if not yet in db
if not movie_details:
movie_details = self.movie_indexer.get_movie_details(wanted_item.imdbid)
if movie_details:
movie_details.path = dirname
self.movie_db.set_movie(movie_details)
# Cache artwork (fullsize and thumbnail) if not yet cached
if movie_details:
# Poster
if movie_details.poster:
if not is_artwork_cached(self.movie_indexer.name, movie_details.imdb_id, 'poster'):
cache_artwork(self.movie_indexer.name, movie_details.imdb_id, 'poster',
movie_details.poster)
if not is_artwork_cached(self.movie_indexer.name, movie_details.imdb_id, 'poster',
thumbnail=True):
cache_artwork(self.movie_indexer.name, movie_details.imdb_id, 'poster',
self.movie_indexer.get_artwork_thumbnail_url(movie_details.poster),
thumbnail=True)
# Check movie details
self._update_movie_details(movie_settings, dirname, filename, wanted_item.imdbid)
def _get_show_path(self, dirname):
path = dirname
# Get root show path (ignore season folders)
while 'season' in safe_lowercase(os.path.normpath(os.path.normcase(path))):
path, _ = os.path.split(path)
return path
def _update_episode_details(self, show_settings, dirname, filename, show_tvdb_id, season, episode):
episode_details = self.show_episodes_db.get_show_episode(show_tvdb_id, season, episode)
# If no episode is found, we need to fetch the episode details of the show
# This is because the show is still on-going and we didn't got all episodes when the show was loaded
if not episode_details:
episode_details = self.show_indexer.get_show_episode(show_tvdb_id, season, episode)
if episode_details:
self.show_episodes_db.set_show_episode(episode_details)
if episode_details:
# Set details
available_subtitles = get_available_subtitles(dirname, filename, autosubliminal.SCANEMBEDDEDSUBS,
autosubliminal.SCANHARDCODEDSUBS)
missing_languages = self._get_missing_subtitle_languages(available_subtitles,
wanted_languages=show_settings.wanted_languages)
episode_details.subtitles = available_subtitles
episode_details.missing_languages = missing_languages
episode_details.path = os.path.abspath(os.path.join(dirname, filename))
# Update details in db
self.show_episodes_db.update_show_episode(episode_details, subtitles=True)
def _update_movie_details(self, movie_settings, dirname, filename, imdb_id):
movie_details = self.movie_db.get_movie(imdb_id)
if movie_details:
# Set details
available_subtitles = get_available_subtitles(dirname, filename, autosubliminal.SCANEMBEDDEDSUBS,
autosubliminal.SCANHARDCODEDSUBS)
missing_languages = self._get_missing_subtitle_languages(available_subtitles,
wanted_languages=movie_settings.wanted_languages)
movie_details.subtitles = available_subtitles
movie_details.missing_languages = missing_languages
movie_details.path = os.path.abspath(os.path.join(dirname, filename))
# Update details in db
self.movie_db.update_movie(movie_details, subtitles=True)
def _get_missing_subtitle_languages(self, available_subtitles, wanted_languages=None):
# Use custom wanted languages or globally configured wanted languages if not provided
if wanted_languages is None:
wanted_languages = get_wanted_languages()
available_languages = []
for subtitle in available_subtitles:
available_languages.append(subtitle.language)
# Return the missing languages (= not in available languages)
return [language for language in wanted_languages if language not in available_languages]
class DiskScanner(ScheduledProcess):
"""Disk scanner.
Scan the specified path for episodes and movies with missing subtitles.
If found, episodes and movies are added to the WANTEDQUEUE.
"""
def __init__(self):
super(DiskScanner, self).__init__()
self.wanted_db = WantedItemsDb()
self.show_settings_db = ShowSettingsDb()
self.movie_settings_db = MovieSettingsDb()
@release_wanted_queue_lock_on_exception
def run(self, force_run):
paths = autosubliminal.VIDEOPATHS
log.info('Starting round of local disk checking at %r', paths)
# Show info message (only when run was forced manually)
if force_run:
send_websocket_notification('Scanning disk...')
# Check if a path exists to scan
if not one_path_exists(paths):
log.error(
'None of the configured video paths (%r) exists, aborting...',
paths)
return
# Walk through paths to search for wanted items
new_wanted_items = []
old_wanted_items = self.wanted_db.get_wanted_items()
for path in paths:
try:
new_wanted_items.extend(self._scan_path(path))
except Exception:
log.exception(
'Could not scan the video path (%s), skipping it', path)
# Cleanup wanted items that have been removed from disk manually but are still stored in the db
log.debug(
'Checking for non existing wanted items in wanted_items database')
for item in old_wanted_items:
if item not in new_wanted_items:
self.wanted_db.delete_wanted_item(item)
log.debug('Deleted non existing wanted item: %s',
item.videopath)
# Populate WANTEDQUEUE with all items from wanted_items database
log.info('Listing videos with missing subtitles:')
autosubliminal.WANTEDQUEUE = []
for item in self.wanted_db.get_wanted_items():
log.info('%s %s', item.videopath, item.languages)
autosubliminal.WANTEDQUEUE.append(item)
# Send home page reload event
send_websocket_event(PAGE_RELOAD, data={'name': 'home'})
log.info('Finished round of local disk checking')
def _scan_path(self, path):
log.info('Scanning video path: %s', path)
wanted_items = []
# Check all folders and files
for dirname, dirnames, filenames in os.walk(os.path.join(path)):
log.debug('Directory: %s', dirname)
# Ignore skipped dirs
if is_skipped_dir(dirname):
continue
# Check files
for filename in filenames:
# Only scan valid video files
if is_valid_video_file(filename):
log.debug('Video file found: %s', filename)
wanted_item = None
try:
wanted_item = self._scan_file(dirname, filename)
except Exception:
log.exception('Error while scanning video file: %s',
os.path.join(dirname, filename))
# Add it to list of wanted items
if wanted_item:
log.debug('Video added to list of wanted items')
wanted_items.append(wanted_item)
return wanted_items
def _scan_file(self, dirname, filename):
# Check if video file has already been processed before, so we don't need to process it again
wanted_item = self.wanted_db.get_wanted_item(
os.path.join(dirname, filename))
if wanted_item:
log.debug(
'Video found in wanted_items database, no need to scan it again'
)
else:
log.debug(
'Video not found in wanted_items database, start scanning it')
# Process file
wanted_item = fileprocessor.process_file(dirname, filename)
if wanted_item:
# Determine wanted languages
wanted_languages = []
if wanted_item.is_episode and wanted_item.tvdbid:
settings = self.show_settings_db.get_show_settings(
wanted_item.tvdbid)
if settings and settings.wanted_languages:
wanted_languages = settings.wanted_languages
elif wanted_item.is_movie and wanted_item.imdbid:
settings = self.movie_settings_db.get_movie_settings(
wanted_item.imdbid)
if settings and settings.wanted_languages:
wanted_languages = settings.wanted_languages
# Check for missing subtitles (scan embedded and detect invalid if configured to do so)
languages = check_missing_subtitle_languages(
dirname,
filename,
scan_embedded=autosubliminal.SCANEMBEDDEDSUBS,
scan_hardcoded=autosubliminal.SCANHARDCODEDSUBS,
detect_invalid=autosubliminal.DETECTINVALIDSUBLANGUAGE,
wanted_languages=wanted_languages)
# Process the video file if there are missing subtitles
if len(languages) > 0:
# Add missing languages and store in wanted_items database
wanted_item.languages = languages
self.wanted_db.set_wanted_item(wanted_item)
else:
log.debug('Video has no missing subtitles')
return None
else:
return None
# Check if we need to skip it and delete it from the database
log.debug('Checking if the video needs to be skipped')
if wanted_item:
# Skip episode check
if wanted_item.is_episode:
title = wanted_item.title
season = wanted_item.season
if skip_show(title, season):
self.wanted_db.delete_wanted_item(wanted_item)
log.info('Skipping %s - Season %s', title, season)
return None
# Skip movie check
if wanted_item.is_movie:
title = wanted_item.title
year = wanted_item.year
if skip_movie(title, year):
self.wanted_db.delete_wanted_item(wanted_item)
log.info('Skipping %s (%s)', title, year)
return None
return wanted_item