def ingest_root(self, path):
scanned = 0
logger_background.info('Scanning {}'.format(path))
for root, dirs, files in os.walk(path):
scanned += len(files)
for file in files:
self.ingest_path(os.path.join(root, file))
logger_background.info('Scanned {} files'.format(scanned))
def auto_watch_new_seasons_task():
"""
look for newly aired seasons that the user wants to automatically watch
"""
nefarious_settings = NefariousSettings.get()
tmdb_client = get_tmdb_client(nefarious_settings)
# cycle through every show that has auto-watch enabled
for watch_show in WatchTVShow.objects.filter(auto_watch=True):
tmdb_show = tmdb_client.TV(watch_show.tmdb_show_id)
show_data = tmdb_show.info()
added_season = False
# find any season with a newer air date than the "auto watch" and queue it up
for season in show_data['seasons']:
air_date = parse_date(season['air_date'] or '')
# air date is newer than our auto watch date
if air_date and watch_show.auto_watch_date_updated and air_date >= watch_show.auto_watch_date_updated:
# season & request params
create_params = dict(watch_tv_show=watch_show,
season_number=season['season_number'],
defaults=dict(
user=watch_show.user,
release_date=air_date,
))
# create a season request instance to keep up with slowly-released episodes
WatchTVSeasonRequest.objects.get_or_create(**create_params)
# also save a watch tv season instance to try and download the whole season immediately
watch_tv_season, was_season_created = WatchTVSeason.objects.get_or_create(
**create_params)
# season was created
if was_season_created:
added_season = True
logger_background.info(
'Automatically watching newly aired season {}'.format(
watch_tv_season))
# send a websocket message for this new season
media_type, data = websocket.get_media_type_and_serialized_watch_media(
watch_tv_season)
send_websocket_message_task.delay(websocket.ACTION_UPDATED,
media_type, data)
# create a task to download the whole season (fallback to individual episodes if it fails)
watch_tv_show_season_task.delay(watch_tv_season.id)
# new season added to show
if added_season:
# update auto watch date requested
watch_show.auto_watch_date_updated = datetime.utcnow().date()
watch_show.save()
def refresh_tmdb_configuration():
logger_background.info('Refreshing TMDB Configuration')
nefarious_settings = NefariousSettings.get()
tmdb_client = get_tmdb_client(nefarious_settings)
configuration = tmdb_client.Configuration()
nefarious_settings.tmdb_configuration = configuration.info()
nefarious_settings.tmdb_languages = configuration.languages()
nefarious_settings.save()
return nefarious_settings.tmdb_configuration
def watch_tv_show_season_task(watch_tv_season_id: int):
processor = WatchTVSeasonProcessor(watch_media_id=watch_tv_season_id)
success = processor.fetch()
watch_tv_season = get_object_or_404(WatchTVSeason, pk=watch_tv_season_id)
# success so update the season request instance as "collected"
if success:
season_request = WatchTVSeasonRequest.objects.filter(
watch_tv_show=watch_tv_season.watch_tv_show,
season_number=watch_tv_season.season_number)
if season_request.exists():
season_request = season_request.first()
season_request.collected = True
season_request.save()
# failed so delete season instance and fallback to trying individual episodes
else:
logger_background.info(
'Failed fetching entire season {} - falling back to individual episodes'
.format(watch_tv_season))
nefarious_settings = NefariousSettings.get()
tmdb = get_tmdb_client(nefarious_settings)
season_request = tmdb.TV_Seasons(
watch_tv_season.watch_tv_show.tmdb_show_id,
watch_tv_season.season_number)
season = season_request.info()
for episode in season['episodes']:
# save individual episode watches
watch_tv_episode, was_created = WatchTVEpisode.objects.get_or_create(
tmdb_episode_id=episode['id'],
# add non-unique constraint fields for the default values
defaults=dict(
user=watch_tv_season.user,
watch_tv_show=watch_tv_season.watch_tv_show,
season_number=watch_tv_season.season_number,
episode_number=episode['episode_number'],
release_date=parse_date(episode.get('air_date') or ''),
))
# queue task to watch episode
watch_tv_episode_task.delay(watch_tv_episode.id)
# remove the "watch season" now that we've requested to fetch all individual episodes
watch_tv_season.delete()
def wanted_media_task():
wanted_kwargs = dict(collected=False,
transmission_torrent_hash__isnull=True)
#
# scan for individual watch media
#
wanted_media_data = {
'movie': {
'query': WatchMovie.objects.filter(**wanted_kwargs),
'task': watch_movie_task,
},
'season': {
'query': WatchTVSeason.objects.filter(**wanted_kwargs),
'task': watch_tv_show_season_task,
},
'episode': {
'query': WatchTVEpisode.objects.filter(**wanted_kwargs),
'task': watch_tv_episode_task,
},
}
today = timezone.now().date()
for media_type, data in wanted_media_data.items():
for media in data['query']:
# media has been released (or it's missing it's release date so try anyway) so create a task to try and fetch it
if not media.release_date or media.release_date <= today:
logger_background.info('Wanted {type}: {media}'.format(
type=media_type, media=media))
# queue task for wanted media
data['task'].delay(media.id)
# media has not been released so skip
else:
logger_background.info(
"Skipping wanted {type} since it hasn't aired yet: {media} "
.format(type=media_type, media=media))
def populate_release_dates_task():
logger_background.info('Populating release dates')
nefarious_settings = NefariousSettings.get()
tmdb_client = get_tmdb_client(nefarious_settings)
kwargs = dict(release_date=None)
for media in WatchMovie.objects.filter(**kwargs):
try:
movie_result = tmdb_client.Movies(media.tmdb_movie_id)
data = movie_result.info()
release_date = parse_date(data.get('release_date', ''))
update_media_release_date(media, release_date)
except Exception as e:
logger_background.exception(e)
for media in WatchTVSeason.objects.filter(**kwargs):
try:
season_result = tmdb_client.TV_Seasons(
media.watch_tv_show.tmdb_show_id, media.season_number)
data = season_result.info()
release_date = parse_date(data.get('air_date', ''))
update_media_release_date(media, release_date)
except Exception as e:
logger_background.exception(e)
for media in WatchTVEpisode.objects.filter(**kwargs):
try:
episode_result = tmdb_client.TV_Episodes(
media.watch_tv_show.tmdb_show_id, media.season_number,
media.episode_number)
data = episode_result.info()
release_date = parse_date(data.get('air_date', ''))
update_media_release_date(media, release_date)
except Exception as e:
logger_background.exception(e)
def ingest_path(self, file_path):
file_name = os.path.basename(file_path)
parser = self._get_parser(file_name)
# match
if parser.match:
file_extension_match = parser.file_extension_regex.search(file_name)
if file_extension_match:
# skip sample files
if parser.sample_file_regex.search(file_name):
logger_background.warning('[NO_MATCH_SAMPLE] Not matching sample file "{}"'.format(file_path))
return False
title = parser.match['title']
if not title:
new_title, parser_match = self._handle_missing_title(parser, file_path)
if new_title:
title = new_title
parser.match.update(parser_match)
else:
logger_background.warning('[NO_MATCH_TITLE] Could not match file without title "{}"'.format(file_path))
return False
file_extension = file_extension_match.group()
if file_extension in video_extensions():
if self._is_parser_exact_match(parser):
if self.media_class.objects.filter(download_path=file_path).exists():
logger_background.info('[SKIP] skipping already-processed file "{}"'.format(file_path))
return False
# get or set tmdb search results for this title in the cache
tmdb_results = cache.get(title)
if not tmdb_results:
try:
tmdb_results = self._get_tmdb_search_results(title)
except HTTPError:
logger_background.error('[ERROR_TMDB] tmdb search exception for title {} on file "{}"'.format(title, file_path))
return False
cache.set(title, tmdb_results, 60 * 60)
# loop over results for the exact match
for tmdb_result in tmdb_results['results']:
# normalize titles and see if they match
if self._is_result_match_title(parser, tmdb_result, title):
watch_media = self._handle_match(parser, tmdb_result, title, file_path)
if watch_media:
logger_background.info('[MATCH] Saved media "{}" from file "{}"'.format(watch_media, file_path))
return watch_media
else: # for/else
logger_background.warning('[NO_MATCH_MEDIA] No media match for title "{}" and file "{}"'.format(title, file_path))
else:
logger_background.warning('[NO_MATCH_EXACT] No exact title match for title "{}" and file "{}"'.format(title, file_path))
else:
logger_background.warning('[NO_MATCH_VIDEO] No valid video file extension for file "{}"'.format(file_path))
else:
logger_background.warning('[NO_MATCH_EXTENSION] No file extension for file "{}"'.format(file_path))
else:
logger_background.info('[NO_MATCH_UNKNOWN] Unknown match for file "{}"'.format(file_path))
return False
def fetch(self):
logger_background.info('Processing request to watch {}'.format(
self.watch_media))
valid_search_results = []
search = self._get_search_results()
# save this attempt date
self.watch_media.last_attempt_date = datetime.utcnow()
self.watch_media.save()
if search.ok:
for result in search.results:
if self.is_match(result['Title']):
valid_search_results.append(result)
else:
logger_background.info('Not matched: {}'.format(
result['Title']))
if valid_search_results:
# trace the "torrent url" (sometimes magnet) in each valid result
valid_search_results = self._results_with_valid_urls(
valid_search_results)
while valid_search_results:
logger_background.info('Valid Search Results: {}'.format(
len(valid_search_results)))
# find the torrent result with the highest weight (i.e seeds)
best_result = self._get_best_torrent_result(
valid_search_results)
transmission_client = get_transmission_client(
self.nefarious_settings)
transmission_session = transmission_client.session_stats()
# add to transmission
torrent = transmission_client.add_torrent(
best_result['torrent_url'],
paused=
True, # start paused to we can verify if the torrent has been blacklisted
download_dir=self._get_download_dir(
transmission_session),
)
# verify it's not blacklisted and save & start this torrent
if not TorrentBlacklist.objects.filter(
hash=torrent.hashString).exists():
logger_background.info('Adding torrent for {}'.format(
self.tmdb_media[self._get_tmdb_title_key()]))
logger_background.info(
'Added torrent {} with {} seeders'.format(
best_result['Title'], best_result['Seeders']))
logger_background.info(
'Starting torrent id: {} and hash {}'.format(
torrent.id, torrent.hashString))
# save torrent details on our watch instance
self._save_torrent_details(torrent)
# start the torrent
if not settings.DEBUG:
torrent.start()
return True
else:
# remove the blacklisted/paused torrent and continue to the next result
logger_background.info(
'BLACKLISTED: {} ({}) - trying next best result'.
format(best_result['Title'], torrent.hashString))
transmission_client.remove_torrent([torrent.id])
valid_search_results.remove(best_result)
continue
else:
logger_background.info('No valid search results for {}'.format(
self.tmdb_media[self._get_tmdb_title_key()]))
else:
logger_background.info('Search error: {}'.format(
search.error_content))
logger_background.info('Unable to find any results for {}'.format(
self.tmdb_media[self._get_tmdb_title_key()]))
return False
def wanted_tv_season_task():
nefarious_settings = NefariousSettings.get()
tmdb = get_tmdb_client(nefarious_settings)
#
# re-check for requested tv seasons that have had new episodes released from TMDB (which was stale previously)
#
for tv_season_request in WatchTVSeasonRequest.objects.filter(
collected=False):
season_request = tmdb.TV_Seasons(
tv_season_request.watch_tv_show.tmdb_show_id,
tv_season_request.season_number)
season = season_request.info()
now = datetime.utcnow()
last_air_date = parse_date(season.get('air_date')
or '') # season air date
# otherwise add any new episodes to our watch list
for episode in season['episodes']:
episode_air_date = parse_date(episode.get('air_date') or '')
# if episode air date exists, use as last air date
if episode_air_date:
last_air_date = episode_air_date if not last_air_date or episode_air_date > last_air_date else last_air_date
try:
watch_tv_episode, was_created = WatchTVEpisode.objects.get_or_create(
tmdb_episode_id=episode['id'],
defaults=dict(
watch_tv_show=tv_season_request.watch_tv_show,
season_number=tv_season_request.season_number,
episode_number=episode['episode_number'],
user=tv_season_request.user,
release_date=episode_air_date,
))
except IntegrityError as e:
logger_background.exception(e)
logger_background.error(
'Failed creating tmdb episode {} when show {}, season #{} and episode #{} already exist'
.format(episode['id'], tv_season_request.watch_tv_show.id,
tv_season_request.season_number,
episode['episode_number']))
continue
if was_created:
logger_background.info(
'adding newly found episode {} for {}'.format(
episode['episode_number'], tv_season_request))
# queue task to watch episode
watch_tv_episode_task.delay(watch_tv_episode.id)
# assume there's no new episodes for anything that's aired this long ago
days_since_aired = (now.date() -
last_air_date).days if last_air_date else 0
if days_since_aired > 30:
logger_background.warning(
'completing old tv season request {}'.format(
tv_season_request))
tv_season_request.collected = True
tv_season_request.save()
def completed_media_task():
nefarious_settings = NefariousSettings.get()
transmission_client = get_transmission_client(nefarious_settings)
incomplete_kwargs = dict(collected=False,
transmission_torrent_hash__isnull=False)
movies = WatchMovie.objects.filter(**incomplete_kwargs)
tv_seasons = WatchTVSeason.objects.filter(**incomplete_kwargs)
tv_episodes = WatchTVEpisode.objects.filter(**incomplete_kwargs)
incomplete_media = list(movies) + list(tv_episodes) + list(tv_seasons)
for media in incomplete_media:
try:
torrent = transmission_client.get_torrent(
media.transmission_torrent_hash)
except KeyError:
# media's torrent reference no longer exists so remove the reference
logger_background.info(
"Media's torrent no longer present, removing reference: {}".
format(media))
media.transmission_torrent_hash = None
media.save()
else:
# download is complete
if torrent.progress == 100:
# flag media as completed
logger_background.info('Media completed: {}'.format(media))
# special handling for tv seasons
if isinstance(media, WatchTVSeason):
# mark season request complete
for season_request in WatchTVSeasonRequest.objects.filter(
watch_tv_show=media.watch_tv_show,
season_number=media.season_number):
season_request.collected = True
season_request.save()
# get the sub path (ie. "movies/", "tv/') so we can move the data from staging
sub_path = (nefarious_settings.transmission_movie_download_dir
if isinstance(media, WatchMovie) else
nefarious_settings.transmission_tv_download_dir
).lstrip('/')
# get the path and updated name for the data
new_path, new_name = get_media_new_path_and_name(
media, torrent.name,
len(torrent.files()) == 1)
relative_path = os.path.join(
sub_path, # i.e "movies" or "tv"
new_path or '',
)
# move the data to a new location
transmission_session = transmission_client.session_stats()
transmission_move_to_path = os.path.join(
transmission_session.download_dir,
relative_path,
)
logger_background.info('Moving torrent data to "{}"'.format(
transmission_move_to_path))
torrent.move_data(transmission_move_to_path)
# rename the data
logger_background.info(
'Renaming torrent file from "{}" to "{}"'.format(
torrent.name, new_name))
transmission_client.rename_torrent_path(
torrent.id, torrent.name, new_name)
# save media as collected
media.collected = True
media.collected_date = timezone.now()
media.save()
# send websocket message media was updated
media_type, data = websocket.get_media_type_and_serialized_watch_media(
media)
websocket.send_message(websocket.ACTION_UPDATED, media_type,
data)
# send complete message through webhook
webhook.send_message('{} was downloaded'.format(media))
# define the import path
import_path = os.path.join(
settings.INTERNAL_DOWNLOAD_PATH,
relative_path,
# new_path will be None if the torrent is already a directory so fall back to the new name
new_path or new_name,
)
# post-tasks
post_tasks = [
# queue import of media to save the actual media paths
import_library_task.si(
'movie' if isinstance(media, WatchMovie) else
'tv', # media type
media.user_id, # user id
import_path,
),
]
# conditionally add subtitles task to post-tasks
if nefarious_settings.should_save_subtitles() and isinstance(
media, (WatchMovie, WatchTVEpisode)):
post_tasks.append(
download_subtitles_task.si(media_type.lower(),
media.id))
# queue post-tasks
chain(*post_tasks)()
def download(self, watch_media):
# downloads the matching subtitle to the media's path
logger_background.info(
'downloading subtitles for {}'.format(watch_media))
if not watch_media.download_path:
logger_background.warning(
'skipping subtitles for media {} since it does not have a download path populated'
.format(watch_media))
return
if not isinstance(watch_media, (WatchMovie, WatchTVEpisode)):
msg = 'error collecting subtitles for media {}: unknown media type'.format(
watch_media)
logger_background.warning(msg)
raise Exception(msg)
# download subtitle
search_result = self.search(
'movie' if isinstance(watch_media, WatchMovie) else 'episode',
watch_media.tmdb_movie_id if isinstance(watch_media, WatchMovie)
else watch_media.tmdb_episode_id,
watch_media.abs_download_path(),
)
# verify a result was found
if not search_result:
logger_background.warning(
'no valid subtitles found for media {}: {}'.format(
watch_media, self.error_message))
return
# retrieve the file id (guaranteed to have a single file from previous validation)
file_id = search_result['attributes']['files'][0]['file_id']
response = requests.post(
self.API_URL_DOWNLOAD,
data={
'file_id': file_id,
},
headers={
'Api-Key':
self.nefarious_settings.open_subtitles_api_key,
'Authorization':
'Bearer: {}'.format(
self.nefarious_settings.open_subtitles_user_token),
},
timeout=30,
)
# validate
if not response.ok:
logger_background.warning(
'error received from opensubtitles: code={}, message={}'.
format(response.status_code, response.content))
response.raise_for_status()
download_result = response.json()
response = requests.get(download_result['link'], timeout=30)
response.raise_for_status()
logger_background.info('found subtitle {} for {}'.format(
search_result.get('attributes', {}).get('url'),
watch_media,
))
# define subtitle extension
extension = '.srt'
file_extension_match = ParserBase.file_extension_regex.search(
download_result['file_name'])
if file_extension_match:
extension = file_extension_match.group().lower()
# subtitle download path, .ie "movies/The.Movie/The.Movie.srt"
subtitle_path = os.path.join(
os.path.dirname(watch_media.abs_download_path()),
'{name}.{language}{extension}'.format(
name=watch_media,
language=self.nefarious_settings.language,
extension=extension,
))
logger_background.info('downloading subtitle {} to {}'.format(
download_result['file_name'], subtitle_path))
# save subtitle
with open(subtitle_path, 'wb') as fh:
fh.write(response.content)
def completed_media_task():
nefarious_settings = NefariousSettings.get()
transmission_client = get_transmission_client(nefarious_settings)
incomplete_kwargs = dict(collected=False,
transmission_torrent_hash__isnull=False)
movies = WatchMovie.objects.filter(**incomplete_kwargs)
tv_seasons = WatchTVSeason.objects.filter(**incomplete_kwargs)
tv_episodes = WatchTVEpisode.objects.filter(**incomplete_kwargs)
incomplete_media = list(movies) + list(tv_episodes) + list(tv_seasons)
for media in incomplete_media:
try:
torrent = transmission_client.get_torrent(
media.transmission_torrent_hash)
except KeyError:
# media's torrent reference no longer exists so remove the reference
logger_background.info(
"Media's torrent no longer present, removing reference: {}".
format(media))
media.transmission_torrent_hash = None
media.save()
else:
# download is complete
if torrent.progress == 100:
# flag media as completed
logger_background.info('Media completed: {}'.format(media))
media.collected = True
media.collected_date = datetime.utcnow()
media.save()
# special handling for tv seasons
if isinstance(media, WatchTVSeason):
# mark season request complete
for season_request in WatchTVSeasonRequest.objects.filter(
watch_tv_show=media.watch_tv_show,
season_number=media.season_number):
season_request.collected = True
season_request.save()
# get the sub path (ie. "movies/", "tv/') so we can move the data from staging
sub_path = (nefarious_settings.transmission_movie_download_dir
if isinstance(media, WatchMovie) else
nefarious_settings.transmission_tv_download_dir
).lstrip('/')
# get the path and updated name for the data
new_path, new_name = get_media_new_path_and_name(
media, torrent.name,
len(torrent.files()) == 1)
# move the data
transmission_session = transmission_client.session_stats()
move_to_path = os.path.join(
transmission_session.download_dir,
sub_path,
new_path or '',
)
logger_background.info(
'Moving torrent data to "{}"'.format(move_to_path))
torrent.move_data(move_to_path)
# rename the data
logger_background.info(
'Renaming torrent file from "{}" to "{}"'.format(
torrent.name, new_name))
transmission_client.rename_torrent_path(
torrent.id, torrent.name, new_name)
# send websocket message media was updated
media_type, data = websocket.get_media_type_and_serialized_watch_media(
media)
websocket.send_message(websocket.ACTION_UPDATED, media_type,
data)
# send complete message through webhook
webhook.send_message('{} was downloaded'.format(media))
