def ingest_root(self, path): scanned = 0 logger_background.info('Scanning {}'.format(path)) for root, dirs, files in os.walk(path): scanned += len(files) for file in files: self.ingest_path(os.path.join(root, file)) logger_background.info('Scanned {} files'.format(scanned))
def auto_watch_new_seasons_task(): """ look for newly aired seasons that the user wants to automatically watch """ nefarious_settings = NefariousSettings.get() tmdb_client = get_tmdb_client(nefarious_settings) # cycle through every show that has auto-watch enabled for watch_show in WatchTVShow.objects.filter(auto_watch=True): tmdb_show = tmdb_client.TV(watch_show.tmdb_show_id) show_data = tmdb_show.info() added_season = False # find any season with a newer air date than the "auto watch" and queue it up for season in show_data['seasons']: air_date = parse_date(season['air_date'] or '') # air date is newer than our auto watch date if air_date and watch_show.auto_watch_date_updated and air_date >= watch_show.auto_watch_date_updated: # season & request params create_params = dict(watch_tv_show=watch_show, season_number=season['season_number'], defaults=dict( user=watch_show.user, release_date=air_date, )) # create a season request instance to keep up with slowly-released episodes WatchTVSeasonRequest.objects.get_or_create(**create_params) # also save a watch tv season instance to try and download the whole season immediately watch_tv_season, was_season_created = WatchTVSeason.objects.get_or_create( **create_params) # season was created if was_season_created: added_season = True logger_background.info( 'Automatically watching newly aired season {}'.format( watch_tv_season)) # send a websocket message for this new season media_type, data = websocket.get_media_type_and_serialized_watch_media( watch_tv_season) send_websocket_message_task.delay(websocket.ACTION_UPDATED, media_type, data) # create a task to download the whole season (fallback to individual episodes if it fails) watch_tv_show_season_task.delay(watch_tv_season.id) # new season added to show if added_season: # update auto watch date requested watch_show.auto_watch_date_updated = datetime.utcnow().date() watch_show.save()
def refresh_tmdb_configuration(): logger_background.info('Refreshing TMDB Configuration') nefarious_settings = NefariousSettings.get() tmdb_client = get_tmdb_client(nefarious_settings) configuration = tmdb_client.Configuration() nefarious_settings.tmdb_configuration = configuration.info() nefarious_settings.tmdb_languages = configuration.languages() nefarious_settings.save() return nefarious_settings.tmdb_configuration
def watch_tv_show_season_task(watch_tv_season_id: int): processor = WatchTVSeasonProcessor(watch_media_id=watch_tv_season_id) success = processor.fetch() watch_tv_season = get_object_or_404(WatchTVSeason, pk=watch_tv_season_id) # success so update the season request instance as "collected" if success: season_request = WatchTVSeasonRequest.objects.filter( watch_tv_show=watch_tv_season.watch_tv_show, season_number=watch_tv_season.season_number) if season_request.exists(): season_request = season_request.first() season_request.collected = True season_request.save() # failed so delete season instance and fallback to trying individual episodes else: logger_background.info( 'Failed fetching entire season {} - falling back to individual episodes' .format(watch_tv_season)) nefarious_settings = NefariousSettings.get() tmdb = get_tmdb_client(nefarious_settings) season_request = tmdb.TV_Seasons( watch_tv_season.watch_tv_show.tmdb_show_id, watch_tv_season.season_number) season = season_request.info() for episode in season['episodes']: # save individual episode watches watch_tv_episode, was_created = WatchTVEpisode.objects.get_or_create( tmdb_episode_id=episode['id'], # add non-unique constraint fields for the default values defaults=dict( user=watch_tv_season.user, watch_tv_show=watch_tv_season.watch_tv_show, season_number=watch_tv_season.season_number, episode_number=episode['episode_number'], release_date=parse_date(episode.get('air_date') or ''), )) # queue task to watch episode watch_tv_episode_task.delay(watch_tv_episode.id) # remove the "watch season" now that we've requested to fetch all individual episodes watch_tv_season.delete()
def wanted_media_task(): wanted_kwargs = dict(collected=False, transmission_torrent_hash__isnull=True) # # scan for individual watch media # wanted_media_data = { 'movie': { 'query': WatchMovie.objects.filter(**wanted_kwargs), 'task': watch_movie_task, }, 'season': { 'query': WatchTVSeason.objects.filter(**wanted_kwargs), 'task': watch_tv_show_season_task, }, 'episode': { 'query': WatchTVEpisode.objects.filter(**wanted_kwargs), 'task': watch_tv_episode_task, }, } today = timezone.now().date() for media_type, data in wanted_media_data.items(): for media in data['query']: # media has been released (or it's missing it's release date so try anyway) so create a task to try and fetch it if not media.release_date or media.release_date <= today: logger_background.info('Wanted {type}: {media}'.format( type=media_type, media=media)) # queue task for wanted media data['task'].delay(media.id) # media has not been released so skip else: logger_background.info( "Skipping wanted {type} since it hasn't aired yet: {media} " .format(type=media_type, media=media))
def populate_release_dates_task(): logger_background.info('Populating release dates') nefarious_settings = NefariousSettings.get() tmdb_client = get_tmdb_client(nefarious_settings) kwargs = dict(release_date=None) for media in WatchMovie.objects.filter(**kwargs): try: movie_result = tmdb_client.Movies(media.tmdb_movie_id) data = movie_result.info() release_date = parse_date(data.get('release_date', '')) update_media_release_date(media, release_date) except Exception as e: logger_background.exception(e) for media in WatchTVSeason.objects.filter(**kwargs): try: season_result = tmdb_client.TV_Seasons( media.watch_tv_show.tmdb_show_id, media.season_number) data = season_result.info() release_date = parse_date(data.get('air_date', '')) update_media_release_date(media, release_date) except Exception as e: logger_background.exception(e) for media in WatchTVEpisode.objects.filter(**kwargs): try: episode_result = tmdb_client.TV_Episodes( media.watch_tv_show.tmdb_show_id, media.season_number, media.episode_number) data = episode_result.info() release_date = parse_date(data.get('air_date', '')) update_media_release_date(media, release_date) except Exception as e: logger_background.exception(e)
def ingest_path(self, file_path): file_name = os.path.basename(file_path) parser = self._get_parser(file_name) # match if parser.match: file_extension_match = parser.file_extension_regex.search(file_name) if file_extension_match: # skip sample files if parser.sample_file_regex.search(file_name): logger_background.warning('[NO_MATCH_SAMPLE] Not matching sample file "{}"'.format(file_path)) return False title = parser.match['title'] if not title: new_title, parser_match = self._handle_missing_title(parser, file_path) if new_title: title = new_title parser.match.update(parser_match) else: logger_background.warning('[NO_MATCH_TITLE] Could not match file without title "{}"'.format(file_path)) return False file_extension = file_extension_match.group() if file_extension in video_extensions(): if self._is_parser_exact_match(parser): if self.media_class.objects.filter(download_path=file_path).exists(): logger_background.info('[SKIP] skipping already-processed file "{}"'.format(file_path)) return False # get or set tmdb search results for this title in the cache tmdb_results = cache.get(title) if not tmdb_results: try: tmdb_results = self._get_tmdb_search_results(title) except HTTPError: logger_background.error('[ERROR_TMDB] tmdb search exception for title {} on file "{}"'.format(title, file_path)) return False cache.set(title, tmdb_results, 60 * 60) # loop over results for the exact match for tmdb_result in tmdb_results['results']: # normalize titles and see if they match if self._is_result_match_title(parser, tmdb_result, title): watch_media = self._handle_match(parser, tmdb_result, title, file_path) if watch_media: logger_background.info('[MATCH] Saved media "{}" from file "{}"'.format(watch_media, file_path)) return watch_media else: # for/else logger_background.warning('[NO_MATCH_MEDIA] No media match for title "{}" and file "{}"'.format(title, file_path)) else: logger_background.warning('[NO_MATCH_EXACT] No exact title match for title "{}" and file "{}"'.format(title, file_path)) else: logger_background.warning('[NO_MATCH_VIDEO] No valid video file extension for file "{}"'.format(file_path)) else: logger_background.warning('[NO_MATCH_EXTENSION] No file extension for file "{}"'.format(file_path)) else: logger_background.info('[NO_MATCH_UNKNOWN] Unknown match for file "{}"'.format(file_path)) return False
def fetch(self): logger_background.info('Processing request to watch {}'.format( self.watch_media)) valid_search_results = [] search = self._get_search_results() # save this attempt date self.watch_media.last_attempt_date = datetime.utcnow() self.watch_media.save() if search.ok: for result in search.results: if self.is_match(result['Title']): valid_search_results.append(result) else: logger_background.info('Not matched: {}'.format( result['Title'])) if valid_search_results: # trace the "torrent url" (sometimes magnet) in each valid result valid_search_results = self._results_with_valid_urls( valid_search_results) while valid_search_results: logger_background.info('Valid Search Results: {}'.format( len(valid_search_results))) # find the torrent result with the highest weight (i.e seeds) best_result = self._get_best_torrent_result( valid_search_results) transmission_client = get_transmission_client( self.nefarious_settings) transmission_session = transmission_client.session_stats() # add to transmission torrent = transmission_client.add_torrent( best_result['torrent_url'], paused= True, # start paused to we can verify if the torrent has been blacklisted download_dir=self._get_download_dir( transmission_session), ) # verify it's not blacklisted and save & start this torrent if not TorrentBlacklist.objects.filter( hash=torrent.hashString).exists(): logger_background.info('Adding torrent for {}'.format( self.tmdb_media[self._get_tmdb_title_key()])) logger_background.info( 'Added torrent {} with {} seeders'.format( best_result['Title'], best_result['Seeders'])) logger_background.info( 'Starting torrent id: {} and hash {}'.format( torrent.id, torrent.hashString)) # save torrent details on our watch instance self._save_torrent_details(torrent) # start the torrent if not settings.DEBUG: torrent.start() return True else: # remove the blacklisted/paused torrent and continue to the next result logger_background.info( 'BLACKLISTED: {} ({}) - trying next best result'. format(best_result['Title'], torrent.hashString)) transmission_client.remove_torrent([torrent.id]) valid_search_results.remove(best_result) continue else: logger_background.info('No valid search results for {}'.format( self.tmdb_media[self._get_tmdb_title_key()])) else: logger_background.info('Search error: {}'.format( search.error_content)) logger_background.info('Unable to find any results for {}'.format( self.tmdb_media[self._get_tmdb_title_key()])) return False
def wanted_tv_season_task(): nefarious_settings = NefariousSettings.get() tmdb = get_tmdb_client(nefarious_settings) # # re-check for requested tv seasons that have had new episodes released from TMDB (which was stale previously) # for tv_season_request in WatchTVSeasonRequest.objects.filter( collected=False): season_request = tmdb.TV_Seasons( tv_season_request.watch_tv_show.tmdb_show_id, tv_season_request.season_number) season = season_request.info() now = datetime.utcnow() last_air_date = parse_date(season.get('air_date') or '') # season air date # otherwise add any new episodes to our watch list for episode in season['episodes']: episode_air_date = parse_date(episode.get('air_date') or '') # if episode air date exists, use as last air date if episode_air_date: last_air_date = episode_air_date if not last_air_date or episode_air_date > last_air_date else last_air_date try: watch_tv_episode, was_created = WatchTVEpisode.objects.get_or_create( tmdb_episode_id=episode['id'], defaults=dict( watch_tv_show=tv_season_request.watch_tv_show, season_number=tv_season_request.season_number, episode_number=episode['episode_number'], user=tv_season_request.user, release_date=episode_air_date, )) except IntegrityError as e: logger_background.exception(e) logger_background.error( 'Failed creating tmdb episode {} when show {}, season #{} and episode #{} already exist' .format(episode['id'], tv_season_request.watch_tv_show.id, tv_season_request.season_number, episode['episode_number'])) continue if was_created: logger_background.info( 'adding newly found episode {} for {}'.format( episode['episode_number'], tv_season_request)) # queue task to watch episode watch_tv_episode_task.delay(watch_tv_episode.id) # assume there's no new episodes for anything that's aired this long ago days_since_aired = (now.date() - last_air_date).days if last_air_date else 0 if days_since_aired > 30: logger_background.warning( 'completing old tv season request {}'.format( tv_season_request)) tv_season_request.collected = True tv_season_request.save()
def completed_media_task(): nefarious_settings = NefariousSettings.get() transmission_client = get_transmission_client(nefarious_settings) incomplete_kwargs = dict(collected=False, transmission_torrent_hash__isnull=False) movies = WatchMovie.objects.filter(**incomplete_kwargs) tv_seasons = WatchTVSeason.objects.filter(**incomplete_kwargs) tv_episodes = WatchTVEpisode.objects.filter(**incomplete_kwargs) incomplete_media = list(movies) + list(tv_episodes) + list(tv_seasons) for media in incomplete_media: try: torrent = transmission_client.get_torrent( media.transmission_torrent_hash) except KeyError: # media's torrent reference no longer exists so remove the reference logger_background.info( "Media's torrent no longer present, removing reference: {}". format(media)) media.transmission_torrent_hash = None media.save() else: # download is complete if torrent.progress == 100: # flag media as completed logger_background.info('Media completed: {}'.format(media)) # special handling for tv seasons if isinstance(media, WatchTVSeason): # mark season request complete for season_request in WatchTVSeasonRequest.objects.filter( watch_tv_show=media.watch_tv_show, season_number=media.season_number): season_request.collected = True season_request.save() # get the sub path (ie. "movies/", "tv/') so we can move the data from staging sub_path = (nefarious_settings.transmission_movie_download_dir if isinstance(media, WatchMovie) else nefarious_settings.transmission_tv_download_dir ).lstrip('/') # get the path and updated name for the data new_path, new_name = get_media_new_path_and_name( media, torrent.name, len(torrent.files()) == 1) relative_path = os.path.join( sub_path, # i.e "movies" or "tv" new_path or '', ) # move the data to a new location transmission_session = transmission_client.session_stats() transmission_move_to_path = os.path.join( transmission_session.download_dir, relative_path, ) logger_background.info('Moving torrent data to "{}"'.format( transmission_move_to_path)) torrent.move_data(transmission_move_to_path) # rename the data logger_background.info( 'Renaming torrent file from "{}" to "{}"'.format( torrent.name, new_name)) transmission_client.rename_torrent_path( torrent.id, torrent.name, new_name) # save media as collected media.collected = True media.collected_date = timezone.now() media.save() # send websocket message media was updated media_type, data = websocket.get_media_type_and_serialized_watch_media( media) websocket.send_message(websocket.ACTION_UPDATED, media_type, data) # send complete message through webhook webhook.send_message('{} was downloaded'.format(media)) # define the import path import_path = os.path.join( settings.INTERNAL_DOWNLOAD_PATH, relative_path, # new_path will be None if the torrent is already a directory so fall back to the new name new_path or new_name, ) # post-tasks post_tasks = [ # queue import of media to save the actual media paths import_library_task.si( 'movie' if isinstance(media, WatchMovie) else 'tv', # media type media.user_id, # user id import_path, ), ] # conditionally add subtitles task to post-tasks if nefarious_settings.should_save_subtitles() and isinstance( media, (WatchMovie, WatchTVEpisode)): post_tasks.append( download_subtitles_task.si(media_type.lower(), media.id)) # queue post-tasks chain(*post_tasks)()
def download(self, watch_media): # downloads the matching subtitle to the media's path logger_background.info( 'downloading subtitles for {}'.format(watch_media)) if not watch_media.download_path: logger_background.warning( 'skipping subtitles for media {} since it does not have a download path populated' .format(watch_media)) return if not isinstance(watch_media, (WatchMovie, WatchTVEpisode)): msg = 'error collecting subtitles for media {}: unknown media type'.format( watch_media) logger_background.warning(msg) raise Exception(msg) # download subtitle search_result = self.search( 'movie' if isinstance(watch_media, WatchMovie) else 'episode', watch_media.tmdb_movie_id if isinstance(watch_media, WatchMovie) else watch_media.tmdb_episode_id, watch_media.abs_download_path(), ) # verify a result was found if not search_result: logger_background.warning( 'no valid subtitles found for media {}: {}'.format( watch_media, self.error_message)) return # retrieve the file id (guaranteed to have a single file from previous validation) file_id = search_result['attributes']['files'][0]['file_id'] response = requests.post( self.API_URL_DOWNLOAD, data={ 'file_id': file_id, }, headers={ 'Api-Key': self.nefarious_settings.open_subtitles_api_key, 'Authorization': 'Bearer: {}'.format( self.nefarious_settings.open_subtitles_user_token), }, timeout=30, ) # validate if not response.ok: logger_background.warning( 'error received from opensubtitles: code={}, message={}'. format(response.status_code, response.content)) response.raise_for_status() download_result = response.json() response = requests.get(download_result['link'], timeout=30) response.raise_for_status() logger_background.info('found subtitle {} for {}'.format( search_result.get('attributes', {}).get('url'), watch_media, )) # define subtitle extension extension = '.srt' file_extension_match = ParserBase.file_extension_regex.search( download_result['file_name']) if file_extension_match: extension = file_extension_match.group().lower() # subtitle download path, .ie "movies/The.Movie/The.Movie.srt" subtitle_path = os.path.join( os.path.dirname(watch_media.abs_download_path()), '{name}.{language}{extension}'.format( name=watch_media, language=self.nefarious_settings.language, extension=extension, )) logger_background.info('downloading subtitle {} to {}'.format( download_result['file_name'], subtitle_path)) # save subtitle with open(subtitle_path, 'wb') as fh: fh.write(response.content)
def completed_media_task(): nefarious_settings = NefariousSettings.get() transmission_client = get_transmission_client(nefarious_settings) incomplete_kwargs = dict(collected=False, transmission_torrent_hash__isnull=False) movies = WatchMovie.objects.filter(**incomplete_kwargs) tv_seasons = WatchTVSeason.objects.filter(**incomplete_kwargs) tv_episodes = WatchTVEpisode.objects.filter(**incomplete_kwargs) incomplete_media = list(movies) + list(tv_episodes) + list(tv_seasons) for media in incomplete_media: try: torrent = transmission_client.get_torrent( media.transmission_torrent_hash) except KeyError: # media's torrent reference no longer exists so remove the reference logger_background.info( "Media's torrent no longer present, removing reference: {}". format(media)) media.transmission_torrent_hash = None media.save() else: # download is complete if torrent.progress == 100: # flag media as completed logger_background.info('Media completed: {}'.format(media)) media.collected = True media.collected_date = datetime.utcnow() media.save() # special handling for tv seasons if isinstance(media, WatchTVSeason): # mark season request complete for season_request in WatchTVSeasonRequest.objects.filter( watch_tv_show=media.watch_tv_show, season_number=media.season_number): season_request.collected = True season_request.save() # get the sub path (ie. "movies/", "tv/') so we can move the data from staging sub_path = (nefarious_settings.transmission_movie_download_dir if isinstance(media, WatchMovie) else nefarious_settings.transmission_tv_download_dir ).lstrip('/') # get the path and updated name for the data new_path, new_name = get_media_new_path_and_name( media, torrent.name, len(torrent.files()) == 1) # move the data transmission_session = transmission_client.session_stats() move_to_path = os.path.join( transmission_session.download_dir, sub_path, new_path or '', ) logger_background.info( 'Moving torrent data to "{}"'.format(move_to_path)) torrent.move_data(move_to_path) # rename the data logger_background.info( 'Renaming torrent file from "{}" to "{}"'.format( torrent.name, new_name)) transmission_client.rename_torrent_path( torrent.id, torrent.name, new_name) # send websocket message media was updated media_type, data = websocket.get_media_type_and_serialized_watch_media( media) websocket.send_message(websocket.ACTION_UPDATED, media_type, data) # send complete message through webhook webhook.send_message('{} was downloaded'.format(media))