def test_is_media_file(self): """ Test is_media_file """ # TODO: Add unicode tests # TODO: Add MAC OS resource fork tests # TODO: Add RARBG release tests # RARBG release intros should be ignored # MAC OS's "resource fork" files should be ignored # Extras should be ignored # and the file extension should be in the list of media extensions # Test all valid media extensions temp_name = 'Show.Name.S01E01.HDTV.x264-RLSGROUP' extension_tests = {'.'.join((temp_name, ext)): True for ext in MEDIA_EXTENSIONS} # ...and some invalid ones other_extensions = ['txt', 'sfv', 'srr', 'rar', 'nfo', 'zip'] extension_tests.update({'.'.join((temp_name, ext)): False for ext in other_extensions + SUBTITLE_EXTENSIONS}) # Samples should be ignored sample_tests = { # Samples should be ignored, valid samples will return False 'Show.Name.S01E01.HDTV.sample.mkv': False, # default case 'Show.Name.S01E01.HDTV.sAmPle.mkv': False, # Ignore case 'Show.Name.S01E01.HDTV.samples.mkv': True, # sample should not be plural 'Show.Name.S01E01.HDTVsample.mkv': True, # no separation, can't identify as sample 'Sample.Show.Name.S01E01.HDTV.mkv': False, # location doesn't matter 'Show.Name.Sample.S01E01.HDTV.sample.mkv': False, # location doesn't matter 'Show.Name.S01E01.HDTV.sample1.mkv': False, # numbered samples are ok 'Show.Name.S01E01.HDTV.sample12.mkv': False, # numbered samples are ok 'Show.Name.S01E01.HDTV.sampleA.mkv': True, # samples should not be indexed alphabetically 'RARBG.mp4': False, 'rarbg.MP4': False, '/TV/Sample.Show.Name.S01E01.HDTV-RARBG/RARBG.mp4': False } edge_cases = { None: False, '': False, 0: False, 1: False, 42: False, 123189274981274: False, 12.23: False, ('this', 'is', 'a tuple'): False, } for cur_test in extension_tests, sample_tests, edge_cases: for cur_name, expected_result in six.iteritems(cur_test): self.assertEqual(helpers.is_media_file(cur_name), expected_result, cur_name)
def process_dir(process_path, release_name=None, process_method=None, force=False, is_priority=None, delete_on=False, failed=False, mode="auto"): """ Scans through the files in process_path and processes whatever media files it finds :param process_path: The folder name to look in :param release_name: The NZB/Torrent name which resulted in this folder being downloaded :param process_method: processing method, copy/move/symlink/link :param force: True to process previously processed files :param is_priority: whether to replace the file even if it exists at higher quality :param delete_on: delete files and folders after they are processed (always happens with move and auto combination) :param failed: Boolean for whether or not the download failed :param mode: Type of postprocessing auto or manual """ result = ProcessResult() # if they passed us a real dir then assume it's the one we want if ek(os.path.isdir, process_path): process_path = ek(os.path.realpath, process_path) result.output += log_helper("Processing in folder {0}".format(process_path), logger.DEBUG) # if the client and SickRage are not on the same machine translate the directory into a network directory elif all([sickbeard.TV_DOWNLOAD_DIR, ek(os.path.isdir, sickbeard.TV_DOWNLOAD_DIR), ek(os.path.normpath, process_path) == ek(os.path.normpath, sickbeard.TV_DOWNLOAD_DIR)]): process_path = ek(os.path.join, sickbeard.TV_DOWNLOAD_DIR, ek(os.path.abspath, process_path).split(os.path.sep)[-1]) result.output += log_helper("Trying to use folder: {0} ".format(process_path), logger.DEBUG) # if we didn't find a real dir then quit if not ek(os.path.isdir, process_path): result.output += log_helper("Unable to figure out what folder to process. " "If your downloader and SickRage aren't on the same PC " "make sure you fill out your TV download dir in the config.", logger.DEBUG) return result.output process_method = process_method or sickbeard.PROCESS_METHOD directories_from_rars = set() # If we have a release name (probably from nzbToMedia), and it is a rar/video, only process that file if release_name and (helpers.is_media_file(release_name) or helpers.is_rar_file(release_name)): result.output += log_helper("Processing {}".format(release_name), logger.INFO) generator_to_use = [(process_path, [], [release_name])] else: result.output += log_helper("Processing {}".format(process_path), logger.INFO) generator_to_use = ek(os.walk, process_path, followlinks=sickbeard.PROCESSOR_FOLLOW_SYMLINKS) for current_directory, directory_names, file_names in generator_to_use: result.result = True file_names = [f for f in file_names if not is_torrent_or_nzb_file(f)] rar_files = [x for x in file_names if helpers.is_rar_file(ek(os.path.join, current_directory, x))] if rar_files: extracted_directories = unrar(current_directory, rar_files, force, result) if extracted_directories: for extracted_directory in extracted_directories: if extracted_directory.split(current_directory)[-1] not in directory_names: result.output += log_helper( "Adding extracted directory to the list of directories to process: {0}".format(extracted_directory), logger.DEBUG ) directories_from_rars.add(extracted_directory) if not validate_dir(current_directory, release_name, failed, result): continue video_files = filter(helpers.is_media_file, file_names) if video_files: process_media(current_directory, video_files, release_name, process_method, force, is_priority, result) else: result.result = False # Delete all file not needed and avoid deleting files if Manual PostProcessing if not(process_method == "move" and result.result) or (mode == "manual" and not delete_on): continue # noinspection PyTypeChecker unwanted_files = filter(lambda x: x in video_files + rar_files, file_names) if unwanted_files: result.output += log_helper("Found unwanted files: {0}".format(unwanted_files), logger.DEBUG) delete_folder(ek(os.path.join, current_directory, '@eaDir'), False) delete_files(current_directory, unwanted_files, result) if delete_folder(current_directory, check_empty=not delete_on): result.output += log_helper("Deleted folder: {0}".format(current_directory), logger.DEBUG) # For processing extracted rars, only allow methods 'move' and 'copy'. # On different methods fall back to 'move'. method_fallback = ('move', process_method)[process_method in ('move', 'copy')] # auto post-processing deletes rar content by default if method is 'move', # sickbeard.DELRARCONTENTS allows to override even if method is NOT 'move' # manual post-processing will only delete when prompted by delete_on delete_rar_contents = any([sickbeard.DELRARCONTENTS and mode != 'manual', not sickbeard.DELRARCONTENTS and mode == 'auto' and method_fallback == 'move', mode == 'manual' and delete_on]) for directory_from_rar in directories_from_rars: process_dir( process_path=directory_from_rar, release_name=ek(os.path.basename, directory_from_rar), process_method=method_fallback, force=force, is_priority=is_priority, delete_on=delete_rar_contents, failed=failed, mode=mode ) # Delete rar file only if the extracted dir was successfully processed if mode == 'auto' and method_fallback == 'move' or mode == 'manual' and delete_on: this_rar = [rar_file for rar_file in rar_files if os.path.basename(directory_from_rar) == rar_file.rpartition('.')[0]] delete_files(current_directory, this_rar, result) # Deletes only if result.result == True result.output += log_helper(("Processing Failed", "Successfully processed")[result.aggresult], (logger.WARNING, logger.INFO)[result.aggresult]) if result.missed_files: result.output += log_helper("Some items were not processed.") for missed_file in result.missed_files: result.output += log_helper(missed_file) return result.output
def download_subtitles(episode, force_lang=None): # pylint: disable=too-many-locals, too-many-branches, too-many-statements existing_subtitles = episode.subtitles if not needs_subtitles(existing_subtitles, force_lang): logger.log( 'Episode already has all needed subtitles, skipping {0} {1}'. format( episode.show.name, episode_num(episode.season, episode.episode) or episode_num( episode.season, episode.episode, numbering='absolute')), logger.DEBUG) return existing_subtitles, None if not force_lang: languages = get_needed_languages(existing_subtitles) else: languages = {from_code(force_lang)} if not languages: logger.log( 'No subtitles needed for {0} {1}'.format( episode.show.name, episode_num(episode.season, episode.episode) or episode_num( episode.season, episode.episode, numbering='absolute')), logger.DEBUG) return existing_subtitles, None subtitles_path = get_subtitles_path(episode.location) video_path = episode.location # Perfect match = hash score - hearing impaired score - resolution score # (subtitle for 720p is the same as for 1080p) # Perfect match = 215 - 1 - 1 = 213 # Non-perfect match = series + year + season + episode # Non-perfect match = 108 + 54 + 18 + 18 = 198 # From latest subliminal code: # episode_scores = {'hash': 215, 'series': 108, 'year': 54, 'season': 18, 'episode': 18, 'release_group': 9, # 'format': 4, 'audio_codec': 2, 'resolution': 1, 'hearing_impaired': 1, 'video_codec': 1} user_score = 213 if sickbeard.SUBTITLES_PERFECT_MATCH else 198 video = get_video(video_path, subtitles_path=subtitles_path, episode=episode) if not video: logger.log( 'Exception caught in subliminal.scan_video for {0} {1}'.format( episode.show.name, episode_num(episode.season, episode.episode) or episode_num( episode.season, episode.episode, numbering='absolute')), logger.DEBUG) return existing_subtitles, None providers = enabled_service_list() pool = SubtitleProviderPool() try: subtitles_list = pool.list_subtitles(video, languages) for provider in providers: if provider in pool.discarded_providers: logger.log( 'Could not search in {0} provider. Discarding for now'. format(provider), logger.DEBUG) if not subtitles_list: logger.log( 'No subtitles found for {0} {1}'.format( episode.show.name, episode_num(episode.season, episode.episode) or episode_num(episode.season, episode.episode, numbering='absolute')), logger.DEBUG) return existing_subtitles, None for subtitle in subtitles_list: score = subliminal.score.compute_score( subtitle, video, hearing_impaired=sickbeard.SUBTITLES_HEARING_IMPAIRED) logger.log( '[{0}] Subtitle score for {1} is: {2} (min={3})'.format( subtitle.provider_name, subtitle.id, score, user_score), logger.DEBUG) found_subtitles = pool.download_best_subtitles( subtitles_list, video, languages=languages, hearing_impaired=sickbeard.SUBTITLES_HEARING_IMPAIRED, min_score=user_score, only_one=not sickbeard.SUBTITLES_MULTI) subliminal.save_subtitles(video, found_subtitles, directory=subtitles_path, single=not sickbeard.SUBTITLES_MULTI) except IOError as error: if 'No space left on device' in ex(error): logger.log('Not enough space on the drive to save subtitles', logger.WARNING) else: logger.log(traceback.format_exc(), logger.WARNING) except Exception: logger.log('Error occurred when downloading subtitles for: {0}'.format( video_path)) logger.log(traceback.format_exc(), logger.ERROR) return existing_subtitles, None for subtitle in found_subtitles: subtitle_path = subliminal.subtitle.get_subtitle_path( video.name, None if not sickbeard.SUBTITLES_MULTI else subtitle.language) if subtitles_path is not None: subtitle_path = os.path.join(subtitles_path, os.path.split(subtitle_path)[1]) sickbeard.helpers.chmodAsParent(subtitle_path) sickbeard.helpers.fixSetGroupID(subtitle_path) if sickbeard.SUBTITLES_HISTORY: logger.log( 'history.logSubtitle {0}, {1}'.format( subtitle.provider_name, subtitle.language.opensubtitles), logger.DEBUG) history.logSubtitle(episode.show.indexerid, episode.season, episode.episode, episode.status, subtitle) if sickbeard.SUBTITLES_EXTRA_SCRIPTS and is_media_file( video_path) and not sickbeard.EMBEDDED_SUBTITLES_ALL: run_subs_extra_scripts(episode, subtitle, video, single=not sickbeard.SUBTITLES_MULTI) new_subtitles = sorted( {subtitle.language.opensubtitles for subtitle in found_subtitles}) current_subtitles = sorted( {subtitle for subtitle in new_subtitles + existing_subtitles}) if existing_subtitles else new_subtitles if not sickbeard.SUBTITLES_MULTI and len(found_subtitles) == 1: new_code = found_subtitles[0].language.opensubtitles if new_code not in existing_subtitles: current_subtitles.remove(new_code) current_subtitles.append('und') return current_subtitles, new_subtitles
def download_subtitles(episode, force_lang=None): # pylint: disable=too-many-locals, too-many-branches, too-many-statements existing_subtitles = episode.subtitles if not needs_subtitles(existing_subtitles, force_lang): logger.log('Episode already has all needed subtitles, skipping {0} {1}'.format (episode.show.name, episode_num(episode.season, episode.episode) or episode_num(episode.season, episode.episode, numbering='absolute')), logger.DEBUG) return existing_subtitles, None if not force_lang: languages = get_needed_languages(existing_subtitles) else: languages = {from_code(force_lang)} if not languages: logger.log('No subtitles needed for {0} {1}'.format (episode.show.name, episode_num(episode.season, episode.episode) or episode_num(episode.season, episode.episode, numbering='absolute')), logger.DEBUG) return existing_subtitles, None subtitles_path = get_subtitles_path(episode.location) video_path = episode.location # Perfect match = hash score - hearing impaired score - resolution score # (subtitle for 720p is the same as for 1080p) # Perfect match = 215 - 1 - 1 = 213 # Non-perfect match = series + year + season + episode # Non-perfect match = 108 + 54 + 18 + 18 = 198 # From latest subliminal code: # episode_scores = {'hash': 215, 'series': 108, 'year': 54, 'season': 18, 'episode': 18, 'release_group': 9, # 'format': 4, 'audio_codec': 2, 'resolution': 1, 'hearing_impaired': 1, 'video_codec': 1} user_score = 213 if sickbeard.SUBTITLES_PERFECT_MATCH else 198 video = get_video(video_path, subtitles_path=subtitles_path, episode=episode) if not video: logger.log('Exception caught in subliminal.scan_video for {0} {1}'.format (episode.show.name, episode_num(episode.season, episode.episode) or episode_num(episode.season, episode.episode, numbering='absolute')), logger.DEBUG) return existing_subtitles, None providers = enabled_service_list() pool = SubtitleProviderPool() try: subtitles_list = pool.list_subtitles(video, languages) for provider in providers: if provider in pool.discarded_providers: logger.log('Could not search in {0} provider. Discarding for now'.format(provider), logger.DEBUG) if not subtitles_list: logger.log('No subtitles found for {0} {1}'.format (episode.show.name, episode_num(episode.season, episode.episode) or episode_num(episode.season, episode.episode, numbering='absolute')), logger.DEBUG) return existing_subtitles, None for subtitle in subtitles_list: score = subliminal.score.compute_score(subtitle, video, hearing_impaired=sickbeard.SUBTITLES_HEARING_IMPAIRED) logger.log('[{0}] Subtitle score for {1} is: {2} (min={3})'.format (subtitle.provider_name, subtitle.id, score, user_score), logger.DEBUG) found_subtitles = pool.download_best_subtitles(subtitles_list, video, languages=languages, hearing_impaired=sickbeard.SUBTITLES_HEARING_IMPAIRED, min_score=user_score, only_one=not sickbeard.SUBTITLES_MULTI) subliminal.save_subtitles(video, found_subtitles, directory=subtitles_path, single=not sickbeard.SUBTITLES_MULTI) except IOError as error: if 'No space left on device' in ex(error): logger.log('Not enough space on the drive to save subtitles', logger.WARNING) else: logger.log(traceback.format_exc(), logger.WARNING) except Exception: logger.log('Error occurred when downloading subtitles for: {0}'.format(video_path)) logger.log(traceback.format_exc(), logger.ERROR) return existing_subtitles, None for subtitle in found_subtitles: subtitle_path = subliminal.subtitle.get_subtitle_path(video.name, None if not sickbeard.SUBTITLES_MULTI else subtitle.language) if subtitles_path is not None: subtitle_path = os.path.join(subtitles_path, os.path.split(subtitle_path)[1]) sickbeard.helpers.chmodAsParent(subtitle_path) sickbeard.helpers.fixSetGroupID(subtitle_path) if sickbeard.SUBTITLES_HISTORY: logger.log('history.logSubtitle {0}, {1}'.format (subtitle.provider_name, subtitle.language.opensubtitles), logger.DEBUG) history.logSubtitle(episode.show.indexerid, episode.season, episode.episode, episode.status, subtitle) if sickbeard.SUBTITLES_EXTRA_SCRIPTS and is_media_file(video_path) and not sickbeard.EMBEDDED_SUBTITLES_ALL: run_subs_extra_scripts(episode, subtitle, video, single=not sickbeard.SUBTITLES_MULTI) new_subtitles = sorted({subtitle.language.opensubtitles for subtitle in found_subtitles}) current_subtitles = sorted({subtitle for subtitle in new_subtitles + existing_subtitles}) if existing_subtitles else new_subtitles if not sickbeard.SUBTITLES_MULTI and len(found_subtitles) == 1: new_code = found_subtitles[0].language.opensubtitles if new_code not in existing_subtitles: current_subtitles.remove(new_code) current_subtitles.append('und') return current_subtitles, new_subtitles