def test_is_media_file(self):
"""
Test isMediaFile
"""
# TODO: Add unicode tests
# TODO: Add MAC OS resource fork tests
# TODO: Add RARBG release tests
# RARBG release intros should be ignored
# MAC OS's "resource fork" files should be ignored
# Extras should be ignored
# and the file extension should be in the list of media extensions
# Test all valid media extensions
temp_name = 'Show.Name.S01E01.HDTV.x264-RLSGROUP'
extension_tests = {
'.'.join((temp_name, ext)): True
for ext in media_extensions
}
# ...and some invalid ones
other_extensions = ['txt', 'sfv', 'srr', 'rar', 'nfo', 'zip']
extension_tests.update({
'.'.join((temp_name, ext)): False
for ext in other_extensions + subtitle_extensions
})
# Samples should be ignored
sample_tests = { # Samples should be ignored, valid samples will return False
'Show.Name.S01E01.HDTV.sample.mkv': False, # default case
'Show.Name.S01E01.HDTV.sAmPle.mkv': False, # Ignore case
'Show.Name.S01E01.HDTV.samples.mkv':
True, # sample should not be plural
'Show.Name.S01E01.HDTVsample.mkv':
True, # no separation, can't identify as sample
'Sample.Show.Name.S01E01.HDTV.mkv':
False, # location doesn't matter
'Show.Name.Sample.S01E01.HDTV.sample.mkv':
False, # location doesn't matter
'Show.Name.S01E01.HDTV.sample1.mkv':
False, # numbered samples are ok
'Show.Name.S01E01.HDTV.sample12.mkv':
False, # numbered samples are ok
'Show.Name.S01E01.HDTV.sampleA.mkv':
True, # samples should not be indexed alphabetically
}
edge_cases = {
None: False,
'': False,
0: False,
1: False,
42: False,
123189274981274: False,
12.23: False,
('this', 'is', 'a tuple'): False,
}
for cur_test in extension_tests, sample_tests, edge_cases:
for cur_name, expected_result in cur_test.items():
self.assertEqual(helpers.isMediaFile(cur_name),
expected_result, cur_name)
def _torrent_has_any_media_files(torrent_info):
"""
Internal function to check if a torrent has any useful media files.
@param torrent_info: (a libtorrent torrent_info object)
@return: (bool) True if any useful media is found, false otherwise.
"""
for f in torrent_info.files():
if isMediaFile(f.path):
return True
return False
def _torrent_has_any_media_files(torrent_info):
"""
Internal function to check if a torrent has any useful media files.
@param torrent_info: (a libtorrent torrent_info object)
@return: (bool) True if any useful media is found, false otherwise.
"""
for f in torrent_info.files():
if isMediaFile(f.path):
return True
return False
def test_is_media_file(self):
"""
Test isMediaFile
"""
# TODO: Add unicode tests
# TODO: Add MAC OS resource fork tests
# TODO: Add RARBG release tests
# RARBG release intros should be ignored
# MAC OS's "resource fork" files should be ignored
# Extras should be ignored
# and the file extension should be in the list of media extensions
# Test all valid media extensions
temp_name = 'Show.Name.S01E01.HDTV.x264-RLSGROUP'
extension_tests = {'.'.join((temp_name, ext)): True for ext in MEDIA_EXTENSIONS}
# ...and some invalid ones
other_extensions = ['txt', 'sfv', 'srr', 'rar', 'nfo', 'zip']
extension_tests.update({'.'.join((temp_name, ext)): False for ext in other_extensions + SUBTITLE_EXTENSIONS})
# Samples should be ignored
sample_tests = { # Samples should be ignored, valid samples will return False
'Show.Name.S01E01.HDTV.sample.mkv': False, # default case
'Show.Name.S01E01.HDTV.sAmPle.mkv': False, # Ignore case
'Show.Name.S01E01.HDTV.samples.mkv': True, # sample should not be plural
'Show.Name.S01E01.HDTVsample.mkv': True, # no separation, can't identify as sample
'Sample.Show.Name.S01E01.HDTV.mkv': False, # location doesn't matter
'Show.Name.Sample.S01E01.HDTV.sample.mkv': False, # location doesn't matter
'Show.Name.S01E01.HDTV.sample1.mkv': False, # numbered samples are ok
'Show.Name.S01E01.HDTV.sample12.mkv': False, # numbered samples are ok
'Show.Name.S01E01.HDTV.sampleA.mkv': True, # samples should not be indexed alphabetically
'RARBG.mp4': False,
'rarbg.MP4': False
}
edge_cases = {
None: False,
'': False,
0: False,
1: False,
42: False,
123189274981274: False,
12.23: False,
('this', 'is', 'a tuple'): False,
}
for cur_test in extension_tests, sample_tests, edge_cases:
for cur_name, expected_result in cur_test.items():
self.assertEqual(helpers.isMediaFile(cur_name), expected_result, cur_name)
def subtitles_download_in_pp(): # pylint: disable=too-many-locals, too-many-branches
logger.log(u'Checking for needed subtitles in Post-Process folder', logger.INFO)
providers = enabled_service_list()
provider_configs = {'addic7ed': {'username': sickbeard.ADDIC7ED_USER,
'password': sickbeard.ADDIC7ED_PASS},
'legendastv': {'username': sickbeard.LEGENDASTV_USER,
'password': sickbeard.LEGENDASTV_PASS},
'opensubtitles': {'username': sickbeard.OPENSUBTITLES_USER,
'password': sickbeard.OPENSUBTITLES_PASS}}
pool = subliminal.api.ProviderPool(providers=providers, provider_configs=provider_configs)
# Search for all wanted languages
languages = {from_code(language) for language in wanted_languages()}
if not languages:
return
run_post_process = False
# Check if PP folder is set
if sickbeard.TV_DOWNLOAD_DIR and ek(os.path.isdir, sickbeard.TV_DOWNLOAD_DIR):
for root, _, files in ek(os.walk, sickbeard.TV_DOWNLOAD_DIR, topdown=False):
for video_filename in sorted(files):
try:
# Remove non release groups from video file. Needed to match subtitles
new_video_filename = remove_non_release_groups(video_filename)
if new_video_filename != video_filename:
os.rename(video_filename, new_video_filename)
video_filename = new_video_filename
except Exception as error:
logger.log(u'Could not remove non release groups from video file. Error: %r'
% ex(error), logger.DEBUG)
if isMediaFile(video_filename):
try:
video = subliminal.scan_video(os.path.join(root, video_filename),
subtitles=False, embedded_subtitles=False)
subtitles_list = pool.list_subtitles(video, languages)
if not subtitles_list:
logger.log(u'No subtitles found for %s'
% ek(os.path.join, root, video_filename), logger.DEBUG)
continue
logger.log(u'Found subtitle(s) canditate(s) for %s' % video_filename, logger.INFO)
hearing_impaired = sickbeard.SUBTITLES_HEARING_IMPAIRED
user_score = 132 if sickbeard.SUBTITLES_PERFECT_MATCH else 111
found_subtitles = pool.download_best_subtitles(subtitles_list, video, languages=languages,
hearing_impaired=hearing_impaired,
min_score=user_score,
only_one=not sickbeard.SUBTITLES_MULTI)
for subtitle in subtitles_list:
matches = subtitle.get_matches(video, hearing_impaired=False)
score = subliminal.subtitle.compute_score(matches, video)
logger.log(u"[%s] Subtitle score for %s is: %s (min=%s)"
% (subtitle.provider_name, subtitle.id, score, user_score), logger.DEBUG)
downloaded_languages = set()
for subtitle in found_subtitles:
logger.log(u"Found subtitle for %s in %s provider with language %s"
% (os.path.join(root, video_filename), subtitle.provider_name,
subtitle.language.opensubtitles), logger.DEBUG)
subliminal.save_subtitles(video, found_subtitles, directory=root,
single=not sickbeard.SUBTITLES_MULTI)
subtitles_multi = not sickbeard.SUBTITLES_MULTI
subtitle_path = subliminal.subtitle.get_subtitle_path(video.name,
None if subtitles_multi else
subtitle.language)
if root is not None:
subtitle_path = ek(os.path.join, root, ek(os.path.split, subtitle_path)[1])
sickbeard.helpers.chmodAsParent(subtitle_path)
sickbeard.helpers.fixSetGroupID(subtitle_path)
downloaded_languages.add(subtitle.language.opensubtitles)
# Don't run post processor unless at least one file has all of the needed subtitles
if not needs_subtitles(downloaded_languages):
run_post_process = True
except Exception as error:
logger.log(u"Error occurred when downloading subtitles for: %s. Error: %r"
% (os.path.join(root, video_filename), ex(error)))
if run_post_process:
logger.log(u"Starting post-process with default settings now that we found subtitles")
processTV.processDir(sickbeard.TV_DOWNLOAD_DIR)
def download_subtitles(subtitles_info): # pylint: disable=too-many-locals, too-many-branches, too-many-statements
existing_subtitles = subtitles_info['subtitles']
if not needs_subtitles(existing_subtitles):
logger.log(u'Episode already has all needed subtitles, skipping %s S%02dE%02d'
% (subtitles_info['show_name'], subtitles_info['season'], subtitles_info['episode']), logger.DEBUG)
return existing_subtitles, None
# Check if we really need subtitles
languages = get_needed_languages(existing_subtitles)
if not languages:
logger.log(u'No subtitles needed for %s S%02dE%02d'
% (subtitles_info['show_name'], subtitles_info['season'],
subtitles_info['episode']), logger.DEBUG)
return existing_subtitles, None
subtitles_path = get_subtitles_path(subtitles_info['location']).encode(sickbeard.SYS_ENCODING)
video_path = subtitles_info['location'].encode(sickbeard.SYS_ENCODING)
user_score = 132 if sickbeard.SUBTITLES_PERFECT_MATCH else 111
video = get_video(video_path, subtitles_path=subtitles_path)
if not video:
logger.log(u'Exception caught in subliminal.scan_video for %s S%02dE%02d'
% (subtitles_info['show_name'], subtitles_info['season'],
subtitles_info['episode']), logger.DEBUG)
return existing_subtitles, None
providers = enabled_service_list()
provider_configs = {'addic7ed': {'username': sickbeard.ADDIC7ED_USER,
'password': sickbeard.ADDIC7ED_PASS},
'legendastv': {'username': sickbeard.LEGENDASTV_USER,
'password': sickbeard.LEGENDASTV_PASS},
'opensubtitles': {'username': sickbeard.OPENSUBTITLES_USER,
'password': sickbeard.OPENSUBTITLES_PASS}}
pool = subliminal.api.ProviderPool(providers=providers, provider_configs=provider_configs)
try:
subtitles_list = pool.list_subtitles(video, languages)
if not subtitles_list:
logger.log(u'No subtitles found for %s S%02dE%02d on any provider'
% (subtitles_info['show_name'], subtitles_info['season'],
subtitles_info['episode']), logger.DEBUG)
return existing_subtitles, None
for subtitle in subtitles_list:
matches = subtitle.get_matches(video, hearing_impaired=False)
score = subliminal.subtitle.compute_score(matches, video)
logger.log(u"[%s] Subtitle score for %s is: %s (min=%s)"
% (subtitle.provider_name, subtitle.id, score, user_score), logger.DEBUG)
found_subtitles = pool.download_best_subtitles(subtitles_list, video, languages=languages,
hearing_impaired=sickbeard.SUBTITLES_HEARING_IMPAIRED,
min_score=user_score, only_one=not sickbeard.SUBTITLES_MULTI)
subliminal.save_subtitles(video, found_subtitles, directory=subtitles_path,
single=not sickbeard.SUBTITLES_MULTI)
except IOError as error:
if 'No space left on device' in ex(error):
logger.log(u'Not enough space on the drive to save subtitles', logger.WARNING)
else:
logger.log(traceback.format_exc(), logger.WARNING)
except Exception:
logger.log(u"Error occurred when downloading subtitles for: %s" % video_path)
logger.log(traceback.format_exc(), logger.ERROR)
return existing_subtitles, None
for subtitle in found_subtitles:
subtitle_path = subliminal.subtitle.get_subtitle_path(video.name,
None if not sickbeard.SUBTITLES_MULTI else
subtitle.language)
if subtitles_path is not None:
subtitle_path = ek(os.path.join, subtitles_path, ek(os.path.split, subtitle_path)[1])
sickbeard.helpers.chmodAsParent(subtitle_path)
sickbeard.helpers.fixSetGroupID(subtitle_path)
if sickbeard.SUBTITLES_HISTORY:
logger.log(u'history.logSubtitle %s, %s' %
(subtitle.provider_name, subtitle.language.opensubtitles), logger.DEBUG)
history.logSubtitle(subtitles_info['show_indexerid'], subtitles_info['season'],
subtitles_info['episode'], subtitles_info['status'], subtitle)
if sickbeard.SUBTITLES_EXTRA_SCRIPTS and isMediaFile(video_path) and not sickbeard.EMBEDDED_SUBTITLES_ALL:
run_subs_extra_scripts(subtitles_info, subtitle, video, single=not sickbeard.SUBTITLES_MULTI)
new_subtitles = sorted({subtitle.language.opensubtitles for subtitle in found_subtitles})
current_subtitles = sorted({subtitle for subtitle in new_subtitles + existing_subtitles}) if existing_subtitles else new_subtitles
if not sickbeard.SUBTITLES_MULTI and len(found_subtitles) == 1:
new_code = found_subtitles[0].language.opensubtitles
if new_code not in existing_subtitles:
current_subtitles.remove(new_code)
current_subtitles.append('und')
return current_subtitles, new_subtitles
def subtitles_download_in_pp(): # pylint: disable=too-many-locals, too-many-branches
logger.log(u'Checking for needed subtitles in Post-Process folder', logger.INFO)
providers = enabled_service_list()
provider_configs = {'addic7ed': {'username': sickbeard.ADDIC7ED_USER,
'password': sickbeard.ADDIC7ED_PASS},
'legendastv': {'username': sickbeard.LEGENDASTV_USER,
'password': sickbeard.LEGENDASTV_PASS},
'opensubtitles': {'username': sickbeard.OPENSUBTITLES_USER,
'password': sickbeard.OPENSUBTITLES_PASS}}
pool = ProviderPool(providers=providers, provider_configs=provider_configs)
# Search for all wanted languages
languages = {from_code(language) for language in wanted_languages()}
if not languages:
return
run_post_process = False
# Check if PP folder is set
if sickbeard.TV_DOWNLOAD_DIR and os.path.isdir(sickbeard.TV_DOWNLOAD_DIR):
for root, _, files in os.walk(sickbeard.TV_DOWNLOAD_DIR, topdown=False):
rar_files = [x for x in files if isRarFile(x)]
if rar_files and sickbeard.UNPACK:
video_files = [x for x in files if isMediaFile(x)]
if u'_UNPACK' not in root and (not video_files or root == sickbeard.TV_DOWNLOAD_DIR):
logger.log(u'Found rar files in post-process folder: {}'.format(rar_files), logger.DEBUG)
result = processTV.ProcessResult()
processTV.unRAR(root, rar_files, False, result)
elif rar_files and not sickbeard.UNPACK:
logger.log(u'Unpack is disabled. Skipping: {}'.format(rar_files), logger.WARNING)
for root, _, files in os.walk(sickbeard.TV_DOWNLOAD_DIR, topdown=False):
for video_filename in sorted(files):
try:
# Remove non release groups from video file. Needed to match subtitles
new_video_filename = remove_non_release_groups(video_filename)
if new_video_filename != video_filename:
os.rename(video_filename, new_video_filename)
video_filename = new_video_filename
except Exception as error:
logger.log(u'Couldn\'t remove non release groups from video file. Error: {}'.format
(ex(error)), logger.DEBUG)
if isMediaFile(video_filename):
try:
video = subliminal.scan_video(os.path.join(root, video_filename),
subtitles=False, embedded_subtitles=False)
subtitles_list = pool.list_subtitles(video, languages)
for provider in providers:
if provider in pool.discarded_providers:
logger.log(u'Could not search in {} provider. Discarding for now'.format(provider), logger.DEBUG)
if not subtitles_list:
logger.log(u'No subtitles found for {}'.format
(os.path.join(root, video_filename)), logger.DEBUG)
continue
logger.log(u'Found subtitle(s) canditate(s) for {}'.format(video_filename), logger.INFO)
hearing_impaired = sickbeard.SUBTITLES_HEARING_IMPAIRED
user_score = 213 if sickbeard.SUBTITLES_PERFECT_MATCH else 204
found_subtitles = pool.download_best_subtitles(subtitles_list, video, languages=languages,
hearing_impaired=hearing_impaired,
min_score=user_score,
only_one=not sickbeard.SUBTITLES_MULTI)
for subtitle in subtitles_list:
score = subliminal.score.compute_score(subtitle, video, hearing_impaired=sickbeard.SUBTITLES_HEARING_IMPAIRED)
logger.log(u'[{}] Subtitle score for {} is: {} (min={})'.format
(subtitle.provider_name, subtitle.id, score, user_score), logger.DEBUG)
downloaded_languages = set()
for subtitle in found_subtitles:
logger.log(u'Found subtitle for {} in {} provider with language {}'.format
(os.path.join(root, video_filename), subtitle.provider_name,
subtitle.language.opensubtitles), logger.DEBUG)
subliminal.save_subtitles(video, found_subtitles, directory=root,
single=not sickbeard.SUBTITLES_MULTI)
subtitles_multi = not sickbeard.SUBTITLES_MULTI
subtitle_path = subliminal.subtitle.get_subtitle_path(video.name,
None if subtitles_multi else
subtitle.language)
if root is not None:
subtitle_path = os.path.join(root, os.path.split(subtitle_path)[1])
sickbeard.helpers.chmodAsParent(subtitle_path)
sickbeard.helpers.fixSetGroupID(subtitle_path)
downloaded_languages.add(subtitle.language.opensubtitles)
# Don't run post processor unless at least one file has all of the needed subtitles
if not needs_subtitles(downloaded_languages):
run_post_process = True
except Exception as error:
logger.log(u'Error occurred when downloading subtitles for: {}. Error: {}'.format
(os.path.join(root, video_filename), ex(error)))
if run_post_process:
logger.log(u'Starting post-process with default settings now that we found subtitles')
processTV.processDir(sickbeard.TV_DOWNLOAD_DIR)
def processDir(dirName,
nzbName=None,
process_method=None,
force=False,
is_priority=None,
delete_on=False,
failed=False,
proc_type="auto",
**kwargs):
"""
Scans through the files in dirName and processes whatever media files it finds
:param dirName: The folder name to look in
:param nzbName: The NZB name which resulted in this folder being downloaded
:param force: True to postprocess already postprocessed files
:param failed: Boolean for whether or not the download failed
:param proc_type: Type of postprocessing auto or manual
"""
result = ProcessResult()
result.output += logHelper("Processing folder %s" % dirName, logging.DEBUG)
result.output += logHelper(
"TV_DOWNLOAD_DIR: %s" % sickbeard.TV_DOWNLOAD_DIR, logging.DEBUG)
postpone = False
# if they passed us a real dir then assume it's the one we want
if ek(os.path.isdir, dirName):
dirName = ek(os.path.realpath, dirName)
# if the client and SickRage are not on the same machine translate the Dir in a network dir
elif sickbeard.TV_DOWNLOAD_DIR and ek(os.path.isdir, sickbeard.TV_DOWNLOAD_DIR) \
and ek(os.path.normpath, dirName) != ek(os.path.normpath, sickbeard.TV_DOWNLOAD_DIR):
dirName = ek(os.path.join, sickbeard.TV_DOWNLOAD_DIR,
ek(os.path.abspath, dirName).split(os.path.sep)[-1])
result.output += logHelper("Trying to use folder %s" % dirName,
logging.DEBUG)
# if we didn't find a real dir then quit
if not ek(os.path.isdir, dirName):
result.output += logHelper(
"Unable to figure out what folder to process. If your downloader and SiCKRAGE aren't on the same PC make sure you fill out your TV download dir in the config.",
logging.DEBUG)
return result.output
path, dirs, files = get_path_dir_files(dirName, nzbName, proc_type)
files = [x for x in files if helpers.notTorNZBFile(x)]
SyncFiles = [x for x in files if helpers.isSyncFile(x)]
# Don't post process if files are still being synced and option is activated
if SyncFiles and sickbeard.POSTPONE_IF_SYNC_FILES:
postpone = True
nzbNameOriginal = nzbName
if not postpone:
result.output += logHelper("PostProcessing Path: %s" % path,
logging.INFO)
result.output += logHelper("PostProcessing Dirs: [%s]" % dirs,
logging.DEBUG)
rarFiles = [x for x in files if helpers.isRarFile(x)]
rarContent = unRAR(path, rarFiles, force, result)
files += rarContent
videoFiles = [x for x in files if helpers.isMediaFile(x)]
videoInRar = [x for x in rarContent if helpers.isMediaFile(x)]
result.output += logHelper(
"PostProcessing Files: [%s]" % ", ".join(files), logging.DEBUG)
result.output += logHelper(
"PostProcessing VideoFiles: [%s]" % ", ".join(videoFiles),
logging.DEBUG)
result.output += logHelper(
"PostProcessing RarContent: [%s]" % ", ".join(rarContent),
logging.DEBUG)
result.output += logHelper(
"PostProcessing VideoInRar: [%s]" % ", ".join(videoInRar),
logging.DEBUG)
# If nzbName is set and there's more than one videofile in the folder, files will be lost (overwritten).
if len(videoFiles) >= 2:
nzbName = None
if not process_method:
process_method = sickbeard.PROCESS_METHOD
result.result = True
# Don't Link media when the media is extracted from a rar in the same path
if process_method in ('hardlink', 'symlink') and videoInRar:
process_media(path, videoInRar, nzbName, 'move', force,
is_priority, result)
delete_files(path, rarContent, result)
for video in set(videoFiles) - set(videoInRar):
process_media(path, [video], nzbName, process_method, force,
is_priority, result)
elif sickbeard.DELRARCONTENTS and videoInRar:
process_media(path, videoInRar, nzbName, process_method, force,
is_priority, result)
delete_files(path, rarContent, result, True)
for video in set(videoFiles) - set(videoInRar):
process_media(path, [video], nzbName, process_method, force,
is_priority, result)
else:
for video in videoFiles:
process_media(path, [video], nzbName, process_method, force,
is_priority, result)
else:
result.output += logHelper(
"Found temporary sync files, skipping post processing for: %s" %
path)
result.output += logHelper("Sync Files: [%s] in path %s" %
(", ".join(SyncFiles), path))
result.missedfiles.append("%s : Syncfiles found" % path)
# Process Video File in all TV Subdir
for curDir in [
x for x in dirs
if validateDir(path, x, nzbNameOriginal, failed, result)
]:
result.result = True
for processPath, _, fileList in ek(os.walk,
ek(os.path.join, path, curDir),
topdown=False):
if not validateDir(path, processPath, nzbNameOriginal, failed,
result):
continue
postpone = False
SyncFiles = [x for x in fileList if helpers.isSyncFile(x)]
# Don't post process if files are still being synced and option is activated
if SyncFiles and sickbeard.POSTPONE_IF_SYNC_FILES:
postpone = True
if not postpone:
rarFiles = [x for x in fileList if helpers.isRarFile(x)]
rarContent = unRAR(processPath, rarFiles, force, result)
fileList = set(fileList + rarContent)
videoFiles = [x for x in fileList if helpers.isMediaFile(x)]
videoInRar = [x for x in rarContent if helpers.isMediaFile(x)]
notwantedFiles = [x for x in fileList if x not in videoFiles]
if notwantedFiles:
result.output += logHelper(
"Found unwanted files: [%s]" %
", ".join(notwantedFiles), logging.DEBUG)
# Don't Link media when the media is extracted from a rar in the same path
if process_method in ('hardlink', 'symlink') and videoInRar:
process_media(processPath, videoInRar, nzbName, 'move',
force, is_priority, result)
process_media(processPath,
set(videoFiles) - set(videoInRar), nzbName,
process_method, force, is_priority, result)
delete_files(processPath, rarContent, result)
elif sickbeard.DELRARCONTENTS and videoInRar:
process_media(processPath, videoInRar, nzbName,
process_method, force, is_priority, result)
process_media(processPath,
set(videoFiles) - set(videoInRar), nzbName,
process_method, force, is_priority, result)
delete_files(processPath, rarContent, result, True)
else:
process_media(processPath, videoFiles, nzbName,
process_method, force, is_priority, result)
# Delete all file not needed
if process_method != "move" or not result.result \
or (
proc_type == "manual" and not delete_on): # Avoid to delete files if is Manual PostProcessing
continue
delete_files(processPath, notwantedFiles, result)
if (not sickbeard.NO_DELETE or proc_type == "manual") and process_method == "move" and \
ek(os.path.normpath, processPath) != ek(os.path.normpath,
sickbeard.TV_DOWNLOAD_DIR):
if delete_folder(processPath, check_empty=True):
result.output += logHelper(
"Deleted folder: %s" % processPath,
logging.DEBUG)
else:
result.output += logHelper(
"Found temporary sync files, skipping post processing for: %s"
% processPath)
result.output += logHelper("Sync Files: [%s] in path %s" %
(", ".join(SyncFiles), processPath))
result.missedfiles.append("%s : Syncfiles found" % processPath)
if result.aggresult:
result.output += logHelper("Processing completed")
if result.missedfiles:
result.output += logHelper(
"I did encounter some unprocessable items: [%s]" %
", ".join(result.missedfiles))
else:
result.output += logHelper(
"Problem(s) during processing, failed the following files/folders: [%s]"
% ", ".join(result.missedfiles), logging.WARNING)
return result.output
def processDir(
dirName,
nzbName=None,
process_method=None,
force=False,
is_priority=None,
delete_on=False,
failed=False,
proc_type="auto",
):
"""
Scans through the files in dirName and processes whatever media files it finds
:param dirName: The folder name to look in
:param nzbName: The NZB name which resulted in this folder being downloaded
:param force: True to postprocess already postprocessed files
:param failed: Boolean for whether or not the download failed
:param proc_type: Type of postprocessing auto or manual
"""
result = ProcessResult()
# if they passed us a real dir then assume it's the one we want
if ek(os.path.isdir, dirName):
dirName = ek(os.path.realpath, dirName)
result.output += logHelper(u"Processing folder %s" % dirName, logger.DEBUG)
# if the client and SickRage are not on the same machine translate the directory into a network directory
elif all(
[
sickbeard.TV_DOWNLOAD_DIR,
ek(os.path.isdir, sickbeard.TV_DOWNLOAD_DIR),
ek(os.path.normpath, dirName) == ek(os.path.normpath, sickbeard.TV_DOWNLOAD_DIR),
]
):
dirName = ek(os.path.join, sickbeard.TV_DOWNLOAD_DIR, ek(os.path.abspath, dirName).split(os.path.sep)[-1])
result.output += logHelper(u"Trying to use folder: %s " % dirName, logger.DEBUG)
# if we didn't find a real dir then quit
if not ek(os.path.isdir, dirName):
result.output += logHelper(
u"Unable to figure out what folder to process. "
u"If your downloader and SickRage aren't on the same PC "
u"make sure you fill out your TV download dir in the config.",
logger.DEBUG,
)
return result.output
path, dirs, files = get_path_dir_files(dirName, nzbName, proc_type)
files = [x for x in files if not is_torrent_or_nzb_file(x)]
SyncFiles = [x for x in files if is_sync_file(x)]
nzbNameOriginal = nzbName
# Don't post process if files are still being synced and option is activated
postpone = SyncFiles and sickbeard.POSTPONE_IF_SYNC_FILES
if not postpone:
result.output += logHelper(u"PostProcessing Path: %s" % path, logger.INFO)
result.output += logHelper(u"PostProcessing Dirs: %s" % str(dirs), logger.DEBUG)
rarFiles = [x for x in files if helpers.isRarFile(x)]
rarContent = unRAR(path, rarFiles, force, result)
files += rarContent
videoFiles = [x for x in files if helpers.isMediaFile(x)]
videoInRar = [x for x in rarContent if helpers.isMediaFile(x)]
result.output += logHelper(u"PostProcessing Files: %s" % files, logger.DEBUG)
result.output += logHelper(u"PostProcessing VideoFiles: %s" % videoFiles, logger.DEBUG)
result.output += logHelper(u"PostProcessing RarContent: %s" % rarContent, logger.DEBUG)
result.output += logHelper(u"PostProcessing VideoInRar: %s" % videoInRar, logger.DEBUG)
# If nzbName is set and there's more than one videofile in the folder, files will be lost (overwritten).
nzbName = None if len(videoFiles) >= 2 else nzbName
process_method = process_method if process_method else sickbeard.PROCESS_METHOD
result.result = True
# Don't Link media when the media is extracted from a rar in the same path
if process_method in (u"hardlink", u"symlink") and videoInRar:
process_media(path, videoInRar, nzbName, u"move", force, is_priority, result)
delete_files(path, rarContent, result)
for video in set(videoFiles) - set(videoInRar):
process_media(path, [video], nzbName, process_method, force, is_priority, result)
elif sickbeard.DELRARCONTENTS and videoInRar:
process_media(path, videoInRar, nzbName, process_method, force, is_priority, result)
delete_files(path, rarContent, result, True)
for video in set(videoFiles) - set(videoInRar):
process_media(path, [video], nzbName, process_method, force, is_priority, result)
else:
for video in videoFiles:
process_media(path, [video], nzbName, process_method, force, is_priority, result)
else:
result.output += logHelper(u"Found temporary sync files: %s in path: %s" % (SyncFiles, path))
result.output += logHelper(u"Skipping post processing for folder: %s" % path)
result.missedfiles.append(u"%s : Syncfiles found" % path)
# Process Video File in all TV Subdir
for curDir in [x for x in dirs if validateDir(path, x, nzbNameOriginal, failed, result)]:
result.result = True
for processPath, _, fileList in ek(os.walk, ek(os.path.join, path, curDir), topdown=False):
if not validateDir(path, processPath, nzbNameOriginal, failed, result):
continue
SyncFiles = [x for x in fileList if is_sync_file(x)]
# Don't post process if files are still being synced and option is activated
postpone = SyncFiles and sickbeard.POSTPONE_IF_SYNC_FILES
if not postpone:
rarFiles = [x for x in fileList if helpers.isRarFile(x)]
rarContent = unRAR(processPath, rarFiles, force, result)
fileList = set(fileList + rarContent)
videoFiles = [x for x in fileList if helpers.isMediaFile(x)]
videoInRar = [x for x in rarContent if helpers.isMediaFile(x)]
notwantedFiles = [x for x in fileList if x not in videoFiles]
if notwantedFiles:
result.output += logHelper(u"Found unwanted files: %s" % notwantedFiles, logger.DEBUG)
# Don't Link media when the media is extracted from a rar in the same path
if process_method in (u"hardlink", u"symlink") and videoInRar:
process_media(processPath, videoInRar, nzbName, u"move", force, is_priority, result)
process_media(
processPath,
set(videoFiles) - set(videoInRar),
nzbName,
process_method,
force,
is_priority,
result,
)
delete_files(processPath, rarContent, result)
elif sickbeard.DELRARCONTENTS and videoInRar:
process_media(processPath, videoInRar, nzbName, process_method, force, is_priority, result)
process_media(
processPath,
set(videoFiles) - set(videoInRar),
nzbName,
process_method,
force,
is_priority,
result,
)
delete_files(processPath, rarContent, result, True)
else:
process_media(processPath, videoFiles, nzbName, process_method, force, is_priority, result)
# Delete all file not needed and avoid deleting files if Manual PostProcessing
if not (process_method == u"move" and result.result) or (proc_type == u"manual" and not delete_on):
continue
delete_folder(ek(os.path.join, processPath, u"@eaDir"))
delete_files(processPath, notwantedFiles, result)
if all(
[
not sickbeard.NO_DELETE or proc_type == u"manual",
process_method == u"move",
ek(os.path.normpath, processPath) != ek(os.path.normpath, sickbeard.TV_DOWNLOAD_DIR),
]
):
if delete_folder(processPath, check_empty=True):
result.output += logHelper(u"Deleted folder: %s" % processPath, logger.DEBUG)
else:
result.output += logHelper(u"Found temporary sync files: %s in path: %s" % (SyncFiles, processPath))
result.output += logHelper(u"Skipping post processing for folder: %s" % processPath)
result.missedfiles.append(u"%s : Syncfiles found" % path)
if result.aggresult:
result.output += logHelper(u"Successfully processed")
if result.missedfiles:
result.output += logHelper(u"I did encounter some unprocessable items: ")
for missedfile in result.missedfiles:
result.output += logHelper(u"[%s]" % missedfile)
else:
result.output += logHelper(
u"Problem(s) during processing, failed the following files/folders: ", logger.WARNING
)
for missedfile in result.missedfiles:
result.output += logHelper(u"[%s]" % missedfile, logger.WARNING)
return result.output
def download_subtitles(subtitles_info): # pylint: disable=too-many-locals, too-many-branches, too-many-statements
existing_subtitles = subtitles_info['subtitles']
if not needs_subtitles(existing_subtitles):
logger.log(
u'Episode already has all needed subtitles, skipping {0} {1}'.
format(
subtitles_info['show_name'],
episode_num(subtitles_info['season'],
subtitles_info['episode'])
or episode_num(subtitles_info['season'],
subtitles_info['episode'],
numbering='absolute')), logger.DEBUG)
return existing_subtitles, None
languages = get_needed_languages(existing_subtitles)
if not languages:
logger.log(
u'No subtitles needed for {0} {1}'.format(
subtitles_info['show_name'],
episode_num(subtitles_info['season'],
subtitles_info['episode'])
or episode_num(subtitles_info['season'],
subtitles_info['episode'],
numbering='absolute')), logger.DEBUG)
return existing_subtitles, None
subtitles_path = get_subtitles_path(subtitles_info['location'])
video_path = subtitles_info['location']
# Perfect match = hash score - hearing impaired score - resolution score (subtitle for 720p is the same as for 1080p)
# Perfect match = 215 - 1 - 1 = 213
# Non-perfect match = series + year + season + episode
# Non-perfect match = 108 + 54 + 18 + 18 = 198
# From latest subliminal code:
# episode_scores = {'hash': 215, 'series': 108, 'year': 54, 'season': 18, 'episode': 18, 'release_group': 9,
# 'format': 4, 'audio_codec': 2, 'resolution': 1, 'hearing_impaired': 1, 'video_codec': 1}
user_score = 213 if sickbeard.SUBTITLES_PERFECT_MATCH else 198
video = get_video(video_path, subtitles_path=subtitles_path)
if not video:
logger.log(
u'Exception caught in subliminal.scan_video for {0} {1}'.format(
subtitles_info['show_name'],
episode_num(subtitles_info['season'],
subtitles_info['episode'])
or episode_num(subtitles_info['season'],
subtitles_info['episode'],
numbering='absolute')), logger.DEBUG)
return existing_subtitles, None
providers = enabled_service_list()
pool = SubtitleProviderPool()
try:
subtitles_list = pool.list_subtitles(video, languages)
for provider in providers:
if provider in pool.discarded_providers:
logger.log(
u'Could not search in {0} provider. Discarding for now'.
format(provider), logger.DEBUG)
if not subtitles_list:
logger.log(
u'No subtitles found for {0} {1}'.format(
subtitles_info['show_name'],
episode_num(subtitles_info['season'],
subtitles_info['episode'])
or episode_num(subtitles_info['season'],
subtitles_info['episode'],
numbering='absolute')), logger.DEBUG)
return existing_subtitles, None
for subtitle in subtitles_list:
score = subliminal.score.compute_score(
subtitle,
video,
hearing_impaired=sickbeard.SUBTITLES_HEARING_IMPAIRED)
logger.log(
u'[{0}] Subtitle score for {1} is: {2} (min={3})'.format(
subtitle.provider_name, subtitle.id, score, user_score),
logger.DEBUG)
found_subtitles = pool.download_best_subtitles(
subtitles_list,
video,
languages=languages,
hearing_impaired=sickbeard.SUBTITLES_HEARING_IMPAIRED,
min_score=user_score,
only_one=not sickbeard.SUBTITLES_MULTI)
subliminal.save_subtitles(video,
found_subtitles,
directory=subtitles_path,
single=not sickbeard.SUBTITLES_MULTI)
except IOError as error:
if 'No space left on device' in ex(error):
logger.log(u'Not enough space on the drive to save subtitles',
logger.WARNING)
else:
logger.log(traceback.format_exc(), logger.WARNING)
except Exception:
logger.log(
u'Error occurred when downloading subtitles for: {0}'.format(
video_path))
logger.log(traceback.format_exc(), logger.ERROR)
return existing_subtitles, None
for subtitle in found_subtitles:
subtitle_path = subliminal.subtitle.get_subtitle_path(
video.name,
None if not sickbeard.SUBTITLES_MULTI else subtitle.language)
if subtitles_path is not None:
subtitle_path = os.path.join(subtitles_path,
os.path.split(subtitle_path)[1])
sickbeard.helpers.chmodAsParent(subtitle_path)
sickbeard.helpers.fixSetGroupID(subtitle_path)
if sickbeard.SUBTITLES_HISTORY:
logger.log(
u'history.logSubtitle {0}, {1}'.format(
subtitle.provider_name, subtitle.language.opensubtitles),
logger.DEBUG)
history.logSubtitle(subtitles_info['show_indexerid'],
subtitles_info['season'],
subtitles_info['episode'],
subtitles_info['status'], subtitle)
if sickbeard.SUBTITLES_EXTRA_SCRIPTS and isMediaFile(
video_path) and not sickbeard.EMBEDDED_SUBTITLES_ALL:
run_subs_extra_scripts(subtitles_info,
subtitle,
video,
single=not sickbeard.SUBTITLES_MULTI)
new_subtitles = sorted(
{subtitle.language.opensubtitles
for subtitle in found_subtitles})
current_subtitles = sorted(
{subtitle
for subtitle in new_subtitles +
existing_subtitles}) if existing_subtitles else new_subtitles
if not sickbeard.SUBTITLES_MULTI and len(found_subtitles) == 1:
new_code = found_subtitles[0].language.opensubtitles
if new_code not in existing_subtitles:
current_subtitles.remove(new_code)
current_subtitles.append('und')
return current_subtitles, new_subtitles
def run(self):
"""
Called every few seconds to handle any running/finished torrents
"""
if not LIBTORRENT_AVAILABLE:
return
if not self.loadedRunningTorrents:
torrent_save_file = _get_running_torrents_pickle_path(False)
if os.path.isfile(torrent_save_file):
logger.log(u'Saved torrents found in %s, loading' % (torrent_save_file), logger.DEBUG)
_load_saved_torrents()
self.loadedRunningTorrents = True
sess = _get_session(False)
if sess is not None:
while 1:
a = sess.pop_alert()
if not a: break
if type(a) == str:
logger.log(a, logger.DEBUG)
else:
logger.log(u'(%s): %s' % (type(a).__name__, ek.fixStupidEncodings(a.message(), True)), logger.DEBUG)
logTorrentStatus = (time.time() - self.lastTorrentStatusLogTS) >= 600
for torrent_data in running_torrents:
if torrent_data['handle'].has_metadata():
ti = torrent_data['handle'].get_torrent_info()
name = ti.name()
torrent_data['name'] = name
torrent_data['total_size'] = ti.total_size()
if not torrent_data['have_torrentFile']:
# if this was a magnet or url, and we now have downloaded the metadata
# for it, best to save it locally in case we need to resume
ti = torrent_data['handle'].get_torrent_info()
torrentFile = lt.create_torrent(ti)
torrent_data['torrent'] = lt.bencode(torrentFile.generate())
torrent_data['have_torrentFile'] = True
logger.log(u'Created torrent file for %s as metadata d/l is now complete' % (name), logger.DEBUG)
if not torrent_data['checkedForMedia']:
with torrent_data['lock']:
# Torrent has metadata, but hasn't been checked for valid media yet. Do so now.
if not _torrent_has_any_media_files(ti):
logger.log(u'Torrent %s has no media files! Deleting it.' % (name), logger.ERROR)
_on_failed_torrent(torrent_data['key'], removeFromRunningTorrents=True,
markEpisodesWanted=True)
break # continue here would be nice, but safer to break b/c we have modified the list
torrent_data['checkedForMedia'] = True
else:
name = '-'
s = torrent_data['handle'].status()
torrent_data['status'] = str(s.state)
torrent_data['progress'] = s.progress
torrent_data['rate_down'] = s.download_rate
torrent_data['rate_up'] = s.upload_rate
torrent_data['paused'] = s.paused
torrent_data['error'] = s.error
#currentRatio = 0.0 if s.total_download == 0 else float(s.total_upload)/float(s.total_download)
currentRatio = 0.0 if s.all_time_download == 0 else float(s.all_time_upload)/float(s.all_time_download)
torrent_data['ratio'] = currentRatio
if s.state in [lt.torrent_status.seeding,
lt.torrent_status.finished]:
with torrent_data['lock']:
# this is the post-processing & removing code, so make sure that there's
# only one thread doing either here, as the two could easily interfere with
# one another
if not torrent_data['post_processed']:
# torrent has just completed download, so we need to do
# post-processing on it.
ti = torrent_data['handle'].get_torrent_info()
any_file_success = False
for f in ti.files():
fullpath = os.path.join(sickbeard.LIBTORRENT_WORKING_DIR, 'data', f.path)
logger.log(u'Post-processing "%s"' % (fullpath), logger.DEBUG)
if isMediaFile(fullpath):
logger.log(u'this is a media file', logger.DEBUG)
try:
processor = postProcessor.PostProcessor(fullpath, name)
if processor.process(forceKeepOriginalFiles=True):
logger.log(u'Success post-processing "%s"' % (fullpath), logger.DEBUG)
any_file_success = True
except exceptions.PostProcessingFailed, e:
logger.log(u'Failed post-processing file "%s" with error "%s"' % (fullpath, ex(e)),
logger.ERROR)
if not any_file_success:
logger.log(u'When post-processing the completed torrent %s, no useful files were found.' % (name), logger.ERROR)
torrent_data['post_processed'] = True
else:
# post-processing has already been performed. So we just
# need to ensure check the ratio and delete the torrent
# if we're good.
if currentRatio >= sickbeard.LIBTORRENT_SEED_TO_RATIO:
logger.log(u'Torrent "%s" has seeded to ratio %f. Removing it.' % (name, currentRatio), logger.MESSAGE)
deleteFilesToo = True
if not torrent_data['post_processed']:
logger.log(u'Torrent has not been post_processed. Keeping files.', logger.MESSAGE)
deleteFilesToo = False
_remove_torrent_by_handle(torrent_data['handle'], deleteFilesToo)
else:
if logTorrentStatus:
self.lastTorrentStatusLogTS = time.time()
logger.log(u'"%s" seeding %0.3f' % (name, currentRatio), logger.DEBUG)
elif s.state == lt.torrent_status.downloading:
if logTorrentStatus:
self.lastTorrentStatusLogTS = time.time()
logger.log(u'"%s" downloading %0.2f' % (name, s.progress * 100.0), logger.DEBUG)
def subtitles_download_in_pp(): # pylint: disable=too-many-locals, too-many-branches, too-many-statements
logger.log(u'Checking for needed subtitles in Post-Process folder', logger.INFO)
providers = enabled_service_list()
pool = SubtitleProviderPool()
# Search for all wanted languages
languages = {from_code(language) for language in wanted_languages()}
if not languages:
return
# Dict of language exceptions to use with subliminal
language_exceptions = {'pt-br': 'pob'}
run_post_process = False
# Check if PP folder is set
if sickbeard.TV_DOWNLOAD_DIR and os.path.isdir(sickbeard.TV_DOWNLOAD_DIR):
for dirpath, dirnames_, files in os.walk(sickbeard.TV_DOWNLOAD_DIR, topdown=False):
rar_files = [rar_file for rar_file in files if isRarFile(rar_file)]
if rar_files and sickbeard.UNPACK:
video_files = [video_file for video_file in files if isMediaFile(video_file)]
if u'_UNPACK' not in dirpath and (not video_files or dirpath == sickbeard.TV_DOWNLOAD_DIR):
logger.log(u'Found rar files in post-process folder: {0}'.format(rar_files), logger.DEBUG)
result = processTV.ProcessResult()
processTV.unRAR(dirpath, rar_files, False, result)
elif rar_files and not sickbeard.UNPACK:
logger.log(u'Unpack is disabled. Skipping: {0}'.format(rar_files), logger.WARNING)
for dirpath, dirnames_, files in os.walk(sickbeard.TV_DOWNLOAD_DIR, topdown=False):
for filename in sorted(files):
try:
# Remove non release groups from video file. Needed to match subtitles
new_filename = remove_non_release_groups(filename)
if new_filename != filename:
os.rename(filename, new_filename)
filename = new_filename
except Exception as error:
logger.log(u"Couldn't remove non release groups from video file. Error: {0}".format
(ex(error)), logger.DEBUG)
# Delete unwanted subtitles before downloading new ones
if sickbeard.SUBTITLES_MULTI and sickbeard.SUBTITLES_KEEP_ONLY_WANTED and filename.rpartition('.')[2] in subtitle_extensions:
subtitle_language = filename.rsplit('.', 2)[1].lower()
if len(subtitle_language) == 2 and subtitle_language in language_converters['opensubtitles'].codes:
subtitle_language = Language.fromcode(subtitle_language, 'alpha2').opensubtitles
elif subtitle_language in language_exceptions:
subtitle_language = language_exceptions.get(subtitle_language, subtitle_language)
elif subtitle_language not in language_converters['opensubtitles'].codes:
subtitle_language = 'unknown'
if subtitle_language not in sickbeard.SUBTITLES_LANGUAGES:
try:
os.remove(os.path.join(dirpath, filename))
logger.log(u"Deleted '{0}' because we don't want subtitle language '{1}'. We only want '{2}' language(s)".format
(filename, subtitle_language, ','.join(sickbeard.SUBTITLES_LANGUAGES)), logger.DEBUG)
except Exception as error:
logger.log(u"Couldn't delete subtitle: {0}. Error: {1}".format(filename, ex(error)), logger.DEBUG)
if isMediaFile(filename) and processTV.subtitles_enabled(filename):
try:
video = get_video(os.path.join(dirpath, filename), subtitles=False, embedded_subtitles=False)
subtitles_list = pool.list_subtitles(video, languages)
for provider in providers:
if provider in pool.discarded_providers:
logger.log(u'Could not search in {0} provider. Discarding for now'.format(provider), logger.DEBUG)
if not subtitles_list:
logger.log(u'No subtitles found for {0}'.format
(os.path.join(dirpath, filename)), logger.DEBUG)
continue
logger.log(u'Found subtitle(s) canditate(s) for {0}'.format(filename), logger.INFO)
hearing_impaired = sickbeard.SUBTITLES_HEARING_IMPAIRED
user_score = 213 if sickbeard.SUBTITLES_PERFECT_MATCH else 198
found_subtitles = pool.download_best_subtitles(subtitles_list, video, languages=languages,
hearing_impaired=hearing_impaired,
min_score=user_score,
only_one=not sickbeard.SUBTITLES_MULTI)
for subtitle in subtitles_list:
score = subliminal.score.compute_score(subtitle, video, hearing_impaired=sickbeard.SUBTITLES_HEARING_IMPAIRED)
logger.log(u'[{0}] Subtitle score for {1} is: {2} (min={3})'.format
(subtitle.provider_name, subtitle.id, score, user_score), logger.DEBUG)
downloaded_languages = set()
for subtitle in found_subtitles:
logger.log(u'Found subtitle for {0} in {1} provider with language {2}'.format
(os.path.join(dirpath, filename), subtitle.provider_name,
subtitle.language.opensubtitles), logger.INFO)
subliminal.save_subtitles(video, found_subtitles, directory=dirpath,
single=not sickbeard.SUBTITLES_MULTI)
subtitles_multi = not sickbeard.SUBTITLES_MULTI
subtitle_path = subliminal.subtitle.get_subtitle_path(video.name,
None if subtitles_multi else
subtitle.language)
if dirpath is not None:
subtitle_path = os.path.join(dirpath, os.path.split(subtitle_path)[1])
sickbeard.helpers.chmodAsParent(subtitle_path)
sickbeard.helpers.fixSetGroupID(subtitle_path)
downloaded_languages.add(subtitle.language.opensubtitles)
# Don't run post processor unless at least one file has all of the needed subtitles
if not needs_subtitles(downloaded_languages):
run_post_process = True
except Exception as error:
logger.log(u'Error occurred when downloading subtitles for: {0}. Error: {1}'.format
(os.path.join(dirpath, filename), ex(error)))
if run_post_process:
logger.log(u'Starting post-process with default settings now that we found subtitles')
processTV.processDir(sickbeard.TV_DOWNLOAD_DIR)
def run(self):
"""
Called every few seconds to handle any running/finished torrents
"""
if not LIBTORRENT_AVAILABLE:
return
if not self.loadedRunningTorrents:
torrent_save_file = _get_running_torrents_pickle_path(False)
if os.path.isfile(torrent_save_file):
logger.log(
u'Saved torrents found in %s, loading' %
(torrent_save_file), logger.DEBUG)
_load_saved_torrents()
self.loadedRunningTorrents = True
sess = _get_session(False)
if sess is not None:
while 1:
a = sess.pop_alert()
if not a: break
if type(a) == str:
logger.log(a, logger.DEBUG)
else:
logger.log(
u'(%s): %s' %
(type(a).__name__,
ek.fixStupidEncodings(a.message(), True)),
logger.DEBUG)
logTorrentStatus = (time.time() -
self.lastTorrentStatusLogTS) >= 600
for torrent_data in running_torrents:
if torrent_data['handle'].has_metadata():
ti = torrent_data['handle'].get_torrent_info()
name = ti.name()
torrent_data['name'] = name
torrent_data['total_size'] = ti.total_size()
if not torrent_data['have_torrentFile']:
# if this was a magnet or url, and we now have downloaded the metadata
# for it, best to save it locally in case we need to resume
ti = torrent_data['handle'].get_torrent_info()
torrentFile = lt.create_torrent(ti)
torrent_data['torrent'] = lt.bencode(
torrentFile.generate())
torrent_data['have_torrentFile'] = True
logger.log(
u'Created torrent file for %s as metadata d/l is now complete'
% (name), logger.DEBUG)
else:
name = '-'
s = torrent_data['handle'].status()
torrent_data['status'] = str(s.state)
torrent_data['progress'] = s.progress
torrent_data['rate_down'] = s.download_rate
torrent_data['rate_up'] = s.upload_rate
torrent_data['paused'] = s.paused
torrent_data['error'] = s.error
#currentRatio = 0.0 if s.total_download == 0 else float(s.total_upload)/float(s.total_download)
currentRatio = 0.0 if s.all_time_download == 0 else float(
s.all_time_upload) / float(s.all_time_download)
torrent_data['ratio'] = currentRatio
if s.state in [
lt.torrent_status.seeding, lt.torrent_status.finished
]:
with torrent_data['lock']:
# this is the post-processing & removing code, so make sure that there's
# only one thread doing either here, as the two could easily interfere with
# one another
if not torrent_data['post_processed']:
# torrent has just completed download, so we need to do
# post-processing on it.
ti = torrent_data['handle'].get_torrent_info()
any_file_success = False
for f in ti.files():
fullpath = os.path.join(
sickbeard.LIBTORRENT_WORKING_DIR, 'data',
f.path)
logger.log(
u'Post-processing "%s"' % (fullpath),
logger.DEBUG)
if isMediaFile(fullpath):
logger.log(u'this is a media file',
logger.DEBUG)
try:
processor = postProcessor.PostProcessor(
fullpath, name)
if processor.process(
forceKeepOriginalFiles=True):
logger.log(
u'Success post-processing "%s"'
% (fullpath), logger.DEBUG)
any_file_success = True
except exceptions.PostProcessingFailed, e:
logger.log(
u'Failed post-processing file "%s" with error "%s"'
% (fullpath, ex(e)), logger.ERROR)
if not any_file_success:
logger.log(
u'When post-processing the completed torrent %s, no useful files were found.'
% (name), logger.ERROR)
torrent_data['post_processed'] = True
else:
# post-processing has already been performed. So we just
# need to ensure check the ratio and delete the torrent
# if we're good.
if currentRatio >= sickbeard.LIBTORRENT_SEED_TO_RATIO:
logger.log(
u'Torrent "%s" has seeded to ratio %f. Removing it.'
% (name, currentRatio), logger.MESSAGE)
deleteFilesToo = True
if not torrent_data['post_processed']:
logger.log(
u'Torrent has not been post_processed. Keeping files.',
logger.MESSAGE)
deleteFilesToo = False
_remove_torrent_by_handle(
torrent_data['handle'], deleteFilesToo)
else:
if logTorrentStatus:
self.lastTorrentStatusLogTS = time.time()
logger.log(
u'"%s" seeding %0.3f' %
(name, currentRatio), logger.DEBUG)
elif s.state == lt.torrent_status.downloading:
if logTorrentStatus:
self.lastTorrentStatusLogTS = time.time()
logger.log(
u'"%s" downloading %0.2f' %
(name, s.progress * 100.0), logger.DEBUG)
def processDir(dirName, nzbName=None, process_method=None, force=False, is_priority=None, delete_on=False, failed=False, proc_type="auto"):
"""
Scans through the files in dirName and processes whatever media files it finds
dirName: The folder name to look in
nzbName: The NZB or torrent which resulted in a folder/file being downloaded
dirName/nzbName == file: Single file torrent downloaded content
dirName == dir : Directory with Torrent or NZB downloaded content
WARNING: Always make sure downloaded content exists, even if failed = true.
'Delete failed' option and 'Move' process method are otherwise not
save to use and may recursively remove source directories. Before
calling *always* make sure dirName exists, and in case of singe
torrents that the file dirName/nzbName exists. An API change is
required to improve on this. For now it remains backwards compatible
and therefore inherits existing issues.
force : True to postprocess already postprocessed files
failed : Boolean for whether or not the download failed
type : Type of postprocessing
- manual: Recursively post-process all file(s)/dir(s) in dirName
(Automatically selected when post-processing TV_DOWNLOAD_DIR.)
- auto : Non-recursively post-process file(s) in dirName. Support for single-file
torrents included.
"""
result = ProcessResult()
result.output += logHelper(u"Processing folder %s" % dirName, logger.DEBUG)
result.output += logHelper(u"TV_DOWNLOAD_DIR: %s" % sickbeard.TV_DOWNLOAD_DIR, logger.DEBUG)
result.output += logHelper(u"Torrent-/NZB-name: " + str(nzbName) , logger.DEBUG)
result.output += logHelper(u"Process method : " + str(process_method), logger.DEBUG)
result.output += logHelper(u"Process type : " + str(type) , logger.DEBUG)
result.output += logHelper(u"Failed download : " + str(failed) , logger.DEBUG)
# Determine torrent type (False if not a torrent)
torrent_type = get_torrent_type(dirName, nzbName)
result.output += logHelper(u"Torrent-type : " + str(torrent_type) , logger.DEBUG)
postpone = False
# if they passed us a real dir then assume it's the one we want
if ek(os.path.isdir,dirName):
dirName = ek(os.path.realpath, dirName)
# if the client and SickRage are not on the same machine translate the Dir in a network dir
elif sickbeard.TV_DOWNLOAD_DIR and ek(os.path.isdir,sickbeard.TV_DOWNLOAD_DIR) \
and ek(os.path.normpath, dirName) != ek(os.path.normpath, sickbeard.TV_DOWNLOAD_DIR):
dirName = ek(os.path.join, sickbeard.TV_DOWNLOAD_DIR, ek(os.path.abspath, dirName).split(os.path.sep)[-1])
result.output += logHelper(u"Trying to use folder %s" % dirName, logger.DEBUG)
# if we didn't find a real dir then quit
if not ek(os.path.isdir,dirName):
result.output += logHelper(
u"Unable to figure out what folder to process. If your downloader and SickRage aren't on the same PC make sure you fill out your TV download dir in the config.",
logger.DEBUG)
return result.output
# Handle failed torrent
if nzbname_is_torrent(nzbName) and failed:
# Mark torrent as failed
process_failed(dirName, nzbName, result)
if result.result:
result.output += logHelper(u"Successfully processed")
else:
result.output += logHelper(u"Problem(s) during processing", logger.WARNING)
return result.output
path, dirs, files = get_path_dir_files(dirName, nzbName, proc_type)
files = [x for x in files if helpers.notTorNZBFile(x)]
SyncFiles = [x for x in files if helpers.isSyncFile(x)]
# Don't post process if files are still being synced and option is activated
if SyncFiles and sickbeard.POSTPONE_IF_SYNC_FILES:
postpone = True
nzbNameOriginal = nzbName
if not postpone:
result.output += logHelper(u"PostProcessing Path: %s" % path, logger.INFO)
result.output += logHelper(u"PostProcessing Dirs: [%s]" % dirs, logger.DEBUG)
rarFiles = [x for x in files if helpers.isRarFile(x)]
rarContent = unRAR(path, rarFiles, force, result)
files += rarContent
videoFiles = [x for x in files if helpers.isMediaFile(x)]
videoInRar = [x for x in rarContent if helpers.isMediaFile(x)]
result.output += logHelper(u"PostProcessing Files: [%s]" % u", ".join(files), logger.DEBUG)
result.output += logHelper(u"PostProcessing VideoFiles: [%s]" % u", ".join(videoFiles), logger.DEBUG)
result.output += logHelper(u"PostProcessing RarContent: [%s]" % u", ".join(rarContent), logger.DEBUG)
result.output += logHelper(u"PostProcessing VideoInRar: [%s]" % u", ".join(videoInRar), logger.DEBUG)
# If nzbName is set and there's more than one videofile in the folder, files will be lost (overwritten).
if len(videoFiles) >= 2:
nzbName = None
if not process_method:
process_method = sickbeard.PROCESS_METHOD
result.result = True
# Don't Link media when the media is extracted from a rar in the same path
if process_method in ('hardlink', 'symlink') and videoInRar:
process_media(path, videoInRar, nzbName, 'move', force, is_priority, result)
delete_files(path, rarContent, result)
for video in set(videoFiles) - set(videoInRar):
process_media(path, [video], nzbName, process_method, force, is_priority, result)
elif sickbeard.DELRARCONTENTS and videoInRar:
process_media(path, videoInRar, nzbName, process_method, force, is_priority, result)
delete_files(path, rarContent, result, True)
for video in set(videoFiles) - set(videoInRar):
process_media(path, [video], nzbName, process_method, force, is_priority, result)
else:
for video in videoFiles:
process_media(path, [video], nzbName, process_method, force, is_priority, result)
else:
result.output += logHelper(u"Found temporary sync files, skipping post processing for: %s" % path)
result.output += logHelper(u"Sync Files: [%s] in path %s" % (u", ".join(SyncFiles), path))
result.missedfiles.append(u"%s : Syncfiles found" % path)
# Process Video File in all TV Subdir
for curDir in [x for x in dirs if validateDir(path, x, nzbNameOriginal, failed, result)]:
result.result = True
for processPath, _, fileList in ek(os.walk, ek(os.path.join, path, curDir), topdown=False):
if not validateDir(path, processPath, nzbNameOriginal, failed, result):
continue
postpone = False
SyncFiles = [x for x in fileList if helpers.isSyncFile(x)]
# Don't post process if files are still being synced and option is activated
if SyncFiles and sickbeard.POSTPONE_IF_SYNC_FILES:
postpone = True
if not postpone:
rarFiles = [x for x in fileList if helpers.isRarFile(x)]
rarContent = unRAR(processPath, rarFiles, force, result)
fileList = set(fileList + rarContent)
videoFiles = [x for x in fileList if helpers.isMediaFile(x)]
videoInRar = [x for x in rarContent if helpers.isMediaFile(x)]
notwantedFiles = [x for x in fileList if x not in videoFiles]
if notwantedFiles:
result.output += logHelper(u"Found unwanted files: [%s]" % u", ".join(notwantedFiles), logger.DEBUG)
# Don't Link media when the media is extracted from a rar in the same path
if process_method in ('hardlink', 'symlink') and videoInRar:
process_media(processPath, videoInRar, nzbName, 'move', force, is_priority, result)
process_media(processPath, set(videoFiles) - set(videoInRar), nzbName, process_method, force,
is_priority, result)
delete_files(processPath, rarContent, result)
elif sickbeard.DELRARCONTENTS and videoInRar:
process_media(processPath, videoInRar, nzbName, process_method, force, is_priority, result)
process_media(processPath, set(videoFiles) - set(videoInRar), nzbName, process_method, force,
is_priority, result)
delete_files(processPath, rarContent, result, True)
else:
process_media(processPath, videoFiles, nzbName, process_method, force, is_priority, result)
# Delete all file not needed
if process_method != "move" or not result.result \
or (proc_type == "manual" and not delete_on): # Avoid to delete files if is Manual PostProcessing
continue
delete_files(processPath, notwantedFiles, result)
if (not sickbeard.NO_DELETE or proc_type == "manual") and process_method == "move" and \
ek(os.path.normpath, processPath) != ek(os.path.normpath, sickbeard.TV_DOWNLOAD_DIR) and \
torrent_type != TorrentType.SINGLE_FILE:
if delete_folder(processPath, check_empty=True):
result.output += logHelper(u"Deleted folder: %s" % processPath, logger.DEBUG)
else:
result.output += logHelper(u"Found temporary sync files, skipping post processing for: %s" % processPath)
result.output += logHelper(u"Sync Files: [%s] in path %s" % (u", ".join(SyncFiles), processPath))
result.missedfiles.append(u"%s : Syncfiles found" % processPath)
if result.aggresult:
result.output += logHelper(u"Processing completed")
if result.missedfiles:
result.output += logHelper(u"I did encounter some unprocessable items: [%s]" % ", ".join(result.missedfiles))
else:
result.output += logHelper(u"Problem(s) during processing, failed the following files/folders: [%s]" % ", ".join(result.missedfiles), logger.WARNING)
return result.output
def download_subtitles(episode, force_lang=None): # pylint: disable=too-many-locals, too-many-branches, too-many-statements
existing_subtitles = episode.subtitles
if not needs_subtitles(existing_subtitles, force_lang):
logger.log(u'Episode already has all needed subtitles, skipping {0} {1}'.format
(episode.show.name, episode_num(episode.season, episode.episode) or
episode_num(episode.season, episode.episode, numbering='absolute')), logger.DEBUG)
return existing_subtitles, None
if not force_lang:
languages = get_needed_languages(existing_subtitles)
else:
languages = {from_code(force_lang)}
if not languages:
logger.log(u'No subtitles needed for {0} {1}'.format
(episode.show.name, episode_num(episode.season, episode.episode) or
episode_num(episode.season, episode.episode, numbering='absolute')), logger.DEBUG)
return existing_subtitles, None
subtitles_path = get_subtitles_path(episode.location)
video_path = episode.location
# Perfect match = hash score - hearing impaired score - resolution score
# (subtitle for 720p is the same as for 1080p)
# Perfect match = 215 - 1 - 1 = 213
# Non-perfect match = series + year + season + episode
# Non-perfect match = 108 + 54 + 18 + 18 = 198
# From latest subliminal code:
# episode_scores = {'hash': 215, 'series': 108, 'year': 54, 'season': 18, 'episode': 18, 'release_group': 9,
# 'format': 4, 'audio_codec': 2, 'resolution': 1, 'hearing_impaired': 1, 'video_codec': 1}
user_score = 213 if sickbeard.SUBTITLES_PERFECT_MATCH else 198
video = get_video(video_path, subtitles_path=subtitles_path, episode=episode)
if not video:
logger.log(u'Exception caught in subliminal.scan_video for {0} {1}'.format
(episode.show.name, episode_num(episode.season, episode.episode) or
episode_num(episode.season, episode.episode, numbering='absolute')), logger.DEBUG)
return existing_subtitles, None
providers = enabled_service_list()
pool = SubtitleProviderPool()
try:
subtitles_list = pool.list_subtitles(video, languages)
for provider in providers:
if provider in pool.discarded_providers:
logger.log(u'Could not search in {0} provider. Discarding for now'.format(provider), logger.DEBUG)
if not subtitles_list:
logger.log(u'No subtitles found for {0} {1}'.format
(episode.show.name, episode_num(episode.season, episode.episode) or
episode_num(episode.season, episode.episode, numbering='absolute')), logger.DEBUG)
return existing_subtitles, None
for subtitle in subtitles_list:
score = subliminal.score.compute_score(subtitle, video,
hearing_impaired=sickbeard.SUBTITLES_HEARING_IMPAIRED)
logger.log(u'[{0}] Subtitle score for {1} is: {2} (min={3})'.format
(subtitle.provider_name, subtitle.id, score, user_score), logger.DEBUG)
found_subtitles = pool.download_best_subtitles(subtitles_list, video, languages=languages,
hearing_impaired=sickbeard.SUBTITLES_HEARING_IMPAIRED,
min_score=user_score, only_one=not sickbeard.SUBTITLES_MULTI)
subliminal.save_subtitles(video, found_subtitles, directory=subtitles_path,
single=not sickbeard.SUBTITLES_MULTI)
except IOError as error:
if 'No space left on device' in ex(error):
logger.log(u'Not enough space on the drive to save subtitles', logger.WARNING)
else:
logger.log(traceback.format_exc(), logger.WARNING)
except Exception:
logger.log(u'Error occurred when downloading subtitles for: {0}'.format(video_path))
logger.log(traceback.format_exc(), logger.ERROR)
return existing_subtitles, None
for subtitle in found_subtitles:
subtitle_path = subliminal.subtitle.get_subtitle_path(video.name,
None if not sickbeard.SUBTITLES_MULTI else
subtitle.language)
if subtitles_path is not None:
subtitle_path = os.path.join(subtitles_path, os.path.split(subtitle_path)[1])
sickbeard.helpers.chmodAsParent(subtitle_path)
sickbeard.helpers.fixSetGroupID(subtitle_path)
if sickbeard.SUBTITLES_HISTORY:
logger.log(u'history.logSubtitle {0}, {1}'.format
(subtitle.provider_name, subtitle.language.opensubtitles), logger.DEBUG)
history.logSubtitle(episode.show.indexerid, episode.season, episode.episode, episode.status, subtitle)
if sickbeard.SUBTITLES_EXTRA_SCRIPTS and isMediaFile(video_path) and not sickbeard.EMBEDDED_SUBTITLES_ALL:
run_subs_extra_scripts(episode, subtitle, video, single=not sickbeard.SUBTITLES_MULTI)
new_subtitles = sorted({subtitle.language.opensubtitles for subtitle in found_subtitles})
current_subtitles = sorted({subtitle for subtitle in new_subtitles + existing_subtitles}) if existing_subtitles else new_subtitles
if not sickbeard.SUBTITLES_MULTI and len(found_subtitles) == 1:
new_code = found_subtitles[0].language.opensubtitles
if new_code not in existing_subtitles:
current_subtitles.remove(new_code)
current_subtitles.append('und')
return current_subtitles, new_subtitles
def _finish_download_process():
"""
Finish downloading a pid
(this is a closure, called in another thread)
"""
last_reported_perc = 0.0
while p.poll() is None:
line = p.stdout.readline()
if line:
line = line.rstrip()
match = re.match(IPLAYER_DL_PROGRESS_PATTERN, line,
re.IGNORECASE)
if match:
dlSize = match.group('size')
dlTime = match.group('time')
dlPerc = float(match.group('perc'))
if dlPerc - last_reported_perc >= 10.0:
# only report progress every 10% or so.
logger.log(u"RUNNING iPLAYER: " + line,
logger.DEBUG)
last_reported_perc = dlPerc
else:
# not a progress line, echo it to debug
logger.log(u"RUNNING iPLAYER: " + line, logger.DEBUG)
logger.log(
u"RUNNING iPLAYER: process has ended, returncode was " +
repr(p.returncode), logger.DEBUG)
# We will need to rename some of the files in the folder to ensure
# that sb is comfortable with them.
videoFiles = listMediaFiles(tmp_dir)
for videoFile in videoFiles:
filePrefix, fileExt = os.path.splitext(videoFile)
if fileExt and fileExt[0] == '.':
fileExt = fileExt[1:]
# split again to get the quality
filePrePrefix, fileQuality = os.path.splitext(filePrefix)
if fileQuality and fileQuality[0] == '.':
fileQuality = fileQuality[1:]
qual_str = cls.iplayer_quality_to_sb_quality_string(
fileQuality)
# reassemble the filename again, with new quality
newFilePrefix = filePrePrefix + '.' + qual_str
newFileName = newFilePrefix + '.' + fileExt
if newFileName != videoFile: # just in case!
logger.log('Renaming %s to %s' % (videoFile, newFileName),
logger.DEBUG)
os.rename(videoFile, newFileName)
# Also need to rename any associated files (nfo and srt)
for otherFile in glob.glob(newFilePrefix + '.*'):
if otherFile == newFileName:
continue
otherFilePrefix, otherFileExt = os.path.splitext(
otherFile)
newOtherFile = newFilePrefix + otherFileExt
logger.log(
'Renaming %s to %s' % (otherFile, newOtherFile),
logger.DEBUG)
os.rename(otherFile, newOtherFile)
# Ok, we're done with *our* post-processing, so let SB do its own.
processResult = processDir(tmp_dir)
#logger.log(u"processDir returned " + processResult , logger.DEBUG) - this is long, and quite useless!
files_remaining = os.listdir(tmp_dir)
can_delete = True
for filename in files_remaining:
fullFilePath = os.path.join(tmp_dir, filename)
isVideo = isMediaFile(fullFilePath)
if isVideo:
can_delete = False # keep the folder - something prob went wrong
logger.log(
'Found a media file after processing, something probably went wrong: '
+ fullFilePath, logger.MESSAGE)
else:
logger.log(
'Extra file left over (will be deleted if no media found): '
+ fullFilePath, logger.DEBUG)
# tidy up - delete our temp dir
if can_delete and os.path.isdir(tmp_dir):
logger.log('Removing temp dir: ' + tmp_dir, logger.DEBUG)
shutil.rmtree(tmp_dir)
def processDir(dirName,
nzbName=None,
process_method=None,
force=False,
is_priority=None,
delete_on=False,
failed=False,
proc_type="auto"):
"""
Scans through the files in dirName and processes whatever media files it finds
:param dirName: The folder name to look in
:param nzbName: The NZB name which resulted in this folder being downloaded
:param force: True to postprocess already postprocessed files
:param failed: Boolean for whether or not the download failed
:param proc_type: Type of postprocessing auto or manual
"""
result = ProcessResult()
# if they passed us a real dir then assume it's the one we want
if ek(os.path.isdir, dirName):
dirName = ek(os.path.realpath, dirName)
result.output += logHelper(u"Processing folder %s" % dirName,
logger.DEBUG)
# if the client and SickRage are not on the same machine translate the directory into a network directory
elif all([
sickbeard.TV_DOWNLOAD_DIR,
ek(os.path.isdir, sickbeard.TV_DOWNLOAD_DIR),
ek(os.path.normpath, dirName) == ek(os.path.normpath,
sickbeard.TV_DOWNLOAD_DIR)
]):
dirName = ek(os.path.join, sickbeard.TV_DOWNLOAD_DIR,
ek(os.path.abspath, dirName).split(os.path.sep)[-1])
result.output += logHelper(u"Trying to use folder: %s " % dirName,
logger.DEBUG)
# if we didn't find a real dir then quit
if not ek(os.path.isdir, dirName):
result.output += logHelper(
u"Unable to figure out what folder to process. "
u"If your downloader and SickRage aren't on the same PC "
u"make sure you fill out your TV download dir in the config.",
logger.DEBUG)
return result.output
path, dirs, files = get_path_dir_files(dirName, nzbName, proc_type)
files = [x for x in files if not is_torrent_or_nzb_file(x)]
SyncFiles = [x for x in files if is_sync_file(x)]
nzbNameOriginal = nzbName
# Don't post process if files are still being synced and option is activated
postpone = SyncFiles and sickbeard.POSTPONE_IF_SYNC_FILES
# Warn user if 'postpone if no subs' is enabled. Will debug possible user issues with PP
if sickbeard.POSTPONE_IF_NO_SUBS:
result.output += logHelper(
u"Feature 'postpone postprocessing if no subtitle available' is enabled",
logger.INFO)
if not postpone:
result.output += logHelper(u"PostProcessing Path: %s" % path,
logger.INFO)
result.output += logHelper(u"PostProcessing Dirs: %s" % str(dirs),
logger.DEBUG)
videoFiles = [x for x in files if helpers.isMediaFile(x)]
rarFiles = [x for x in files if helpers.isRarFile(x)]
rarContent = ""
if rarFiles and not (sickbeard.POSTPONE_IF_NO_SUBS and videoFiles):
# Unpack only if video file was not already extracted by 'postpone if no subs' feature
rarContent = unRAR(path, rarFiles, force, result)
files += rarContent
videoInRar = [x for x in rarContent
if helpers.isMediaFile(x)] if rarContent else ''
result.output += logHelper(u"PostProcessing Files: %s" % files,
logger.DEBUG)
result.output += logHelper(
u"PostProcessing VideoFiles: %s" % videoFiles, logger.DEBUG)
result.output += logHelper(
u"PostProcessing RarContent: %s" % rarContent, logger.DEBUG)
result.output += logHelper(
u"PostProcessing VideoInRar: %s" % videoInRar, logger.DEBUG)
# If nzbName is set and there's more than one videofile in the folder, files will be lost (overwritten).
nzbName = None if len(videoFiles) >= 2 else nzbName
process_method = process_method if process_method else sickbeard.PROCESS_METHOD
result.result = True
# Don't Link media when the media is extracted from a rar in the same path
if process_method in (u'hardlink', u'symlink') and videoInRar:
process_media(path, videoInRar, nzbName, u'move', force,
is_priority, result)
delete_files(path, rarContent, result)
for video in set(videoFiles) - set(videoInRar):
process_media(path, [video], nzbName, process_method, force,
is_priority, result)
elif sickbeard.DELRARCONTENTS and videoInRar:
process_media(path, videoInRar, nzbName, process_method, force,
is_priority, result)
delete_files(path, rarContent, result, True)
for video in set(videoFiles) - set(videoInRar):
process_media(path, [video], nzbName, process_method, force,
is_priority, result)
else:
for video in videoFiles:
process_media(path, [video], nzbName, process_method, force,
is_priority, result)
else:
result.output += logHelper(
u"Found temporary sync files: %s in path: %s" % (SyncFiles, path))
result.output += logHelper(u"Skipping post processing for folder: %s" %
path)
result.missedfiles.append(u"%s : Syncfiles found" % path)
# Process Video File in all TV Subdir
for curDir in [
x for x in dirs
if validateDir(path, x, nzbNameOriginal, failed, result)
]:
result.result = True
for processPath, _, fileList in ek(os.walk,
ek(os.path.join, path, curDir),
topdown=False):
if not validateDir(path, processPath, nzbNameOriginal, failed,
result):
continue
SyncFiles = [x for x in fileList if is_sync_file(x)]
# Don't post process if files are still being synced and option is activated
postpone = SyncFiles and sickbeard.POSTPONE_IF_SYNC_FILES
if not postpone:
videoFiles = [x for x in fileList if helpers.isMediaFile(x)]
rarFiles = [x for x in fileList if helpers.isRarFile(x)]
rarContent = ""
if rarFiles and not (sickbeard.POSTPONE_IF_NO_SUBS
and videoFiles):
# Unpack only if video file was not already extracted by 'postpone if no subs' feature
rarContent = unRAR(processPath, rarFiles, force, result)
fileList = set(fileList + rarContent)
videoInRar = [x for x in rarContent
if helpers.isMediaFile(x)] if rarContent else ''
notwantedFiles = [x for x in fileList if x not in videoFiles]
if notwantedFiles:
result.output += logHelper(
u"Found unwanted files: %s" % notwantedFiles,
logger.DEBUG)
# Don't Link media when the media is extracted from a rar in the same path
if process_method in (u'hardlink', u'symlink') and videoInRar:
process_media(processPath, videoInRar, nzbName, u'move',
force, is_priority, result)
process_media(processPath,
set(videoFiles) - set(videoInRar), nzbName,
process_method, force, is_priority, result)
delete_files(processPath, rarContent, result)
elif sickbeard.DELRARCONTENTS and videoInRar:
process_media(processPath, videoInRar, nzbName,
process_method, force, is_priority, result)
process_media(processPath,
set(videoFiles) - set(videoInRar), nzbName,
process_method, force, is_priority, result)
delete_files(processPath, rarContent, result, True)
else:
process_media(processPath, videoFiles, nzbName,
process_method, force, is_priority, result)
# Delete all file not needed and avoid deleting files if Manual PostProcessing
if not (process_method == u"move" and result.result) or (
proc_type == u"manual" and not delete_on):
continue
delete_folder(ek(os.path.join, processPath, u'@eaDir'))
delete_files(processPath, notwantedFiles, result)
if all([
not sickbeard.NO_DELETE or proc_type == u"manual",
process_method == u"move",
ek(os.path.normpath, processPath) != ek(
os.path.normpath, sickbeard.TV_DOWNLOAD_DIR)
]):
if delete_folder(processPath, check_empty=True):
result.output += logHelper(
u"Deleted folder: %s" % processPath,
logger.DEBUG)
else:
result.output += logHelper(
u"Found temporary sync files: %s in path: %s" %
(SyncFiles, processPath))
result.output += logHelper(
u"Skipping post processing for folder: %s" % processPath)
result.missedfiles.append(u"%s : Syncfiles found" % path)
if result.aggresult:
result.output += logHelper(u"Successfully processed")
if result.missedfiles:
result.output += logHelper(
u"I did encounter some unprocessable items: ")
for missedfile in result.missedfiles:
result.output += logHelper(u"[%s]" % missedfile)
else:
result.output += logHelper(
u"Problem(s) during processing, failed the following files/folders: ",
logger.WARNING)
for missedfile in result.missedfiles:
result.output += logHelper(u"[%s]" % missedfile, logger.WARNING)
return result.output
def run(self):
"""
Called every few seconds to handle any running/finished torrents
"""
if not LIBTORRENT_AVAILABLE:
return
if not self.loadedRunningTorrents:
torrent_save_file = _get_running_torrents_pickle_path(False)
if os.path.isfile(torrent_save_file):
logger.log(u"Saved torrents found in %s, loading" % (torrent_save_file), logger.DEBUG)
_load_saved_torrents()
self.loadedRunningTorrents = True
sess = _get_session(False)
if sess is not None:
while 1:
a = sess.pop_alert()
if not a:
break
if type(a) == str:
logger.log(a, logger.DEBUG)
else:
logger.log(u"(%s): %s" % (type(a).__name__, ek.fixStupidEncodings(a.message(), True)), logger.DEBUG)
logTorrentStatus = (time.time() - self.lastTorrentStatusLogTS) >= 600
for torrent_data in running_torrents:
if torrent_data["handle"].has_metadata():
ti = torrent_data["handle"].get_torrent_info()
name = ti.name()
torrent_data["name"] = name
torrent_data["total_size"] = ti.total_size()
if not torrent_data["have_torrentFile"]:
# if this was a magnet or url, and we now have downloaded the metadata
# for it, best to save it locally in case we need to resume
ti = torrent_data["handle"].get_torrent_info()
torrentFile = lt.create_torrent(ti)
torrent_data["torrent"] = lt.bencode(torrentFile.generate())
torrent_data["have_torrentFile"] = True
logger.log(
u"Created torrent file for %s as metadata d/l is now complete" % (name), logger.DEBUG
)
else:
name = "-"
s = torrent_data["handle"].status()
torrent_data["status"] = str(s.state)
torrent_data["progress"] = s.progress
torrent_data["rate_down"] = s.download_rate
torrent_data["rate_up"] = s.upload_rate
torrent_data["paused"] = s.paused
torrent_data["error"] = s.error
# currentRatio = 0.0 if s.total_download == 0 else float(s.total_upload)/float(s.total_download)
currentRatio = (
0.0 if s.all_time_download == 0 else float(s.all_time_upload) / float(s.all_time_download)
)
torrent_data["ratio"] = currentRatio
if s.state in [lt.torrent_status.seeding, lt.torrent_status.finished]:
with torrent_data["lock"]:
# this is the post-processing & removing code, so make sure that there's
# only one thread doing either here, as the two could easily interfere with
# one another
if not torrent_data["post_processed"]:
# torrent has just completed download, so we need to do
# post-processing on it.
ti = torrent_data["handle"].get_torrent_info()
any_file_success = False
for f in ti.files():
fullpath = os.path.join(sickbeard.LIBTORRENT_WORKING_DIR, "data", f.path)
logger.log(u'Post-processing "%s"' % (fullpath), logger.DEBUG)
if isMediaFile(fullpath):
logger.log(u"this is a media file", logger.DEBUG)
try:
processor = postProcessor.PostProcessor(fullpath, name)
if processor.process(forceKeepOriginalFiles=True):
logger.log(u'Success post-processing "%s"' % (fullpath), logger.DEBUG)
any_file_success = True
except exceptions.PostProcessingFailed, e:
logger.log(
u'Failed post-processing file "%s" with error "%s"' % (fullpath, ex(e)),
logger.ERROR,
)
## Richard Hills [email protected] 2013-07-20 : if your file is downloaded but does not match we remove torrent and leave files for manual processing.
## any file success can match a single file from a multi file torrent - we will lose some data regardless if we have 1 file in a multi file torrent which matches and the others fail
if not any_file_success:
logger.log(
u'Post processing found no useful information for file "%s" please process this manually (rename/move and manually trigger processing)'
% (fullpath),
logger.ERROR,
)
_remove_torrent_by_handle(torrent_data["handle"], False)
else:
torrent_data["post_processed"] = True
else:
# post-processing has already been performed. So we just
# need to ensure check the ratio and delete the torrent
# if we're good.
if currentRatio >= sickbeard.LIBTORRENT_SEED_TO_RATIO:
logger.log(
u'Torrent "%s" has seeded to ratio %f. Removing it.' % (name, currentRatio),
logger.MESSAGE,
)
deleteFilesToo = True
if not torrent_data["post_processed"]:
logger.log(u"Torrent has not been post_processed. Keeping files.", logger.MESSAGE)
deleteFilesToo = False
_remove_torrent_by_handle(torrent_data["handle"], deleteFilesToo)
else:
if logTorrentStatus:
self.lastTorrentStatusLogTS = time.time()
logger.log(u'"%s" seeding %0.3f' % (name, currentRatio), logger.DEBUG)
elif s.state == lt.torrent_status.downloading:
if logTorrentStatus:
self.lastTorrentStatusLogTS = time.time()
logger.log(u'"%s" downloading %0.2f' % (name, s.progress * 100.0), logger.DEBUG)
def subtitles_download_in_pp(): # pylint: disable=too-many-locals, too-many-branches, too-many-statements
logger.log(u'Checking for needed subtitles in Post-Process folder',
logger.INFO)
providers = enabled_service_list()
pool = SubtitleProviderPool()
# Search for all wanted languages
languages = {from_code(language) for language in wanted_languages()}
if not languages:
return
# Dict of language exceptions to use with subliminal
language_exceptions = {'pt-br': 'pob'}
run_post_process = False
# Check if PP folder is set
if sickbeard.TV_DOWNLOAD_DIR and os.path.isdir(
sickbeard.TV_DOWNLOAD_DIR):
for root, _, files in os.walk(sickbeard.TV_DOWNLOAD_DIR,
topdown=False):
rar_files = [
rar_file for rar_file in files if isRarFile(rar_file)
]
if rar_files and sickbeard.UNPACK:
video_files = [
video_file for video_file in files
if isMediaFile(video_file)
]
if u'_UNPACK' not in root and (
not video_files
or root == sickbeard.TV_DOWNLOAD_DIR):
logger.log(
u'Found rar files in post-process folder: {0}'.
format(rar_files), logger.DEBUG)
result = processTV.ProcessResult()
processTV.unRAR(root, rar_files, False, result)
elif rar_files and not sickbeard.UNPACK:
logger.log(
u'Unpack is disabled. Skipping: {0}'.format(rar_files),
logger.WARNING)
for root, _, files in os.walk(sickbeard.TV_DOWNLOAD_DIR,
topdown=False):
for filename in sorted(files):
try:
# Remove non release groups from video file. Needed to match subtitles
new_filename = remove_non_release_groups(filename)
if new_filename != filename:
os.rename(filename, new_filename)
filename = new_filename
except Exception as error:
logger.log(
u"Couldn't remove non release groups from video file. Error: {0}"
.format(ex(error)), logger.DEBUG)
# Delete unwanted subtitles before downloading new ones
if sickbeard.SUBTITLES_MULTI and sickbeard.SUBTITLES_KEEP_ONLY_WANTED and filename.rpartition(
'.')[2] in subtitle_extensions:
subtitle_language = filename.rsplit('.', 2)[1].lower()
if len(
subtitle_language
) == 2 and subtitle_language in language_converters[
'opensubtitles'].codes:
subtitle_language = Language.fromcode(
subtitle_language, 'alpha2').opensubtitles
elif subtitle_language in language_exceptions:
subtitle_language = language_exceptions.get(
subtitle_language, subtitle_language)
elif subtitle_language not in language_converters[
'opensubtitles'].codes:
subtitle_language = 'unknown'
if subtitle_language not in sickbeard.SUBTITLES_LANGUAGES:
try:
os.remove(os.path.join(root, filename))
logger.log(
u"Deleted '{0}' because we don't want subtitle language '{1}'. We only want '{2}' language(s)"
.format(
filename, subtitle_language, ','.join(
sickbeard.SUBTITLES_LANGUAGES)),
logger.DEBUG)
except Exception as error:
logger.log(
u"Couldn't delete subtitle: {0}. Error: {1}"
.format(filename, ex(error)), logger.DEBUG)
if isMediaFile(filename) and processTV.subtitles_enabled(
filename):
try:
video = get_video(os.path.join(root, filename),
subtitles=False,
embedded_subtitles=False)
subtitles_list = pool.list_subtitles(
video, languages)
for provider in providers:
if provider in pool.discarded_providers:
logger.log(
u'Could not search in {0} provider. Discarding for now'
.format(provider), logger.DEBUG)
if not subtitles_list:
logger.log(
u'No subtitles found for {0}'.format(
os.path.join(root, filename)),
logger.DEBUG)
continue
logger.log(
u'Found subtitle(s) canditate(s) for {0}'.
format(filename), logger.INFO)
hearing_impaired = sickbeard.SUBTITLES_HEARING_IMPAIRED
user_score = 213 if sickbeard.SUBTITLES_PERFECT_MATCH else 198
found_subtitles = pool.download_best_subtitles(
subtitles_list,
video,
languages=languages,
hearing_impaired=hearing_impaired,
min_score=user_score,
only_one=not sickbeard.SUBTITLES_MULTI)
for subtitle in subtitles_list:
score = subliminal.score.compute_score(
subtitle,
video,
hearing_impaired=sickbeard.
SUBTITLES_HEARING_IMPAIRED)
logger.log(
u'[{0}] Subtitle score for {1} is: {2} (min={3})'
.format(subtitle.provider_name,
subtitle.id, score, user_score),
logger.DEBUG)
downloaded_languages = set()
for subtitle in found_subtitles:
logger.log(
u'Found subtitle for {0} in {1} provider with language {2}'
.format(os.path.join(root, filename),
subtitle.provider_name,
subtitle.language.opensubtitles),
logger.INFO)
subliminal.save_subtitles(
video,
found_subtitles,
directory=root,
single=not sickbeard.SUBTITLES_MULTI)
subtitles_multi = not sickbeard.SUBTITLES_MULTI
subtitle_path = subliminal.subtitle.get_subtitle_path(
video.name, None
if subtitles_multi else subtitle.language)
if root is not None:
subtitle_path = os.path.join(
root,
os.path.split(subtitle_path)[1])
sickbeard.helpers.chmodAsParent(subtitle_path)
sickbeard.helpers.fixSetGroupID(subtitle_path)
downloaded_languages.add(
subtitle.language.opensubtitles)
# Don't run post processor unless at least one file has all of the needed subtitles
if not needs_subtitles(downloaded_languages):
run_post_process = True
except Exception as error:
logger.log(
u'Error occurred when downloading subtitles for: {0}. Error: {1}'
.format(os.path.join(root, filename),
ex(error)))
if run_post_process:
logger.log(
u'Starting post-process with default settings now that we found subtitles'
)
processTV.processDir(sickbeard.TV_DOWNLOAD_DIR)
def _finish_download_process():
"""
Finish downloading a pid
(this is a closure, called in another thread)
"""
last_reported_perc = 0.0
while p.poll() is None:
line = p.stdout.readline()
if line:
line = line.rstrip()
match = re.match(IPLAYER_DL_PROGRESS_PATTERN, line, re.IGNORECASE)
if match:
dlSize = match.group('size')
dlTime = match.group('time')
dlPerc = float(match.group('perc'))
if dlPerc - last_reported_perc >= 10.0:
# only report progress every 10% or so.
logger.log(u"RUNNING iPLAYER: "+line, logger.DEBUG)
last_reported_perc = dlPerc
else:
# not a progress line, echo it to debug
logger.log(u"RUNNING iPLAYER: "+line, logger.DEBUG)
logger.log(u"RUNNING iPLAYER: process has ended, returncode was " +
repr(p.returncode) , logger.DEBUG)
# We will need to rename some of the files in the folder to ensure
# that sb is comfortable with them.
videoFiles = listMediaFiles(tmp_dir)
for videoFile in videoFiles:
filePrefix, fileExt = os.path.splitext(videoFile)
if fileExt and fileExt[0] == '.':
fileExt = fileExt[1:]
# split again to get the quality
filePrePrefix, fileQuality = os.path.splitext(filePrefix)
if fileQuality and fileQuality[0] == '.':
fileQuality = fileQuality[1:]
qual_str = cls.iplayer_quality_to_sb_quality_string(fileQuality)
# reassemble the filename again, with new quality
newFilePrefix = filePrePrefix + '.' + qual_str
newFileName = newFilePrefix + '.' + fileExt
if newFileName != videoFile: # just in case!
logger.log('Renaming {0} to {1}'.format(videoFile, newFileName), logger.DEBUG)
os.rename(videoFile, newFileName)
# Also need to rename any associated files (nfo and srt)
for otherFile in glob.glob(newFilePrefix + '.*'):
if otherFile == newFileName:
continue
otherFilePrefix, otherFileExt = os.path.splitext(otherFile)
newOtherFile = newFilePrefix + otherFileExt
logger.log('Renaming {0} to {1}'.format(otherFile, newOtherFile), logger.DEBUG)
os.rename(otherFile, newOtherFile)
# Ok, we're done with *our* post-processing, so let SB do its own.
processResult = processDir(tmp_dir)
#logger.log(u"processDir returned " + processResult , logger.DEBUG) - this is long, and quite useless!
files_remaining = os.listdir(tmp_dir)
can_delete = True
for filename in files_remaining:
fullFilePath = os.path.join(tmp_dir, filename)
isVideo = isMediaFile(fullFilePath)
if isVideo:
can_delete = False # keep the folder - something prob went wrong
logger.log('Found a media file after processing, something probably went wrong: ' + fullFilePath, logger.MESSAGE)
else:
logger.log('Extra file left over (will be deleted if no media found): ' + fullFilePath, logger.DEBUG)
# tidy up - delete our temp dir
if can_delete and os.path.isdir(tmp_dir):
logger.log('Removing temp dir: ' + tmp_dir, logger.DEBUG)
shutil.rmtree(tmp_dir)
def validateDir(
path, dirName, nzbNameOriginal, failed, result
): # pylint: disable=too-many-locals,too-many-branches,too-many-return-statements
"""
Check if directory is valid for processing
:param path: Path to use
:param dirName: Directory to check
:param nzbNameOriginal: Original NZB name
:param failed: Previously failed objects
:param result: Previous results
:return: True if dir is valid for processing, False if not
"""
dirName = ss(dirName)
IGNORED_FOLDERS = [u".AppleDouble", u".@__thumb", u"@eaDir"]
folder_name = ek(os.path.basename, dirName)
if folder_name in IGNORED_FOLDERS:
return False
result.output += logHelper(u"Processing folder " + dirName, logger.DEBUG)
if folder_name.startswith(u"_FAILED_"):
result.output += logHelper(u"The directory name indicates it failed to extract.", logger.DEBUG)
failed = True
elif folder_name.startswith(u"_UNDERSIZED_"):
result.output += logHelper(
u"The directory name indicates that it was previously rejected for being undersized.", logger.DEBUG
)
failed = True
elif folder_name.upper().startswith(u"_UNPACK"):
result.output += logHelper(
u"The directory name indicates that this release is in the process of being unpacked.", logger.DEBUG
)
result.missedfiles.append(u"%s : Being unpacked" % dirName)
return False
if failed:
process_failed(ek(os.path.join, path, dirName), nzbNameOriginal, result)
result.missedfiles.append(u"%s : Failed download" % dirName)
return False
if helpers.is_hidden_folder(ek(os.path.join, path, dirName)):
result.output += logHelper(u"Ignoring hidden folder: %s" % dirName, logger.DEBUG)
result.missedfiles.append(u"%s : Hidden folder" % dirName)
return False
# make sure the dir isn't inside a show dir
main_db_con = db.DBConnection()
sql_results = main_db_con.select("SELECT location FROM tv_shows")
for sqlShow in sql_results:
if (
dirName.lower().startswith(ek(os.path.realpath, sqlShow["location"]).lower() + os.sep)
or dirName.lower() == ek(os.path.realpath, sqlShow["location"]).lower()
):
result.output += logHelper(
u"Cannot process an episode that's already been moved to its show dir, skipping " + dirName,
logger.WARNING,
)
return False
# Get the videofile list for the next checks
allFiles = []
allDirs = []
for _, processdir, fileList in ek(os.walk, ek(os.path.join, path, dirName), topdown=False):
allDirs += processdir
allFiles += fileList
videoFiles = [x for x in allFiles if helpers.isMediaFile(x)]
allDirs.append(dirName)
# check if the dir have at least one tv video file
for video in videoFiles:
try:
NameParser().parse(video, cache_result=False)
return True
except (InvalidNameException, InvalidShowException):
pass
for proc_dir in allDirs:
try:
NameParser().parse(proc_dir, cache_result=False)
return True
except (InvalidNameException, InvalidShowException):
pass
if sickbeard.UNPACK:
# Search for packed release
packedFiles = [x for x in allFiles if helpers.isRarFile(x)]
for packed in packedFiles:
try:
NameParser().parse(packed, cache_result=False)
return True
except (InvalidNameException, InvalidShowException):
pass
result.output += logHelper(u"%s : No processable items found in folder" % dirName, logger.DEBUG)
return False
def download_subtitles(subtitles_info): # pylint: disable=too-many-locals, too-many-branches, too-many-statements
existing_subtitles = subtitles_info['subtitles']
if not needs_subtitles(existing_subtitles):
logger.log(u'Episode already has all needed subtitles, skipping {} {}'.format
(subtitles_info['show_name'], episode_num(subtitles_info['season'], subtitles_info['episode']) or
episode_num(subtitles_info['season'], subtitles_info['episode'], numbering='absolute')), logger.DEBUG)
return existing_subtitles, None
# Check if we really need subtitles
languages = get_needed_languages(existing_subtitles)
if not languages:
logger.log(u'No subtitles needed for {} {}'.format
(subtitles_info['show_name'], episode_num(subtitles_info['season'], subtitles_info['episode']) or
episode_num(subtitles_info['season'], subtitles_info['episode'], numbering='absolute')), logger.DEBUG)
return existing_subtitles, None
subtitles_path = get_subtitles_path(subtitles_info['location'])
video_path = subtitles_info['location']
# Perfect match = hash score - hearing impaired score - resolution score (subtitle for 720p its the same for 1080p)
# Perfect match = 215 -1 -1 = 213
# No-perfect match = hash score - hearing impaired score - resolution score - release_group score
# No-perfect match = 215 -1 -1 -9 = 204
# From latest subliminal code:
# episode_scores = {'hash': 215, 'series': 108, 'year': 54, 'season': 18, 'episode': 18, 'release_group': 9,
# 'format': 4, 'audio_codec': 2, 'resolution': 1, 'hearing_impaired': 1, 'video_codec': 1}
user_score = 213 if sickbeard.SUBTITLES_PERFECT_MATCH else 204
video = get_video(video_path, subtitles_path=subtitles_path)
if not video:
logger.log(u'Exception caught in subliminal.scan_video for {} {}'.format
(subtitles_info['show_name'], episode_num(subtitles_info['season'], subtitles_info['episode']) or
episode_num(subtitles_info['season'], subtitles_info['episode'], numbering='absolute')), logger.DEBUG)
return existing_subtitles, None
providers = enabled_service_list()
provider_configs = {'addic7ed': {'username': sickbeard.ADDIC7ED_USER,
'password': sickbeard.ADDIC7ED_PASS},
'legendastv': {'username': sickbeard.LEGENDASTV_USER,
'password': sickbeard.LEGENDASTV_PASS},
'opensubtitles': {'username': sickbeard.OPENSUBTITLES_USER,
'password': sickbeard.OPENSUBTITLES_PASS}}
pool = ProviderPool(providers=providers, provider_configs=provider_configs)
try:
subtitles_list = pool.list_subtitles(video, languages)
for provider in providers:
if provider in pool.discarded_providers:
logger.log(u'Could not search in {} provider. Discarding for now'.format(provider), logger.DEBUG)
if not subtitles_list:
logger.log(u'No subtitles found for {} {}'.format
(subtitles_info['show_name'], episode_num(subtitles_info['season'], subtitles_info['episode']) or
episode_num(subtitles_info['season'], subtitles_info['episode'], numbering='absolute')), logger.DEBUG)
return existing_subtitles, None
for subtitle in subtitles_list:
score = subliminal.score.compute_score(subtitle, video, hearing_impaired=sickbeard.SUBTITLES_HEARING_IMPAIRED)
logger.log(u'[{}] Subtitle score for {} is: {} (min={})'.format
(subtitle.provider_name, subtitle.id, score, user_score), logger.DEBUG)
found_subtitles = pool.download_best_subtitles(subtitles_list, video, languages=languages,
hearing_impaired=sickbeard.SUBTITLES_HEARING_IMPAIRED,
min_score=user_score, only_one=not sickbeard.SUBTITLES_MULTI)
subliminal.save_subtitles(video, found_subtitles, directory=subtitles_path,
single=not sickbeard.SUBTITLES_MULTI)
except IOError as error:
if 'No space left on device' in ex(error):
logger.log(u'Not enough space on the drive to save subtitles', logger.WARNING)
else:
logger.log(traceback.format_exc(), logger.WARNING)
except Exception:
logger.log(u'Error occurred when downloading subtitles for: {}'.format(video_path))
logger.log(traceback.format_exc(), logger.ERROR)
return existing_subtitles, None
for subtitle in found_subtitles:
subtitle_path = subliminal.subtitle.get_subtitle_path(video.name,
None if not sickbeard.SUBTITLES_MULTI else
subtitle.language)
if subtitles_path is not None:
subtitle_path = os.path.join(subtitles_path, os.path.split(subtitle_path)[1])
sickbeard.helpers.chmodAsParent(subtitle_path)
sickbeard.helpers.fixSetGroupID(subtitle_path)
if sickbeard.SUBTITLES_HISTORY:
logger.log(u'history.logSubtitle {}, {}'.format
(subtitle.provider_name, subtitle.language.opensubtitles), logger.DEBUG)
history.logSubtitle(subtitles_info['show_indexerid'], subtitles_info['season'],
subtitles_info['episode'], subtitles_info['status'], subtitle)
if sickbeard.SUBTITLES_EXTRA_SCRIPTS and isMediaFile(video_path) and not sickbeard.EMBEDDED_SUBTITLES_ALL:
run_subs_extra_scripts(subtitles_info, subtitle, video, single=not sickbeard.SUBTITLES_MULTI)
new_subtitles = sorted({subtitle.language.opensubtitles for subtitle in found_subtitles})
current_subtitles = sorted({subtitle for subtitle in new_subtitles + existing_subtitles}) if existing_subtitles else new_subtitles
if not sickbeard.SUBTITLES_MULTI and len(found_subtitles) == 1:
new_code = found_subtitles[0].language.opensubtitles
if new_code not in existing_subtitles:
current_subtitles.remove(new_code)
current_subtitles.append('und')
return current_subtitles, new_subtitles
def validateDir(path, dirName, nzbNameOriginal, failed, result):
"""
Check if directory is valid for processing
:param path: Path to use
:param dirName: Directory to check
:param nzbNameOriginal: Original NZB name
:param failed: Previously failed objects
:param result: Previous results
:return: True if dir is valid for processing, False if not
"""
IGNORED_FOLDERS = ['.AppleDouble', '.@__thumb', '@eaDir']
folder_name = ek(os.path.basename, dirName)
if folder_name in IGNORED_FOLDERS:
return False
result.output += logHelper("Processing folder " + dirName, logging.DEBUG)
if folder_name.startswith('_FAILED_'):
result.output += logHelper(
"The directory name indicates it failed to extract.",
logging.DEBUG)
failed = True
elif folder_name.startswith('_UNDERSIZED_'):
result.output += logHelper(
"The directory name indicates that it was previously rejected for being undersized.",
logging.DEBUG)
failed = True
elif folder_name.upper().startswith('_UNPACK'):
result.output += logHelper(
"The directory name indicates that this release is in the process of being unpacked.",
logging.DEBUG)
result.missedfiles.append(dirName + " : Being unpacked")
return False
if failed:
process_failed(ek(os.path.join, path, dirName), nzbNameOriginal,
result)
result.missedfiles.append(dirName + " : Failed download")
return False
if helpers.is_hidden_folder(ek(os.path.join, path, dirName)):
result.output += logHelper("Ignoring hidden folder: " + dirName,
logging.DEBUG)
result.missedfiles.append(dirName + " : Hidden folder")
return False
# make sure the dir isn't inside a show dir
myDB = db.DBConnection()
sqlResults = myDB.select("SELECT * FROM tv_shows")
for sqlShow in sqlResults:
if dirName.lower().startswith(ek(os.path.realpath, sqlShow[b"location"]).lower() + os.sep) or \
dirName.lower() == ek(os.path.realpath, sqlShow[b"location"]).lower():
result.output += logHelper(
"Cannot process an episode that's already been moved to its show dir, skipping "
+ dirName, logging.WARNING)
return False
# Get the videofile list for the next checks
allFiles = []
allDirs = []
for _, processdir, fileList in ek(os.walk,
ek(os.path.join, path, dirName),
topdown=False):
allDirs += processdir
allFiles += fileList
videoFiles = [x for x in allFiles if helpers.isMediaFile(x)]
allDirs.append(dirName)
# check if the dir have at least one tv video file
for video in videoFiles:
try:
NameParser().parse(video, cache_result=False)
return True
except (InvalidNameException, InvalidShowException):
pass
for proc_dir in allDirs:
try:
NameParser().parse(proc_dir, cache_result=False)
return True
except (InvalidNameException, InvalidShowException):
pass
if sickbeard.UNPACK:
# Search for packed release
packedFiles = [x for x in allFiles if helpers.isRarFile(x)]
for packed in packedFiles:
try:
NameParser().parse(packed, cache_result=False)
return True
except (InvalidNameException, InvalidShowException):
pass
result.output += logHelper(
dirName + " : No processable items found in folder", logging.DEBUG)
return False
def processDir(dirName, nzbName=None, process_method=None, force=False, is_priority=None, delete_on=False, failed=False, proc_type="auto"): # pylint: disable=too-many-arguments,too-many-branches,too-many-statements,too-many-locals
"""
Scans through the files in dirName and processes whatever media files it finds
:param dirName: The folder name to look in
:param nzbName: The NZB name which resulted in this folder being downloaded
:param force: True to postprocess already postprocessed files
:param failed: Boolean for whether or not the download failed
:param proc_type: Type of postprocessing auto or manual
"""
result = ProcessResult()
result.output += logHelper(u"Processing folder " + dirName, logger.DEBUG)
result.output += logHelper(u"TV_DOWNLOAD_DIR: " + sickbeard.TV_DOWNLOAD_DIR, logger.DEBUG)
postpone = False
# if they passed us a real dir then assume it's the one we want
if ek(os.path.isdir, dirName):
dirName = ek(os.path.realpath, dirName)
# if the client and SickRage are not on the same machine translate the Dir in a network dir
elif sickbeard.TV_DOWNLOAD_DIR and ek(os.path.isdir, sickbeard.TV_DOWNLOAD_DIR) \
and ek(os.path.normpath, dirName) != ek(os.path.normpath, sickbeard.TV_DOWNLOAD_DIR):
dirName = ek(os.path.join, sickbeard.TV_DOWNLOAD_DIR, ek(os.path.abspath, dirName).split(os.path.sep)[-1])
result.output += logHelper(u"Trying to use folder " + dirName, logger.DEBUG)
# if we didn't find a real dir then quit
if not ek(os.path.isdir, dirName):
result.output += logHelper(
u"Unable to figure out what folder to process. If your downloader and SickRage aren't on the same PC make sure you fill out your TV download dir in the config.",
logger.DEBUG)
return result.output
path, dirs, files = get_path_dir_files(dirName, nzbName, proc_type)
files = [x for x in files if not is_torrent_or_nzb_file(x)]
SyncFiles = [x for x in files if is_sync_file(x)]
# Don't post process if files are still being synced and option is activated
if SyncFiles and sickbeard.POSTPONE_IF_SYNC_FILES:
postpone = True
nzbNameOriginal = nzbName
if not postpone:
result.output += logHelper(u"PostProcessing Path: " + path, logger.INFO)
result.output += logHelper(u"PostProcessing Dirs: " + str(dirs), logger.DEBUG)
rarFiles = [x for x in files if helpers.isRarFile(x)]
rarContent = unRAR(path, rarFiles, force, result)
files += rarContent
videoFiles = [x for x in files if helpers.isMediaFile(x)]
videoInRar = [x for x in rarContent if helpers.isMediaFile(x)]
result.output += logHelper(u"PostProcessing Files: " + str(files), logger.DEBUG)
result.output += logHelper(u"PostProcessing VideoFiles: " + str(videoFiles), logger.DEBUG)
result.output += logHelper(u"PostProcessing RarContent: " + str(rarContent), logger.DEBUG)
result.output += logHelper(u"PostProcessing VideoInRar: " + str(videoInRar), logger.DEBUG)
# If nzbName is set and there's more than one videofile in the folder, files will be lost (overwritten).
if len(videoFiles) >= 2:
nzbName = None
if not process_method:
process_method = sickbeard.PROCESS_METHOD
result.result = True
# Don't Link media when the media is extracted from a rar in the same path
if process_method in ('hardlink', 'symlink') and videoInRar:
process_media(path, videoInRar, nzbName, 'move', force, is_priority, result)
delete_files(path, rarContent, result)
for video in set(videoFiles) - set(videoInRar):
process_media(path, [video], nzbName, process_method, force, is_priority, result)
elif sickbeard.DELRARCONTENTS and videoInRar:
process_media(path, videoInRar, nzbName, process_method, force, is_priority, result)
delete_files(path, rarContent, result, True)
for video in set(videoFiles) - set(videoInRar):
process_media(path, [video], nzbName, process_method, force, is_priority, result)
else:
for video in videoFiles:
process_media(path, [video], nzbName, process_method, force, is_priority, result)
else:
result.output += logHelper(u"Found temporary sync files, skipping post processing for folder " + str(path))
result.output += logHelper(u"Sync Files: " + str(SyncFiles) + " in path: " + path)
result.missedfiles.append(path + " : Syncfiles found")
# Process Video File in all TV Subdir
for curDir in [x for x in dirs if validateDir(path, x, nzbNameOriginal, failed, result)]:
result.result = True
for processPath, _, fileList in ek(os.walk, ek(os.path.join, path, curDir), topdown=False):
if not validateDir(path, processPath, nzbNameOriginal, failed, result):
continue
postpone = False
SyncFiles = [x for x in fileList if is_sync_file(x)]
# Don't post process if files are still being synced and option is activated
if SyncFiles and sickbeard.POSTPONE_IF_SYNC_FILES:
postpone = True
if not postpone:
rarFiles = [x for x in fileList if helpers.isRarFile(x)]
rarContent = unRAR(processPath, rarFiles, force, result)
fileList = set(fileList + rarContent)
videoFiles = [x for x in fileList if helpers.isMediaFile(x)]
videoInRar = [x for x in rarContent if helpers.isMediaFile(x)]
notwantedFiles = [x for x in fileList if x not in videoFiles]
if notwantedFiles:
result.output += logHelper(u"Found unwanted files: " + str(notwantedFiles), logger.DEBUG)
# Don't Link media when the media is extracted from a rar in the same path
if process_method in ('hardlink', 'symlink') and videoInRar:
process_media(processPath, videoInRar, nzbName, 'move', force, is_priority, result)
process_media(processPath, set(videoFiles) - set(videoInRar), nzbName, process_method, force,
is_priority, result)
delete_files(processPath, rarContent, result)
elif sickbeard.DELRARCONTENTS and videoInRar:
process_media(processPath, videoInRar, nzbName, process_method, force, is_priority, result)
process_media(processPath, set(videoFiles) - set(videoInRar), nzbName, process_method, force,
is_priority, result)
delete_files(processPath, rarContent, result, True)
else:
process_media(processPath, videoFiles, nzbName, process_method, force, is_priority, result)
# Delete all file not needed
if process_method != "move" or not result.result \
or (proc_type == "manual" and not delete_on): # Avoid to delete files if is Manual PostProcessing
continue
delete_files(processPath, notwantedFiles, result)
if (not sickbeard.NO_DELETE or proc_type == "manual") and process_method == "move" and \
ek(os.path.normpath, processPath) != ek(os.path.normpath, sickbeard.TV_DOWNLOAD_DIR):
if delete_folder(processPath, check_empty=True):
result.output += logHelper(u"Deleted folder: " + processPath, logger.DEBUG)
else:
result.output += logHelper(u"Found temporary sync files, skipping post processing for folder: " + str(processPath))
result.output += logHelper(u"Sync Files: " + str(SyncFiles) + " in path: " + processPath)
result.missedfiles.append(processPath + " : Syncfiles found")
if result.aggresult:
result.output += logHelper(u"Processing completed")
if result.missedfiles:
result.output += logHelper(u"I did encounter some unprocessable items: ")
for missedfile in result.missedfiles:
result.output += logHelper(u"[" + missedfile + "]")
else:
result.output += logHelper(u"Problem(s) during processing, failed the following files/folders: ", logger.WARNING)
for missedfile in result.missedfiles:
result.output += logHelper(u"[" + missedfile + "]", logger.WARNING)
return result.output
