def unrar(path, rar_files, force, result):
"""
Extracts RAR files
:param path: Path to look for files in
:param rar_files: Names of RAR files
:param force: process currently processing items
:param result: Previous results
:return: List of unpacked file names
"""
unpacked_dirs = []
if sickrage.app.config.unpack == 1 and rar_files:
result.output += logHelper("Packed Releases detected: {0}".format(rar_files), sickrage.app.log.DEBUG)
for archive in rar_files:
failure = None
rar_handle = None
try:
archive_path = os.path.join(path, archive)
if already_postprocessed(path, archive, force, result):
result.output += logHelper("Archive file already post-processed, extraction skipped: {}".format
(archive_path), sickrage.app.log.DEBUG)
continue
if not is_rar_file(archive_path):
continue
result.output += logHelper("Checking if archive is valid and contains a video: {}".format(archive_path),
sickrage.app.log.DEBUG)
rar_handle = rarfile.RarFile(archive_path)
if rar_handle.needs_password():
# TODO: Add support in settings for a list of passwords to try here with rar_handle.set_password(x)
result.output += logHelper('Archive needs a password, skipping: {0}'.format(archive_path))
continue
# If there are no video files in the rar, don't extract it
rar_media_files = filter(is_media_file, rar_handle.namelist())
if not rar_media_files:
continue
rar_release_name = archive.rpartition('.')[0]
# Choose the directory we'll unpack to:
if sickrage.app.config.unpack_dir and os.path.isdir(sickrage.app.config.unpack_dir):
unpack_base_dir = sickrage.app.config.unpack_dir
else:
unpack_base_dir = path
if sickrage.app.config.unpack_dir: # Let user know if we can't unpack there
result.output += logHelper('Unpack directory cannot be verified. Using {}'.format(path),
sickrage.app.log.DEBUG)
# Fix up the list for checking if already processed
rar_media_files = [os.path.join(unpack_base_dir, rar_release_name, rar_media_file) for rar_media_file in
rar_media_files]
skip_rar = False
for rar_media_file in rar_media_files:
check_path, check_file = os.path.split(rar_media_file)
if already_postprocessed(check_path, check_file, force, result):
result.output += logHelper(
"Archive file already post-processed, extraction skipped: {0}".format
(rar_media_file), sickrage.app.log.DEBUG)
skip_rar = True
break
if skip_rar:
continue
rar_extract_path = os.path.join(unpack_base_dir, rar_release_name)
result.output += logHelper("Unpacking archive: {0}".format(archive), sickrage.app.log.DEBUG)
rar_handle.extractall(path=rar_extract_path)
unpacked_dirs.append(rar_extract_path)
except rarfile.RarCRCError:
failure = ('Archive Broken', 'Unpacking failed because of a CRC error')
except rarfile.RarWrongPassword:
failure = ('Incorrect RAR Password', 'Unpacking failed because of an Incorrect Rar Password')
except rarfile.PasswordRequired:
failure = ('Rar is password protected', 'Unpacking failed because it needs a password')
except rarfile.RarOpenError:
failure = ('Rar Open Error, check the parent folder and destination file permissions.',
'Unpacking failed with a File Open Error (file permissions?)')
except rarfile.RarExecError:
failure = ('Invalid Rar Archive Usage',
'Unpacking Failed with Invalid Rar Archive Usage. Is unrar installed and on the system PATH?')
except rarfile.BadRarFile:
failure = ('Invalid Rar Archive', 'Unpacking Failed with an Invalid Rar Archive Error')
except rarfile.NeedFirstVolume:
continue
except (Exception, rarfile.Error) as e:
failure = (e, 'Unpacking failed')
finally:
if rar_handle:
del rar_handle
if failure:
result.output += logHelper('Failed to extract the archive {}: {}'.format(archive, failure[0]),
sickrage.app.log.WARNING)
result.missed_files.append('{} : Unpacking failed: {}'.format(archive, failure[1]))
result.result = False
continue
return unpacked_dirs
def processDir(dirName, nzbName=None, process_method=None, force=False, is_priority=None, delete_on=False, failed=False,
proc_type="auto", **kwargs):
"""
Scans through the files in dirName and processes whatever media files it finds
:param dirName: The folder name to look in
:param nzbName: The NZB name which resulted in this folder being downloaded
:param force: True to postprocess already postprocessed files
:param is_priority: whether to replace the file even if it exists at higher quality
:param delete_on: delete files and folders after they are processed (always happens with move and auto combination)
:param failed: Boolean for whether or not the download failed
:param proc_type: Type of postprocessing auto or manual
"""
result = ProcessResult()
# if they passed us a real dir then assume it's the one we want
if os.path.isdir(dirName):
dirName = os.path.realpath(dirName)
result.output += logHelper("Processing in folder {0}".format(dirName), sickrage.app.log.DEBUG)
# if the client and SickRage are not on the same machine translate the directory into a network directory
elif all([sickrage.app.config.tv_download_dir,
os.path.isdir(sickrage.app.config.tv_download_dir),
os.path.normpath(dirName) == os.path.normpath(sickrage.app.config.tv_download_dir)]):
dirName = os.path.join(sickrage.app.config.tv_download_dir, os.path.abspath(dirName).split(os.path.sep)[-1])
result.output += logHelper("Trying to use folder: {0} ".format(dirName), sickrage.app.log.DEBUG)
# if we didn't find a real dir then quit
if not os.path.isdir(dirName):
result.output += logHelper("Unable to figure out what folder to process. "
"If your downloader and SiCKRAGE aren't on the same PC "
"make sure you fill out your TV download dir in the config.",
sickrage.app.log.DEBUG)
return result
process_method = process_method or sickrage.app.config.process_method
directories_from_rars = set()
# If we have a release name (probably from nzbToMedia), and it is a rar/video, only process that file
if nzbName and (is_media_file(nzbName) or is_rar_file(nzbName)):
result.output += logHelper("Processing {}".format(nzbName), sickrage.app.log.INFO)
generator_to_use = [(dirName, [], [nzbName])]
else:
result.output += logHelper("Processing {}".format(dirName), sickrage.app.log.INFO)
generator_to_use = os.walk(dirName, followlinks=sickrage.app.config.processor_follow_symlinks)
rar_files = []
for current_directory, directory_names, file_names in generator_to_use:
result.result = True
file_names = [f for f in file_names if not is_torrent_or_nzb_file(f)]
rar_files = [x for x in file_names if is_rar_file(os.path.join(current_directory, x))]
if rar_files:
extracted_directories = unrar(current_directory, rar_files, force, result)
if extracted_directories:
for extracted_directory in extracted_directories:
if extracted_directory.split(current_directory)[-1] not in directory_names:
result.output += logHelper(
"Adding extracted directory to the list of directories to process: {0}".format(
extracted_directory), sickrage.app.log.DEBUG
)
directories_from_rars.add(extracted_directory)
if not validateDir(current_directory, nzbName, failed, result):
continue
video_files = filter(is_media_file, file_names)
if video_files:
try:
process_media(current_directory, video_files, nzbName, process_method, force, is_priority, result)
except NoFreeSpaceException:
continue
else:
result.result = False
# Delete all file not needed and avoid deleting files if Manual PostProcessing
if not (process_method == "move" and result.result) or (proc_type == "manual" and not delete_on):
continue
# Check for unwanted files
unwanted_files = filter(lambda x: x in video_files + rar_files, file_names)
if unwanted_files:
result.output += logHelper("Found unwanted files: {0}".format(unwanted_files), sickrage.app.log.DEBUG)
delete_folder(os.path.join(current_directory, '@eaDir'), False)
delete_files(current_directory, unwanted_files, result)
if delete_folder(current_directory, check_empty=not delete_on):
result.output += logHelper("Deleted folder: {0}".format(current_directory), sickrage.app.log.DEBUG)
# For processing extracted rars, only allow methods 'move' and 'copy'.
# On different methods fall back to 'move'.
method_fallback = ('move', process_method)[process_method in ('move', 'copy')]
# auto post-processing deletes rar content by default if method is 'move',
# sickbeard.DELRARCONTENTS allows to override even if method is NOT 'move'
# manual post-processing will only delete when prompted by delete_on
delete_rar_contents = any([sickrage.app.config.delrarcontents and proc_type != 'manual',
not sickrage.app.config.delrarcontents and proc_type == 'auto' and method_fallback == 'move',
proc_type == 'manual' and delete_on])
for directory_from_rar in directories_from_rars:
processDir(
dirName=directory_from_rar,
nzbName=os.path.basename(directory_from_rar),
process_method=method_fallback,
force=force,
is_priority=is_priority,
delete_on=delete_rar_contents,
failed=failed,
proc_type=proc_type
)
# Delete rar file only if the extracted dir was successfully processed
if proc_type == 'auto' and method_fallback == 'move' or proc_type == 'manual' and delete_on:
this_rar = [rar_file for rar_file in rar_files if
os.path.basename(directory_from_rar) == rar_file.rpartition('.')[0]]
delete_files(dirName, this_rar, result) # Deletes only if result.result == True
result.output += logHelper(("Processing Failed", "Successfully processed")[result.agg_result],
(sickrage.app.log.WARNING, sickrage.app.log.INFO)[result.agg_result])
if result.missed_files:
result.output += logHelper("Some items were not processed.")
for missed_file in result.missed_files:
result.output += logHelper(missed_file)
return result
def process(self, nzbName=None, force=False, is_priority=None, delete_on=False, failed=False):
"""
Scans through the files in dir_name and processes whatever media files it finds
:param nzbName: The NZB name which resulted in this folder being downloaded
:param force: True to postprocess already postprocessed files
:param is_priority: whether to replace the file even if it exists at higher quality
:param delete_on: delete files and folders after they are processed (always happens with move and auto combination)
:param failed: Boolean for whether or not the download failed
"""
self.clear_log()
directories_from_rars = set()
# If we have a release name (probably from nzbToMedia), and it is a rar/video, only process that file
if nzbName and (is_media_file(nzbName) or is_rar_file(nzbName)):
self.log("Processing {}".format(nzbName), sickrage.app.log.INFO)
generator_to_use = [(self.path, [], [nzbName])]
else:
self.log("Processing {}".format(self.path), sickrage.app.log.INFO)
generator_to_use = os.walk(self.path, followlinks=sickrage.app.config.processor_follow_symlinks)
rar_files = []
for current_directory, directory_names, file_names in generator_to_use:
self.result = True
file_names = [f for f in file_names if not is_torrent_or_nzb_file(f)]
rar_files = [x for x in file_names if is_rar_file(os.path.join(current_directory, x))]
if rar_files:
extracted_directories = self.unrar(current_directory, rar_files, force)
if extracted_directories:
for extracted_directory in extracted_directories:
if extracted_directory.split(current_directory)[-1] not in directory_names:
self.log(
"Adding extracted directory to the list of directories to process: {0}".format(
extracted_directory), sickrage.app.log.DEBUG
)
directories_from_rars.add(extracted_directory)
if not self.validateDir(current_directory, nzbName, failed):
continue
video_files = list(filter(is_media_file, file_names))
if video_files:
try:
self.process_media(current_directory, video_files, nzbName, self.process_method, force, is_priority)
except NoFreeSpaceException:
continue
else:
self.result = False
# Delete all file not needed and avoid deleting files if Manual PostProcessing
if not (self.process_method == "move" and self.result) or (self.process_type == "manual" and not delete_on):
continue
# Check for unwanted files
unwanted_files = list(
filter(
lambda x: x not in video_files and get_extension(x) not in sickrage.app.config.allowed_extensions,
file_names)
)
if unwanted_files:
self.log("Found unwanted files: {0}".format(unwanted_files), sickrage.app.log.DEBUG)
self.delete_folder(os.path.join(current_directory, '@eaDir'), False)
self.delete_files(current_directory, unwanted_files)
if self.delete_folder(current_directory, check_empty=not delete_on):
self.log("Deleted folder: {0}".format(current_directory), sickrage.app.log.DEBUG)
method_fallback = ('move', self.process_method)[self.process_method in ('move', 'copy')]
delete_rar_contents = any([sickrage.app.config.delrarcontents and self.process_type != 'manual',
not sickrage.app.config.delrarcontents and self.process_type == 'auto' and method_fallback == 'move',
self.process_type == 'manual' and delete_on])
for directory_from_rar in directories_from_rars:
ProcessResult(directories_from_rars, self.process_method, self.process_type).process(
nzbName=os.path.basename(directory_from_rar),
force=force,
is_priority=is_priority,
delete_on=delete_rar_contents,
failed=failed
)
# Delete rar file only if the extracted dir was successfully processed
if self.process_type == 'auto' and method_fallback == 'move' or self.process_type == 'manual' and delete_on:
this_rar = [rar_file for rar_file in rar_files if
os.path.basename(directory_from_rar) == rar_file.rpartition('.')[0]]
self.delete_files(self.path, this_rar)
self.log(("Processing Failed", "Successfully processed")[self.succeeded],
(sickrage.app.log.WARNING, sickrage.app.log.INFO)[self.succeeded])
if self.missed_files:
self.log("Some items were not processed.")
for missed_file in self.missed_files:
self.log(missed_file)
return self.output
def unrar(self, path, rar_files, force):
"""
Extracts RAR files
:param path: Path to look for files in
:param rar_files: Names of RAR files
:param force: process currently processing items
:return: List of unpacked file names
"""
unpacked_dirs = []
if sickrage.app.config.unpack == 1 and rar_files:
self.log("Packed Releases detected: {0}".format(rar_files), sickrage.app.log.DEBUG)
for archive in rar_files:
failure = None
rar_handle = None
try:
archive_path = os.path.join(path, archive)
if self.already_postprocessed(path, archive, force):
self.log("Archive file already post-processed, extraction skipped: {}".format
(archive_path), sickrage.app.log.DEBUG)
continue
if not is_rar_file(archive_path):
continue
self.log(
"Checking if archive is valid and contains a video: {}".format(archive_path),
sickrage.app.log.DEBUG)
rar_handle = rarfile.RarFile(archive_path)
if rar_handle.needs_password():
# TODO: Add support in settings for a list of passwords to try here with rar_handle.set_password(x)
self.log('Archive needs a password, skipping: {0}'.format(archive_path))
continue
# If there are no video files in the rar, don't extract it
rar_media_files = list(filter(is_media_file, rar_handle.namelist()))
if not rar_media_files:
continue
rar_release_name = archive.rpartition('.')[0]
# Choose the directory we'll unpack to:
if sickrage.app.config.unpack_dir and os.path.isdir(sickrage.app.config.unpack_dir):
unpack_base_dir = sickrage.app.config.unpack_dir
else:
unpack_base_dir = path
if sickrage.app.config.unpack_dir: # Let user know if we can't unpack there
self.log('Unpack directory cannot be verified. Using {}'.format(path),
sickrage.app.log.DEBUG)
# Fix up the list for checking if already processed
rar_media_files = [os.path.join(unpack_base_dir, rar_release_name, rar_media_file) for
rar_media_file in
rar_media_files]
skip_rar = False
for rar_media_file in rar_media_files:
check_path, check_file = os.path.split(rar_media_file)
if self.already_postprocessed(check_path, check_file, force):
self.log(
"Archive file already post-processed, extraction skipped: {0}".format
(rar_media_file), sickrage.app.log.DEBUG)
skip_rar = True
break
if skip_rar:
continue
rar_extract_path = os.path.join(unpack_base_dir, rar_release_name)
self.log("Unpacking archive: {0}".format(archive), sickrage.app.log.DEBUG)
rar_handle.extractall(path=rar_extract_path)
unpacked_dirs.append(rar_extract_path)
except rarfile.RarCRCError:
failure = ('Archive Broken', 'Unpacking failed because of a CRC error')
except rarfile.RarWrongPassword:
failure = ('Incorrect RAR Password', 'Unpacking failed because of an Incorrect Rar Password')
except rarfile.PasswordRequired:
failure = ('Rar is password protected', 'Unpacking failed because it needs a password')
except rarfile.RarOpenError:
failure = ('Rar Open Error, check the parent folder and destination file permissions.',
'Unpacking failed with a File Open Error (file permissions?)')
except rarfile.RarExecError:
failure = ('Invalid Rar Archive Usage',
'Unpacking Failed with Invalid Rar Archive Usage. Is unrar installed and on the system '
'PATH?')
except rarfile.BadRarFile:
failure = ('Invalid Rar Archive', 'Unpacking Failed with an Invalid Rar Archive Error')
except rarfile.NeedFirstVolume:
continue
except (Exception, rarfile.Error) as e:
failure = (e, 'Unpacking failed')
finally:
if rar_handle:
del rar_handle
if failure:
self.log('Failed to extract the archive {}: {}'.format(archive, failure[0]),
sickrage.app.log.WARNING)
self.missed_files.append('{} : Unpacking failed: {}'.format(archive, failure[1]))
self.result = False
continue
return unpacked_dirs
def processDir(dirName,
nzbName=None,
process_method=None,
force=False,
is_priority=None,
delete_on=False,
failed=False,
proc_type="auto",
**kwargs):
"""
Scans through the files in dirName and processes whatever media files it finds
:param dirName: The folder name to look in
:param nzbName: The NZB name which resulted in this folder being downloaded
:param force: True to postprocess already postprocessed files
:param is_priority: whether to replace the file even if it exists at higher quality
:param delete_on: delete files and folders after they are processed (always happens with move and auto combination)
:param failed: Boolean for whether or not the download failed
:param proc_type: Type of postprocessing auto or manual
"""
result = ProcessResult()
# if they passed us a real dir then assume it's the one we want
if os.path.isdir(dirName):
dirName = os.path.realpath(dirName)
result.output += logHelper("Processing in folder {0}".format(dirName),
sickrage.app.log.DEBUG)
# if the client and SickRage are not on the same machine translate the directory into a network directory
elif all([
sickrage.app.config.tv_download_dir,
os.path.isdir(sickrage.app.config.tv_download_dir),
os.path.normpath(dirName) == os.path.normpath(
sickrage.app.config.tv_download_dir)
]):
dirName = os.path.join(sickrage.app.config.tv_download_dir,
os.path.abspath(dirName).split(os.path.sep)[-1])
result.output += logHelper(
"Trying to use folder: {0} ".format(dirName),
sickrage.app.log.DEBUG)
# if we didn't find a real dir then quit
if not os.path.isdir(dirName):
result.output += logHelper(
"Unable to figure out what folder to process. "
"If your downloader and SiCKRAGE aren't on the same PC "
"make sure you fill out your TV download dir in the config.",
sickrage.app.log.DEBUG)
return result
process_method = process_method or sickrage.app.config.process_method
directories_from_rars = set()
# If we have a release name (probably from nzbToMedia), and it is a rar/video, only process that file
if nzbName and (is_media_file(nzbName) or is_rar_file(nzbName)):
result.output += logHelper("Processing {}".format(nzbName),
sickrage.app.log.INFO)
generator_to_use = [(dirName, [], [nzbName])]
else:
result.output += logHelper("Processing {}".format(dirName),
sickrage.app.log.INFO)
generator_to_use = os.walk(
dirName, followlinks=sickrage.app.config.processor_follow_symlinks)
rar_files = []
for current_directory, directory_names, file_names in generator_to_use:
result.result = True
file_names = [f for f in file_names if not is_torrent_or_nzb_file(f)]
rar_files = [
x for x in file_names
if is_rar_file(os.path.join(current_directory, x))
]
if rar_files:
extracted_directories = unrar(current_directory, rar_files, force,
result)
if extracted_directories:
for extracted_directory in extracted_directories:
if extracted_directory.split(
current_directory)[-1] not in directory_names:
result.output += logHelper(
"Adding extracted directory to the list of directories to process: {0}"
.format(extracted_directory),
sickrage.app.log.DEBUG)
directories_from_rars.add(extracted_directory)
if not validateDir(current_directory, nzbName, failed, result):
continue
video_files = filter(is_media_file, file_names)
if video_files:
try:
process_media(current_directory, video_files, nzbName,
process_method, force, is_priority, result)
except NoFreeSpaceException:
continue
else:
result.result = False
# Delete all file not needed and avoid deleting files if Manual PostProcessing
if not (process_method == "move" and result.result) or (
proc_type == "manual" and not delete_on):
continue
# Check for unwanted files
unwanted_files = filter(lambda x: x in video_files + rar_files,
file_names)
if unwanted_files:
result.output += logHelper(
"Found unwanted files: {0}".format(unwanted_files),
sickrage.app.log.DEBUG)
delete_folder(os.path.join(current_directory, '@eaDir'), False)
delete_files(current_directory, unwanted_files, result)
if delete_folder(current_directory, check_empty=not delete_on):
result.output += logHelper(
"Deleted folder: {0}".format(current_directory),
sickrage.app.log.DEBUG)
# For processing extracted rars, only allow methods 'move' and 'copy'.
# On different methods fall back to 'move'.
method_fallback = ('move', process_method)[process_method in ('move',
'copy')]
# auto post-processing deletes rar content by default if method is 'move',
# sickbeard.DELRARCONTENTS allows to override even if method is NOT 'move'
# manual post-processing will only delete when prompted by delete_on
delete_rar_contents = any([
sickrage.app.config.delrarcontents and proc_type != 'manual',
not sickrage.app.config.delrarcontents and proc_type == 'auto'
and method_fallback == 'move', proc_type == 'manual' and delete_on
])
for directory_from_rar in directories_from_rars:
processDir(dirName=directory_from_rar,
nzbName=os.path.basename(directory_from_rar),
process_method=method_fallback,
force=force,
is_priority=is_priority,
delete_on=delete_rar_contents,
failed=failed,
proc_type=proc_type)
# Delete rar file only if the extracted dir was successfully processed
if proc_type == 'auto' and method_fallback == 'move' or proc_type == 'manual' and delete_on:
this_rar = [
rar_file for rar_file in rar_files if os.path.basename(
directory_from_rar) == rar_file.rpartition('.')[0]
]
delete_files(dirName, this_rar,
result) # Deletes only if result.result == True
result.output += logHelper(
("Processing Failed", "Successfully processed")[result.agg_result],
(sickrage.app.log.WARNING, sickrage.app.log.INFO)[result.agg_result])
if result.missed_files:
result.output += logHelper("Some items were not processed.")
for missed_file in result.missed_files:
result.output += logHelper(missed_file)
return result