def is_archive(path):
""" Check if file in path is an ZIP, RAR or 7z file
:param path: path to file
:return: (zf, status, expected_extension)
status: -1==Error/Retry, 0==OK, 1==Ignore
"""
if zipfile.is_zipfile(path):
try:
zf = zipfile.ZipFile(path)
return 0, zf, '.zip'
except:
logging.info(T('Cannot read %s'), path, exc_info=True)
return -1, None, ''
elif rarfile.is_rarfile(path):
try:
# Set path to tool to open it
rarfile.UNRAR_TOOL = sabnzbd.newsunpack.RAR_COMMAND
zf = rarfile.RarFile(path)
return 0, zf, '.rar'
except:
logging.info(T('Cannot read %s'), path, exc_info=True)
return -1, None, ''
elif is_sevenfile(path):
try:
zf = SevenZip(path)
return 0, zf, '.7z'
except:
logging.info(T('Cannot read %s'), path, exc_info=True)
return -1, None, ''
else:
logging.info('Archive %s is not a real archive!',
os.path.basename(path))
return 1, None, ''
def is_archive(path):
""" Check if file in path is an ZIP, RAR or 7z file
:param path: path to file
:return: (zf, status, expected_extension)
status: -1==Error/Retry, 0==OK, 1==Ignore
"""
if zipfile.is_zipfile(path):
try:
zf = zipfile.ZipFile(path)
return 0, zf, '.zip'
except:
return -1, None, ''
elif rarfile.is_rarfile(path):
try:
zf = rarfile.RarFile(path)
# Set path to tool to open it
rarfile.UNRAR_TOOL = sabnzbd.newsunpack.RAR_COMMAND
return 0, zf, '.rar'
except:
return -1, None, ''
elif is_sevenfile(path):
try:
zf = SevenZip(path)
return 0, zf, '.7z'
except:
return -1, None, ''
else:
return 1, None, ''
def is_archive(path: str) -> Tuple[int, Any, str]:
"""Check if file in path is an ZIP, RAR or 7z file
:param path: path to file
:return: (zf, status, expected_extension)
status: -1==Error/Retry, 0==OK, 1==Ignore
"""
if zipfile.is_zipfile(path):
try:
zf = zipfile.ZipFile(path)
return 0, zf, ".zip"
except:
logging.info(T("Cannot read %s"), path, exc_info=True)
return -1, None, ""
elif rarfile.is_rarfile(path):
try:
# Set path to tool to open it
rarfile.UNRAR_TOOL = sabnzbd.newsunpack.RAR_COMMAND
zf = rarfile.RarFile(path)
return 0, zf, ".rar"
except:
logging.info(T("Cannot read %s"), path, exc_info=True)
return -1, None, ""
elif sabnzbd.newsunpack.is_sevenfile(path):
try:
zf = sabnzbd.newsunpack.SevenZip(path)
return 0, zf, ".7z"
except:
logging.info(T("Cannot read %s"), path, exc_info=True)
return -1, None, ""
else:
logging.info("Archive %s is not a real archive!",
os.path.basename(path))
return 1, None, ""
def is_archive(path):
""" Check if file in path is an ZIP, RAR or 7z file
:param path: path to file
:return: (zf, status, expected_extension)
status: -1==Error/Retry, 0==OK, 1==Ignore
"""
if zipfile.is_zipfile(path):
try:
zf = zipfile.ZipFile(path)
return 0, zf, '.zip'
except:
return -1, None, ''
elif is_rarfile(path):
try:
zf = RarFile(path)
return 0, zf, '.rar'
except:
return -1, None, ''
elif is_sevenfile(path):
try:
zf = SevenZip(path)
return 0, zf, '.7z'
except:
return -1, None, ''
else:
return 1, None, ''
def is_archive(path):
""" Check if file in path is an ZIP, RAR or 7z file
:param path: path to file
:return: (zf, status, expected_extension)
status: -1==Error/Retry, 0==OK, 1==Ignore
"""
if zipfile.is_zipfile(path):
try:
zf = zipfile.ZipFile(path)
return 0, zf, '.zip'
except:
return -1, None, ''
elif is_rarfile(path):
try:
zf = RarFile(path)
return 0, zf, '.rar'
except:
return -1, None, ''
elif is_sevenfile(path):
try:
zf = SevenZip(path)
return 0, zf, '.7z'
except:
return -1, None, ''
else:
return 1, None, ''
def is_archive(path):
""" Check if file in path is an ZIP, RAR or 7z file
:param path: path to file
:return: (zf, status, expected_extension)
status: -1==Error/Retry, 0==OK, 1==Ignore
"""
if zipfile.is_zipfile(path):
try:
zf = zipfile.ZipFile(path)
return 0, zf, '.zip'
except:
return -1, None, ''
elif rarfile.is_rarfile(path):
try:
zf = rarfile.RarFile(path)
# Set path to tool to open it
rarfile.UNRAR_TOOL = sabnzbd.newsunpack.RAR_COMMAND
return 0, zf, '.rar'
except:
return -1, None, ''
elif is_sevenfile(path):
try:
zf = SevenZip(path)
return 0, zf, '.7z'
except:
return -1, None, ''
else:
return 1, None, ''
def check_encrypted_rar(nzo, filepath):
""" Check if file is rar and is encrypted """
encrypted = False
if not nzo.password and cfg.pause_on_pwrar() and is_rarfile(filepath):
try:
zf = RarFile(filepath)
encrypted = zf.encrypted
if encrypted and int(nzo.encrypted) < 2:
nzo.encrypted = 1
else:
encrypted = False
zf.close()
del zf
except:
logging.debug('RAR file %s cannot be inspected', filepath)
return encrypted
def check_encrypted_rar(nzo, filepath):
""" Check if file is rar and is encrypted """
encrypted = False
if not nzo.password and cfg.pause_on_pwrar() and is_rarfile(filepath):
try:
zf = RarFile(filepath, all_names=True)
encrypted = zf.encrypted or is_cloaked(filepath, zf.namelist())
if encrypted and int(nzo.encrypted) < 2:
nzo.encrypted = 1
else:
encrypted = False
zf.close()
del zf
except:
logging.debug('RAR file %s cannot be inspected', filepath)
return encrypted
def check_encrypted_rar(nzo, filepath):
""" Check if file is rar and is encrypted """
encrypted = False
if not nzo.password and not nzo.meta.get(
'password') and cfg.pause_on_pwrar() and is_rarfile(filepath):
try:
zf = RarFile(filepath, all_names=True)
encrypted = zf.encrypted or is_cloaked(filepath, zf.namelist())
if encrypted and int(nzo.encrypted) < 2 and not nzo.reuse:
nzo.encrypted = 1
else:
encrypted = False
zf.close()
del zf
except:
logging.debug('RAR file %s cannot be inspected', filepath)
return encrypted
def rar_contains_unwanted_file(filepath):
# checks for unwanted extensions in the rar file 'filepath'
# ... unwanted extensions are defined in global variable cfg.unwanted_extensions()
# returns False if no unwanted extensions are found in the rar file
# returns name of file if unwanted extension is found in the rar file
unwanted = None
if is_rarfile(filepath):
#logging.debug('rar file to check: %s',filepath)
#logging.debug('unwanted extensions are: %s', cfg.unwanted_extensions())
try:
zf = RarFile(filepath, all_names=True)
#logging.debug('files in rar file: %s', zf.namelist())
for somefile in zf.namelist() :
logging.debug('file in rar file: %s', somefile)
if os.path.splitext(somefile)[1].replace('.', '').lower() in cfg.unwanted_extensions():
logging.debug('Unwanted file %s', somefile)
unwanted = somefile
zf.close()
except:
logging.debug('RAR file %s cannot be inspected.', filepath)
return unwanted
def rar_contains_unwanted_file(filepath):
# checks for unwanted extensions in the rar file 'filepath'
# ... unwanted extensions are defined in global variable cfg.unwanted_extensions()
# returns False if no unwanted extensions are found in the rar file
# returns name of file if unwanted extension is found in the rar file
unwanted = None
if cfg.unwanted_extensions() and is_rarfile(filepath):
# logging.debug('rar file to check: %s',filepath)
# logging.debug('unwanted extensions are: %s', cfg.unwanted_extensions())
try:
zf = RarFile(filepath, all_names=True)
# logging.debug('files in rar file: %s', zf.namelist())
for somefile in zf.namelist():
logging.debug("file in rar file: %s", somefile)
if os.path.splitext(somefile)[1].replace(".", "").lower() in cfg.unwanted_extensions():
logging.debug("Unwanted file %s", somefile)
unwanted = somefile
zf.close()
except:
logging.debug("RAR file %s cannot be inspected.", filepath)
return unwanted
def check_encrypted_rar(nzo, filepath):
""" Check if file is rar and is encrypted """
encrypted = False
if (
nzo.encrypted == 0
and not nzo.password
and not nzo.meta.get("password")
and cfg.pause_on_pwrar()
and is_rarfile(filepath)
):
try:
zf = RarFile(filepath, all_names=True)
encrypted = zf.encrypted or is_cloaked(filepath, zf.namelist())
if encrypted and not nzo.reuse:
nzo.encrypted = 1
else:
# Don't check other files
nzo.encrypted = -1
encrypted = False
zf.close()
del zf
except:
logging.debug("RAR file %s cannot be inspected", filepath)
return encrypted
def get_rar_extension(myrarfile):
"""
Find out orginal extension of a rar file. Returns "" in case of file problems
So ... returns: "part001.rar", ... "part005.rar"
or old number scheme (can only happen for rar3/rar4 files): "rar", r00, ... r89
"""
# When things go wrong
volumenumber = -1
org_extension = False
try:
rar_ver = rarfile.is_rarfile(myrarfile)
with open(myrarfile, "rb") as fh:
if rar_ver.endswith("3"):
# As it's rar3, let's first find the numbering scheme: old (rNN) or new (partNN.rar)
mybuf = fh.read(100) # first 100 bytes is enough
HEAD_FLAGS_LSB = mybuf[10] # LSB = Least Significant Byte
newnumbering = HEAD_FLAGS_LSB & 0x10
# For the volume number, At the end of the file, we need about 20 bytes
fh.seek(-20, os.SEEK_END)
mybuf = fh.read()
volumenumber = 1 + mybuf[-9] + 256 * mybuf[-8]
if newnumbering:
org_extension = "part%02d.rar" % volumenumber
else:
# 1, 2, 3, 4 resp refers to .rar, .r00, .r01, .r02 ...
if volumenumber == 1:
org_extension = "rar"
else:
org_extension = "r%02d" % (volumenumber - 2)
elif rar_ver.endswith("5"):
mybuf = fh.read(100) # first 100 bytes is enough
# Get (and skip) the first 8 + 4 bytes
rar5sig, newpos = rarfile.load_bytes(mybuf, 8,
0) # Rar5 signature
crc32, newpos = rarfile.load_bytes(mybuf, 4, newpos) # crc32
# Then get the VINT values (with variable size, so parse them all):
headersize, newpos = rarfile.load_vint(mybuf, newpos)
headertype, newpos = rarfile.load_vint(mybuf, newpos)
headerflags, newpos = rarfile.load_vint(mybuf, newpos)
extraareasize, newpos = rarfile.load_vint(mybuf, newpos)
archiveflags, newpos = rarfile.load_vint(mybuf, newpos)
# Now we're ready for the volume number:
if archiveflags & 2:
value, newpos = rarfile.load_vint(mybuf, newpos)
volumenumber = value + 1
else:
# first volume, aka 1
volumenumber = 1
# Combine into the extension
org_extension = "part%03d.rar" % volumenumber
except:
pass
return volumenumber, org_extension
def check_encrypted_and_unwanted_files(nzo, filepath):
""" Combines check for unwanted and encrypted files to save on CPU and IO """
encrypted = False
unwanted = None
if (cfg.unwanted_extensions() and cfg.action_on_unwanted_extensions()) or (nzo.encrypted == 0 and cfg.pause_on_pwrar()):
# These checks should not break the assembler
try:
# Rarfile freezes on Windows special names, so don't try those!
if sabnzbd.WIN32 and has_win_device(filepath):
return encrypted, unwanted
# Is it even a rarfile?
if rarfile.is_rarfile(filepath):
# Open the rar
rarfile.UNRAR_TOOL = sabnzbd.newsunpack.RAR_COMMAND
zf = rarfile.RarFile(filepath, all_names=True)
# Check for encryption
if nzo.encrypted == 0 and cfg.pause_on_pwrar() and (zf.needs_password() or is_cloaked(nzo, filepath, zf.namelist())):
# Load all passwords
passwords = get_all_passwords(nzo)
# Cloaked job?
if is_cloaked(nzo, filepath, zf.namelist()):
encrypted = True
elif not sabnzbd.HAVE_CRYPTOGRAPHY and not passwords:
# if no cryptography installed, only error when no password was set
logging.info(T('%s missing'), 'Python Cryptography')
nzo.encrypted = 1
encrypted = True
elif sabnzbd.HAVE_CRYPTOGRAPHY:
# Lets test if any of the password work
password_hit = False
for password in passwords:
if password:
logging.info('Trying password "%s" on job "%s"', password, nzo.final_name)
try:
zf.setpassword(password)
except:
# On weird passwords the setpassword() will fail
# but the actual rartest() will work
pass
try:
zf.testrar()
password_hit = password
break
except rarfile.RarCRCError:
# On CRC error we can continue!
password_hit = password
break
except Exception as e:
# Did we start from the right volume?
if 'need to start extraction from a previous volume' in e[0]:
return encrypted, unwanted
# This one failed
pass
# Did any work?
if password_hit:
# Don't check other files
logging.info('Password "%s" matches for job "%s"', password_hit, nzo.final_name)
nzo.encrypted = -1
encrypted = False
else:
# Encrypted and none of them worked
nzo.encrypted = 1
encrypted = True
else:
# Don't check other files
nzo.encrypted = -1
encrypted = False
# Check for unwanted extensions
if cfg.unwanted_extensions() and cfg.action_on_unwanted_extensions():
for somefile in zf.namelist():
logging.debug('File contains: %s', somefile)
if os.path.splitext(somefile)[1].replace('.', '').lower() in cfg.unwanted_extensions():
logging.debug('Unwanted file %s', somefile)
unwanted = somefile
zf.close()
del zf
except:
logging.info('Error during inspection of RAR-file %s', filepath, exc_info=True)
return encrypted, unwanted
def ProcessArchiveFile(filename, path, pp=None, script=None, cat=None, catdir=None, keep=False,
priority=None, url='', nzbname=None, password=None, nzo_id=None):
""" Analyse ZIP file and create job(s).
Accepts ZIP files with ONLY nzb/nfo/folder files in it.
returns (status, nzo_ids)
status: -1==Error/Retry, 0==OK, 1==Ignore
"""
from sabnzbd.nzbqueue import add_nzo
nzo_ids = []
if catdir is None:
catdir = cat
filename, cat = name_to_cat(filename, catdir)
if zipfile.is_zipfile(path):
try:
zf = zipfile.ZipFile(path)
except:
return -1, []
elif is_rarfile(path):
try:
zf = RarFile(path)
except:
return -1, []
elif is_sevenfile(path):
try:
zf = SevenZip(path)
except:
return -1, []
else:
return 1, []
status = 1
names = zf.namelist()
names.sort()
nzbcount = 0
for name in names:
name = name.lower()
if not (name.endswith('.nzb') or name.endswith('.nfo') or name.endswith('/')):
status = 1
break
elif name.endswith('.nzb'):
status = 0
nzbcount += 1
if status == 0:
if nzbcount != 1:
nzbname = None
for name in names:
if name.lower().endswith('.nzb'):
try:
data = zf.read(name)
except:
zf.close()
return -1, []
name = os.path.basename(name)
if data:
try:
nzo = nzbstuff.NzbObject(name, pp, script, data, cat=cat, url=url,
priority=priority, nzbname=nzbname)
if not nzo.password:
nzo.password = password
except:
nzo = None
if nzo:
if nzo_id:
# Re-use existing nzo_id, when a "future" job gets it payload
sabnzbd.nzbqueue.NzbQueue.do.remove(nzo_id, add_to_history=False)
nzo.nzo_id = nzo_id
nzo_ids.append(add_nzo(nzo))
nzo.update_rating()
zf.close()
try:
if not keep:
os.remove(path)
except:
logging.error(T('Error removing %s'), misc.clip_path(path))
logging.info("Traceback: ", exc_info=True)
status = 1
else:
zf.close()
status = 1
return status, nzo_ids
def ProcessArchiveFile(filename, path, pp=None, script=None, cat=None, catdir=None, keep=False,
priority=None, url='', nzbname=None):
""" Analyse ZIP file and create job(s).
Accepts ZIP files with ONLY nzb/nfo/folder files in it.
returns (status, nzo_ids)
status: -1==Error/Retry, 0==OK, 1==Ignore
"""
from sabnzbd.nzbqueue import add_nzo
nzo_ids = []
if catdir is None:
catdir = cat
filename, cat = name_to_cat(filename, catdir)
if zipfile.is_zipfile(path):
try:
zf = zipfile.ZipFile(path)
except:
return -1, []
elif is_rarfile(path):
try:
zf = RarFile(path)
except:
return -1, []
else:
return 1, []
status = 1
names = zf.namelist()
names.sort()
nzbcount = 0
for name in names:
name = name.lower()
if not (name.endswith('.nzb') or name.endswith('.nfo') or name.endswith('/')):
status = 1
break
elif name.endswith('.nzb'):
status = 0
nzbcount += 1
if status == 0:
if nzbcount != 1:
nzbname = None
for name in names:
if name.lower().endswith('.nzb'):
try:
data = zf.read(name)
except:
zf.close()
return -1, []
name = re.sub(r'\[.*nzbmatrix.com\]', '', name)
name = os.path.basename(name)
name = misc.sanitize_foldername(name)
if data:
try:
nzo = nzbstuff.NzbObject(name, 0, pp, script, data, cat=cat, url=url,
priority=priority, nzbname=nzbname)
except:
nzo = None
if nzo:
nzo_ids.append(add_nzo(nzo))
nzo.update_rating()
zf.close()
try:
if not keep: os.remove(path)
except:
logging.error(Ta('Error removing %s'), path)
logging.info("Traceback: ", exc_info = True)
status = 1
else:
zf.close()
status = 1
return status, nzo_ids
def check_encrypted_and_unwanted_files(nzo: NzbObject, filepath: str) -> Tuple[bool, Optional[str]]:
""" Combines check for unwanted and encrypted files to save on CPU and IO """
encrypted = False
unwanted = None
if (cfg.unwanted_extensions() and cfg.action_on_unwanted_extensions()) or (
nzo.encrypted == 0 and cfg.pause_on_pwrar()
):
# These checks should not break the assembler
try:
# Rarfile freezes on Windows special names, so don't try those!
if sabnzbd.WIN32 and has_win_device(filepath):
return encrypted, unwanted
# Is it even a rarfile?
if rarfile.is_rarfile(filepath):
# Open the rar
rarfile.UNRAR_TOOL = sabnzbd.newsunpack.RAR_COMMAND
zf = rarfile.RarFile(filepath, single_file_check=True)
# Check for encryption
if (
nzo.encrypted == 0
and cfg.pause_on_pwrar()
and (zf.needs_password() or is_cloaked(nzo, filepath, zf.namelist()))
):
# Load all passwords
passwords = get_all_passwords(nzo)
# Cloaked job?
if is_cloaked(nzo, filepath, zf.namelist()):
encrypted = True
elif not passwords:
# Only error when no password was set
nzo.encrypted = 1
encrypted = True
else:
# Lets test if any of the password work
password_hit = False
for password in passwords:
if password:
logging.info('Trying password "%s" on job "%s"', password, nzo.final_name)
try:
zf.setpassword(password)
except rarfile.Error:
# On weird passwords the setpassword() will fail
# but the actual testrar() will work
pass
try:
zf.testrar()
password_hit = password
break
except rarfile.RarWrongPassword:
# This one really didn't work
pass
except rarfile.RarCRCError as e:
# CRC errors can be thrown for wrong password or
# missing the next volume (with correct password)
if "cannot find volume" in str(e).lower():
# We assume this one worked!
password_hit = password
break
# This one didn't work
pass
except:
# All the other errors we skip, they might be fixable in post-proc.
# For example starting from the wrong volume, or damaged files
# This will cause the check to be performed again for the next rar, might
# be disk-intensive! Could be removed later and just accept the password.
return encrypted, unwanted
# Did any work?
if password_hit:
# We always trust the user's input
if not nzo.password:
nzo.password = password_hit
# Don't check other files
logging.info('Password "%s" matches for job "%s"', password_hit, nzo.final_name)
nzo.encrypted = -1
encrypted = False
else:
# Encrypted and none of them worked
nzo.encrypted = 1
encrypted = True
# Check for unwanted extensions
if cfg.unwanted_extensions() and cfg.action_on_unwanted_extensions():
for somefile in zf.namelist():
logging.debug("File contains: %s", somefile)
if get_ext(somefile).replace(".", "").lower() in cfg.unwanted_extensions():
logging.debug("Unwanted file %s", somefile)
unwanted = somefile
zf.close()
del zf
except:
logging.info("Error during inspection of RAR-file %s", filepath)
logging.debug("Traceback: ", exc_info=True)
return encrypted, unwanted
def run(self):
while 1:
# Set NzbObject and NzbFile objects to None so references
# from this thread do not keep the objects alive (see #1628)
nzo = nzf = None
nzo, nzf, file_done = self.queue.get()
if not nzo:
logging.info("Shutting down")
break
if nzf:
# Check if enough disk space is free after each file is done
# If not enough space left, pause downloader and send email
if file_done and not sabnzbd.Downloader.paused:
freespace = diskspace(force=True)
full_dir = None
required_space = (cfg.download_free.get_float() + nzf.bytes) / GIGI
if freespace["download_dir"][1] < required_space:
full_dir = "download_dir"
# Enough space in download_dir, check complete_dir
complete_free = cfg.complete_free.get_float()
if complete_free > 0 and not full_dir:
required_space = 0
if cfg.direct_unpack():
required_space = (complete_free + nzo.bytes_downloaded) / GIGI
else:
# Continue downloading until 95% complete before checking
if nzo.bytes_tried > (nzo.bytes - nzo.bytes_par2) * 0.95:
required_space = (complete_free + nzo.bytes) / GIGI
if required_space and freespace["complete_dir"][1] < required_space:
full_dir = "complete_dir"
if full_dir:
logging.warning(T("Too little diskspace forcing PAUSE"))
# Pause downloader, but don't save, since the disk is almost full!
sabnzbd.Downloader.pause()
if cfg.fulldisk_autoresume():
sabnzbd.Scheduler.plan_diskspace_resume(full_dir, required_space)
sabnzbd.emailer.diskfull_mail()
# Prepare filepath
filepath = nzf.prepare_filepath()
if filepath:
logging.debug("Decoding part of %s", filepath)
try:
self.assemble(nzf, file_done)
except IOError as err:
# If job was deleted or in active post-processing, ignore error
if not nzo.deleted and not nzo.is_gone() and not nzo.pp_active:
# 28 == disk full => pause downloader
if err.errno == 28:
logging.error(T("Disk full! Forcing Pause"))
else:
logging.error(T("Disk error on creating file %s"), clip_path(filepath))
# Log traceback
logging.info("Traceback: ", exc_info=True)
# Pause without saving
sabnzbd.Downloader.pause()
continue
except:
logging.error(T("Fatal error in Assembler"), exc_info=True)
break
# Continue after partly written data
if not file_done:
continue
# Clean-up admin data
logging.info("Decoding finished %s", filepath)
nzf.remove_admin()
# Do rar-related processing
if rarfile.is_rarfile(filepath):
# Encryption and unwanted extension detection
rar_encrypted, unwanted_file = check_encrypted_and_unwanted_files(nzo, filepath)
if rar_encrypted:
if cfg.pause_on_pwrar() == 1:
logging.warning(
T(
'Paused job "%s" because of encrypted RAR file (if supplied, all passwords were tried)'
),
nzo.final_name,
)
nzo.pause()
else:
logging.warning(
T(
'Aborted job "%s" because of encrypted RAR file (if supplied, all passwords were tried)'
),
nzo.final_name,
)
nzo.fail_msg = T("Aborted, encryption detected")
sabnzbd.NzbQueue.end_job(nzo)
if unwanted_file:
# Don't repeat the warning after a user override of an unwanted extension pause
if nzo.unwanted_ext == 0:
logging.warning(
T('In "%s" unwanted extension in RAR file. Unwanted file is %s '),
nzo.final_name,
unwanted_file,
)
logging.debug(T("Unwanted extension is in rar file %s"), filepath)
if cfg.action_on_unwanted_extensions() == 1 and nzo.unwanted_ext == 0:
logging.debug("Unwanted extension ... pausing")
nzo.unwanted_ext = 1
nzo.pause()
if cfg.action_on_unwanted_extensions() == 2:
logging.debug("Unwanted extension ... aborting")
nzo.fail_msg = T("Aborted, unwanted extension detected")
sabnzbd.NzbQueue.end_job(nzo)
# Add to direct unpack
nzo.add_to_direct_unpacker(nzf)
elif par2file.is_parfile(filepath):
# Parse par2 files, cloaked or not
nzo.handle_par2(nzf, filepath)
filter_output, reason = nzo_filtered_by_rating(nzo)
if filter_output == 1:
logging.warning(
T('Paused job "%s" because of rating (%s)'),
nzo.final_name,
reason,
)
nzo.pause()
elif filter_output == 2:
logging.warning(
T('Aborted job "%s" because of rating (%s)'),
nzo.final_name,
reason,
)
nzo.fail_msg = T("Aborted, rating filter matched (%s)") % reason
sabnzbd.NzbQueue.end_job(nzo)
else:
sabnzbd.NzbQueue.remove(nzo.nzo_id, cleanup=False)
sabnzbd.PostProcessor.process(nzo)
def run(self):
while 1:
job = self.queue.get()
if not job:
logging.info("Shutting down")
break
nzo, nzf, file_done = job
if nzf:
# Check if enough disk space is free after each file is done
# If not enough space left, pause downloader and send email
if (file_done and diskspace(force=True)["download_dir"][1] <
(cfg.download_free.get_float() + nzf.bytes) / GIGI):
# Only warn and email once
if not sabnzbd.downloader.Downloader.do.paused:
logging.warning(
T("Too little diskspace forcing PAUSE"))
# Pause downloader, but don't save, since the disk is almost full!
sabnzbd.downloader.Downloader.do.pause()
sabnzbd.emailer.diskfull_mail()
# Abort all direct unpackers, just to be sure
sabnzbd.directunpacker.abort_all()
# Prepare filepath
filepath = nzf.prepare_filepath()
if filepath:
logging.debug("Decoding part of %s", filepath)
try:
self.assemble(nzf, file_done)
except IOError as err:
# If job was deleted or in active post-processing, ignore error
if not nzo.deleted and not nzo.is_gone(
) and not nzo.pp_active:
# 28 == disk full => pause downloader
if err.errno == 28:
logging.error(T("Disk full! Forcing Pause"))
else:
logging.error(
T("Disk error on creating file %s"),
clip_path(filepath))
# Log traceback
logging.info("Traceback: ", exc_info=True)
# Pause without saving
sabnzbd.downloader.Downloader.do.pause()
continue
except:
logging.error(T("Fatal error in Assembler"),
exc_info=True)
break
# Continue after partly written data
if not file_done:
continue
# Clean-up admin data
logging.info("Decoding finished %s", filepath)
nzf.remove_admin()
# Do rar-related processing
if rarfile.is_rarfile(filepath):
# Encryption and unwanted extension detection
rar_encrypted, unwanted_file = check_encrypted_and_unwanted_files(
nzo, filepath)
if rar_encrypted:
if cfg.pause_on_pwrar() == 1:
logging.warning(
T('Paused job "%s" because of encrypted RAR file (if supplied, all passwords were tried)'
),
nzo.final_name,
)
nzo.pause()
else:
logging.warning(
T('Aborted job "%s" because of encrypted RAR file (if supplied, all passwords were tried)'
),
nzo.final_name,
)
nzo.fail_msg = T(
"Aborted, encryption detected")
sabnzbd.nzbqueue.NzbQueue.do.end_job(nzo)
if unwanted_file:
logging.warning(
T('In "%s" unwanted extension in RAR file. Unwanted file is %s '
),
nzo.final_name,
unwanted_file,
)
logging.debug(
T("Unwanted extension is in rar file %s"),
filepath)
if cfg.action_on_unwanted_extensions(
) == 1 and nzo.unwanted_ext == 0:
logging.debug("Unwanted extension ... pausing")
nzo.unwanted_ext = 1
nzo.pause()
if cfg.action_on_unwanted_extensions() == 2:
logging.debug(
"Unwanted extension ... aborting")
nzo.fail_msg = T(
"Aborted, unwanted extension detected")
sabnzbd.nzbqueue.NzbQueue.do.end_job(nzo)
# Add to direct unpack
nzo.add_to_direct_unpacker(nzf)
elif par2file.is_parfile(filepath):
# Parse par2 files, cloaked or not
nzo.handle_par2(nzf, filepath)
filter_output, reason = nzo_filtered_by_rating(nzo)
if filter_output == 1:
logging.warning(
T('Paused job "%s" because of rating (%s)'),
nzo.final_name,
reason,
)
nzo.pause()
elif filter_output == 2:
logging.warning(
T('Aborted job "%s" because of rating (%s)'),
nzo.final_name,
reason,
)
nzo.fail_msg = T(
"Aborted, rating filter matched (%s)") % reason
sabnzbd.nzbqueue.NzbQueue.do.end_job(nzo)
else:
sabnzbd.nzbqueue.NzbQueue.do.remove(nzo.nzo_id,
add_to_history=False,
cleanup=False)
PostProcessor.do.process(nzo)
def check_encrypted_and_unwanted_files(nzo, filepath):
""" Combines check for unwanted and encrypted files to save on CPU and IO """
encrypted = False
unwanted = None
if (cfg.unwanted_extensions() and cfg.action_on_unwanted_extensions()) or (
nzo.encrypted == 0 and cfg.pause_on_pwrar()):
# These checks should not break the assembler
try:
# Rarfile freezes on Windows special names, so don't try those!
if sabnzbd.WIN32 and has_win_device(filepath):
return encrypted, unwanted
# Is it even a rarfile?
if rarfile.is_rarfile(filepath):
# Open the rar
rarfile.UNRAR_TOOL = sabnzbd.newsunpack.RAR_COMMAND
zf = rarfile.RarFile(filepath, all_names=True)
# Check for encryption
if nzo.encrypted == 0 and cfg.pause_on_pwrar() and (
zf.needs_password()
or is_cloaked(nzo, filepath, zf.namelist())):
# Load all passwords
passwords = get_all_passwords(nzo)
# Cloaked job?
if is_cloaked(nzo, filepath, zf.namelist()):
encrypted = True
elif not sabnzbd.HAVE_CRYPTOGRAPHY and not passwords:
# if no cryptography installed, only error when no password was set
logging.info(T('%s missing'), 'Python Cryptography')
nzo.encrypted = 1
encrypted = True
elif sabnzbd.HAVE_CRYPTOGRAPHY:
# Lets test if any of the password work
password_hit = False
for password in passwords:
if password:
logging.info(
'Trying password "%s" on job "%s"',
password, nzo.final_name)
try:
zf.setpassword(password)
except:
# On weird passwords the setpassword() will fail
# but the actual rartest() will work
pass
try:
zf.testrar()
password_hit = password
break
except rarfile.RarCRCError:
# On CRC error we can continue!
password_hit = password
break
except Exception as e:
# Did we start from the right volume?
if 'need to start extraction from a previous volume' in e[
0]:
return encrypted, unwanted
# This one failed
pass
# Did any work?
if password_hit:
# We always trust the user's input
if not nzo.password:
nzo.password = password_hit
# Don't check other files
logging.info('Password "%s" matches for job "%s"',
password_hit, nzo.final_name)
nzo.encrypted = -1
encrypted = False
else:
# Encrypted and none of them worked
nzo.encrypted = 1
encrypted = True
else:
# Don't check other files
nzo.encrypted = -1
encrypted = False
# Check for unwanted extensions
if cfg.unwanted_extensions(
) and cfg.action_on_unwanted_extensions():
for somefile in zf.namelist():
logging.debug('File contains: %s', somefile)
if get_ext(somefile).replace(
'.', '').lower() in cfg.unwanted_extensions():
logging.debug('Unwanted file %s', somefile)
unwanted = somefile
zf.close()
del zf
except:
logging.info('Error during inspection of RAR-file %s', filepath)
logging.debug('Traceback: ', exc_info=True)
return encrypted, unwanted
class Assembler(Thread):
do = None # Link to the instance of this method
def __init__(self, queue=None):
Thread.__init__(self)
if queue:
self.queue = queue
else:
self.queue = Queue.Queue()
Assembler.do = self
def stop(self):
self.process(None)
def process(self, job):
self.queue.put(job)
def run(self):
while 1:
job = self.queue.get()
if not job:
logging.info("Shutting down")
break
nzo, nzf = job
if nzf:
# Check if enough disk space is free, if not pause downloader and send email
if diskspace(force=True)['download_dir'][1] < (
cfg.download_free.get_float() + nzf.bytes) / GIGI:
# Only warn and email once
if not sabnzbd.downloader.Downloader.do.paused:
logging.warning(
T('Too little diskspace forcing PAUSE'))
# Pause downloader, but don't save, since the disk is almost full!
sabnzbd.downloader.Downloader.do.pause()
sabnzbd.emailer.diskfull()
# Abort all direct unpackers, just to be sure
sabnzbd.directunpacker.abort_all()
# Place job back in queue and wait 30 seconds to hope it gets resolved
self.process(job)
sleep(30)
continue
# Prepare filename
nzo.verify_nzf_filename(nzf)
nzf.filename = sanitize_filename(nzf.filename)
filepath = get_filepath(long_path(cfg.download_dir.get_path()),
nzo, nzf.filename)
nzf.filename = get_filename(filepath)
if filepath:
logging.info('Decoding %s %s', filepath, nzf.type)
try:
filepath = self.assemble(nzf, filepath)
except IOError, (errno, strerror):
# If job was deleted or in active post-processing, ignore error
if not nzo.deleted and not nzo.is_gone(
) and not nzo.pp_active:
# 28 == disk full => pause downloader
if errno == 28:
logging.error(T('Disk full! Forcing Pause'))
else:
logging.error(
T('Disk error on creating file %s'),
clip_path(filepath))
# Log traceback
logging.info('Traceback: ', exc_info=True)
# Pause without saving
sabnzbd.downloader.Downloader.do.pause()
continue
except:
logging.error(T('Fatal error in Assembler'),
exc_info=True)
break
# Clean-up admin data
nzf.remove_admin()
# Do rar-related processing
if rarfile.is_rarfile(filepath):
# Encryption and unwanted extension detection
rar_encrypted, unwanted_file = check_encrypted_and_unwanted_files(
nzo, filepath)
if rar_encrypted:
if cfg.pause_on_pwrar() == 1:
logging.warning(
remove_warning_label(
T('WARNING: Paused job "%s" because of encrypted RAR file (if supplied, all passwords were tried)'
)), nzo.final_name)
nzo.pause()
else:
logging.warning(
remove_warning_label(
T('WARNING: Aborted job "%s" because of encrypted RAR file (if supplied, all passwords were tried)'
)), nzo.final_name)
nzo.fail_msg = T(
'Aborted, encryption detected')
sabnzbd.nzbqueue.NzbQueue.do.end_job(nzo)
if unwanted_file:
logging.warning(
remove_warning_label(
T('WARNING: In "%s" unwanted extension in RAR file. Unwanted file is %s '
)), nzo.final_name, unwanted_file)
logging.debug(
T('Unwanted extension is in rar file %s'),
filepath)
if cfg.action_on_unwanted_extensions(
) == 1 and nzo.unwanted_ext == 0:
logging.debug('Unwanted extension ... pausing')
nzo.unwanted_ext = 1
nzo.pause()
if cfg.action_on_unwanted_extensions() == 2:
logging.debug(
'Unwanted extension ... aborting')
nzo.fail_msg = T(
'Aborted, unwanted extension detected')
sabnzbd.nzbqueue.NzbQueue.do.end_job(nzo)
# Add to direct unpack
nzo.add_to_direct_unpacker(nzf)
elif par2file.is_parfile(filepath):
# Parse par2 files, cloaked or not
nzo.handle_par2(nzf, filepath)
filter, reason = nzo_filtered_by_rating(nzo)
if filter == 1:
logging.warning(
remove_warning_label(
T('WARNING: Paused job "%s" because of rating (%s)'
)), nzo.final_name, reason)
nzo.pause()
elif filter == 2:
logging.warning(
remove_warning_label(
T('WARNING: Aborted job "%s" because of rating (%s)'
)), nzo.final_name, reason)
nzo.fail_msg = T(
'Aborted, rating filter matched (%s)') % reason
sabnzbd.nzbqueue.NzbQueue.do.end_job(nzo)
def check_encrypted_and_unwanted_files(nzo, filepath):
""" Combines check for unwanted and encrypted files to save on CPU and IO """
encrypted = False
unwanted = None
if cfg.unwanted_extensions() or (nzo.encrypted == 0 and cfg.pause_on_pwrar()):
# Safe-format for Windows
# RarFile requires de-unicoded filenames for zf.testrar()
filepath_split = os.path.split(filepath)
workdir_short = short_path(filepath_split[0])
filepath = deunicode(os.path.join(workdir_short, filepath_split[1]))
# Is it even a rarfile?
if rarfile.is_rarfile(filepath):
try:
zf = rarfile.RarFile(filepath, all_names=True)
# Check for encryption
if nzo.encrypted == 0 and cfg.pause_on_pwrar() and (zf.needs_password() or is_cloaked(filepath, zf.namelist())):
# Load all passwords
passwords = get_all_passwords(nzo)
# if no cryptography installed, only error when no password was set
if not sabnzbd.HAVE_CRYPTOGRAPHY and not passwords:
logging.info(T('%s missing'), 'Python Cryptography')
nzo.encrypted = 1
encrypted = True
elif sabnzbd.HAVE_CRYPTOGRAPHY:
# Lets test if any of the password work
password_hit = False
rarfile.UNRAR_TOOL = sabnzbd.newsunpack.RAR_COMMAND
for password in passwords:
if password:
logging.info('Trying password "%s" on job "%s"', password, nzo.final_name)
try:
zf.setpassword(password)
zf.testrar()
password_hit = password
break
except rarfile.RarCRCError:
# On CRC error we can continue!
password_hit = password
break
except:
pass
# Did any work?
if password_hit:
# Don't check other files
logging.info('Password "%s" matches for job "%s"', password_hit, nzo.final_name)
nzo.encrypted = -1
encrypted = False
else:
# Encrypted and none of them worked
nzo.encrypted = 1
encrypted = True
else:
# Don't check other files
nzo.encrypted = -1
encrypted = False
# Check for unwanted extensions
if cfg.unwanted_extensions():
for somefile in zf.namelist():
logging.debug('File contains: %s', somefile)
if os.path.splitext(somefile)[1].replace('.', '').lower() in cfg.unwanted_extensions():
logging.debug('Unwanted file %s', somefile)
unwanted = somefile
zf.close()
zf.close()
del zf
except:
logging.debug('RAR file %s cannot be inspected', filepath)
return encrypted, unwanted