def deobfuscate_list(filelist, usefulname):
""" Check all files in filelist, and if wanted, deobfuscate """
# to be sure, only keep really exsiting files:
filelist = [f for f in filelist if os.path.exists(f)]
# Search for par2 files in the filelist
par2_files = [f for f in filelist if f.endswith(".par2")]
# Found any par2 files we can use?
run_renamer = True
if not par2_files:
logging.debug("No par2 files found to process, running renamer.")
else:
# Run par2 from SABnzbd on them
for par2_file in par2_files:
# Analyse data and analyse result
logging.debug("Deobfuscate par2: handling %s", par2_file)
if decode_par2(par2_file):
logging.debug("Deobfuscate par2 repair/verify finished.")
run_renamer = False
else:
logging.debug(
"Deobfuscate par2 repair/verify did not find anything to rename."
)
# No par2 files? Then we try to rename qualifying (big, not-excluded, obfuscated) files to the job-name
if run_renamer:
logging.debug(
"Trying to see if there are qualifying files to be deobfuscated")
for filename in filelist:
logging.debug("Deobfuscate inspecting %s", filename)
file_size = os.path.getsize(filename)
# Do we need to rename this file?
# Criteria: big, not-excluded extension, obfuscated (in that order)
if (file_size > MIN_FILE_SIZE
and get_ext(filename) not in EXCLUDED_FILE_EXTS
and is_probably_obfuscated(
filename
) # this as last test to avoid unnecessary analysis
):
# OK, rename
path, file = os.path.split(filename)
new_name = get_unique_filename(
"%s%s" %
(os.path.join(path, usefulname), get_ext(filename)))
logging.info("Deobfuscate renaming %s to %s", filename,
new_name)
# Rename and make sure the new filename is unique
renamer(filename, new_name)
else:
logging.info("No qualifying files found to deobfuscate")
def nzb_redirect(wdir, nzbname, pp, script, cat, priority):
"""Check if this job contains only NZB files,
if so send to queue and remove if on clean-up list
Returns list of processed NZB's
"""
files = listdir_full(wdir)
for nzb_file in files:
if get_ext(nzb_file) != ".nzb":
return None
# For multiple NZBs, cannot use the current job name
if len(files) != 1:
nzbname = None
# Process all NZB files
for nzb_file in files:
process_single_nzb(
get_filename(nzb_file),
nzb_file,
pp=pp,
script=script,
cat=cat,
priority=priority,
dup_check=False,
nzbname=nzbname,
)
return files
def download_nzb(self, nzb_dir, file_output):
# Verify if the server was setup before we start
self.is_server_configured()
# Create NZB
nzb_path = create_nzb(nzb_dir)
# Add NZB
test_job_name = "testfile_%s" % time.time()
api_result = get_api_result("addlocalfile",
extra_arguments={
"name": nzb_path,
"nzbname": test_job_name
})
assert api_result["status"]
# Remove NZB-file
os.remove(nzb_path)
# See how it's doing
self.open_page("http://%s:%s/sabnzbd/" % (SAB_HOST, SAB_PORT))
# We wait for 20 seconds to let it complete
for _ in range(20):
try:
# Locate status of our job
status_text = self.driver.find_element_by_xpath(
'//div[@id="history-tab"]//tr[td/div/span[contains(text(), "%s")]]/td[contains(@class, "status")]'
% test_job_name).text
if status_text == "Completed":
break
else:
time.sleep(1)
except WebDriverException:
time.sleep(1)
else:
pytest.fail("Download did not complete")
# Check if there is only 1 of the expected file
# Sometimes par2 can also be included, but we accept that. For example when small
# par2 files get assembled in after the download already finished (see #1509)
assert [file_output] == filesystem.globber(
os.path.join(SAB_COMPLETE_DIR, test_job_name),
"*" + filesystem.get_ext(file_output))
# Verify if the garbage collection works (see #1628)
# We need to give it a second to calm down and clear the variables
time.sleep(2)
gc_results = get_api_result("gc_stats")["value"]
if gc_results:
pytest.fail(
f"Objects were left in memory after the job finished! {gc_results}"
)
def filter_files(_file, current_path):
if is_full_path(_file):
filepath = os.path.normpath(_file)
else:
filepath = os.path.normpath(os.path.join(current_path, _file))
if os.path.exists(filepath):
size = os.stat(filepath).st_size
if (
size >= cfg.movie_rename_limit.get_int()
and not RE_SAMPLE.search(_file)
and get_ext(_file) not in EXCLUDED_FILE_EXTS
):
return True
return False
def rename(self, files, current_path):
""" Rename for Series """
logging.debug("Renaming Series")
largest = (None, None, 0)
def to_filepath(f, current_path):
if is_full_path(f):
filepath = os.path.normpath(f)
else:
filepath = os.path.normpath(os.path.join(current_path, f))
return filepath
# Create a generator of filepaths, ignore sample files and excluded files (vobs ect)
filepaths = (
(file, to_filepath(file, current_path))
for file in files
if not RE_SAMPLE.search(file) and get_ext(file) not in EXCLUDED_FILE_EXTS
)
# Find the largest existing file
for file, fp in filepaths:
# If for some reason the file no longer exists, skip
if not os.path.exists(fp):
continue
size = os.stat(fp).st_size
f_file, f_fp, f_size = largest
if size > f_size:
largest = (file, fp, size)
file, filepath, size = largest
# >20MB
if filepath and size > 20971520:
self.fname, self.ext = os.path.splitext(os.path.split(file)[1])
newname = "%s%s" % (self.filename_set, self.ext)
# Replace %fn with the original filename
newname = newname.replace("%fn", self.fname)
newpath = os.path.join(current_path, newname)
# Replace %ext with extension
newpath = newpath.replace("%ext", self.ext)
try:
logging.debug("Rename: %s to %s", filepath, newpath)
renamer(filepath, newpath)
except:
logging.error(T("Failed to rename: %s to %s"), clip_path(current_path), clip_path(newpath))
logging.info("Traceback: ", exc_info=True)
rename_similar(current_path, self.ext, self.filename_set, ())
else:
logging.debug("Nothing to rename, %s", files)
def add_nzbfile(
nzbfile,
pp=None,
script=None,
cat=None,
catdir=None,
priority=DEFAULT_PRIORITY,
nzbname=None,
nzo_info=None,
url=None,
keep=None,
reuse=None,
password=None,
nzo_id=None,
):
"""Add file, either a single NZB-file or an archive.
All other parameters are passed to the NZO-creation.
"""
if pp == "-1":
pp = None
if script and script.lower() == "default":
script = None
if cat and cat.lower() == "default":
cat = None
if isinstance(nzbfile, str):
# File coming from queue repair or local file-path
path = nzbfile
filename = os.path.basename(path)
keep_default = True
if not sabnzbd.WIN32:
# If windows client sends file to Unix server backslashes may
# be included, so convert these
path = path.replace("\\", "/")
logging.info("Attempting to add %s [%s]", filename, path)
else:
# File from file-upload object
# CherryPy mangles unicode-filenames: https://github.com/cherrypy/cherrypy/issues/1766
filename = encoding.correct_unknown_encoding(nzbfile.filename)
logging.info("Attempting to add %s", filename)
keep_default = False
try:
# We have to create a copy, because we can't re-use the CherryPy temp-file
# Just to be sure we add the extension to detect file type later on
nzb_temp_file, path = tempfile.mkstemp(suffix=filesystem.get_ext(filename))
os.write(nzb_temp_file, nzbfile.file.read())
os.close(nzb_temp_file)
except OSError:
logging.error(T("Cannot create temp file for %s"), filename)
logging.info("Traceback: ", exc_info=True)
return None
# Externally defined if we should keep the file?
if keep is None:
keep = keep_default
if filesystem.get_ext(filename) in VALID_ARCHIVES:
return nzbparser.process_nzb_archive_file(
filename,
path=path,
pp=pp,
script=script,
cat=cat,
catdir=catdir,
priority=priority,
nzbname=nzbname,
keep=keep,
reuse=reuse,
nzo_info=nzo_info,
url=url,
password=password,
nzo_id=nzo_id,
)
else:
return nzbparser.process_single_nzb(
filename,
path=path,
pp=pp,
script=script,
cat=cat,
catdir=catdir,
priority=priority,
nzbname=nzbname,
keep=keep,
reuse=reuse,
nzo_info=nzo_info,
url=url,
password=password,
nzo_id=nzo_id,
)
def run_dir(folder, catdir):
try:
files = os.listdir(folder)
except OSError:
if not self.error_reported and not catdir:
logging.error(T("Cannot read Watched Folder %s"),
filesystem.clip_path(folder))
self.error_reported = True
files = []
for filename in files:
path = os.path.join(folder, filename)
if os.path.isdir(
path) or path in self.ignored or filename[0] == ".":
continue
if filesystem.get_ext(
path) in VALID_NZB_FILES + VALID_ARCHIVES:
try:
stat_tuple = os.stat(path)
except OSError:
continue
else:
self.ignored[path] = 1
continue
if path in self.suspected:
if compare_stat_tuple(self.suspected[path], stat_tuple):
# Suspected file still has the same attributes
continue
else:
del self.suspected[path]
if stat_tuple.st_size > 0:
logging.info("Trying to import %s", path)
# Wait until the attributes are stable for 1 second, but give up after 3 sec
# This indicates that the file is fully written to disk
for n in range(3):
time.sleep(1.0)
try:
stat_tuple_tmp = os.stat(path)
except OSError:
continue
if compare_stat_tuple(stat_tuple, stat_tuple_tmp):
break
stat_tuple = stat_tuple_tmp
else:
# Not stable
continue
# Add the NZB's
res, _ = sabnzbd.add_nzbfile(path,
catdir=catdir,
keep=False)
if res < 0:
# Retry later, for example when we can't read the file
self.suspected[path] = stat_tuple
elif res == 0:
self.error_reported = False
else:
self.ignored[path] = 1
# Remove files from the bookkeeping that are no longer on the disk
clean_file_list(self.ignored, folder, files)
clean_file_list(self.suspected, folder, files)
def check_encrypted_and_unwanted_files(nzo, filepath):
""" Combines check for unwanted and encrypted files to save on CPU and IO """
encrypted = False
unwanted = None
if (cfg.unwanted_extensions() and cfg.action_on_unwanted_extensions()) or (
nzo.encrypted == 0 and cfg.pause_on_pwrar()):
# These checks should not break the assembler
try:
# Rarfile freezes on Windows special names, so don't try those!
if sabnzbd.WIN32 and has_win_device(filepath):
return encrypted, unwanted
# Is it even a rarfile?
if rarfile.is_rarfile(filepath):
# Open the rar
rarfile.UNRAR_TOOL = sabnzbd.newsunpack.RAR_COMMAND
zf = rarfile.RarFile(filepath, single_file_check=True)
# Check for encryption
if (nzo.encrypted == 0 and cfg.pause_on_pwrar()
and (zf.needs_password()
or is_cloaked(nzo, filepath, zf.namelist()))):
# Load all passwords
passwords = get_all_passwords(nzo)
# Cloaked job?
if is_cloaked(nzo, filepath, zf.namelist()):
encrypted = True
elif not passwords:
# Only error when no password was set
nzo.encrypted = 1
encrypted = True
else:
# Lets test if any of the password work
password_hit = False
for password in passwords:
if password:
logging.info(
'Trying password "%s" on job "%s"',
password, nzo.final_name)
try:
zf.setpassword(password)
except rarfile.Error:
# On weird passwords the setpassword() will fail
# but the actual rartest() will work
pass
try:
zf.testrar()
password_hit = password
break
except rarfile.RarCRCError:
# On CRC error we can continue!
password_hit = password
break
except Exception as e:
# Did we start from the right volume?
if "need to start extraction from a previous volume" in str(
e):
return encrypted, unwanted
# This one failed
pass
# Did any work?
if password_hit:
# We always trust the user's input
if not nzo.password:
nzo.password = password_hit
# Don't check other files
logging.info('Password "%s" matches for job "%s"',
password_hit, nzo.final_name)
nzo.encrypted = -1
encrypted = False
else:
# Encrypted and none of them worked
nzo.encrypted = 1
encrypted = True
# Check for unwanted extensions
if cfg.unwanted_extensions(
) and cfg.action_on_unwanted_extensions():
for somefile in zf.namelist():
logging.debug("File contains: %s", somefile)
if get_ext(somefile).replace(
".", "").lower() in cfg.unwanted_extensions():
logging.debug("Unwanted file %s", somefile)
unwanted = somefile
zf.close()
del zf
except:
logging.info("Error during inspection of RAR-file %s", filepath)
logging.debug("Traceback: ", exc_info=True)
return encrypted, unwanted
def check_encrypted_and_unwanted_files(nzo: NzbObject, filepath: str) -> Tuple[bool, Optional[str]]:
""" Combines check for unwanted and encrypted files to save on CPU and IO """
encrypted = False
unwanted = None
if (cfg.unwanted_extensions() and cfg.action_on_unwanted_extensions()) or (
nzo.encrypted == 0 and cfg.pause_on_pwrar()
):
# These checks should not break the assembler
try:
# Rarfile freezes on Windows special names, so don't try those!
if sabnzbd.WIN32 and has_win_device(filepath):
return encrypted, unwanted
# Is it even a rarfile?
if rarfile.is_rarfile(filepath):
# Open the rar
rarfile.UNRAR_TOOL = sabnzbd.newsunpack.RAR_COMMAND
zf = rarfile.RarFile(filepath, single_file_check=True)
# Check for encryption
if (
nzo.encrypted == 0
and cfg.pause_on_pwrar()
and (zf.needs_password() or is_cloaked(nzo, filepath, zf.namelist()))
):
# Load all passwords
passwords = get_all_passwords(nzo)
# Cloaked job?
if is_cloaked(nzo, filepath, zf.namelist()):
encrypted = True
elif not passwords:
# Only error when no password was set
nzo.encrypted = 1
encrypted = True
else:
# Lets test if any of the password work
password_hit = False
for password in passwords:
if password:
logging.info('Trying password "%s" on job "%s"', password, nzo.final_name)
try:
zf.setpassword(password)
except rarfile.Error:
# On weird passwords the setpassword() will fail
# but the actual testrar() will work
pass
try:
zf.testrar()
password_hit = password
break
except rarfile.RarWrongPassword:
# This one really didn't work
pass
except rarfile.RarCRCError as e:
# CRC errors can be thrown for wrong password or
# missing the next volume (with correct password)
if "cannot find volume" in str(e).lower():
# We assume this one worked!
password_hit = password
break
# This one didn't work
pass
except:
# All the other errors we skip, they might be fixable in post-proc.
# For example starting from the wrong volume, or damaged files
# This will cause the check to be performed again for the next rar, might
# be disk-intensive! Could be removed later and just accept the password.
return encrypted, unwanted
# Did any work?
if password_hit:
# We always trust the user's input
if not nzo.password:
nzo.password = password_hit
# Don't check other files
logging.info('Password "%s" matches for job "%s"', password_hit, nzo.final_name)
nzo.encrypted = -1
encrypted = False
else:
# Encrypted and none of them worked
nzo.encrypted = 1
encrypted = True
# Check for unwanted extensions
if cfg.unwanted_extensions() and cfg.action_on_unwanted_extensions():
for somefile in zf.namelist():
logging.debug("File contains: %s", somefile)
if get_ext(somefile).replace(".", "").lower() in cfg.unwanted_extensions():
logging.debug("Unwanted file %s", somefile)
unwanted = somefile
zf.close()
del zf
except:
logging.info("Error during inspection of RAR-file %s", filepath)
logging.debug("Traceback: ", exc_info=True)
return encrypted, unwanted
def deobfuscate_list(filelist, usefulname):
""" Check all files in filelist, and if wanted, deobfuscate: rename to filename based on usefulname"""
# to be sure, only keep really exsiting files:
filelist = [f for f in filelist if os.path.exists(f)]
# Search for par2 files in the filelist
par2_files = [f for f in filelist if f.endswith(".par2")]
# Found any par2 files we can use?
run_renamer = True
if not par2_files:
logging.debug("No par2 files found to process, running renamer.")
else:
# Run par2 from SABnzbd on them
for par2_file in par2_files:
# Analyse data and analyse result
logging.debug("Deobfuscate par2: handling %s", par2_file)
if decode_par2(par2_file):
logging.debug("Deobfuscate par2 repair/verify finished.")
run_renamer = False
else:
logging.debug(
"Deobfuscate par2 repair/verify did not find anything to rename."
)
# No par2 files? Then we try to rename qualifying (big, not-excluded, obfuscated) files to the job-name
if run_renamer:
excluded_file_exts = EXCLUDED_FILE_EXTS
# If there is a collection with bigger files with the same extension, we don't want to rename it
extcounter = {}
for file in filelist:
if os.path.getsize(file) < MIN_FILE_SIZE:
# too small to care
continue
_, ext = os.path.splitext(file)
if ext in extcounter:
extcounter[ext] += 1
else:
extcounter[ext] = 1
if extcounter[ext] >= 3 and ext not in excluded_file_exts:
# collection, and extension not yet in excluded_file_exts, so add it
excluded_file_exts = (*excluded_file_exts, ext)
logging.debug(
"Found a collection of at least %s files with extension %s, so not renaming those files",
extcounter[ext],
ext,
)
logging.debug(
"Trying to see if there are qualifying files to be deobfuscated")
# We start with he biggest file ... probably the most important file
filelist = sorted(filelist, key=os.path.getsize, reverse=True)
for filename in filelist:
# check that file is still there (and not renamed by the secondary renaming process below)
if not os.path.isfile(filename):
continue
logging.debug("Deobfuscate inspecting %s", filename)
# Do we need to rename this file?
# Criteria: big, not-excluded extension, obfuscated (in that order)
if (os.path.getsize(filename) > MIN_FILE_SIZE
and get_ext(filename) not in excluded_file_exts
and is_probably_obfuscated(
filename
) # this as last test to avoid unnecessary analysis
):
# Rename and make sure the new filename is unique
path, file = os.path.split(filename)
# construct new_name: <path><usefulname><extension>
new_name = get_unique_filename(
"%s%s" %
(os.path.join(path, usefulname), get_ext(filename)))
logging.info("Deobfuscate renaming %s to %s", filename,
new_name)
renamer(filename, new_name)
# find other files with the same basename in filelist, and rename them in the same way:
basedirfile, _ = os.path.splitext(
filename) # something like "/home/this/myiso"
for otherfile in filelist:
if otherfile.startswith(basedirfile +
".") and os.path.isfile(otherfile):
# yes, same basedirfile, only different extension
remainingextension = otherfile.replace(
basedirfile,
"") # might be long ext, like ".dut.srt"
new_name = get_unique_filename("%s%s" % (os.path.join(
path, usefulname), remainingextension))
logging.info("Deobfuscate renaming %s to %s",
otherfile, new_name)
# Rename and make sure the new filename is unique
renamer(otherfile, new_name)
else:
logging.info("No qualifying files found to deobfuscate")
