def decode_par2(parfile):
""" Parse a par2 file and rename files listed in the par2 to their real name """
# Check if really a par2 file
if not is_parfile(parfile):
logging.info("Par2 file %s was not really a par2 file")
return False
# Parse the par2 file
md5of16k = {}
parse_par2_file(parfile, md5of16k)
# Parse all files in the folder
dirname = os.path.dirname(parfile)
result = False
for fn in os.listdir(dirname):
filepath = os.path.join(dirname, fn)
# Only check files
if os.path.isfile(filepath):
with open(filepath, "rb") as fileToMatch:
first16k_data = fileToMatch.read(16384)
# Check if we have this hash
file_md5of16k = hashlib.md5(first16k_data).digest()
if file_md5of16k in md5of16k:
new_path = os.path.join(dirname, md5of16k[file_md5of16k])
# Make sure it's a unique name
renamer(filepath, get_unique_filename(new_path))
result = True
return result
def rename_and_collapse_folder(oldpath, newpath, files):
"""Rename folder, collapsing when there's just a single subfolder
oldpath --> newpath OR oldpath/subfolder --> newpath
Modify list of filenames accordingly
"""
orgpath = oldpath
items = globber(oldpath)
if len(items) == 1:
folder = items[0]
folder_path = os.path.join(oldpath, folder)
if os.path.isdir(folder_path) and folder not in ("VIDEO_TS",
"AUDIO_TS"):
logging.info("Collapsing %s", os.path.join(newpath, folder))
oldpath = folder_path
oldpath = os.path.normpath(oldpath)
newpath = os.path.normpath(newpath)
files = [os.path.normpath(f).replace(oldpath, newpath) for f in files]
renamer(oldpath, newpath)
try:
remove_dir(orgpath)
except:
pass
return files
def move(self, workdir_complete):
ok = True
if self.type == "movie":
move_to_parent = True
# check if we should leave the files inside an extra folder
if cfg.movie_extra_folders():
# if there is a folder in the download, leave it in an extra folder
move_to_parent = not check_for_folder(workdir_complete)
if move_to_parent:
workdir_complete, ok = move_to_parent_folder(workdir_complete)
else:
workdir_complete, ok = move_to_parent_folder(workdir_complete)
if not ok:
return workdir_complete, False
path, part = os.path.split(workdir_complete)
if "%fn" in part and self.sorter.fname:
old = workdir_complete
workdir_complete = os.path.join(path, part.replace("%fn", self.sorter.fname))
workdir_complete = get_unique_path(workdir_complete, create_dir=False)
try:
renamer(old, workdir_complete)
except:
logging.error(T("Cannot create directory %s"), clip_path(workdir_complete))
workdir_complete = old
ok = False
return workdir_complete, ok
def rename(self, files, current_path):
""" Renaming Date file """
logging.debug("Renaming Date file")
# find the master file to rename
for file in files:
if is_full_path(file):
filepath = os.path.normpath(file)
else:
filepath = os.path.normpath(os.path.join(current_path, file))
if os.path.exists(filepath):
size = os.stat(filepath).st_size
if size > cfg.movie_rename_limit.get_int():
if "sample" not in file:
self.fname, ext = os.path.splitext(os.path.split(file)[1])
newname = "%s%s" % (self.filename_set, ext)
newname = newname.replace("%fn", self.fname)
newpath = os.path.join(current_path, newname)
if not os.path.exists(newpath):
try:
logging.debug("Rename: %s to %s", filepath, newpath)
renamer(filepath, newpath)
except:
logging.error(
T("Failed to rename: %s to %s"), clip_path(current_path), clip_path(newpath)
)
logging.info("Traceback: ", exc_info=True)
rename_similar(current_path, ext, self.filename_set, ())
break
def rename_similar(folder, skip_ext, name, skipped_files):
""" Rename all other files in the 'folder' hierarchy after 'name'
and move them to the root of 'folder'.
Files having extension 'skip_ext' will be moved, but not renamed.
Don't touch files in list `skipped_files`
"""
logging.debug('Give files in set "%s" matching names.', name)
folder = os.path.normpath(folder)
skip_ext = skip_ext.lower()
for root, dirs, files in os.walk(folder):
for f in files:
path = os.path.join(root, f)
if path in skipped_files:
continue
org, ext = os.path.splitext(f)
if ext.lower() == skip_ext:
# Move file, but do not rename
newpath = os.path.join(folder, f)
else:
# Move file and rename
newname = "%s%s" % (name, ext)
newname = newname.replace("%fn", org)
newpath = os.path.join(folder, newname)
if path != newpath:
newpath = get_unique_filename(newpath)
try:
logging.debug("Rename: %s to %s", path, newpath)
renamer(path, newpath)
except:
logging.error(T("Failed to rename similar file: %s to %s"), clip_path(path), clip_path(newpath))
logging.info("Traceback: ", exc_info=True)
cleanup_empty_directories(folder)
def prepare_extraction_path(nzo):
"""Based on the information that we have, generate
the extraction path and create the directory.
Separated so it can be called from DirectUnpacker
"""
one_folder = False
marker_file = None
# Determine class directory
catdir = config.get_categories(nzo.cat).dir()
if catdir.endswith("*"):
catdir = catdir.strip("*")
one_folder = True
complete_dir = real_path(cfg.complete_dir.get_path(), catdir)
complete_dir = long_path(complete_dir)
# TV/Movie/Date Renaming code part 1 - detect and construct paths
if cfg.enable_meta():
file_sorter = Sorter(nzo, nzo.cat)
else:
file_sorter = Sorter(None, nzo.cat)
complete_dir = file_sorter.detect(nzo.final_name, complete_dir)
if file_sorter.sort_file:
one_folder = False
complete_dir = sanitize_and_trim_path(complete_dir)
if one_folder:
workdir_complete = create_all_dirs(complete_dir, apply_umask=True)
else:
workdir_complete = get_unique_path(os.path.join(
complete_dir, nzo.final_name),
create_dir=True)
marker_file = set_marker(workdir_complete)
if not workdir_complete or not os.path.exists(workdir_complete):
logging.error(
T("Cannot create final folder %s") %
os.path.join(complete_dir, nzo.final_name))
raise IOError
if cfg.folder_rename() and not one_folder:
prefixed_path = prefix(workdir_complete, "_UNPACK_")
tmp_workdir_complete = get_unique_path(prefix(workdir_complete,
"_UNPACK_"),
create_dir=False)
try:
renamer(workdir_complete, tmp_workdir_complete)
except:
pass # On failure, just use the original name
# Is the unique path different? Then we also need to modify the final path
if prefixed_path != tmp_workdir_complete:
workdir_complete = workdir_complete + os.path.splitext(
tmp_workdir_complete)[1]
else:
tmp_workdir_complete = workdir_complete
return tmp_workdir_complete, workdir_complete, file_sorter, one_folder, marker_file
def rename_with_ext(self, workdir_complete):
""" Special renamer for %ext """
if self.sorter.rename_or_not and "%ext" in workdir_complete and self.ext:
# Replace %ext with extension
newpath = workdir_complete.replace("%ext", self.ext)
try:
renamer(workdir_complete, newpath)
except:
return newpath, False
return newpath, True
else:
return workdir_complete, True
def deobfuscate_list(filelist, usefulname):
""" Check all files in filelist, and if wanted, deobfuscate """
# to be sure, only keep really exsiting files:
filelist = [f for f in filelist if os.path.exists(f)]
# Search for par2 files in the filelist
par2_files = [f for f in filelist if f.endswith(".par2")]
# Found any par2 files we can use?
run_renamer = True
if not par2_files:
logging.debug("No par2 files found to process, running renamer.")
else:
# Run par2 from SABnzbd on them
for par2_file in par2_files:
# Analyse data and analyse result
logging.debug("Deobfuscate par2: handling %s", par2_file)
if decode_par2(par2_file):
logging.debug("Deobfuscate par2 repair/verify finished.")
run_renamer = False
else:
logging.debug(
"Deobfuscate par2 repair/verify did not find anything to rename."
)
# No par2 files? Then we try to rename qualifying (big, not-excluded, obfuscated) files to the job-name
if run_renamer:
logging.debug(
"Trying to see if there are qualifying files to be deobfuscated")
for filename in filelist:
logging.debug("Deobfuscate inspecting %s", filename)
file_size = os.path.getsize(filename)
# Do we need to rename this file?
# Criteria: big, not-excluded extension, obfuscated (in that order)
if (file_size > MIN_FILE_SIZE
and get_ext(filename) not in EXCLUDED_FILE_EXTS
and is_probably_obfuscated(
filename
) # this as last test to avoid unnecessary analysis
):
# OK, rename
path, file = os.path.split(filename)
new_name = get_unique_filename(
"%s%s" %
(os.path.join(path, usefulname), get_ext(filename)))
logging.info("Deobfuscate renaming %s to %s", filename,
new_name)
# Rename and make sure the new filename is unique
renamer(filename, new_name)
else:
logging.info("No qualifying files found to deobfuscate")
def rename(self, files, current_path):
""" Rename for Series """
logging.debug("Renaming Series")
largest = (None, None, 0)
def to_filepath(f, current_path):
if is_full_path(f):
filepath = os.path.normpath(f)
else:
filepath = os.path.normpath(os.path.join(current_path, f))
return filepath
# Create a generator of filepaths, ignore sample files and excluded files (vobs ect)
filepaths = (
(file, to_filepath(file, current_path))
for file in files
if not RE_SAMPLE.search(file) and get_ext(file) not in EXCLUDED_FILE_EXTS
)
# Find the largest existing file
for file, fp in filepaths:
# If for some reason the file no longer exists, skip
if not os.path.exists(fp):
continue
size = os.stat(fp).st_size
f_file, f_fp, f_size = largest
if size > f_size:
largest = (file, fp, size)
file, filepath, size = largest
# >20MB
if filepath and size > 20971520:
self.fname, self.ext = os.path.splitext(os.path.split(file)[1])
newname = "%s%s" % (self.filename_set, self.ext)
# Replace %fn with the original filename
newname = newname.replace("%fn", self.fname)
newpath = os.path.join(current_path, newname)
# Replace %ext with extension
newpath = newpath.replace("%ext", self.ext)
try:
logging.debug("Rename: %s to %s", filepath, newpath)
renamer(filepath, newpath)
except:
logging.error(T("Failed to rename: %s to %s"), clip_path(current_path), clip_path(newpath))
logging.info("Traceback: ", exc_info=True)
rename_similar(current_path, self.ext, self.filename_set, ())
else:
logging.debug("Nothing to rename, %s", files)
def rar_renamer(nzo, workdir):
""" Deobfuscate rar file names: Use header and content information to give RAR-files decent names """
nzo.status = Status.VERIFYING
nzo.set_unpack_info("Repair", T("Trying RAR-based verification"))
nzo.set_action_line(T("Trying RAR-based verification"), "...")
renamed_files = 0
# This is the most important datastructure (in case of mixed obfuscated rarsets)
rarvolnr = {}
# rarvolnr will contain per rar vol number the rarfilenames and their respective contents (and maybe other characteristics, like filesizes).
# for example: rarvolnr[6]['somerandomfilename.rar']={'readme.txt', 'linux.iso'},
# which means 'somerandomfilename.rar' has rarvolnumber 6, and contents 'readme.txt' and 'linux.iso'
# if we find a rarfile with rarvolnumber 7, and 'linux.iso' in it, we have a match!
# The volume number and real extension of a (obfuscated) rar file
# so volnrext['dfakjldfalkjdfl.blabla'] = (14, 'part014.rar') or (2, 'r000')
# Not really needed, but handy to avoid a second lookup at the renaming
volnrext = {}
# Scan rar files in workdir, but not subdirs
workdir_files = os.listdir(workdir)
for file_to_check in workdir_files:
file_to_check = os.path.join(workdir, file_to_check)
# We only want files:
if not (os.path.isfile(file_to_check)):
continue
# The function will check if it's a RAR-file
# We do a sanity-check for the returned number
rar_vol, new_extension = rarvolinfo.get_rar_extension(file_to_check)
if 0 < rar_vol < 1000:
logging.debug("Detected volume-number %s from RAR-header: %s ",
rar_vol, file_to_check)
volnrext[file_to_check] = (rar_vol, new_extension)
# The files inside rar file
rar_contents = rarfile.RarFile(os.path.join(
workdir, file_to_check),
single_file_check=True).filelist()
try:
rarvolnr[rar_vol]
except:
# does not yet exist, so create:
rarvolnr[rar_vol] = {}
rarvolnr[rar_vol][
file_to_check] = rar_contents # store them for matching (if needed)
else:
logging.debug("No RAR-volume-number found in %s", file_to_check)
logging.debug("Deobfuscate: rarvolnr is: %s", rarvolnr)
logging.debug("Deobfuscate: volnrext is: %s", volnrext)
# Could be that there are no rar-files, we stop
if not len(rarvolnr):
return renamed_files
# Check number of different obfuscated rar sets:
numberofrarsets = len(rarvolnr[1])
if numberofrarsets == 1:
# Just one obfuscated rarset
logging.debug("Deobfuscate: Just one obfuscated rarset")
for filename in volnrext:
new_rar_name = "%s.%s" % (nzo.final_name, volnrext[filename][1])
new_rar_name = os.path.join(workdir, new_rar_name)
new_rar_name = get_unique_filename(new_rar_name)
logging.debug("Deobfuscate: Renaming %s to %s" %
(filename, new_rar_name))
renamer(filename, new_rar_name)
renamed_files += 1
else:
# More than one obfuscated rarset, so we must do matching based of files inside the rar files
logging.debug("Number of obfuscated rarsets: %s", numberofrarsets)
# Assign (random) rar set names
rarsetname = {
} # in which rar set it should be, so rar set 'A', or 'B', or ...
mychar = "A"
# First things first: Assigning a rarsetname to the rar file which have volume number 1
for base_obfuscated_filename in rarvolnr[1]:
rarsetname[
base_obfuscated_filename] = mychar + "--" + nzo.final_name
mychar = chr(ord(mychar) + 1)
logging.debug("Deobfuscate: rarsetname %s", rarsetname)
# Do the matching, layer by layer (read: rarvolnumber)
# So, all rar files with rarvolnr 1, find the contents (files inside the rar),
# and match with rarfiles with rarvolnr 2, and put them in the correct rarset.
# And so on, until the highest rarvolnr minus 1 matched against highest rarvolnr
for n in range(1, len(rarvolnr.keys())):
logging.debug(
"Deobfuscate: Finding matches between rar sets %s and %s" %
(n, n + 1))
for base_obfuscated_filename in rarvolnr[n]:
matchcounter = 0
for next_obfuscated_filename in rarvolnr[n + 1]:
# set() method with intersection (less strict): set(rarvolnr[n][base_obfuscated_filename]).intersection(set(rarvolnr[n+1][next_obfuscated_filename]))
# check if the last filename inside the existing rar matches with the first filename in the following rar
if rarvolnr[n][base_obfuscated_filename][-1] == rarvolnr[
n + 1][next_obfuscated_filename][0]:
try:
rarsetname[next_obfuscated_filename] = rarsetname[
base_obfuscated_filename]
matchcounter += 1
except KeyError:
logging.warning(
T("No matching earlier rar file for %s"),
next_obfuscated_filename)
if matchcounter > 1:
logging.info(
"Deobfuscate: more than one match, so risk on false positive matching."
)
# Do the renaming:
for filename in rarsetname:
new_rar_name = "%s.%s" % (rarsetname[filename],
volnrext[filename][1])
new_rar_name = os.path.join(workdir, new_rar_name)
new_rar_name = get_unique_filename(new_rar_name)
logging.debug("Deobfuscate: Renaming %s to %s" %
(filename, new_rar_name))
renamer(filename, new_rar_name)
renamed_files += 1
# Done: The obfuscated rar files have now been renamed to regular formatted filenames
return renamed_files
def test_renamer(self):
# First of all, create a working directory (with a random name)
dirname = os.path.join(SAB_DATA_DIR,
"testdir" + str(random.randint(10000, 99999)))
os.mkdir(dirname)
# base case: rename file within directory
filename = os.path.join(dirname, "myfile.txt")
Path(filename).touch() # create file
newfilename = os.path.join(dirname, "newfile.txt")
filesystem.renamer(filename,
newfilename) # rename() does not return a value ...
assert not os.path.isfile(filename)
assert os.path.isfile(newfilename)
# standard behaviour: renaming (moving) into an exiting other directory *is* allowed
filename = os.path.join(dirname, "myfile.txt")
Path(filename).touch() # create file
sameleveldirname = os.path.join(
SAB_DATA_DIR, "othertestdir" + str(random.randint(10000, 99999)))
os.mkdir(sameleveldirname)
newfilename = os.path.join(sameleveldirname, "newfile.txt")
filesystem.renamer(filename, newfilename)
assert not os.path.isfile(filename)
assert os.path.isfile(newfilename)
shutil.rmtree(sameleveldirname)
# Default: renaming into a non-existing subdirectory not allowed
Path(filename).touch() # create file
newfilename = os.path.join(dirname, "nonexistingsubdir", "newfile.txt")
try:
filesystem.renamer(
filename, newfilename) # rename() does not return a value ...
except:
pass
assert os.path.isfile(filename)
assert not os.path.isfile(newfilename)
# Creation of subdirectory is allowed if create_local_directories=True
Path(filename).touch()
newfilename = os.path.join(dirname, "newsubdir", "newfile.txt")
try:
filesystem.renamer(filename,
newfilename,
create_local_directories=True)
except:
pass
assert not os.path.isfile(filename)
assert os.path.isfile(newfilename)
# Creation of subdirectory plus deeper sudbdir is allowed if create_local_directories=True
Path(filename).touch()
newfilename = os.path.join(dirname, "newsubdir", "deepersubdir",
"newfile.txt")
try:
filesystem.renamer(filename,
newfilename,
create_local_directories=True)
except:
pass
assert not os.path.isfile(filename)
assert os.path.isfile(newfilename)
# ... escaping the directory plus subdir creation is not allowed
Path(filename).touch()
newfilename = os.path.join(dirname, "..", "newsubdir", "newfile.txt")
try:
filesystem.renamer(filename,
newfilename,
create_local_directories=True)
except:
pass
assert os.path.isfile(filename)
assert not os.path.isfile(newfilename)
# Cleanup working directory
shutil.rmtree(dirname)
def rename(self, _files, current_path):
""" Rename for Generic files """
logging.debug("Renaming Generic file")
def filter_files(_file, current_path):
if is_full_path(_file):
filepath = os.path.normpath(_file)
else:
filepath = os.path.normpath(os.path.join(current_path, _file))
if os.path.exists(filepath):
size = os.stat(filepath).st_size
if (
size >= cfg.movie_rename_limit.get_int()
and not RE_SAMPLE.search(_file)
and get_ext(_file) not in EXCLUDED_FILE_EXTS
):
return True
return False
# remove any files below the limit from this list
files = [_file for _file in _files if filter_files(_file, current_path)]
length = len(files)
# Single File Handling
if length == 1:
file = files[0]
if is_full_path(file):
filepath = os.path.normpath(file)
else:
filepath = os.path.normpath(os.path.join(current_path, file))
if os.path.exists(filepath):
self.fname, ext = os.path.splitext(os.path.split(file)[1])
newname = "%s%s" % (self.filename_set, ext)
newname = newname.replace("%fn", self.fname)
newpath = os.path.join(current_path, newname)
try:
logging.debug("Rename: %s to %s", filepath, newpath)
renamer(filepath, newpath)
except:
logging.error(T("Failed to rename: %s to %s"), clip_path(filepath), clip_path(newpath))
logging.info("Traceback: ", exc_info=True)
rename_similar(current_path, ext, self.filename_set, ())
# Sequence File Handling
# if there is more than one extracted file check for CD1/1/A in the title
elif self.extra:
matched_files = check_for_multiple(files)
# rename files marked as in a set
if matched_files:
logging.debug("Renaming a series of generic files (%s)", matched_files)
renamed = list(matched_files.values())
for index, file in matched_files.items():
filepath = os.path.join(current_path, file)
renamed.append(filepath)
self.fname, ext = os.path.splitext(os.path.split(file)[1])
name = "%s%s" % (self.filename_set, self.extra)
name = name.replace("%1", str(index)).replace("%fn", self.fname)
name = name + ext
newpath = os.path.join(current_path, name)
try:
logging.debug("Rename: %s to %s", filepath, newpath)
renamer(filepath, newpath)
except:
logging.error(T("Failed to rename: %s to %s"), clip_path(filepath), clip_path(newpath))
logging.info("Traceback: ", exc_info=True)
rename_similar(current_path, ext, self.filename_set, renamed)
else:
logging.debug("Movie files not in sequence %s", _files)
def save_config(force=False):
""" Update Setup file with current option values """
global CFG, database, modified
if not (modified or force):
return True
if sabnzbd.cfg.configlock():
logging.warning(T("Configuration locked, cannot save settings"))
return False
for section in database:
if section in ("servers", "categories", "rss"):
try:
CFG[section]
except KeyError:
CFG[section] = {}
for subsec in database[section]:
if section == "servers":
subsec_mod = subsec.replace("[", "{").replace("]", "}")
else:
subsec_mod = subsec
try:
CFG[section][subsec_mod]
except KeyError:
CFG[section][subsec_mod] = {}
items = database[section][subsec].get_dict()
CFG[section][subsec_mod] = items
else:
for option in database[section]:
sec, kw = database[section][option].ident()
sec = sec[-1]
try:
CFG[sec]
except KeyError:
CFG[sec] = {}
value = database[section][option]()
# bool is a subclass of int, check first
if isinstance(value, bool):
# convert bool to int when saving so we store 0 or 1
CFG[sec][kw] = str(int(value))
elif isinstance(value, int):
CFG[sec][kw] = str(value)
else:
CFG[sec][kw] = value
res = False
filename = CFG.filename
bakname = filename + ".bak"
# Check if file is writable
if not is_writable(filename):
logging.error(T("Cannot write to INI file %s"), filename)
return res
# copy current file to backup
try:
shutil.copyfile(filename, bakname)
shutil.copymode(filename, bakname)
except:
# Something wrong with the backup,
logging.error(T("Cannot create backup file for %s"), bakname)
logging.info("Traceback: ", exc_info=True)
return res
# Write new config file
try:
logging.info("Writing settings to INI file %s", filename)
CFG.write()
shutil.copymode(bakname, filename)
modified = False
res = True
except:
logging.error(T("Cannot write to INI file %s"), filename)
logging.info("Traceback: ", exc_info=True)
try:
remove_file(filename)
except:
pass
# Restore INI file from backup
renamer(bakname, filename)
return res
def rar_renamer(nzo: NzbObject, workdir):
""" Deobfuscate rar file names: Use header and content information to give RAR-files decent names """
nzo.status = Status.VERIFYING
nzo.set_unpack_info("Repair", T("Trying RAR renamer"))
nzo.set_action_line(T("Trying RAR renamer"), "...")
renamed_files = 0
# This is the most important datastructure (in case of mixed obfuscated rarsets)
rarvolnr = {}
# rarvolnr will contain per rar vol number the rarfilenames and their respective contents (and maybe other characteristics, like filesizes).
# for example: rarvolnr[6]['somerandomfilename.rar']={'readme.txt', 'linux.iso'},
# which means 'somerandomfilename.rar' has rarvolnumber 6, and contents 'readme.txt' and 'linux.iso'
# if we find a rarfile with rarvolnumber 7, and 'linux.iso' in it, we have a match!
# The volume number and real extension of a (obfuscated) rar file
# so volnrext['dfakjldfalkjdfl.blabla'] = (14, 'part014.rar') or (2, 'r000')
# Not really needed, but handy to avoid a second lookup at the renaming
volnrext = {}
# Scan rar files in workdir, but not subdirs
workdir_files = os.listdir(workdir)
for file_to_check in workdir_files:
file_to_check = os.path.join(workdir, file_to_check)
# We only want files:
if not (os.path.isfile(file_to_check)):
continue
# The function will check if it's a RAR-file
# We do a sanity-check for the returned number
rar_vol, new_extension = rarvolinfo.get_rar_extension(file_to_check)
if 0 < rar_vol < 1000:
logging.debug("Detected volume-number %s from RAR-header: %s ",
rar_vol, file_to_check)
volnrext[file_to_check] = (rar_vol, new_extension)
# The files inside rar file
rar_contents = rarfile.RarFile(os.path.join(
workdir, file_to_check),
single_file_check=True).filelist()
try:
rarvolnr[rar_vol]
except:
# does not yet exist, so create:
rarvolnr[rar_vol] = {}
rarvolnr[rar_vol][
file_to_check] = rar_contents # store them for matching (if needed)
else:
logging.debug("No RAR-volume-number found in %s", file_to_check)
logging.debug("Deobfuscate: rarvolnr is: %s", rarvolnr)
logging.debug("Deobfuscate: volnrext is: %s", volnrext)
# Could be that there are no rar-files, we stop
if not len(rarvolnr):
return renamed_files
# this can probably done with a max-key-lambda oneliner, but ... how?
numberofrarsets = 0
for mykey in rarvolnr.keys():
numberofrarsets = max(numberofrarsets, len(rarvolnr[mykey]))
logging.debug("Number of rarset is %s", numberofrarsets)
if numberofrarsets == 1:
# Just one obfuscated rarset ... that's easy
logging.debug("Deobfuscate: Just one obfuscated rarset")
for filename in volnrext:
new_rar_name = "%s.%s" % (nzo.final_name, volnrext[filename][1])
new_rar_name = os.path.join(workdir, new_rar_name)
new_rar_name = get_unique_filename(new_rar_name)
logging.debug("Deobfuscate: Renaming %s to %s" %
(filename, new_rar_name))
renamer(filename, new_rar_name)
renamed_files += 1
return renamed_files
# numberofrarsets bigger than 1, so a mixed rar set, so we need pre-checking
# Sanity check of the rar set
# Get the highest rar part number (that's the upper limit):
highest_rar = sorted(rarvolnr.keys())[-1]
# A staircase check: number of rarsets should no go up, but stay the same or go down
how_many_previous = 1000 # 1000 rarset mixed ... should be enough ... typical is 1, 2 or maybe 3
# Start at part001.rar and go the highest
for rar_set_number in range(1, highest_rar + 1):
try:
how_many_here = len(rarvolnr[rar_set_number])
except:
# rarset does not exist at all
logging.warning(
"rarset %s is missing completely, so I can't deobfuscate.",
rar_set_number)
return 0
# OK, it exists, now let's check it's not higher
if how_many_here > how_many_previous:
# this should not happen: higher number of rarset than previous number of rarset
logging.warning(
"no staircase! rarset %s is higher than previous, so I can't deobfuscate.",
rar_set_number)
return 0
how_many_previous = how_many_here
# OK, that looked OK (a declining staircase), so we can safely proceed
# More than one obfuscated rarset, so we must do matching based of files inside the rar files
# Assign (random) rar set names, first come first serve basis
rarsetname = {
} # in which rar set it should be, so rar set 'A', or 'B', or ...
mychar = "A"
# First things first: Assigning a rarsetname to the rar file which have volume number 1
for base_obfuscated_filename in rarvolnr[1]:
rarsetname[base_obfuscated_filename] = mychar + "--" + nzo.final_name
mychar = chr(ord(mychar) + 1)
logging.debug("Deobfuscate: rarsetname %s", rarsetname)
# Do the matching, layer by layer (read: rarvolnumber)
# So, all rar files with rarvolnr 1, find the contents (files inside the rar),
# and match with rarfiles with rarvolnr 2, and put them in the correct rarset.
# And so on, until the highest rarvolnr minus 1 matched against highest rarvolnr
for n in range(1, len(rarvolnr)):
logging.debug(
"Deobfuscate: Finding matches between rar sets %s and %s" %
(n, n + 1))
for base_obfuscated_filename in rarvolnr[n]:
matchcounter = 0
for next_obfuscated_filename in rarvolnr[n + 1]:
# set() method with intersection (less strict): set(rarvolnr[n][base_obfuscated_filename]).intersection(set(rarvolnr[n+1][next_obfuscated_filename]))
# check if the last filename inside the existing rar matches with the first filename in the following rar
if rarvolnr[n][base_obfuscated_filename][-1] == rarvolnr[
n + 1][next_obfuscated_filename][0]:
try:
rarsetname[next_obfuscated_filename] = rarsetname[
base_obfuscated_filename]
matchcounter += 1
except KeyError:
logging.warning(
T("No matching earlier rar file for %s"),
next_obfuscated_filename)
if matchcounter > 1:
logging.info(
"Deobfuscate: more than one match, so risk on false positive matching."
)
# Do the renaming:
for filename in rarsetname:
new_rar_name = "%s.%s" % (rarsetname[filename], volnrext[filename][1])
new_rar_name = os.path.join(workdir, new_rar_name)
new_rar_name = get_unique_filename(new_rar_name)
logging.debug("Deobfuscate: Renaming %s to %s" %
(filename, new_rar_name))
renamer(filename, new_rar_name)
renamed_files += 1
# Done: The obfuscated rar files have now been renamed to regular formatted filenames
return renamed_files
def deobfuscate_list(filelist, usefulname):
""" Check all files in filelist, and if wanted, deobfuscate: rename to filename based on usefulname"""
# to be sure, only keep really exsiting files:
filelist = [f for f in filelist if os.path.exists(f)]
# Search for par2 files in the filelist
par2_files = [f for f in filelist if f.endswith(".par2")]
# Found any par2 files we can use?
run_renamer = True
if not par2_files:
logging.debug("No par2 files found to process, running renamer.")
else:
# Run par2 from SABnzbd on them
for par2_file in par2_files:
# Analyse data and analyse result
logging.debug("Deobfuscate par2: handling %s", par2_file)
if decode_par2(par2_file):
logging.debug("Deobfuscate par2 repair/verify finished.")
run_renamer = False
else:
logging.debug(
"Deobfuscate par2 repair/verify did not find anything to rename."
)
# No par2 files? Then we try to rename qualifying (big, not-excluded, obfuscated) files to the job-name
if run_renamer:
excluded_file_exts = EXCLUDED_FILE_EXTS
# If there is a collection with bigger files with the same extension, we don't want to rename it
extcounter = {}
for file in filelist:
if os.path.getsize(file) < MIN_FILE_SIZE:
# too small to care
continue
_, ext = os.path.splitext(file)
if ext in extcounter:
extcounter[ext] += 1
else:
extcounter[ext] = 1
if extcounter[ext] >= 3 and ext not in excluded_file_exts:
# collection, and extension not yet in excluded_file_exts, so add it
excluded_file_exts = (*excluded_file_exts, ext)
logging.debug(
"Found a collection of at least %s files with extension %s, so not renaming those files",
extcounter[ext],
ext,
)
logging.debug(
"Trying to see if there are qualifying files to be deobfuscated")
# We start with he biggest file ... probably the most important file
filelist = sorted(filelist, key=os.path.getsize, reverse=True)
for filename in filelist:
# check that file is still there (and not renamed by the secondary renaming process below)
if not os.path.isfile(filename):
continue
logging.debug("Deobfuscate inspecting %s", filename)
# Do we need to rename this file?
# Criteria: big, not-excluded extension, obfuscated (in that order)
if (os.path.getsize(filename) > MIN_FILE_SIZE
and get_ext(filename) not in excluded_file_exts
and is_probably_obfuscated(
filename
) # this as last test to avoid unnecessary analysis
):
# Rename and make sure the new filename is unique
path, file = os.path.split(filename)
# construct new_name: <path><usefulname><extension>
new_name = get_unique_filename(
"%s%s" %
(os.path.join(path, usefulname), get_ext(filename)))
logging.info("Deobfuscate renaming %s to %s", filename,
new_name)
renamer(filename, new_name)
# find other files with the same basename in filelist, and rename them in the same way:
basedirfile, _ = os.path.splitext(
filename) # something like "/home/this/myiso"
for otherfile in filelist:
if otherfile.startswith(basedirfile +
".") and os.path.isfile(otherfile):
# yes, same basedirfile, only different extension
remainingextension = otherfile.replace(
basedirfile,
"") # might be long ext, like ".dut.srt"
new_name = get_unique_filename("%s%s" % (os.path.join(
path, usefulname), remainingextension))
logging.info("Deobfuscate renaming %s to %s",
otherfile, new_name)
# Rename and make sure the new filename is unique
renamer(otherfile, new_name)
else:
logging.info("No qualifying files found to deobfuscate")
