def decode_par2(parfile): """ Parse a par2 file and rename files listed in the par2 to their real name """ # Check if really a par2 file if not is_parfile(parfile): logging.info("Par2 file %s was not really a par2 file") return False # Parse the par2 file md5of16k = {} parse_par2_file(parfile, md5of16k) # Parse all files in the folder dirname = os.path.dirname(parfile) result = False for fn in os.listdir(dirname): filepath = os.path.join(dirname, fn) # Only check files if os.path.isfile(filepath): with open(filepath, "rb") as fileToMatch: first16k_data = fileToMatch.read(16384) # Check if we have this hash file_md5of16k = hashlib.md5(first16k_data).digest() if file_md5of16k in md5of16k: new_path = os.path.join(dirname, md5of16k[file_md5of16k]) # Make sure it's a unique name renamer(filepath, get_unique_filename(new_path)) result = True return result
def rename_and_collapse_folder(oldpath, newpath, files): """Rename folder, collapsing when there's just a single subfolder oldpath --> newpath OR oldpath/subfolder --> newpath Modify list of filenames accordingly """ orgpath = oldpath items = globber(oldpath) if len(items) == 1: folder = items[0] folder_path = os.path.join(oldpath, folder) if os.path.isdir(folder_path) and folder not in ("VIDEO_TS", "AUDIO_TS"): logging.info("Collapsing %s", os.path.join(newpath, folder)) oldpath = folder_path oldpath = os.path.normpath(oldpath) newpath = os.path.normpath(newpath) files = [os.path.normpath(f).replace(oldpath, newpath) for f in files] renamer(oldpath, newpath) try: remove_dir(orgpath) except: pass return files
def move(self, workdir_complete): ok = True if self.type == "movie": move_to_parent = True # check if we should leave the files inside an extra folder if cfg.movie_extra_folders(): # if there is a folder in the download, leave it in an extra folder move_to_parent = not check_for_folder(workdir_complete) if move_to_parent: workdir_complete, ok = move_to_parent_folder(workdir_complete) else: workdir_complete, ok = move_to_parent_folder(workdir_complete) if not ok: return workdir_complete, False path, part = os.path.split(workdir_complete) if "%fn" in part and self.sorter.fname: old = workdir_complete workdir_complete = os.path.join(path, part.replace("%fn", self.sorter.fname)) workdir_complete = get_unique_path(workdir_complete, create_dir=False) try: renamer(old, workdir_complete) except: logging.error(T("Cannot create directory %s"), clip_path(workdir_complete)) workdir_complete = old ok = False return workdir_complete, ok
def rename(self, files, current_path): """ Renaming Date file """ logging.debug("Renaming Date file") # find the master file to rename for file in files: if is_full_path(file): filepath = os.path.normpath(file) else: filepath = os.path.normpath(os.path.join(current_path, file)) if os.path.exists(filepath): size = os.stat(filepath).st_size if size > cfg.movie_rename_limit.get_int(): if "sample" not in file: self.fname, ext = os.path.splitext(os.path.split(file)[1]) newname = "%s%s" % (self.filename_set, ext) newname = newname.replace("%fn", self.fname) newpath = os.path.join(current_path, newname) if not os.path.exists(newpath): try: logging.debug("Rename: %s to %s", filepath, newpath) renamer(filepath, newpath) except: logging.error( T("Failed to rename: %s to %s"), clip_path(current_path), clip_path(newpath) ) logging.info("Traceback: ", exc_info=True) rename_similar(current_path, ext, self.filename_set, ()) break
def rename_similar(folder, skip_ext, name, skipped_files): """ Rename all other files in the 'folder' hierarchy after 'name' and move them to the root of 'folder'. Files having extension 'skip_ext' will be moved, but not renamed. Don't touch files in list `skipped_files` """ logging.debug('Give files in set "%s" matching names.', name) folder = os.path.normpath(folder) skip_ext = skip_ext.lower() for root, dirs, files in os.walk(folder): for f in files: path = os.path.join(root, f) if path in skipped_files: continue org, ext = os.path.splitext(f) if ext.lower() == skip_ext: # Move file, but do not rename newpath = os.path.join(folder, f) else: # Move file and rename newname = "%s%s" % (name, ext) newname = newname.replace("%fn", org) newpath = os.path.join(folder, newname) if path != newpath: newpath = get_unique_filename(newpath) try: logging.debug("Rename: %s to %s", path, newpath) renamer(path, newpath) except: logging.error(T("Failed to rename similar file: %s to %s"), clip_path(path), clip_path(newpath)) logging.info("Traceback: ", exc_info=True) cleanup_empty_directories(folder)
def prepare_extraction_path(nzo): """Based on the information that we have, generate the extraction path and create the directory. Separated so it can be called from DirectUnpacker """ one_folder = False marker_file = None # Determine class directory catdir = config.get_categories(nzo.cat).dir() if catdir.endswith("*"): catdir = catdir.strip("*") one_folder = True complete_dir = real_path(cfg.complete_dir.get_path(), catdir) complete_dir = long_path(complete_dir) # TV/Movie/Date Renaming code part 1 - detect and construct paths if cfg.enable_meta(): file_sorter = Sorter(nzo, nzo.cat) else: file_sorter = Sorter(None, nzo.cat) complete_dir = file_sorter.detect(nzo.final_name, complete_dir) if file_sorter.sort_file: one_folder = False complete_dir = sanitize_and_trim_path(complete_dir) if one_folder: workdir_complete = create_all_dirs(complete_dir, apply_umask=True) else: workdir_complete = get_unique_path(os.path.join( complete_dir, nzo.final_name), create_dir=True) marker_file = set_marker(workdir_complete) if not workdir_complete or not os.path.exists(workdir_complete): logging.error( T("Cannot create final folder %s") % os.path.join(complete_dir, nzo.final_name)) raise IOError if cfg.folder_rename() and not one_folder: prefixed_path = prefix(workdir_complete, "_UNPACK_") tmp_workdir_complete = get_unique_path(prefix(workdir_complete, "_UNPACK_"), create_dir=False) try: renamer(workdir_complete, tmp_workdir_complete) except: pass # On failure, just use the original name # Is the unique path different? Then we also need to modify the final path if prefixed_path != tmp_workdir_complete: workdir_complete = workdir_complete + os.path.splitext( tmp_workdir_complete)[1] else: tmp_workdir_complete = workdir_complete return tmp_workdir_complete, workdir_complete, file_sorter, one_folder, marker_file
def rename_with_ext(self, workdir_complete): """ Special renamer for %ext """ if self.sorter.rename_or_not and "%ext" in workdir_complete and self.ext: # Replace %ext with extension newpath = workdir_complete.replace("%ext", self.ext) try: renamer(workdir_complete, newpath) except: return newpath, False return newpath, True else: return workdir_complete, True
def deobfuscate_list(filelist, usefulname): """ Check all files in filelist, and if wanted, deobfuscate """ # to be sure, only keep really exsiting files: filelist = [f for f in filelist if os.path.exists(f)] # Search for par2 files in the filelist par2_files = [f for f in filelist if f.endswith(".par2")] # Found any par2 files we can use? run_renamer = True if not par2_files: logging.debug("No par2 files found to process, running renamer.") else: # Run par2 from SABnzbd on them for par2_file in par2_files: # Analyse data and analyse result logging.debug("Deobfuscate par2: handling %s", par2_file) if decode_par2(par2_file): logging.debug("Deobfuscate par2 repair/verify finished.") run_renamer = False else: logging.debug( "Deobfuscate par2 repair/verify did not find anything to rename." ) # No par2 files? Then we try to rename qualifying (big, not-excluded, obfuscated) files to the job-name if run_renamer: logging.debug( "Trying to see if there are qualifying files to be deobfuscated") for filename in filelist: logging.debug("Deobfuscate inspecting %s", filename) file_size = os.path.getsize(filename) # Do we need to rename this file? # Criteria: big, not-excluded extension, obfuscated (in that order) if (file_size > MIN_FILE_SIZE and get_ext(filename) not in EXCLUDED_FILE_EXTS and is_probably_obfuscated( filename ) # this as last test to avoid unnecessary analysis ): # OK, rename path, file = os.path.split(filename) new_name = get_unique_filename( "%s%s" % (os.path.join(path, usefulname), get_ext(filename))) logging.info("Deobfuscate renaming %s to %s", filename, new_name) # Rename and make sure the new filename is unique renamer(filename, new_name) else: logging.info("No qualifying files found to deobfuscate")
def rename(self, files, current_path): """ Rename for Series """ logging.debug("Renaming Series") largest = (None, None, 0) def to_filepath(f, current_path): if is_full_path(f): filepath = os.path.normpath(f) else: filepath = os.path.normpath(os.path.join(current_path, f)) return filepath # Create a generator of filepaths, ignore sample files and excluded files (vobs ect) filepaths = ( (file, to_filepath(file, current_path)) for file in files if not RE_SAMPLE.search(file) and get_ext(file) not in EXCLUDED_FILE_EXTS ) # Find the largest existing file for file, fp in filepaths: # If for some reason the file no longer exists, skip if not os.path.exists(fp): continue size = os.stat(fp).st_size f_file, f_fp, f_size = largest if size > f_size: largest = (file, fp, size) file, filepath, size = largest # >20MB if filepath and size > 20971520: self.fname, self.ext = os.path.splitext(os.path.split(file)[1]) newname = "%s%s" % (self.filename_set, self.ext) # Replace %fn with the original filename newname = newname.replace("%fn", self.fname) newpath = os.path.join(current_path, newname) # Replace %ext with extension newpath = newpath.replace("%ext", self.ext) try: logging.debug("Rename: %s to %s", filepath, newpath) renamer(filepath, newpath) except: logging.error(T("Failed to rename: %s to %s"), clip_path(current_path), clip_path(newpath)) logging.info("Traceback: ", exc_info=True) rename_similar(current_path, self.ext, self.filename_set, ()) else: logging.debug("Nothing to rename, %s", files)
def rar_renamer(nzo, workdir): """ Deobfuscate rar file names: Use header and content information to give RAR-files decent names """ nzo.status = Status.VERIFYING nzo.set_unpack_info("Repair", T("Trying RAR-based verification")) nzo.set_action_line(T("Trying RAR-based verification"), "...") renamed_files = 0 # This is the most important datastructure (in case of mixed obfuscated rarsets) rarvolnr = {} # rarvolnr will contain per rar vol number the rarfilenames and their respective contents (and maybe other characteristics, like filesizes). # for example: rarvolnr[6]['somerandomfilename.rar']={'readme.txt', 'linux.iso'}, # which means 'somerandomfilename.rar' has rarvolnumber 6, and contents 'readme.txt' and 'linux.iso' # if we find a rarfile with rarvolnumber 7, and 'linux.iso' in it, we have a match! # The volume number and real extension of a (obfuscated) rar file # so volnrext['dfakjldfalkjdfl.blabla'] = (14, 'part014.rar') or (2, 'r000') # Not really needed, but handy to avoid a second lookup at the renaming volnrext = {} # Scan rar files in workdir, but not subdirs workdir_files = os.listdir(workdir) for file_to_check in workdir_files: file_to_check = os.path.join(workdir, file_to_check) # We only want files: if not (os.path.isfile(file_to_check)): continue # The function will check if it's a RAR-file # We do a sanity-check for the returned number rar_vol, new_extension = rarvolinfo.get_rar_extension(file_to_check) if 0 < rar_vol < 1000: logging.debug("Detected volume-number %s from RAR-header: %s ", rar_vol, file_to_check) volnrext[file_to_check] = (rar_vol, new_extension) # The files inside rar file rar_contents = rarfile.RarFile(os.path.join( workdir, file_to_check), single_file_check=True).filelist() try: rarvolnr[rar_vol] except: # does not yet exist, so create: rarvolnr[rar_vol] = {} rarvolnr[rar_vol][ file_to_check] = rar_contents # store them for matching (if needed) else: logging.debug("No RAR-volume-number found in %s", file_to_check) logging.debug("Deobfuscate: rarvolnr is: %s", rarvolnr) logging.debug("Deobfuscate: volnrext is: %s", volnrext) # Could be that there are no rar-files, we stop if not len(rarvolnr): return renamed_files # Check number of different obfuscated rar sets: numberofrarsets = len(rarvolnr[1]) if numberofrarsets == 1: # Just one obfuscated rarset logging.debug("Deobfuscate: Just one obfuscated rarset") for filename in volnrext: new_rar_name = "%s.%s" % (nzo.final_name, volnrext[filename][1]) new_rar_name = os.path.join(workdir, new_rar_name) new_rar_name = get_unique_filename(new_rar_name) logging.debug("Deobfuscate: Renaming %s to %s" % (filename, new_rar_name)) renamer(filename, new_rar_name) renamed_files += 1 else: # More than one obfuscated rarset, so we must do matching based of files inside the rar files logging.debug("Number of obfuscated rarsets: %s", numberofrarsets) # Assign (random) rar set names rarsetname = { } # in which rar set it should be, so rar set 'A', or 'B', or ... mychar = "A" # First things first: Assigning a rarsetname to the rar file which have volume number 1 for base_obfuscated_filename in rarvolnr[1]: rarsetname[ base_obfuscated_filename] = mychar + "--" + nzo.final_name mychar = chr(ord(mychar) + 1) logging.debug("Deobfuscate: rarsetname %s", rarsetname) # Do the matching, layer by layer (read: rarvolnumber) # So, all rar files with rarvolnr 1, find the contents (files inside the rar), # and match with rarfiles with rarvolnr 2, and put them in the correct rarset. # And so on, until the highest rarvolnr minus 1 matched against highest rarvolnr for n in range(1, len(rarvolnr.keys())): logging.debug( "Deobfuscate: Finding matches between rar sets %s and %s" % (n, n + 1)) for base_obfuscated_filename in rarvolnr[n]: matchcounter = 0 for next_obfuscated_filename in rarvolnr[n + 1]: # set() method with intersection (less strict): set(rarvolnr[n][base_obfuscated_filename]).intersection(set(rarvolnr[n+1][next_obfuscated_filename])) # check if the last filename inside the existing rar matches with the first filename in the following rar if rarvolnr[n][base_obfuscated_filename][-1] == rarvolnr[ n + 1][next_obfuscated_filename][0]: try: rarsetname[next_obfuscated_filename] = rarsetname[ base_obfuscated_filename] matchcounter += 1 except KeyError: logging.warning( T("No matching earlier rar file for %s"), next_obfuscated_filename) if matchcounter > 1: logging.info( "Deobfuscate: more than one match, so risk on false positive matching." ) # Do the renaming: for filename in rarsetname: new_rar_name = "%s.%s" % (rarsetname[filename], volnrext[filename][1]) new_rar_name = os.path.join(workdir, new_rar_name) new_rar_name = get_unique_filename(new_rar_name) logging.debug("Deobfuscate: Renaming %s to %s" % (filename, new_rar_name)) renamer(filename, new_rar_name) renamed_files += 1 # Done: The obfuscated rar files have now been renamed to regular formatted filenames return renamed_files
def test_renamer(self): # First of all, create a working directory (with a random name) dirname = os.path.join(SAB_DATA_DIR, "testdir" + str(random.randint(10000, 99999))) os.mkdir(dirname) # base case: rename file within directory filename = os.path.join(dirname, "myfile.txt") Path(filename).touch() # create file newfilename = os.path.join(dirname, "newfile.txt") filesystem.renamer(filename, newfilename) # rename() does not return a value ... assert not os.path.isfile(filename) assert os.path.isfile(newfilename) # standard behaviour: renaming (moving) into an exiting other directory *is* allowed filename = os.path.join(dirname, "myfile.txt") Path(filename).touch() # create file sameleveldirname = os.path.join( SAB_DATA_DIR, "othertestdir" + str(random.randint(10000, 99999))) os.mkdir(sameleveldirname) newfilename = os.path.join(sameleveldirname, "newfile.txt") filesystem.renamer(filename, newfilename) assert not os.path.isfile(filename) assert os.path.isfile(newfilename) shutil.rmtree(sameleveldirname) # Default: renaming into a non-existing subdirectory not allowed Path(filename).touch() # create file newfilename = os.path.join(dirname, "nonexistingsubdir", "newfile.txt") try: filesystem.renamer( filename, newfilename) # rename() does not return a value ... except: pass assert os.path.isfile(filename) assert not os.path.isfile(newfilename) # Creation of subdirectory is allowed if create_local_directories=True Path(filename).touch() newfilename = os.path.join(dirname, "newsubdir", "newfile.txt") try: filesystem.renamer(filename, newfilename, create_local_directories=True) except: pass assert not os.path.isfile(filename) assert os.path.isfile(newfilename) # Creation of subdirectory plus deeper sudbdir is allowed if create_local_directories=True Path(filename).touch() newfilename = os.path.join(dirname, "newsubdir", "deepersubdir", "newfile.txt") try: filesystem.renamer(filename, newfilename, create_local_directories=True) except: pass assert not os.path.isfile(filename) assert os.path.isfile(newfilename) # ... escaping the directory plus subdir creation is not allowed Path(filename).touch() newfilename = os.path.join(dirname, "..", "newsubdir", "newfile.txt") try: filesystem.renamer(filename, newfilename, create_local_directories=True) except: pass assert os.path.isfile(filename) assert not os.path.isfile(newfilename) # Cleanup working directory shutil.rmtree(dirname)
def rename(self, _files, current_path): """ Rename for Generic files """ logging.debug("Renaming Generic file") def filter_files(_file, current_path): if is_full_path(_file): filepath = os.path.normpath(_file) else: filepath = os.path.normpath(os.path.join(current_path, _file)) if os.path.exists(filepath): size = os.stat(filepath).st_size if ( size >= cfg.movie_rename_limit.get_int() and not RE_SAMPLE.search(_file) and get_ext(_file) not in EXCLUDED_FILE_EXTS ): return True return False # remove any files below the limit from this list files = [_file for _file in _files if filter_files(_file, current_path)] length = len(files) # Single File Handling if length == 1: file = files[0] if is_full_path(file): filepath = os.path.normpath(file) else: filepath = os.path.normpath(os.path.join(current_path, file)) if os.path.exists(filepath): self.fname, ext = os.path.splitext(os.path.split(file)[1]) newname = "%s%s" % (self.filename_set, ext) newname = newname.replace("%fn", self.fname) newpath = os.path.join(current_path, newname) try: logging.debug("Rename: %s to %s", filepath, newpath) renamer(filepath, newpath) except: logging.error(T("Failed to rename: %s to %s"), clip_path(filepath), clip_path(newpath)) logging.info("Traceback: ", exc_info=True) rename_similar(current_path, ext, self.filename_set, ()) # Sequence File Handling # if there is more than one extracted file check for CD1/1/A in the title elif self.extra: matched_files = check_for_multiple(files) # rename files marked as in a set if matched_files: logging.debug("Renaming a series of generic files (%s)", matched_files) renamed = list(matched_files.values()) for index, file in matched_files.items(): filepath = os.path.join(current_path, file) renamed.append(filepath) self.fname, ext = os.path.splitext(os.path.split(file)[1]) name = "%s%s" % (self.filename_set, self.extra) name = name.replace("%1", str(index)).replace("%fn", self.fname) name = name + ext newpath = os.path.join(current_path, name) try: logging.debug("Rename: %s to %s", filepath, newpath) renamer(filepath, newpath) except: logging.error(T("Failed to rename: %s to %s"), clip_path(filepath), clip_path(newpath)) logging.info("Traceback: ", exc_info=True) rename_similar(current_path, ext, self.filename_set, renamed) else: logging.debug("Movie files not in sequence %s", _files)
def save_config(force=False): """ Update Setup file with current option values """ global CFG, database, modified if not (modified or force): return True if sabnzbd.cfg.configlock(): logging.warning(T("Configuration locked, cannot save settings")) return False for section in database: if section in ("servers", "categories", "rss"): try: CFG[section] except KeyError: CFG[section] = {} for subsec in database[section]: if section == "servers": subsec_mod = subsec.replace("[", "{").replace("]", "}") else: subsec_mod = subsec try: CFG[section][subsec_mod] except KeyError: CFG[section][subsec_mod] = {} items = database[section][subsec].get_dict() CFG[section][subsec_mod] = items else: for option in database[section]: sec, kw = database[section][option].ident() sec = sec[-1] try: CFG[sec] except KeyError: CFG[sec] = {} value = database[section][option]() # bool is a subclass of int, check first if isinstance(value, bool): # convert bool to int when saving so we store 0 or 1 CFG[sec][kw] = str(int(value)) elif isinstance(value, int): CFG[sec][kw] = str(value) else: CFG[sec][kw] = value res = False filename = CFG.filename bakname = filename + ".bak" # Check if file is writable if not is_writable(filename): logging.error(T("Cannot write to INI file %s"), filename) return res # copy current file to backup try: shutil.copyfile(filename, bakname) shutil.copymode(filename, bakname) except: # Something wrong with the backup, logging.error(T("Cannot create backup file for %s"), bakname) logging.info("Traceback: ", exc_info=True) return res # Write new config file try: logging.info("Writing settings to INI file %s", filename) CFG.write() shutil.copymode(bakname, filename) modified = False res = True except: logging.error(T("Cannot write to INI file %s"), filename) logging.info("Traceback: ", exc_info=True) try: remove_file(filename) except: pass # Restore INI file from backup renamer(bakname, filename) return res
def rar_renamer(nzo: NzbObject, workdir): """ Deobfuscate rar file names: Use header and content information to give RAR-files decent names """ nzo.status = Status.VERIFYING nzo.set_unpack_info("Repair", T("Trying RAR renamer")) nzo.set_action_line(T("Trying RAR renamer"), "...") renamed_files = 0 # This is the most important datastructure (in case of mixed obfuscated rarsets) rarvolnr = {} # rarvolnr will contain per rar vol number the rarfilenames and their respective contents (and maybe other characteristics, like filesizes). # for example: rarvolnr[6]['somerandomfilename.rar']={'readme.txt', 'linux.iso'}, # which means 'somerandomfilename.rar' has rarvolnumber 6, and contents 'readme.txt' and 'linux.iso' # if we find a rarfile with rarvolnumber 7, and 'linux.iso' in it, we have a match! # The volume number and real extension of a (obfuscated) rar file # so volnrext['dfakjldfalkjdfl.blabla'] = (14, 'part014.rar') or (2, 'r000') # Not really needed, but handy to avoid a second lookup at the renaming volnrext = {} # Scan rar files in workdir, but not subdirs workdir_files = os.listdir(workdir) for file_to_check in workdir_files: file_to_check = os.path.join(workdir, file_to_check) # We only want files: if not (os.path.isfile(file_to_check)): continue # The function will check if it's a RAR-file # We do a sanity-check for the returned number rar_vol, new_extension = rarvolinfo.get_rar_extension(file_to_check) if 0 < rar_vol < 1000: logging.debug("Detected volume-number %s from RAR-header: %s ", rar_vol, file_to_check) volnrext[file_to_check] = (rar_vol, new_extension) # The files inside rar file rar_contents = rarfile.RarFile(os.path.join( workdir, file_to_check), single_file_check=True).filelist() try: rarvolnr[rar_vol] except: # does not yet exist, so create: rarvolnr[rar_vol] = {} rarvolnr[rar_vol][ file_to_check] = rar_contents # store them for matching (if needed) else: logging.debug("No RAR-volume-number found in %s", file_to_check) logging.debug("Deobfuscate: rarvolnr is: %s", rarvolnr) logging.debug("Deobfuscate: volnrext is: %s", volnrext) # Could be that there are no rar-files, we stop if not len(rarvolnr): return renamed_files # this can probably done with a max-key-lambda oneliner, but ... how? numberofrarsets = 0 for mykey in rarvolnr.keys(): numberofrarsets = max(numberofrarsets, len(rarvolnr[mykey])) logging.debug("Number of rarset is %s", numberofrarsets) if numberofrarsets == 1: # Just one obfuscated rarset ... that's easy logging.debug("Deobfuscate: Just one obfuscated rarset") for filename in volnrext: new_rar_name = "%s.%s" % (nzo.final_name, volnrext[filename][1]) new_rar_name = os.path.join(workdir, new_rar_name) new_rar_name = get_unique_filename(new_rar_name) logging.debug("Deobfuscate: Renaming %s to %s" % (filename, new_rar_name)) renamer(filename, new_rar_name) renamed_files += 1 return renamed_files # numberofrarsets bigger than 1, so a mixed rar set, so we need pre-checking # Sanity check of the rar set # Get the highest rar part number (that's the upper limit): highest_rar = sorted(rarvolnr.keys())[-1] # A staircase check: number of rarsets should no go up, but stay the same or go down how_many_previous = 1000 # 1000 rarset mixed ... should be enough ... typical is 1, 2 or maybe 3 # Start at part001.rar and go the highest for rar_set_number in range(1, highest_rar + 1): try: how_many_here = len(rarvolnr[rar_set_number]) except: # rarset does not exist at all logging.warning( "rarset %s is missing completely, so I can't deobfuscate.", rar_set_number) return 0 # OK, it exists, now let's check it's not higher if how_many_here > how_many_previous: # this should not happen: higher number of rarset than previous number of rarset logging.warning( "no staircase! rarset %s is higher than previous, so I can't deobfuscate.", rar_set_number) return 0 how_many_previous = how_many_here # OK, that looked OK (a declining staircase), so we can safely proceed # More than one obfuscated rarset, so we must do matching based of files inside the rar files # Assign (random) rar set names, first come first serve basis rarsetname = { } # in which rar set it should be, so rar set 'A', or 'B', or ... mychar = "A" # First things first: Assigning a rarsetname to the rar file which have volume number 1 for base_obfuscated_filename in rarvolnr[1]: rarsetname[base_obfuscated_filename] = mychar + "--" + nzo.final_name mychar = chr(ord(mychar) + 1) logging.debug("Deobfuscate: rarsetname %s", rarsetname) # Do the matching, layer by layer (read: rarvolnumber) # So, all rar files with rarvolnr 1, find the contents (files inside the rar), # and match with rarfiles with rarvolnr 2, and put them in the correct rarset. # And so on, until the highest rarvolnr minus 1 matched against highest rarvolnr for n in range(1, len(rarvolnr)): logging.debug( "Deobfuscate: Finding matches between rar sets %s and %s" % (n, n + 1)) for base_obfuscated_filename in rarvolnr[n]: matchcounter = 0 for next_obfuscated_filename in rarvolnr[n + 1]: # set() method with intersection (less strict): set(rarvolnr[n][base_obfuscated_filename]).intersection(set(rarvolnr[n+1][next_obfuscated_filename])) # check if the last filename inside the existing rar matches with the first filename in the following rar if rarvolnr[n][base_obfuscated_filename][-1] == rarvolnr[ n + 1][next_obfuscated_filename][0]: try: rarsetname[next_obfuscated_filename] = rarsetname[ base_obfuscated_filename] matchcounter += 1 except KeyError: logging.warning( T("No matching earlier rar file for %s"), next_obfuscated_filename) if matchcounter > 1: logging.info( "Deobfuscate: more than one match, so risk on false positive matching." ) # Do the renaming: for filename in rarsetname: new_rar_name = "%s.%s" % (rarsetname[filename], volnrext[filename][1]) new_rar_name = os.path.join(workdir, new_rar_name) new_rar_name = get_unique_filename(new_rar_name) logging.debug("Deobfuscate: Renaming %s to %s" % (filename, new_rar_name)) renamer(filename, new_rar_name) renamed_files += 1 # Done: The obfuscated rar files have now been renamed to regular formatted filenames return renamed_files
def deobfuscate_list(filelist, usefulname): """ Check all files in filelist, and if wanted, deobfuscate: rename to filename based on usefulname""" # to be sure, only keep really exsiting files: filelist = [f for f in filelist if os.path.exists(f)] # Search for par2 files in the filelist par2_files = [f for f in filelist if f.endswith(".par2")] # Found any par2 files we can use? run_renamer = True if not par2_files: logging.debug("No par2 files found to process, running renamer.") else: # Run par2 from SABnzbd on them for par2_file in par2_files: # Analyse data and analyse result logging.debug("Deobfuscate par2: handling %s", par2_file) if decode_par2(par2_file): logging.debug("Deobfuscate par2 repair/verify finished.") run_renamer = False else: logging.debug( "Deobfuscate par2 repair/verify did not find anything to rename." ) # No par2 files? Then we try to rename qualifying (big, not-excluded, obfuscated) files to the job-name if run_renamer: excluded_file_exts = EXCLUDED_FILE_EXTS # If there is a collection with bigger files with the same extension, we don't want to rename it extcounter = {} for file in filelist: if os.path.getsize(file) < MIN_FILE_SIZE: # too small to care continue _, ext = os.path.splitext(file) if ext in extcounter: extcounter[ext] += 1 else: extcounter[ext] = 1 if extcounter[ext] >= 3 and ext not in excluded_file_exts: # collection, and extension not yet in excluded_file_exts, so add it excluded_file_exts = (*excluded_file_exts, ext) logging.debug( "Found a collection of at least %s files with extension %s, so not renaming those files", extcounter[ext], ext, ) logging.debug( "Trying to see if there are qualifying files to be deobfuscated") # We start with he biggest file ... probably the most important file filelist = sorted(filelist, key=os.path.getsize, reverse=True) for filename in filelist: # check that file is still there (and not renamed by the secondary renaming process below) if not os.path.isfile(filename): continue logging.debug("Deobfuscate inspecting %s", filename) # Do we need to rename this file? # Criteria: big, not-excluded extension, obfuscated (in that order) if (os.path.getsize(filename) > MIN_FILE_SIZE and get_ext(filename) not in excluded_file_exts and is_probably_obfuscated( filename ) # this as last test to avoid unnecessary analysis ): # Rename and make sure the new filename is unique path, file = os.path.split(filename) # construct new_name: <path><usefulname><extension> new_name = get_unique_filename( "%s%s" % (os.path.join(path, usefulname), get_ext(filename))) logging.info("Deobfuscate renaming %s to %s", filename, new_name) renamer(filename, new_name) # find other files with the same basename in filelist, and rename them in the same way: basedirfile, _ = os.path.splitext( filename) # something like "/home/this/myiso" for otherfile in filelist: if otherfile.startswith(basedirfile + ".") and os.path.isfile(otherfile): # yes, same basedirfile, only different extension remainingextension = otherfile.replace( basedirfile, "") # might be long ext, like ".dut.srt" new_name = get_unique_filename("%s%s" % (os.path.join( path, usefulname), remainingextension)) logging.info("Deobfuscate renaming %s to %s", otherfile, new_name) # Rename and make sure the new filename is unique renamer(otherfile, new_name) else: logging.info("No qualifying files found to deobfuscate")