def deobfuscate_list(filelist, usefulname): """ Check all files in filelist, and if wanted, deobfuscate """ # to be sure, only keep really exsiting files: filelist = [f for f in filelist if os.path.exists(f)] # Search for par2 files in the filelist par2_files = [f for f in filelist if f.endswith(".par2")] # Found any par2 files we can use? run_renamer = True if not par2_files: logging.debug("No par2 files found to process, running renamer.") else: # Run par2 from SABnzbd on them for par2_file in par2_files: # Analyse data and analyse result logging.debug("Deobfuscate par2: handling %s", par2_file) if decode_par2(par2_file): logging.debug("Deobfuscate par2 repair/verify finished.") run_renamer = False else: logging.debug( "Deobfuscate par2 repair/verify did not find anything to rename." ) # No par2 files? Then we try to rename qualifying (big, not-excluded, obfuscated) files to the job-name if run_renamer: logging.debug( "Trying to see if there are qualifying files to be deobfuscated") for filename in filelist: logging.debug("Deobfuscate inspecting %s", filename) file_size = os.path.getsize(filename) # Do we need to rename this file? # Criteria: big, not-excluded extension, obfuscated (in that order) if (file_size > MIN_FILE_SIZE and get_ext(filename) not in EXCLUDED_FILE_EXTS and is_probably_obfuscated( filename ) # this as last test to avoid unnecessary analysis ): # OK, rename path, file = os.path.split(filename) new_name = get_unique_filename( "%s%s" % (os.path.join(path, usefulname), get_ext(filename))) logging.info("Deobfuscate renaming %s to %s", filename, new_name) # Rename and make sure the new filename is unique renamer(filename, new_name) else: logging.info("No qualifying files found to deobfuscate")
def nzb_redirect(wdir, nzbname, pp, script, cat, priority): """Check if this job contains only NZB files, if so send to queue and remove if on clean-up list Returns list of processed NZB's """ files = listdir_full(wdir) for nzb_file in files: if get_ext(nzb_file) != ".nzb": return None # For multiple NZBs, cannot use the current job name if len(files) != 1: nzbname = None # Process all NZB files for nzb_file in files: process_single_nzb( get_filename(nzb_file), nzb_file, pp=pp, script=script, cat=cat, priority=priority, dup_check=False, nzbname=nzbname, ) return files
def download_nzb(self, nzb_dir, file_output): # Verify if the server was setup before we start self.is_server_configured() # Create NZB nzb_path = create_nzb(nzb_dir) # Add NZB test_job_name = "testfile_%s" % time.time() api_result = get_api_result("addlocalfile", extra_arguments={ "name": nzb_path, "nzbname": test_job_name }) assert api_result["status"] # Remove NZB-file os.remove(nzb_path) # See how it's doing self.open_page("http://%s:%s/sabnzbd/" % (SAB_HOST, SAB_PORT)) # We wait for 20 seconds to let it complete for _ in range(20): try: # Locate status of our job status_text = self.driver.find_element_by_xpath( '//div[@id="history-tab"]//tr[td/div/span[contains(text(), "%s")]]/td[contains(@class, "status")]' % test_job_name).text if status_text == "Completed": break else: time.sleep(1) except WebDriverException: time.sleep(1) else: pytest.fail("Download did not complete") # Check if there is only 1 of the expected file # Sometimes par2 can also be included, but we accept that. For example when small # par2 files get assembled in after the download already finished (see #1509) assert [file_output] == filesystem.globber( os.path.join(SAB_COMPLETE_DIR, test_job_name), "*" + filesystem.get_ext(file_output)) # Verify if the garbage collection works (see #1628) # We need to give it a second to calm down and clear the variables time.sleep(2) gc_results = get_api_result("gc_stats")["value"] if gc_results: pytest.fail( f"Objects were left in memory after the job finished! {gc_results}" )
def filter_files(_file, current_path): if is_full_path(_file): filepath = os.path.normpath(_file) else: filepath = os.path.normpath(os.path.join(current_path, _file)) if os.path.exists(filepath): size = os.stat(filepath).st_size if ( size >= cfg.movie_rename_limit.get_int() and not RE_SAMPLE.search(_file) and get_ext(_file) not in EXCLUDED_FILE_EXTS ): return True return False
def rename(self, files, current_path): """ Rename for Series """ logging.debug("Renaming Series") largest = (None, None, 0) def to_filepath(f, current_path): if is_full_path(f): filepath = os.path.normpath(f) else: filepath = os.path.normpath(os.path.join(current_path, f)) return filepath # Create a generator of filepaths, ignore sample files and excluded files (vobs ect) filepaths = ( (file, to_filepath(file, current_path)) for file in files if not RE_SAMPLE.search(file) and get_ext(file) not in EXCLUDED_FILE_EXTS ) # Find the largest existing file for file, fp in filepaths: # If for some reason the file no longer exists, skip if not os.path.exists(fp): continue size = os.stat(fp).st_size f_file, f_fp, f_size = largest if size > f_size: largest = (file, fp, size) file, filepath, size = largest # >20MB if filepath and size > 20971520: self.fname, self.ext = os.path.splitext(os.path.split(file)[1]) newname = "%s%s" % (self.filename_set, self.ext) # Replace %fn with the original filename newname = newname.replace("%fn", self.fname) newpath = os.path.join(current_path, newname) # Replace %ext with extension newpath = newpath.replace("%ext", self.ext) try: logging.debug("Rename: %s to %s", filepath, newpath) renamer(filepath, newpath) except: logging.error(T("Failed to rename: %s to %s"), clip_path(current_path), clip_path(newpath)) logging.info("Traceback: ", exc_info=True) rename_similar(current_path, self.ext, self.filename_set, ()) else: logging.debug("Nothing to rename, %s", files)
def add_nzbfile( nzbfile, pp=None, script=None, cat=None, catdir=None, priority=DEFAULT_PRIORITY, nzbname=None, nzo_info=None, url=None, keep=None, reuse=None, password=None, nzo_id=None, ): """Add file, either a single NZB-file or an archive. All other parameters are passed to the NZO-creation. """ if pp == "-1": pp = None if script and script.lower() == "default": script = None if cat and cat.lower() == "default": cat = None if isinstance(nzbfile, str): # File coming from queue repair or local file-path path = nzbfile filename = os.path.basename(path) keep_default = True if not sabnzbd.WIN32: # If windows client sends file to Unix server backslashes may # be included, so convert these path = path.replace("\\", "/") logging.info("Attempting to add %s [%s]", filename, path) else: # File from file-upload object # CherryPy mangles unicode-filenames: https://github.com/cherrypy/cherrypy/issues/1766 filename = encoding.correct_unknown_encoding(nzbfile.filename) logging.info("Attempting to add %s", filename) keep_default = False try: # We have to create a copy, because we can't re-use the CherryPy temp-file # Just to be sure we add the extension to detect file type later on nzb_temp_file, path = tempfile.mkstemp(suffix=filesystem.get_ext(filename)) os.write(nzb_temp_file, nzbfile.file.read()) os.close(nzb_temp_file) except OSError: logging.error(T("Cannot create temp file for %s"), filename) logging.info("Traceback: ", exc_info=True) return None # Externally defined if we should keep the file? if keep is None: keep = keep_default if filesystem.get_ext(filename) in VALID_ARCHIVES: return nzbparser.process_nzb_archive_file( filename, path=path, pp=pp, script=script, cat=cat, catdir=catdir, priority=priority, nzbname=nzbname, keep=keep, reuse=reuse, nzo_info=nzo_info, url=url, password=password, nzo_id=nzo_id, ) else: return nzbparser.process_single_nzb( filename, path=path, pp=pp, script=script, cat=cat, catdir=catdir, priority=priority, nzbname=nzbname, keep=keep, reuse=reuse, nzo_info=nzo_info, url=url, password=password, nzo_id=nzo_id, )
def run_dir(folder, catdir): try: files = os.listdir(folder) except OSError: if not self.error_reported and not catdir: logging.error(T("Cannot read Watched Folder %s"), filesystem.clip_path(folder)) self.error_reported = True files = [] for filename in files: path = os.path.join(folder, filename) if os.path.isdir( path) or path in self.ignored or filename[0] == ".": continue if filesystem.get_ext( path) in VALID_NZB_FILES + VALID_ARCHIVES: try: stat_tuple = os.stat(path) except OSError: continue else: self.ignored[path] = 1 continue if path in self.suspected: if compare_stat_tuple(self.suspected[path], stat_tuple): # Suspected file still has the same attributes continue else: del self.suspected[path] if stat_tuple.st_size > 0: logging.info("Trying to import %s", path) # Wait until the attributes are stable for 1 second, but give up after 3 sec # This indicates that the file is fully written to disk for n in range(3): time.sleep(1.0) try: stat_tuple_tmp = os.stat(path) except OSError: continue if compare_stat_tuple(stat_tuple, stat_tuple_tmp): break stat_tuple = stat_tuple_tmp else: # Not stable continue # Add the NZB's res, _ = sabnzbd.add_nzbfile(path, catdir=catdir, keep=False) if res < 0: # Retry later, for example when we can't read the file self.suspected[path] = stat_tuple elif res == 0: self.error_reported = False else: self.ignored[path] = 1 # Remove files from the bookkeeping that are no longer on the disk clean_file_list(self.ignored, folder, files) clean_file_list(self.suspected, folder, files)
def check_encrypted_and_unwanted_files(nzo, filepath): """ Combines check for unwanted and encrypted files to save on CPU and IO """ encrypted = False unwanted = None if (cfg.unwanted_extensions() and cfg.action_on_unwanted_extensions()) or ( nzo.encrypted == 0 and cfg.pause_on_pwrar()): # These checks should not break the assembler try: # Rarfile freezes on Windows special names, so don't try those! if sabnzbd.WIN32 and has_win_device(filepath): return encrypted, unwanted # Is it even a rarfile? if rarfile.is_rarfile(filepath): # Open the rar rarfile.UNRAR_TOOL = sabnzbd.newsunpack.RAR_COMMAND zf = rarfile.RarFile(filepath, single_file_check=True) # Check for encryption if (nzo.encrypted == 0 and cfg.pause_on_pwrar() and (zf.needs_password() or is_cloaked(nzo, filepath, zf.namelist()))): # Load all passwords passwords = get_all_passwords(nzo) # Cloaked job? if is_cloaked(nzo, filepath, zf.namelist()): encrypted = True elif not passwords: # Only error when no password was set nzo.encrypted = 1 encrypted = True else: # Lets test if any of the password work password_hit = False for password in passwords: if password: logging.info( 'Trying password "%s" on job "%s"', password, nzo.final_name) try: zf.setpassword(password) except rarfile.Error: # On weird passwords the setpassword() will fail # but the actual rartest() will work pass try: zf.testrar() password_hit = password break except rarfile.RarCRCError: # On CRC error we can continue! password_hit = password break except Exception as e: # Did we start from the right volume? if "need to start extraction from a previous volume" in str( e): return encrypted, unwanted # This one failed pass # Did any work? if password_hit: # We always trust the user's input if not nzo.password: nzo.password = password_hit # Don't check other files logging.info('Password "%s" matches for job "%s"', password_hit, nzo.final_name) nzo.encrypted = -1 encrypted = False else: # Encrypted and none of them worked nzo.encrypted = 1 encrypted = True # Check for unwanted extensions if cfg.unwanted_extensions( ) and cfg.action_on_unwanted_extensions(): for somefile in zf.namelist(): logging.debug("File contains: %s", somefile) if get_ext(somefile).replace( ".", "").lower() in cfg.unwanted_extensions(): logging.debug("Unwanted file %s", somefile) unwanted = somefile zf.close() del zf except: logging.info("Error during inspection of RAR-file %s", filepath) logging.debug("Traceback: ", exc_info=True) return encrypted, unwanted
def check_encrypted_and_unwanted_files(nzo: NzbObject, filepath: str) -> Tuple[bool, Optional[str]]: """ Combines check for unwanted and encrypted files to save on CPU and IO """ encrypted = False unwanted = None if (cfg.unwanted_extensions() and cfg.action_on_unwanted_extensions()) or ( nzo.encrypted == 0 and cfg.pause_on_pwrar() ): # These checks should not break the assembler try: # Rarfile freezes on Windows special names, so don't try those! if sabnzbd.WIN32 and has_win_device(filepath): return encrypted, unwanted # Is it even a rarfile? if rarfile.is_rarfile(filepath): # Open the rar rarfile.UNRAR_TOOL = sabnzbd.newsunpack.RAR_COMMAND zf = rarfile.RarFile(filepath, single_file_check=True) # Check for encryption if ( nzo.encrypted == 0 and cfg.pause_on_pwrar() and (zf.needs_password() or is_cloaked(nzo, filepath, zf.namelist())) ): # Load all passwords passwords = get_all_passwords(nzo) # Cloaked job? if is_cloaked(nzo, filepath, zf.namelist()): encrypted = True elif not passwords: # Only error when no password was set nzo.encrypted = 1 encrypted = True else: # Lets test if any of the password work password_hit = False for password in passwords: if password: logging.info('Trying password "%s" on job "%s"', password, nzo.final_name) try: zf.setpassword(password) except rarfile.Error: # On weird passwords the setpassword() will fail # but the actual testrar() will work pass try: zf.testrar() password_hit = password break except rarfile.RarWrongPassword: # This one really didn't work pass except rarfile.RarCRCError as e: # CRC errors can be thrown for wrong password or # missing the next volume (with correct password) if "cannot find volume" in str(e).lower(): # We assume this one worked! password_hit = password break # This one didn't work pass except: # All the other errors we skip, they might be fixable in post-proc. # For example starting from the wrong volume, or damaged files # This will cause the check to be performed again for the next rar, might # be disk-intensive! Could be removed later and just accept the password. return encrypted, unwanted # Did any work? if password_hit: # We always trust the user's input if not nzo.password: nzo.password = password_hit # Don't check other files logging.info('Password "%s" matches for job "%s"', password_hit, nzo.final_name) nzo.encrypted = -1 encrypted = False else: # Encrypted and none of them worked nzo.encrypted = 1 encrypted = True # Check for unwanted extensions if cfg.unwanted_extensions() and cfg.action_on_unwanted_extensions(): for somefile in zf.namelist(): logging.debug("File contains: %s", somefile) if get_ext(somefile).replace(".", "").lower() in cfg.unwanted_extensions(): logging.debug("Unwanted file %s", somefile) unwanted = somefile zf.close() del zf except: logging.info("Error during inspection of RAR-file %s", filepath) logging.debug("Traceback: ", exc_info=True) return encrypted, unwanted
def deobfuscate_list(filelist, usefulname): """ Check all files in filelist, and if wanted, deobfuscate: rename to filename based on usefulname""" # to be sure, only keep really exsiting files: filelist = [f for f in filelist if os.path.exists(f)] # Search for par2 files in the filelist par2_files = [f for f in filelist if f.endswith(".par2")] # Found any par2 files we can use? run_renamer = True if not par2_files: logging.debug("No par2 files found to process, running renamer.") else: # Run par2 from SABnzbd on them for par2_file in par2_files: # Analyse data and analyse result logging.debug("Deobfuscate par2: handling %s", par2_file) if decode_par2(par2_file): logging.debug("Deobfuscate par2 repair/verify finished.") run_renamer = False else: logging.debug( "Deobfuscate par2 repair/verify did not find anything to rename." ) # No par2 files? Then we try to rename qualifying (big, not-excluded, obfuscated) files to the job-name if run_renamer: excluded_file_exts = EXCLUDED_FILE_EXTS # If there is a collection with bigger files with the same extension, we don't want to rename it extcounter = {} for file in filelist: if os.path.getsize(file) < MIN_FILE_SIZE: # too small to care continue _, ext = os.path.splitext(file) if ext in extcounter: extcounter[ext] += 1 else: extcounter[ext] = 1 if extcounter[ext] >= 3 and ext not in excluded_file_exts: # collection, and extension not yet in excluded_file_exts, so add it excluded_file_exts = (*excluded_file_exts, ext) logging.debug( "Found a collection of at least %s files with extension %s, so not renaming those files", extcounter[ext], ext, ) logging.debug( "Trying to see if there are qualifying files to be deobfuscated") # We start with he biggest file ... probably the most important file filelist = sorted(filelist, key=os.path.getsize, reverse=True) for filename in filelist: # check that file is still there (and not renamed by the secondary renaming process below) if not os.path.isfile(filename): continue logging.debug("Deobfuscate inspecting %s", filename) # Do we need to rename this file? # Criteria: big, not-excluded extension, obfuscated (in that order) if (os.path.getsize(filename) > MIN_FILE_SIZE and get_ext(filename) not in excluded_file_exts and is_probably_obfuscated( filename ) # this as last test to avoid unnecessary analysis ): # Rename and make sure the new filename is unique path, file = os.path.split(filename) # construct new_name: <path><usefulname><extension> new_name = get_unique_filename( "%s%s" % (os.path.join(path, usefulname), get_ext(filename))) logging.info("Deobfuscate renaming %s to %s", filename, new_name) renamer(filename, new_name) # find other files with the same basename in filelist, and rename them in the same way: basedirfile, _ = os.path.splitext( filename) # something like "/home/this/myiso" for otherfile in filelist: if otherfile.startswith(basedirfile + ".") and os.path.isfile(otherfile): # yes, same basedirfile, only different extension remainingextension = otherfile.replace( basedirfile, "") # might be long ext, like ".dut.srt" new_name = get_unique_filename("%s%s" % (os.path.join( path, usefulname), remainingextension)) logging.info("Deobfuscate renaming %s to %s", otherfile, new_name) # Rename and make sure the new filename is unique renamer(otherfile, new_name) else: logging.info("No qualifying files found to deobfuscate")