def parse_par2_file_packet( f, header) -> Tuple[Optional[str], Optional[bytes], Optional[bytes]]: """ Look up and analyze a FileDesc package """ nothing = None, None, None if header != PAR_PKT_ID: return nothing # Length must be multiple of 4 and at least 20 pack_len = struct.unpack("<Q", f.read(8))[0] if int(pack_len / 4) * 4 != pack_len or pack_len < 20: return nothing # Next 16 bytes is md5sum of this packet md5sum = f.read(16) # Read and check the data data = f.read(pack_len - 32) md5 = hashlib.md5() md5.update(data) if md5sum != md5.digest(): return nothing # The FileDesc packet looks like: # 16 : "PAR 2.0\0FileDesc" # 16 : FileId # 16 : Hash for full file ** # 16 : Hash for first 16K # 8 : File length # xx : Name (multiple of 4, padded with \0 if needed) ** # See if it's the right packet and get name + hash for offset in range(0, pack_len, 8): if data[offset:offset + 16] == PAR_FILE_ID: filehash = data[offset + 32:offset + 48] hash16k = data[offset + 48:offset + 64] filename = correct_unknown_encoding(data[offset + 72:].strip(b"\0")) return filename, filehash, hash16k elif data[offset:offset + 15] == PAR_CREATOR_ID: # From here until the end is the creator-text # Useful in case of bugs in the par2-creating software par2creator = data[offset + 16:].strip( b"\0") # Remove any trailing \0 logging.debug("Par2-creator of %s is: %s", os.path.basename(f.name), correct_unknown_encoding(par2creator)) return nothing
def fix_unix_encoding(folder): """ Fix bad name encoding for Unix systems This happens for example when files are created on Windows but unpacked/repaired on linux """ if not sabnzbd.WIN32 and not sabnzbd.DARWIN: for root, dirs, files in os.walk(folder): for name in files: new_name = correct_unknown_encoding(name) if name != new_name: try: renamer(os.path.join(root, name), os.path.join(root, new_name)) except: logging.info("Cannot correct name of %s", os.path.join(root, name))
def add_nzbfile( nzbfile, pp=None, script=None, cat=None, catdir=None, priority=DEFAULT_PRIORITY, nzbname=None, nzo_info=None, url=None, keep=None, reuse=None, password=None, nzo_id=None, ): """Add file, either a single NZB-file or an archive. All other parameters are passed to the NZO-creation. """ if pp == "-1": pp = None if script and script.lower() == "default": script = None if cat and cat.lower() == "default": cat = None if isinstance(nzbfile, str): # File coming from queue repair or local file-path path = nzbfile filename = os.path.basename(path) keep_default = True if not sabnzbd.WIN32: # If windows client sends file to Unix server backslashes may # be included, so convert these path = path.replace("\\", "/") logging.info("Attempting to add %s [%s]", filename, path) else: # File from file-upload object # CherryPy mangles unicode-filenames: https://github.com/cherrypy/cherrypy/issues/1766 filename = encoding.correct_unknown_encoding(nzbfile.filename) logging.info("Attempting to add %s", filename) keep_default = False try: # We have to create a copy, because we can't re-use the CherryPy temp-file # Just to be sure we add the extension to detect file type later on nzb_temp_file, path = tempfile.mkstemp(suffix=filesystem.get_ext(filename)) os.write(nzb_temp_file, nzbfile.file.read()) os.close(nzb_temp_file) except OSError: logging.error(T("Cannot create temp file for %s"), filename) logging.info("Traceback: ", exc_info=True) return None # Externally defined if we should keep the file? if keep is None: keep = keep_default if filesystem.get_ext(filename) in VALID_ARCHIVES: return nzbparser.process_nzb_archive_file( filename, path=path, pp=pp, script=script, cat=cat, catdir=catdir, priority=priority, nzbname=nzbname, keep=keep, reuse=reuse, nzo_info=nzo_info, url=url, password=password, nzo_id=nzo_id, ) else: return nzbparser.process_single_nzb( filename, path=path, pp=pp, script=script, cat=cat, catdir=catdir, priority=priority, nzbname=nzbname, keep=keep, reuse=reuse, nzo_info=nzo_info, url=url, password=password, nzo_id=nzo_id, )
def process_single_nzb( filename, path, pp=None, script=None, cat=None, catdir=None, keep=False, priority=None, nzbname=None, reuse=None, nzo_info=None, dup_check=True, url=None, password=None, nzo_id=None, ): """Analyze file and create a job from it Supports NZB, NZB.BZ2, NZB.GZ and GZ.NZB-in-disguise returns (status, nzo_ids) status: -2==Error/retry, -1==Error, 0==OK """ nzo_ids = [] if catdir is None: catdir = cat try: with open(path, "rb") as nzb_file: check_bytes = nzb_file.read(2) if check_bytes == b"\x1f\x8b": # gzip file or gzip in disguise filename = filename.replace(".nzb.gz", ".nzb") nzb_reader_handler = gzip.GzipFile elif check_bytes == b"BZ": # bz2 file or bz2 in disguise filename = filename.replace(".nzb.bz2", ".nzb") nzb_reader_handler = bz2.BZ2File else: nzb_reader_handler = open # Let's get some data and hope we can decode it with nzb_reader_handler(path, "rb") as nzb_file: data = correct_unknown_encoding(nzb_file.read()) except OSError: logging.warning(T("Cannot read %s"), filesystem.clip_path(path)) logging.info("Traceback: ", exc_info=True) return -2, nzo_ids if filename: filename, cat = name_to_cat(filename, catdir) # The name is used as the name of the folder, so sanitize it using folder specific santization if not nzbname: # Prevent embedded password from being damaged by sanitize and trimming nzbname = get_filename(filename) try: nzo = nzbstuff.NzbObject( filename, pp=pp, script=script, nzb=data, cat=cat, url=url, priority=priority, nzbname=nzbname, nzo_info=nzo_info, reuse=reuse, dup_check=dup_check, ) if not nzo.password: nzo.password = password except TypeError: # Duplicate, ignore if nzo_id: sabnzbd.NzbQueue.remove(nzo_id) nzo = None except ValueError: # Empty return 1, nzo_ids except: if data.find("<nzb") >= 0 > data.find("</nzb"): # Looks like an incomplete file, retry return -2, nzo_ids else: # Something else is wrong, show error logging.error(T("Error while adding %s, removing"), filename, exc_info=True) return -1, nzo_ids if nzo: if nzo_id: # Re-use existing nzo_id, when a "future" job gets it payload sabnzbd.NzbQueue.remove(nzo_id, delete_all_data=False) nzo.nzo_id = nzo_id nzo_ids.append(sabnzbd.NzbQueue.add(nzo, quiet=reuse)) nzo.update_rating() try: if not keep: filesystem.remove_file(path) except OSError: # Job was still added to the queue, so throw error but don't report failed add logging.error(T("Error removing %s"), filesystem.clip_path(path)) logging.info("Traceback: ", exc_info=True) return 0, nzo_ids
def process_nzb_archive_file( filename, path, pp=None, script=None, cat=None, catdir=None, keep=False, priority=None, nzbname=None, reuse=None, nzo_info=None, dup_check=True, url=None, password=None, nzo_id=None, ): """Analyse ZIP file and create job(s). Accepts ZIP files with ONLY nzb/nfo/folder files in it. returns (status, nzo_ids) status: -1==Error, 0==OK, 1==Ignore """ nzo_ids = [] if catdir is None: catdir = cat filename, cat = name_to_cat(filename, catdir) # Returns -1==Error/Retry, 0==OK, 1==Ignore status, zf, extension = is_archive(path) if status != 0: return status, [] status = 1 names = zf.namelist() nzbcount = 0 for name in names: name = name.lower() if name.endswith(".nzb"): status = 0 nzbcount += 1 if status == 0: if nzbcount != 1: nzbname = None for name in names: if name.lower().endswith(".nzb"): try: data = correct_unknown_encoding(zf.read(name)) except OSError: logging.error(T("Cannot read %s"), name, exc_info=True) zf.close() return -1, [] name = filesystem.setname_from_path(name) if data: nzo = None try: nzo = nzbstuff.NzbObject( name, pp=pp, script=script, nzb=data, cat=cat, url=url, priority=priority, nzbname=nzbname, nzo_info=nzo_info, reuse=reuse, dup_check=dup_check, ) if not nzo.password: nzo.password = password except (TypeError, ValueError): # Duplicate or empty, ignore pass except: # Something else is wrong, show error logging.error(T("Error while adding %s, removing"), name, exc_info=True) if nzo: if nzo_id: # Re-use existing nzo_id, when a "future" job gets it payload sabnzbd.NzbQueue.remove(nzo_id, delete_all_data=False) nzo.nzo_id = nzo_id nzo_id = None nzo_ids.append(sabnzbd.NzbQueue.add(nzo)) nzo.update_rating() zf.close() try: if not keep: filesystem.remove_file(path) except OSError: logging.error(T("Error removing %s"), filesystem.clip_path(path)) logging.info("Traceback: ", exc_info=True) else: zf.close() status = 1 return status, nzo_ids
def test_correct_unknown_encoding(self): # Windows encoding in bytes assert "frènch_german_demö" == enc.correct_unknown_encoding(b"fr\xe8nch_german_dem\xf6") # Windows encoding in string that's already UTF8 assert "demotöwers" == enc.correct_unknown_encoding("demot\udcf6wers")