def parse_par2_file_packet(
f, header) -> Tuple[Optional[str], Optional[bytes], Optional[bytes]]:
""" Look up and analyze a FileDesc package """
nothing = None, None, None
if header != PAR_PKT_ID:
return nothing
# Length must be multiple of 4 and at least 20
pack_len = struct.unpack("<Q", f.read(8))[0]
if int(pack_len / 4) * 4 != pack_len or pack_len < 20:
return nothing
# Next 16 bytes is md5sum of this packet
md5sum = f.read(16)
# Read and check the data
data = f.read(pack_len - 32)
md5 = hashlib.md5()
md5.update(data)
if md5sum != md5.digest():
return nothing
# The FileDesc packet looks like:
# 16 : "PAR 2.0\0FileDesc"
# 16 : FileId
# 16 : Hash for full file **
# 16 : Hash for first 16K
# 8 : File length
# xx : Name (multiple of 4, padded with \0 if needed) **
# See if it's the right packet and get name + hash
for offset in range(0, pack_len, 8):
if data[offset:offset + 16] == PAR_FILE_ID:
filehash = data[offset + 32:offset + 48]
hash16k = data[offset + 48:offset + 64]
filename = correct_unknown_encoding(data[offset +
72:].strip(b"\0"))
return filename, filehash, hash16k
elif data[offset:offset + 15] == PAR_CREATOR_ID:
# From here until the end is the creator-text
# Useful in case of bugs in the par2-creating software
par2creator = data[offset + 16:].strip(
b"\0") # Remove any trailing \0
logging.debug("Par2-creator of %s is: %s",
os.path.basename(f.name),
correct_unknown_encoding(par2creator))
return nothing
def fix_unix_encoding(folder):
""" Fix bad name encoding for Unix systems
This happens for example when files are created
on Windows but unpacked/repaired on linux
"""
if not sabnzbd.WIN32 and not sabnzbd.DARWIN:
for root, dirs, files in os.walk(folder):
for name in files:
new_name = correct_unknown_encoding(name)
if name != new_name:
try:
renamer(os.path.join(root, name), os.path.join(root, new_name))
except:
logging.info("Cannot correct name of %s", os.path.join(root, name))
def add_nzbfile(
nzbfile,
pp=None,
script=None,
cat=None,
catdir=None,
priority=DEFAULT_PRIORITY,
nzbname=None,
nzo_info=None,
url=None,
keep=None,
reuse=None,
password=None,
nzo_id=None,
):
"""Add file, either a single NZB-file or an archive.
All other parameters are passed to the NZO-creation.
"""
if pp == "-1":
pp = None
if script and script.lower() == "default":
script = None
if cat and cat.lower() == "default":
cat = None
if isinstance(nzbfile, str):
# File coming from queue repair or local file-path
path = nzbfile
filename = os.path.basename(path)
keep_default = True
if not sabnzbd.WIN32:
# If windows client sends file to Unix server backslashes may
# be included, so convert these
path = path.replace("\\", "/")
logging.info("Attempting to add %s [%s]", filename, path)
else:
# File from file-upload object
# CherryPy mangles unicode-filenames: https://github.com/cherrypy/cherrypy/issues/1766
filename = encoding.correct_unknown_encoding(nzbfile.filename)
logging.info("Attempting to add %s", filename)
keep_default = False
try:
# We have to create a copy, because we can't re-use the CherryPy temp-file
# Just to be sure we add the extension to detect file type later on
nzb_temp_file, path = tempfile.mkstemp(suffix=filesystem.get_ext(filename))
os.write(nzb_temp_file, nzbfile.file.read())
os.close(nzb_temp_file)
except OSError:
logging.error(T("Cannot create temp file for %s"), filename)
logging.info("Traceback: ", exc_info=True)
return None
# Externally defined if we should keep the file?
if keep is None:
keep = keep_default
if filesystem.get_ext(filename) in VALID_ARCHIVES:
return nzbparser.process_nzb_archive_file(
filename,
path=path,
pp=pp,
script=script,
cat=cat,
catdir=catdir,
priority=priority,
nzbname=nzbname,
keep=keep,
reuse=reuse,
nzo_info=nzo_info,
url=url,
password=password,
nzo_id=nzo_id,
)
else:
return nzbparser.process_single_nzb(
filename,
path=path,
pp=pp,
script=script,
cat=cat,
catdir=catdir,
priority=priority,
nzbname=nzbname,
keep=keep,
reuse=reuse,
nzo_info=nzo_info,
url=url,
password=password,
nzo_id=nzo_id,
)
def process_single_nzb(
filename,
path,
pp=None,
script=None,
cat=None,
catdir=None,
keep=False,
priority=None,
nzbname=None,
reuse=None,
nzo_info=None,
dup_check=True,
url=None,
password=None,
nzo_id=None,
):
"""Analyze file and create a job from it
Supports NZB, NZB.BZ2, NZB.GZ and GZ.NZB-in-disguise
returns (status, nzo_ids)
status: -2==Error/retry, -1==Error, 0==OK
"""
nzo_ids = []
if catdir is None:
catdir = cat
try:
with open(path, "rb") as nzb_file:
check_bytes = nzb_file.read(2)
if check_bytes == b"\x1f\x8b":
# gzip file or gzip in disguise
filename = filename.replace(".nzb.gz", ".nzb")
nzb_reader_handler = gzip.GzipFile
elif check_bytes == b"BZ":
# bz2 file or bz2 in disguise
filename = filename.replace(".nzb.bz2", ".nzb")
nzb_reader_handler = bz2.BZ2File
else:
nzb_reader_handler = open
# Let's get some data and hope we can decode it
with nzb_reader_handler(path, "rb") as nzb_file:
data = correct_unknown_encoding(nzb_file.read())
except OSError:
logging.warning(T("Cannot read %s"), filesystem.clip_path(path))
logging.info("Traceback: ", exc_info=True)
return -2, nzo_ids
if filename:
filename, cat = name_to_cat(filename, catdir)
# The name is used as the name of the folder, so sanitize it using folder specific santization
if not nzbname:
# Prevent embedded password from being damaged by sanitize and trimming
nzbname = get_filename(filename)
try:
nzo = nzbstuff.NzbObject(
filename,
pp=pp,
script=script,
nzb=data,
cat=cat,
url=url,
priority=priority,
nzbname=nzbname,
nzo_info=nzo_info,
reuse=reuse,
dup_check=dup_check,
)
if not nzo.password:
nzo.password = password
except TypeError:
# Duplicate, ignore
if nzo_id:
sabnzbd.NzbQueue.remove(nzo_id)
nzo = None
except ValueError:
# Empty
return 1, nzo_ids
except:
if data.find("<nzb") >= 0 > data.find("</nzb"):
# Looks like an incomplete file, retry
return -2, nzo_ids
else:
# Something else is wrong, show error
logging.error(T("Error while adding %s, removing"),
filename,
exc_info=True)
return -1, nzo_ids
if nzo:
if nzo_id:
# Re-use existing nzo_id, when a "future" job gets it payload
sabnzbd.NzbQueue.remove(nzo_id, delete_all_data=False)
nzo.nzo_id = nzo_id
nzo_ids.append(sabnzbd.NzbQueue.add(nzo, quiet=reuse))
nzo.update_rating()
try:
if not keep:
filesystem.remove_file(path)
except OSError:
# Job was still added to the queue, so throw error but don't report failed add
logging.error(T("Error removing %s"), filesystem.clip_path(path))
logging.info("Traceback: ", exc_info=True)
return 0, nzo_ids
def process_nzb_archive_file(
filename,
path,
pp=None,
script=None,
cat=None,
catdir=None,
keep=False,
priority=None,
nzbname=None,
reuse=None,
nzo_info=None,
dup_check=True,
url=None,
password=None,
nzo_id=None,
):
"""Analyse ZIP file and create job(s).
Accepts ZIP files with ONLY nzb/nfo/folder files in it.
returns (status, nzo_ids)
status: -1==Error, 0==OK, 1==Ignore
"""
nzo_ids = []
if catdir is None:
catdir = cat
filename, cat = name_to_cat(filename, catdir)
# Returns -1==Error/Retry, 0==OK, 1==Ignore
status, zf, extension = is_archive(path)
if status != 0:
return status, []
status = 1
names = zf.namelist()
nzbcount = 0
for name in names:
name = name.lower()
if name.endswith(".nzb"):
status = 0
nzbcount += 1
if status == 0:
if nzbcount != 1:
nzbname = None
for name in names:
if name.lower().endswith(".nzb"):
try:
data = correct_unknown_encoding(zf.read(name))
except OSError:
logging.error(T("Cannot read %s"), name, exc_info=True)
zf.close()
return -1, []
name = filesystem.setname_from_path(name)
if data:
nzo = None
try:
nzo = nzbstuff.NzbObject(
name,
pp=pp,
script=script,
nzb=data,
cat=cat,
url=url,
priority=priority,
nzbname=nzbname,
nzo_info=nzo_info,
reuse=reuse,
dup_check=dup_check,
)
if not nzo.password:
nzo.password = password
except (TypeError, ValueError):
# Duplicate or empty, ignore
pass
except:
# Something else is wrong, show error
logging.error(T("Error while adding %s, removing"),
name,
exc_info=True)
if nzo:
if nzo_id:
# Re-use existing nzo_id, when a "future" job gets it payload
sabnzbd.NzbQueue.remove(nzo_id,
delete_all_data=False)
nzo.nzo_id = nzo_id
nzo_id = None
nzo_ids.append(sabnzbd.NzbQueue.add(nzo))
nzo.update_rating()
zf.close()
try:
if not keep:
filesystem.remove_file(path)
except OSError:
logging.error(T("Error removing %s"), filesystem.clip_path(path))
logging.info("Traceback: ", exc_info=True)
else:
zf.close()
status = 1
return status, nzo_ids
def test_correct_unknown_encoding(self):
# Windows encoding in bytes
assert "frènch_german_demö" == enc.correct_unknown_encoding(b"fr\xe8nch_german_dem\xf6")
# Windows encoding in string that's already UTF8
assert "demotöwers" == enc.correct_unknown_encoding("demot\udcf6wers")
