def decode_par2(parfile):
""" Parse a par2 file and rename files listed in the par2 to their real name """
# Check if really a par2 file
if not is_parfile(parfile):
logging.info("Par2 file %s was not really a par2 file")
return False
# Parse the par2 file
md5of16k = {}
parse_par2_file(parfile, md5of16k)
# Parse all files in the folder
dirname = os.path.dirname(parfile)
result = False
for fn in os.listdir(dirname):
filepath = os.path.join(dirname, fn)
# Only check files
if os.path.isfile(filepath):
with open(filepath, "rb") as fileToMatch:
first16k_data = fileToMatch.read(16384)
# Check if we have this hash
file_md5of16k = hashlib.md5(first16k_data).digest()
if file_md5of16k in md5of16k:
new_path = os.path.join(dirname, md5of16k[file_md5of16k])
# Make sure it's a unique name
renamer(filepath, get_unique_filename(new_path))
result = True
return result
class Assembler(Thread):
do = None # Link to the instance of this method
def __init__(self, queue=None):
Thread.__init__(self)
if queue:
self.queue = queue
else:
self.queue = Queue.Queue()
Assembler.do = self
def stop(self):
self.process(None)
def process(self, job):
self.queue.put(job)
def run(self):
while 1:
job = self.queue.get()
if not job:
logging.info("Shutting down")
break
nzo, nzf = job
if nzf:
# Check if enough disk space is free, if not pause downloader and send email
if diskspace(force=True)['download_dir'][1] < (
cfg.download_free.get_float() + nzf.bytes) / GIGI:
# Only warn and email once
if not sabnzbd.downloader.Downloader.do.paused:
logging.warning(
T('Too little diskspace forcing PAUSE'))
# Pause downloader, but don't save, since the disk is almost full!
sabnzbd.downloader.Downloader.do.pause()
sabnzbd.emailer.diskfull()
# Abort all direct unpackers, just to be sure
sabnzbd.directunpacker.abort_all()
# Place job back in queue and wait 30 seconds to hope it gets resolved
self.process(job)
sleep(30)
continue
# Prepare filename
nzo.verify_nzf_filename(nzf)
nzf.filename = sanitize_filename(nzf.filename)
filepath = get_filepath(long_path(cfg.download_dir.get_path()),
nzo, nzf.filename)
nzf.filename = get_filename(filepath)
if filepath:
logging.info('Decoding %s %s', filepath, nzf.type)
try:
filepath = self.assemble(nzf, filepath)
except IOError, (errno, strerror):
# If job was deleted or in active post-processing, ignore error
if not nzo.deleted and not nzo.is_gone(
) and not nzo.pp_active:
# 28 == disk full => pause downloader
if errno == 28:
logging.error(T('Disk full! Forcing Pause'))
else:
logging.error(
T('Disk error on creating file %s'),
clip_path(filepath))
# Log traceback
logging.info('Traceback: ', exc_info=True)
# Pause without saving
sabnzbd.downloader.Downloader.do.pause()
continue
except:
logging.error(T('Fatal error in Assembler'),
exc_info=True)
break
# Clean-up admin data
nzf.remove_admin()
# Do rar-related processing
if rarfile.is_rarfile(filepath):
# Encryption and unwanted extension detection
rar_encrypted, unwanted_file = check_encrypted_and_unwanted_files(
nzo, filepath)
if rar_encrypted:
if cfg.pause_on_pwrar() == 1:
logging.warning(
remove_warning_label(
T('WARNING: Paused job "%s" because of encrypted RAR file (if supplied, all passwords were tried)'
)), nzo.final_name)
nzo.pause()
else:
logging.warning(
remove_warning_label(
T('WARNING: Aborted job "%s" because of encrypted RAR file (if supplied, all passwords were tried)'
)), nzo.final_name)
nzo.fail_msg = T(
'Aborted, encryption detected')
sabnzbd.nzbqueue.NzbQueue.do.end_job(nzo)
if unwanted_file:
logging.warning(
remove_warning_label(
T('WARNING: In "%s" unwanted extension in RAR file. Unwanted file is %s '
)), nzo.final_name, unwanted_file)
logging.debug(
T('Unwanted extension is in rar file %s'),
filepath)
if cfg.action_on_unwanted_extensions(
) == 1 and nzo.unwanted_ext == 0:
logging.debug('Unwanted extension ... pausing')
nzo.unwanted_ext = 1
nzo.pause()
if cfg.action_on_unwanted_extensions() == 2:
logging.debug(
'Unwanted extension ... aborting')
nzo.fail_msg = T(
'Aborted, unwanted extension detected')
sabnzbd.nzbqueue.NzbQueue.do.end_job(nzo)
# Add to direct unpack
nzo.add_to_direct_unpacker(nzf)
elif par2file.is_parfile(filepath):
# Parse par2 files, cloaked or not
nzo.handle_par2(nzf, filepath)
filter, reason = nzo_filtered_by_rating(nzo)
if filter == 1:
logging.warning(
remove_warning_label(
T('WARNING: Paused job "%s" because of rating (%s)'
)), nzo.final_name, reason)
nzo.pause()
elif filter == 2:
logging.warning(
remove_warning_label(
T('WARNING: Aborted job "%s" because of rating (%s)'
)), nzo.final_name, reason)
nzo.fail_msg = T(
'Aborted, rating filter matched (%s)') % reason
sabnzbd.nzbqueue.NzbQueue.do.end_job(nzo)
def run(self):
while 1:
job = self.queue.get()
if not job:
logging.info("Shutting down")
break
nzo, nzf, file_done = job
if nzf:
# Check if enough disk space is free after each file is done
# If not enough space left, pause downloader and send email
if (file_done and diskspace(force=True)["download_dir"][1] <
(cfg.download_free.get_float() + nzf.bytes) / GIGI):
# Only warn and email once
if not sabnzbd.downloader.Downloader.do.paused:
logging.warning(
T("Too little diskspace forcing PAUSE"))
# Pause downloader, but don't save, since the disk is almost full!
sabnzbd.downloader.Downloader.do.pause()
sabnzbd.emailer.diskfull_mail()
# Abort all direct unpackers, just to be sure
sabnzbd.directunpacker.abort_all()
# Prepare filepath
filepath = nzf.prepare_filepath()
if filepath:
logging.debug("Decoding part of %s", filepath)
try:
self.assemble(nzf, file_done)
except IOError as err:
# If job was deleted or in active post-processing, ignore error
if not nzo.deleted and not nzo.is_gone(
) and not nzo.pp_active:
# 28 == disk full => pause downloader
if err.errno == 28:
logging.error(T("Disk full! Forcing Pause"))
else:
logging.error(
T("Disk error on creating file %s"),
clip_path(filepath))
# Log traceback
logging.info("Traceback: ", exc_info=True)
# Pause without saving
sabnzbd.downloader.Downloader.do.pause()
continue
except:
logging.error(T("Fatal error in Assembler"),
exc_info=True)
break
# Continue after partly written data
if not file_done:
continue
# Clean-up admin data
logging.info("Decoding finished %s", filepath)
nzf.remove_admin()
# Do rar-related processing
if rarfile.is_rarfile(filepath):
# Encryption and unwanted extension detection
rar_encrypted, unwanted_file = check_encrypted_and_unwanted_files(
nzo, filepath)
if rar_encrypted:
if cfg.pause_on_pwrar() == 1:
logging.warning(
T('Paused job "%s" because of encrypted RAR file (if supplied, all passwords were tried)'
),
nzo.final_name,
)
nzo.pause()
else:
logging.warning(
T('Aborted job "%s" because of encrypted RAR file (if supplied, all passwords were tried)'
),
nzo.final_name,
)
nzo.fail_msg = T(
"Aborted, encryption detected")
sabnzbd.nzbqueue.NzbQueue.do.end_job(nzo)
if unwanted_file:
logging.warning(
T('In "%s" unwanted extension in RAR file. Unwanted file is %s '
),
nzo.final_name,
unwanted_file,
)
logging.debug(
T("Unwanted extension is in rar file %s"),
filepath)
if cfg.action_on_unwanted_extensions(
) == 1 and nzo.unwanted_ext == 0:
logging.debug("Unwanted extension ... pausing")
nzo.unwanted_ext = 1
nzo.pause()
if cfg.action_on_unwanted_extensions() == 2:
logging.debug(
"Unwanted extension ... aborting")
nzo.fail_msg = T(
"Aborted, unwanted extension detected")
sabnzbd.nzbqueue.NzbQueue.do.end_job(nzo)
# Add to direct unpack
nzo.add_to_direct_unpacker(nzf)
elif par2file.is_parfile(filepath):
# Parse par2 files, cloaked or not
nzo.handle_par2(nzf, filepath)
filter_output, reason = nzo_filtered_by_rating(nzo)
if filter_output == 1:
logging.warning(
T('Paused job "%s" because of rating (%s)'),
nzo.final_name,
reason,
)
nzo.pause()
elif filter_output == 2:
logging.warning(
T('Aborted job "%s" because of rating (%s)'),
nzo.final_name,
reason,
)
nzo.fail_msg = T(
"Aborted, rating filter matched (%s)") % reason
sabnzbd.nzbqueue.NzbQueue.do.end_job(nzo)
else:
sabnzbd.nzbqueue.NzbQueue.do.remove(nzo.nzo_id,
add_to_history=False,
cleanup=False)
PostProcessor.do.process(nzo)
def run(self):
while 1:
# Set NzbObject and NzbFile objects to None so references
# from this thread do not keep the objects alive (see #1628)
nzo = nzf = None
nzo, nzf, file_done = self.queue.get()
if not nzo:
logging.info("Shutting down")
break
if nzf:
# Check if enough disk space is free after each file is done
# If not enough space left, pause downloader and send email
if file_done and not sabnzbd.Downloader.paused:
freespace = diskspace(force=True)
full_dir = None
required_space = (cfg.download_free.get_float() + nzf.bytes) / GIGI
if freespace["download_dir"][1] < required_space:
full_dir = "download_dir"
# Enough space in download_dir, check complete_dir
complete_free = cfg.complete_free.get_float()
if complete_free > 0 and not full_dir:
required_space = 0
if cfg.direct_unpack():
required_space = (complete_free + nzo.bytes_downloaded) / GIGI
else:
# Continue downloading until 95% complete before checking
if nzo.bytes_tried > (nzo.bytes - nzo.bytes_par2) * 0.95:
required_space = (complete_free + nzo.bytes) / GIGI
if required_space and freespace["complete_dir"][1] < required_space:
full_dir = "complete_dir"
if full_dir:
logging.warning(T("Too little diskspace forcing PAUSE"))
# Pause downloader, but don't save, since the disk is almost full!
sabnzbd.Downloader.pause()
if cfg.fulldisk_autoresume():
sabnzbd.Scheduler.plan_diskspace_resume(full_dir, required_space)
sabnzbd.emailer.diskfull_mail()
# Prepare filepath
filepath = nzf.prepare_filepath()
if filepath:
logging.debug("Decoding part of %s", filepath)
try:
self.assemble(nzf, file_done)
except IOError as err:
# If job was deleted or in active post-processing, ignore error
if not nzo.deleted and not nzo.is_gone() and not nzo.pp_active:
# 28 == disk full => pause downloader
if err.errno == 28:
logging.error(T("Disk full! Forcing Pause"))
else:
logging.error(T("Disk error on creating file %s"), clip_path(filepath))
# Log traceback
logging.info("Traceback: ", exc_info=True)
# Pause without saving
sabnzbd.Downloader.pause()
continue
except:
logging.error(T("Fatal error in Assembler"), exc_info=True)
break
# Continue after partly written data
if not file_done:
continue
# Clean-up admin data
logging.info("Decoding finished %s", filepath)
nzf.remove_admin()
# Do rar-related processing
if rarfile.is_rarfile(filepath):
# Encryption and unwanted extension detection
rar_encrypted, unwanted_file = check_encrypted_and_unwanted_files(nzo, filepath)
if rar_encrypted:
if cfg.pause_on_pwrar() == 1:
logging.warning(
T(
'Paused job "%s" because of encrypted RAR file (if supplied, all passwords were tried)'
),
nzo.final_name,
)
nzo.pause()
else:
logging.warning(
T(
'Aborted job "%s" because of encrypted RAR file (if supplied, all passwords were tried)'
),
nzo.final_name,
)
nzo.fail_msg = T("Aborted, encryption detected")
sabnzbd.NzbQueue.end_job(nzo)
if unwanted_file:
# Don't repeat the warning after a user override of an unwanted extension pause
if nzo.unwanted_ext == 0:
logging.warning(
T('In "%s" unwanted extension in RAR file. Unwanted file is %s '),
nzo.final_name,
unwanted_file,
)
logging.debug(T("Unwanted extension is in rar file %s"), filepath)
if cfg.action_on_unwanted_extensions() == 1 and nzo.unwanted_ext == 0:
logging.debug("Unwanted extension ... pausing")
nzo.unwanted_ext = 1
nzo.pause()
if cfg.action_on_unwanted_extensions() == 2:
logging.debug("Unwanted extension ... aborting")
nzo.fail_msg = T("Aborted, unwanted extension detected")
sabnzbd.NzbQueue.end_job(nzo)
# Add to direct unpack
nzo.add_to_direct_unpacker(nzf)
elif par2file.is_parfile(filepath):
# Parse par2 files, cloaked or not
nzo.handle_par2(nzf, filepath)
filter_output, reason = nzo_filtered_by_rating(nzo)
if filter_output == 1:
logging.warning(
T('Paused job "%s" because of rating (%s)'),
nzo.final_name,
reason,
)
nzo.pause()
elif filter_output == 2:
logging.warning(
T('Aborted job "%s" because of rating (%s)'),
nzo.final_name,
reason,
)
nzo.fail_msg = T("Aborted, rating filter matched (%s)") % reason
sabnzbd.NzbQueue.end_job(nzo)
else:
sabnzbd.NzbQueue.remove(nzo.nzo_id, cleanup=False)
sabnzbd.PostProcessor.process(nzo)