def bad_fetch(nzo, url, msg='', content=False):
""" Create History entry for failed URL Fetch
msg : message to be logged
retry : make retry link in history
content : report in history that cause is a bad NZB file
"""
if msg:
msg = unicoder(msg)
else:
msg = ''
nzo.status = Status.FAILED
if url:
nzo.filename = url
nzo.final_name = url.strip()
if content:
# Bad content
msg = T('Unusable NZB file')
else:
# Failed fetch
msg = T('URL Fetching failed; %s') % msg
nzo.fail_msg = msg
notifier.send_notification(T('URL Fetching failed; %s') % '', '%s\n%s' % (msg, url), 'other')
if cfg.email_endjob() > 0:
emailer.badfetch_mail(msg, url)
NzbQueue.do.remove(nzo.nzo_id, add_to_history=True)
def bad_fetch(nzo, url, msg='', content=False):
""" Create History entry for failed URL Fetch
msg : message to be logged
retry : make retry link in history
content : report in history that cause is a bad NZB file
"""
if msg:
msg = unicoder(msg)
else:
msg = ''
nzo.status = Status.FAILED
if url:
nzo.filename = url
nzo.final_name = url.strip()
if content:
# Bad content
msg = T('Unusable NZB file')
else:
# Failed fetch
msg = T('URL Fetching failed; %s') % msg
nzo.fail_msg = msg
notifier.send_notification(T('URL Fetching failed; %s') % '', '%s\n%s' % (msg, url), 'other', nzo.cat)
if cfg.email_endjob() > 0:
emailer.badfetch_mail(msg, url)
NzbQueue.do.remove(nzo.nzo_id, add_to_history=True)
def fail_to_history(nzo: NzbObject, url: str, msg="", content=False):
"""Create History entry for failed URL Fetch
msg: message to be logged
content: report in history that cause is a bad NZB file
"""
# Remove the "Trying to fetch" part
if url:
nzo.filename = url
nzo.final_name = url.strip()
if content:
# Bad content
msg = T("Unusable NZB file")
else:
# Failed fetch
msg = T("URL Fetching failed; %s") % msg
# Mark as failed
nzo.set_unpack_info("Source", msg)
nzo.fail_msg = msg
notifier.send_notification(
T("URL Fetching failed; %s") % "", "%s\n%s" % (msg, url), "failed",
nzo.cat)
if cfg.email_endjob() > 0:
emailer.badfetch_mail(msg, url)
# Parse category to make sure script is set correctly after a grab
nzo.cat, _, nzo.script, _ = misc.cat_to_opts(nzo.cat,
script=nzo.script)
# Add to history and run script if desired
sabnzbd.NzbQueue.remove(nzo.nzo_id)
sabnzbd.PostProcessor.process(nzo)
def fail_to_history(self, nzo, url, msg='', content=False):
""" Create History entry for failed URL Fetch
msg: message to be logged
content: report in history that cause is a bad NZB file
"""
# Remove the "Trying to fetch" part
if url:
nzo.filename = url
nzo.final_name = url.strip()
if content:
# Bad content
msg = T('Unusable NZB file')
else:
# Failed fetch
msg = T('URL Fetching failed; %s') % unicoder(msg)
# Mark as failed
nzo.status = Status.FAILED
nzo.fail_msg = msg
notifier.send_notification(T('URL Fetching failed; %s') % '', '%s\n%s' % (msg, url), 'other', nzo.cat)
if cfg.email_endjob() > 0:
emailer.badfetch_mail(msg, url)
# Parse category to make sure script is set correctly after a grab
nzo.cat, _, nzo.script, _ = misc.cat_to_opts(nzo.cat, script=nzo.script)
# Add to history and run script if desired
NzbQueue.do.remove(nzo.nzo_id, add_to_history=False)
PostProcessor.do.process(nzo)
def resume(self):
# Do not notify when SABnzbd is still starting
if self.paused and sabnzbd.WEB_DIR:
logging.info("Resuming")
notifier.send_notification("SABnzbd", T("Resuming"),
"pause_resume")
self.paused = False
def add(self, nzo, save=True, quiet=False):
if not nzo.nzo_id:
nzo.nzo_id = sabnzbd.get_new_id("nzo", nzo.workpath,
self.__nzo_table)
# If no files are to be downloaded anymore, send to postproc
if not nzo.files and not nzo.futuretype:
self.end_job(nzo)
return nzo.nzo_id
# Reset try_lists
nzo.reset_try_list()
if nzo.nzo_id:
nzo.deleted = False
priority = nzo.priority
if sabnzbd.scheduler.analyse(False, priority):
nzo.status = Status.PAUSED
self.__nzo_table[nzo.nzo_id] = nzo
if priority > HIGH_PRIORITY:
# Top and repair priority items are added to the top of the queue
self.__nzo_list.insert(0, nzo)
elif priority == LOW_PRIORITY:
self.__nzo_list.append(nzo)
else:
# for high priority we need to add the item at the bottom
# of any other high priority items above the normal priority
# for normal priority we need to add the item at the bottom
# of the normal priority items above the low priority
if self.__nzo_list:
pos = 0
added = False
for position in self.__nzo_list:
if position.priority < priority:
self.__nzo_list.insert(pos, nzo)
added = True
break
pos += 1
if not added:
# if there are no other items classed as a lower priority
# then it will be added to the bottom of the queue
self.__nzo_list.append(nzo)
else:
# if the queue is empty then simple append the item to the bottom
self.__nzo_list.append(nzo)
if save:
self.save(nzo)
if not (quiet or nzo.status == Status.FETCHING):
notifier.send_notification(T("NZB added to queue"),
nzo.filename, "download", nzo.cat)
if not quiet and cfg.auto_sort():
try:
field, direction = cfg.auto_sort().split()
self.sort_queue(field, direction)
except ValueError:
pass
return nzo.nzo_id
def fail_to_history(self, nzo, url, msg='', content=False):
""" Create History entry for failed URL Fetch
msg: message to be logged
content: report in history that cause is a bad NZB file
"""
# Remove the "Trying to fetch" part
if url:
nzo.filename = url
nzo.final_name = url.strip()
if content:
# Bad content
msg = T('Unusable NZB file')
else:
# Failed fetch
msg = T('URL Fetching failed; %s') % unicoder(msg)
# Mark as failed
nzo.status = Status.FAILED
nzo.fail_msg = msg
notifier.send_notification(
T('URL Fetching failed; %s') % '', '%s\n%s' % (msg, url), 'other',
nzo.cat)
if cfg.email_endjob() > 0:
emailer.badfetch_mail(msg, url)
# Parse category to make sure script is set correctly after a grab
nzo.cat, _, nzo.script, _ = misc.cat_to_opts(nzo.cat,
script=nzo.script)
# Add to history and run script if desired
NzbQueue.do.remove(nzo.nzo_id, add_to_history=False)
PostProcessor.do.process(nzo)
def add(self, nzo, save=True, quiet=False):
if 0:
assert isinstance(nzo, NzbObject) # Assert only for debug purposes
if not nzo.nzo_id:
nzo.nzo_id = sabnzbd.get_new_id('nzo', nzo.workpath,
self.__nzo_table)
# If no files are to be downloaded anymore, send to postproc
if not nzo.files and not nzo.futuretype:
self.end_job(nzo)
return ''
# Reset try_lists
nzo.reset_try_list()
self.reset_try_list()
if nzo.nzo_id:
nzo.deleted = False
priority = nzo.priority
if sabnzbd.scheduler.analyse(False, priority):
nzo.status = Status.PAUSED
self.__nzo_table[nzo.nzo_id] = nzo
if priority > HIGH_PRIORITY:
# Top and repair priority items are added to the top of the queue
self.__nzo_list.insert(0, nzo)
elif priority == LOW_PRIORITY:
self.__nzo_list.append(nzo)
else:
# for high priority we need to add the item at the bottom
# of any other high priority items above the normal priority
# for normal priority we need to add the item at the bottom
# of the normal priority items above the low priority
if self.__nzo_list:
pos = 0
added = False
for position in self.__nzo_list:
if position.priority < priority:
self.__nzo_list.insert(pos, nzo)
added = True
break
pos += 1
if not added:
# if there are no other items classed as a lower priority
# then it will be added to the bottom of the queue
self.__nzo_list.append(nzo)
else:
# if the queue is empty then simple append the item to the bottom
self.__nzo_list.append(nzo)
if save:
self.save(nzo)
if not (quiet or nzo.status in ('Fetching', )):
notifier.send_notification(T('NZB added to queue'),
nzo.filename, 'download')
if cfg.auto_sort():
self.sort_by_avg_age()
return nzo.nzo_id
def versionUpdate(self):
try:
if sabnzbd.NEW_VERSION and self.version_notify:
# logging.info("[osx] New Version : %s" % (sabnzbd.NEW_VERSION))
new_release, _new_rel_url = sabnzbd.NEW_VERSION
notifier.send_notification("SABnzbd", "%s : %s" % (T('New release available'), new_release), 'other')
self.version_notify = 0
except:
logging.info("[osx] versionUpdate Exception %s" % (sys.exc_info()[0]))
def versionUpdate(self):
try:
if sabnzbd.NEW_VERSION and self.version_notify:
# logging.info("[osx] New Version : %s" % (sabnzbd.NEW_VERSION))
new_release, _new_rel_url = sabnzbd.NEW_VERSION
notifier.send_notification("SABnzbd", "%s : %s" % (T('New release available'), new_release), 'other')
self.version_notify = 0
except:
logging.info("[osx] versionUpdate Exception %s" % (sys.exc_info()[0]))
def add(self, nzo, save=True, quiet=False):
if 0: assert isinstance(nzo, NzbObject) # Assert only for debug purposes
if not nzo.nzo_id:
nzo.nzo_id = sabnzbd.get_new_id('nzo', nzo.workpath, self.__nzo_table)
# If no files are to be downloaded anymore, send to postproc
if not nzo.files and not nzo.futuretype:
self.end_job(nzo)
return ''
# Reset try_lists
nzo.reset_try_list()
self.reset_try_list()
if nzo.nzo_id:
nzo.deleted = False
priority = nzo.priority
if sabnzbd.scheduler.analyse(False, priority):
nzo.status = Status.PAUSED
self.__nzo_table[nzo.nzo_id] = nzo
if priority > HIGH_PRIORITY:
# Top and repair priority items are added to the top of the queue
self.__nzo_list.insert(0, nzo)
elif priority == LOW_PRIORITY:
self.__nzo_list.append(nzo)
else:
# for high priority we need to add the item at the bottom
# of any other high priority items above the normal priority
# for normal priority we need to add the item at the bottom
# of the normal priority items above the low priority
if self.__nzo_list:
pos = 0
added = False
for position in self.__nzo_list:
if position.priority < priority:
self.__nzo_list.insert(pos, nzo)
added = True
break
pos += 1
if not added:
# if there are no other items classed as a lower priority
# then it will be added to the bottom of the queue
self.__nzo_list.append(nzo)
else:
# if the queue is empty then simple append the item to the bottom
self.__nzo_list.append(nzo)
if save:
self.save(nzo)
if not (quiet or nzo.status in ('Fetching',)):
notifier.send_notification(T('NZB added to queue'), nzo.filename, 'download')
if cfg.auto_sort():
self.sort_by_avg_age()
return nzo.nzo_id
def pause(self):
""" Pause the downloader, optionally saving admin """
if not self.paused:
self.paused = True
logging.info("Pausing")
notifier.send_notification("SABnzbd", T('Paused'), 'download')
if self.is_paused():
BPSMeter.do.reset()
if cfg.autodisconnect():
self.disconnect()
def pause(self):
""" Pause the downloader, optionally saving admin """
if not self.paused:
self.paused = True
logging.info("Pausing")
notifier.send_notification("SABnzbd", T('Paused'), 'download')
if self.is_paused():
BPSMeter.do.reset()
if cfg.autodisconnect():
self.disconnect()
def parring(nzo, workdir):
""" Perform par processing. Returns: (par_error, re_add) """
if 0: assert isinstance(nzo, sabnzbd.nzbstuff.NzbObject) # Assert only for debug purposes
filename = nzo.final_name
notifier.send_notification(T('Post-processing'), filename, 'pp')
logging.info('Starting verification and repair of %s', filename)
# Get verification status of sets
verified = sabnzbd.load_data(VERIFIED_FILE, nzo.workpath, remove=False) or {}
# Collect the par files
if nzo.partable:
par_table = nzo.partable.copy()
else:
par_table = {}
repair_sets = par_table.keys()
re_add = False
par_error = False
single = len(repair_sets) == 1
if repair_sets:
for setname in repair_sets:
if cfg.ignore_samples() and 'sample' in setname.lower():
continue
if not verified.get(setname, False):
logging.info("Running verification and repair on set %s", setname)
parfile_nzf = par_table[setname]
if os.path.exists(os.path.join(nzo.downpath, parfile_nzf.filename)) or parfile_nzf.extrapars:
need_re_add, res = par2_repair(parfile_nzf, nzo, workdir, setname, single=single)
re_add = re_add or need_re_add
if not res and not need_re_add and cfg.sfv_check():
res = try_sfv_check(nzo, workdir, setname)
verified[setname] = res
else:
continue
par_error = par_error or not res
else:
logging.info("No par2 sets for %s", filename)
nzo.set_unpack_info('Repair', T('[%s] No par2 sets') % unicoder(filename))
if cfg.sfv_check() and not verified.get('', False):
par_error = not try_sfv_check(nzo, workdir, '')
verified[''] = not par_error
if re_add:
logging.info('Readded %s to queue', filename)
if nzo.priority != TOP_PRIORITY:
nzo.priority = REPAIR_PRIORITY
sabnzbd.nzbqueue.add_nzo(nzo)
sabnzbd.downloader.Downloader.do.resume_from_postproc()
sabnzbd.save_data(verified, VERIFIED_FILE, nzo.workpath)
logging.info('Verification and repair finished for %s', filename)
return par_error, re_add
def parring(nzo, workdir):
""" Perform par processing. Returns: (par_error, re_add) """
if 0: assert isinstance(nzo, sabnzbd.nzbstuff.NzbObject) # Assert only for debug purposes
filename = nzo.final_name
notifier.send_notification(T('Post-processing'), filename, 'pp')
logging.info('Starting verification and repair of %s', filename)
# Get verification status of sets
verified = sabnzbd.load_data(VERIFIED_FILE, nzo.workpath, remove=False) or {}
# Collect the par files
if nzo.partable:
par_table = nzo.partable.copy()
else:
par_table = {}
repair_sets = par_table.keys()
re_add = False
par_error = False
single = len(repair_sets) == 1
if repair_sets:
for setname in repair_sets:
if cfg.ignore_samples() and 'sample' in setname.lower():
continue
if not verified.get(setname, False):
logging.info("Running verification and repair on set %s", setname)
parfile_nzf = par_table[setname]
if os.path.exists(os.path.join(nzo.downpath, parfile_nzf.filename)) or parfile_nzf.extrapars:
need_re_add, res = par2_repair(parfile_nzf, nzo, workdir, setname, single=single)
re_add = re_add or need_re_add
if not res and not need_re_add and cfg.sfv_check():
res = try_sfv_check(nzo, workdir, setname)
verified[setname] = res
else:
continue
par_error = par_error or not res
else:
logging.info("No par2 sets for %s", filename)
nzo.set_unpack_info('Repair', T('[%s] No par2 sets') % unicoder(filename))
if cfg.sfv_check() and not verified.get('', False):
par_error = not try_sfv_check(nzo, workdir, '')
verified[''] = not par_error
if re_add:
logging.info('Readded %s to queue', filename)
if nzo.priority != TOP_PRIORITY:
nzo.priority = REPAIR_PRIORITY
sabnzbd.nzbqueue.add_nzo(nzo)
sabnzbd.downloader.Downloader.do.resume_from_postproc()
sabnzbd.save_data(verified, VERIFIED_FILE, nzo.workpath)
logging.info('Verification and repair finished for %s', filename)
return par_error, re_add
def pause(self, save=True):
""" Pause the downloader, optionally saving admin """
if not self.paused:
self.paused = True
self.can_be_slowed = None
logging.info("Pausing")
notifier.send_notification("SABnzbd", T('Paused'), 'download')
if self.is_paused():
BPSMeter.do.reset()
if cfg.autodisconnect():
self.disconnect()
if save:
ArticleCache.do.flush_articles()
def pause(self, save=True):
""" Pause the downloader, optionally saving admin """
if not self.paused:
self.paused = True
self.can_be_slowed = None
logging.info("Pausing")
notifier.send_notification("SABnzbd", T('Paused'), 'download')
if self.is_paused():
BPSMeter.do.reset()
if cfg.autodisconnect():
self.disconnect()
if save:
ArticleCache.do.flush_articles()
def applicationShouldTerminate_(self, sender):
logging.info('[osx] application terminating')
self.setMenuTitle_("\n\n%s\n" % (T('Stopping...')))
self.status_item.setHighlightMode_(NO)
self.osx_icon = False
sabnzbd.shutdown_program()
try:
notifier.send_notification('SABnzbd', T('SABnzbd shutdown finished'), notifier.NOTIFICATION['other'])
except AttributeError:
# Fails for the OSX binary
pass
logging.info('Leaving SABnzbd')
sys.stderr.flush()
sys.stdout.flush()
return NSTerminateNow
def applicationShouldTerminate_(self, sender):
logging.info('[osx] application terminating')
self.setMenuTitle_("\n\n%s\n" % (T('Stopping...')))
self.status_item.setHighlightMode_(NO)
self.osx_icon = False
sabnzbd.shutdown_program()
try:
notifier.send_notification('SABnzbd', T('SABnzbd shutdown finished'), notifier.NOTIFICATION['other'])
except AttributeError:
# Fails for the OSX binary
pass
logging.info('Leaving SABnzbd')
sys.stderr.flush()
sys.stdout.flush()
return NSTerminateNow
def handle_empty_queue():
""" Check if empty queue calls for action """
if sabnzbd.nzbqueue.NzbQueue.do.actives() == 0:
sabnzbd.save_state()
notifier.send_notification("SABnzbd", T("Queue finished"),
"queue_done")
# Perform end-of-queue action when one is set
if sabnzbd.QUEUECOMPLETEACTION:
logging.info("Queue has finished, launching: %s (%s)",
sabnzbd.QUEUECOMPLETEACTION, sabnzbd.QUEUECOMPLETEARG)
if sabnzbd.QUEUECOMPLETEARG:
sabnzbd.QUEUECOMPLETEACTION(sabnzbd.QUEUECOMPLETEARG)
else:
Thread(target=sabnzbd.QUEUECOMPLETEACTION).start()
sabnzbd.change_queue_complete_action(cfg.queue_complete(),
new=False)
def handle_empty_queue():
""" Check if empty queue calls for action """
if sabnzbd.NzbQueue.actives() == 0:
sabnzbd.save_state()
notifier.send_notification("SABnzbd", T("Queue finished"),
"queue_done")
# Perform end-of-queue action when one is set
if sabnzbd.QUEUECOMPLETEACTION:
logging.info("Queue has finished, launching: %s (%s)",
sabnzbd.QUEUECOMPLETEACTION, sabnzbd.QUEUECOMPLETEARG)
if sabnzbd.QUEUECOMPLETEARG:
sabnzbd.QUEUECOMPLETEACTION(sabnzbd.QUEUECOMPLETEARG)
else:
Thread(target=sabnzbd.QUEUECOMPLETEACTION).start()
sabnzbd.change_queue_complete_action(cfg.queue_complete(),
new=False)
# Trigger garbage collection and release of memory
logging.debug("Triggering garbage collection and release of memory")
gc.collect()
if sabnzbd.LIBC:
sabnzbd.LIBC.malloc_trim(0)
def resume(self):
# Do not notify when SABnzbd is still starting
if self.paused and sabnzbd.WEB_DIR:
logging.info("Resuming")
notifier.send_notification("SABnzbd", T('Resuming'), 'download')
self.paused = False
def stop(self):
self.shutdown = True
notifier.send_notification("SABnzbd", T('Shutting down'), 'startup')
def parring(nzo, workdir):
""" Perform par processing. Returns: (par_error, re_add) """
job_name = nzo.final_name
notifier.send_notification(T("Post-processing"), job_name, "pp", nzo.cat)
logging.info("Starting verification and repair of %s", job_name)
# Get verification status of sets
verified = sabnzbd.load_data(VERIFIED_FILE, nzo.workpath,
remove=False) or {}
re_add = False
par_error = False
single = len(nzo.extrapars) == 1
if nzo.extrapars:
# Need to make a copy because it can change during iteration
for setname in list(nzo.extrapars):
if cfg.ignore_samples() and RE_SAMPLE.search(setname.lower()):
continue
# Skip sets that were already tried
if not verified.get(setname, False):
logging.info("Running verification and repair on set %s",
setname)
parfile_nzf = nzo.partable[setname]
# Check if file maybe wasn't deleted and if we maybe have more files in the parset
if os.path.exists(
os.path.join(
nzo.downpath,
parfile_nzf.filename)) or nzo.extrapars[setname]:
need_re_add, res = par2_repair(parfile_nzf,
nzo,
workdir,
setname,
single=single)
# Was it aborted?
if not nzo.pp_active:
re_add = False
par_error = True
break
re_add = re_add or need_re_add
verified[setname] = res
else:
continue
par_error = par_error or not res
elif not verified.get("", False):
# No par2-sets found, skipped if already tried before
logging.info("No par2 sets for %s", job_name)
nzo.set_unpack_info("Repair", T("[%s] No par2 sets") % job_name)
# Try SFV-based verification and rename
sfv_check_result = None
if cfg.sfv_check() and not verified.get("", False):
sfv_check_result = try_sfv_check(nzo, workdir)
par_error = sfv_check_result is False
# If no luck with SFV, do RAR-check or RAR-rename
if sfv_check_result is None and cfg.enable_unrar():
# Check for RAR's with a sensible extension
_, _, rars, _, _ = build_filelists(workdir, check_rar=False)
# If there's no RAR's, they might be super-obfuscated
if not rars:
# Returns number of renamed RAR's
if rar_renamer(nzo, workdir):
# Re-parse the files so we can do RAR-check
_, _, rars, _, _ = build_filelists(workdir)
if rars:
par_error = not try_rar_check(nzo, rars)
# Save that we already tried SFV/RAR-verification
verified[""] = not par_error
if re_add:
logging.info("Re-added %s to queue", job_name)
if nzo.priority != TOP_PRIORITY:
nzo.priority = REPAIR_PRIORITY
nzo.status = Status.FETCHING
sabnzbd.nzbqueue.NzbQueue.do.add(nzo)
sabnzbd.downloader.Downloader.do.resume_from_postproc()
sabnzbd.save_data(verified, VERIFIED_FILE, nzo.workpath)
logging.info("Verification and repair finished for %s", job_name)
return par_error, re_add
def parring(nzo, workdir):
""" Perform par processing. Returns: (par_error, re_add) """
filename = nzo.final_name
notifier.send_notification(T('Post-processing'), filename, 'pp', nzo.cat)
logging.info('Starting verification and repair of %s', filename)
# Get verification status of sets
verified = sabnzbd.load_data(VERIFIED_FILE, nzo.workpath,
remove=False) or {}
repair_sets = nzo.extrapars.keys()
re_add = False
par_error = False
single = len(repair_sets) == 1
if repair_sets:
for setname in repair_sets:
if cfg.ignore_samples() and RE_SAMPLE.search(setname.lower()):
continue
if not verified.get(setname, False):
logging.info("Running verification and repair on set %s",
setname)
parfile_nzf = nzo.partable[setname]
# Check if file maybe wasn't deleted and if we maybe have more files in the parset
if os.path.exists(
os.path.join(
nzo.downpath,
parfile_nzf.filename)) or nzo.extrapars[setname]:
need_re_add, res = par2_repair(parfile_nzf,
nzo,
workdir,
setname,
single=single)
# Was it aborted?
if not nzo.pp_active:
re_add = False
par_error = True
break
re_add = re_add or need_re_add
verified[setname] = res
else:
continue
par_error = par_error or not res
else:
# We must not have found any par2..
logging.info("No par2 sets for %s", filename)
nzo.set_unpack_info('Repair',
T('[%s] No par2 sets') % unicoder(filename))
if cfg.sfv_check() and not verified.get('', False):
par_error = not try_sfv_check(nzo, workdir, '')
verified[''] = not par_error
# If still no success, do RAR-check
if not par_error and cfg.enable_unrar():
par_error = not try_rar_check(nzo, workdir, '')
verified[''] = not par_error
if re_add:
logging.info('Re-added %s to queue', filename)
if nzo.priority != TOP_PRIORITY:
nzo.priority = REPAIR_PRIORITY
nzo.status = Status.FETCHING
sabnzbd.nzbqueue.NzbQueue.do.add(nzo)
sabnzbd.downloader.Downloader.do.resume_from_postproc()
sabnzbd.save_data(verified, VERIFIED_FILE, nzo.workpath)
logging.info('Verification and repair finished for %s', filename)
return par_error, re_add
def process_job(nzo):
""" Process one job """
if 0: assert isinstance(nzo, sabnzbd.nzbstuff.NzbObject) # Assert only for debug purposes
start = time.time()
# keep track of whether we can continue
all_ok = True
# keep track of par problems
par_error = False
# keep track of any unpacking errors
unpack_error = False
# Signal empty download, for when 'empty_postproc' is enabled
empty = False
nzb_list = []
# These need to be initialized in case of a crash
workdir_complete = ''
postproc_time = 0 # @UnusedVariable -- pep8 bug?
script_log = ''
script_line = ''
crash_msg = ''
# Get the job flags
nzo.save_attribs()
flag_repair, flag_unpack, flag_delete = nzo.repair_opts
# Normalize PP
if flag_delete:
flag_unpack = True
if flag_unpack:
flag_repair = True
# Get the NZB name
filename = nzo.final_name
if cfg.allow_streaming() and not (flag_repair or flag_unpack or flag_delete):
# After streaming, force +D
nzo.set_pp(3)
nzo.status = Status.FAILED
nzo.save_attribs()
all_ok = False
if nzo.fail_msg: # Special case: aborted due to too many missing data
nzo.status = Status.FAILED
nzo.save_attribs()
all_ok = False
par_error = True
unpack_error = 1
try:
# Get the folder containing the download result
workdir = nzo.downpath
tmp_workdir_complete = None
# if no files are present (except __admin__), fail the job
if all_ok and len(globber(workdir)) < 2:
if nzo.precheck:
_enough, ratio = nzo.check_quality()
req_ratio = float(cfg.req_completion_rate()) / 100.0
# Make sure that rounded ratio doesn't equal required ratio
# when it is actually below required
if (ratio < req_ratio) and (req_ratio - ratio) < 0.001:
ratio = req_ratio - 0.001
emsg = '%.1f%%' % (ratio * 100.0)
emsg2 = '%.1f%%' % float(cfg.req_completion_rate())
emsg = T('Download might fail, only %s of required %s available') % (emsg, emsg2)
else:
emsg = T('Download failed - Not on your server(s)')
empty = True
nzo.fail_msg = emsg
nzo.set_unpack_info('Fail', emsg)
nzo.status = Status.FAILED
# do not run unpacking or parity verification
flag_repair = flag_unpack = False
all_ok = cfg.empty_postproc() and empty
if not all_ok:
par_error = True
unpack_error = 1
script = nzo.script
cat = nzo.cat
logging.info('Starting Post-Processing on %s' +
' => Repair:%s, Unpack:%s, Delete:%s, Script:%s, Cat:%s',
filename, flag_repair, flag_unpack, flag_delete, script, cat)
# Set complete dir to workdir in case we need to abort
workdir_complete = workdir
dirname = nzo.final_name
marker_file = None
# Par processing, if enabled
if all_ok and flag_repair:
if not check_win_maxpath(workdir):
crash_msg = T('Path exceeds 260, repair by "par2" is not possible')
raise WindowsError
par_error, re_add = parring(nzo, workdir)
if re_add:
# Try to get more par files
return False
# Check if user allows unsafe post-processing
if flag_repair and cfg.safe_postproc():
all_ok = all_ok and not par_error
if all_ok:
fix_unix_encoding(workdir)
one_folder = False
# Determine class directory
if cfg.create_group_folders():
complete_dir = addPrefixes(cfg.complete_dir.get_path(), nzo.dirprefix)
complete_dir = create_dirs(complete_dir)
else:
catdir = config.get_categories(cat).dir()
if catdir.endswith('*'):
catdir = catdir.strip('*')
one_folder = True
complete_dir = real_path(cfg.complete_dir.get_path(), catdir)
complete_dir = long_path(complete_dir)
# TV/Movie/Date Renaming code part 1 - detect and construct paths
if cfg.enable_meta():
file_sorter = Sorter(nzo, cat)
else:
file_sorter = Sorter(None, cat)
complete_dir = file_sorter.detect(dirname, complete_dir)
if file_sorter.sort_file:
one_folder = False
complete_dir = sanitize_and_trim_path(complete_dir)
if one_folder:
workdir_complete = create_dirs(complete_dir)
else:
workdir_complete = get_unique_path(os.path.join(complete_dir, dirname), create_dir=True)
marker_file = set_marker(workdir_complete)
if not workdir_complete or not os.path.exists(workdir_complete):
crash_msg = T('Cannot create final folder %s') % unicoder(os.path.join(complete_dir, dirname))
raise IOError
if cfg.folder_rename() and not one_folder:
tmp_workdir_complete = prefix(workdir_complete, '_UNPACK_')
try:
renamer(workdir_complete, tmp_workdir_complete)
except:
pass # On failure, just use the original name
else:
tmp_workdir_complete = workdir_complete
newfiles = []
# Run Stage 2: Unpack
if flag_unpack:
if all_ok:
# set the current nzo status to "Extracting...". Used in History
nzo.status = Status.EXTRACTING
logging.info("Running unpack_magic on %s", filename)
short_complete = short_path(tmp_workdir_complete)
unpack_error, newfiles = unpack_magic(nzo, short_path(workdir), short_complete, flag_delete, one_folder, (), (), (), (), ())
if short_complete != tmp_workdir_complete:
newfiles = [f.replace(short_complete, tmp_workdir_complete) for f in newfiles]
logging.info("unpack_magic finished on %s", filename)
else:
nzo.set_unpack_info('Unpack', T('No post-processing because of failed verification'))
if cfg.safe_postproc():
all_ok = all_ok and not unpack_error
if all_ok:
# Move any (left-over) files to destination
nzo.status = Status.MOVING
nzo.set_action_line(T('Moving'), '...')
for root, _dirs, files in os.walk(workdir):
if not root.endswith(JOB_ADMIN):
for file_ in files:
path = os.path.join(root, file_)
new_path = path.replace(workdir, tmp_workdir_complete)
ok, new_path = move_to_path(path, new_path)
newfiles.append(new_path)
if not ok:
nzo.set_unpack_info('Unpack', T('Failed moving %s to %s') % (unicoder(path), unicoder(new_path)))
all_ok = False
break
# Set permissions right
set_permissions(tmp_workdir_complete)
if all_ok and marker_file:
del_marker(os.path.join(tmp_workdir_complete, marker_file))
remove_from_list(marker_file, newfiles)
if all_ok:
# Remove files matching the cleanup list
cleanup_list(tmp_workdir_complete, True)
# Check if this is an NZB-only download, if so redirect to queue
# except when PP was Download-only
if flag_repair:
nzb_list = nzb_redirect(tmp_workdir_complete, nzo.final_name, nzo.pp, script, cat, priority=nzo.priority)
else:
nzb_list = None
if nzb_list:
nzo.set_unpack_info('Download', T('Sent %s to queue') % unicoder(nzb_list))
cleanup_empty_directories(tmp_workdir_complete)
else:
cleanup_list(tmp_workdir_complete, False)
script_output = ''
script_ret = 0
if not nzb_list:
# Give destination its final name
if cfg.folder_rename() and tmp_workdir_complete and not one_folder:
if all_ok:
try:
newfiles = rename_and_collapse_folder(tmp_workdir_complete, workdir_complete, newfiles)
except:
logging.error(T('Error renaming "%s" to "%s"'), clip_path(tmp_workdir_complete), clip_path(workdir_complete))
logging.info('Traceback: ', exc_info=True)
# Better disable sorting because filenames are all off now
file_sorter.sort_file = None
else:
workdir_complete = tmp_workdir_complete.replace('_UNPACK_', '_FAILED_')
workdir_complete = get_unique_path(workdir_complete, n=0, create_dir=False)
workdir_complete = workdir_complete
if empty:
job_result = -1
else:
job_result = int(par_error) + int(bool(unpack_error)) * 2
if cfg.ignore_samples():
remove_samples(workdir_complete)
# TV/Movie/Date Renaming code part 2 - rename and move files to parent folder
if all_ok and file_sorter.sort_file:
if newfiles:
file_sorter.rename(newfiles, workdir_complete)
workdir_complete, ok = file_sorter.move(workdir_complete)
else:
workdir_complete, ok = file_sorter.rename_with_ext(workdir_complete)
if not ok:
nzo.set_unpack_info('Unpack', T('Failed to move files'))
all_ok = False
# Run the user script
script_path = make_script_path(script)
if (all_ok or not cfg.safe_postproc()) and (not nzb_list) and script_path:
# set the current nzo status to "Ext Script...". Used in History
nzo.status = Status.RUNNING
nzo.set_action_line(T('Running script'), unicoder(script))
nzo.set_unpack_info('Script', T('Running user script %s') % unicoder(script), unique=True)
script_log, script_ret = external_processing(short_path(script_path, False), short_path(workdir_complete, False), nzo.filename,
dirname, cat, nzo.group, job_result,
nzo.nzo_info.get('failure', ''))
script_line = get_last_line(script_log)
if script_log:
script_output = nzo.nzo_id
if script_line:
nzo.set_unpack_info('Script', unicoder(script_line), unique=True)
else:
nzo.set_unpack_info('Script', T('Ran %s') % unicoder(script), unique=True)
else:
script = ""
script_line = ""
script_ret = 0
# Maybe bad script result should fail job
if script_ret and cfg.script_can_fail():
script_error = True
all_ok = False
nzo.fail_msg = T('Script exit code is %s') % script_ret
else:
script_error = False
# Email the results
if (not nzb_list) and cfg.email_endjob():
if (cfg.email_endjob() == 1) or (cfg.email_endjob() == 2 and (unpack_error or par_error or script_error)):
emailer.endjob(dirname, cat, all_ok, workdir_complete, nzo.bytes_downloaded,
nzo.fail_msg, nzo.unpack_info, script, TRANS(script_log), script_ret)
if script_output:
# Can do this only now, otherwise it would show up in the email
if script_ret:
script_ret = 'Exit(%s) ' % script_ret
else:
script_ret = ''
if len(script_log.rstrip().split('\n')) > 1:
nzo.set_unpack_info('Script',
u'%s%s <a href="./scriptlog?name=%s">(%s)</a>' % (script_ret, xml.sax.saxutils.escape(script_line),
xml.sax.saxutils.escape(script_output), T('More')), unique=True)
else:
# No '(more)' button needed
nzo.set_unpack_info('Script', u'%s%s ' % (script_ret, xml.sax.saxutils.escape(script_line)), unique=True)
# Cleanup again, including NZB files
if all_ok:
cleanup_list(workdir_complete, False)
# Force error for empty result
all_ok = all_ok and not empty
# Update indexer with results
if cfg.rating_enable():
if nzo.encrypted > 0:
Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_ENCRYPTED)
if empty:
hosts = map(lambda s: s.host, sabnzbd.downloader.Downloader.do.nzo_servers(nzo))
if not hosts:
hosts = [None]
for host in hosts:
Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_EXPIRED, host)
# Show final status in history
if all_ok:
notifier.send_notification(T('Download Completed'), filename, 'complete')
nzo.status = Status.COMPLETED
else:
notifier.send_notification(T('Download Failed'), filename, 'failed')
nzo.status = Status.FAILED
except:
logging.error(T('Post Processing Failed for %s (%s)'), filename, crash_msg)
if not crash_msg:
logging.info("Traceback: ", exc_info=True)
crash_msg = T('see logfile')
nzo.fail_msg = T('PostProcessing was aborted (%s)') % unicoder(crash_msg)
notifier.send_notification(T('Download Failed'), filename, 'failed')
nzo.status = Status.FAILED
par_error = True
all_ok = False
if cfg.email_endjob():
emailer.endjob(dirname, cat, all_ok, clip_path(workdir_complete), nzo.bytes_downloaded,
nzo.fail_msg, nzo.unpack_info, '', '', 0)
if all_ok:
# If the folder only contains one file OR folder, have that as the path
# Be aware that series/generic/date sorting may move a single file into a folder containing other files
workdir_complete = one_file_or_folder(workdir_complete)
workdir_complete = os.path.normpath(workdir_complete)
# Log the overall time taken for postprocessing
postproc_time = int(time.time() - start)
# Create the history DB instance
history_db = database.HistoryDB()
# Add the nzo to the database. Only the path, script and time taken is passed
# Other information is obtained from the nzo
history_db.add_history_db(nzo, clip_path(workdir_complete), nzo.downpath, postproc_time, script_log, script_line)
# The connection is only used once, so close it here
history_db.close()
# Clean up the NZO
try:
logging.info('Cleaning up %s (keep_basic=%s)', filename, str(not all_ok))
sabnzbd.nzbqueue.NzbQueue.do.cleanup_nzo(nzo, keep_basic=not all_ok)
except:
logging.error(T('Cleanup of %s failed.'), nzo.final_name)
logging.info("Traceback: ", exc_info=True)
# Remove download folder
if all_ok:
try:
if os.path.exists(workdir):
logging.debug('Removing workdir %s', workdir)
remove_all(workdir, recursive=True)
except:
logging.error(T('Error removing workdir (%s)'), clip_path(workdir))
logging.info("Traceback: ", exc_info=True)
# Use automatic retry link on par2 errors and encrypted/bad RARs
if par_error or unpack_error in (2, 3):
try_alt_nzb(nzo)
# Update the last check time
sabnzbd.LAST_HISTORY_UPDATE = time.time()
return True
def stop(self):
self.shutdown = True
notifier.send_notification("SABnzbd", T("Shutting down"), "startup")
def process_job(nzo):
""" Process one job """
if 0: assert isinstance(nzo, sabnzbd.nzbstuff.NzbObject) # Assert only for debug purposes
start = time.time()
# keep track of whether we can continue
all_ok = True
# keep track of par problems
par_error = False
# keep track of any unpacking errors
unpack_error = False
# Signal empty download, for when 'empty_postproc' is enabled
empty = False
nzb_list = []
# These need to be initialized in case of a crash
workdir_complete = ''
postproc_time = 0 # @UnusedVariable -- pep8 bug?
script_log = ''
script_line = ''
crash_msg = ''
# Get the job flags
nzo.save_attribs()
flag_repair, flag_unpack, flag_delete = nzo.repair_opts
# Normalize PP
if flag_delete:
flag_unpack = True
if flag_unpack:
flag_repair = True
# Get the NZB name
filename = nzo.final_name
if cfg.allow_streaming() and not (flag_repair or flag_unpack or flag_delete):
# After streaming, force +D
nzo.set_pp(3)
nzo.status = Status.FAILED
nzo.save_attribs()
all_ok = False
if nzo.fail_msg: # Special case: aborted due to too many missing data
nzo.status = Status.FAILED
nzo.save_attribs()
all_ok = False
par_error = True
unpack_error = 1
try:
# Get the folder containing the download result
workdir = nzo.downpath
tmp_workdir_complete = None
# if no files are present (except __admin__), fail the job
if all_ok and len(globber(workdir)) < 2:
if nzo.precheck:
_enough, ratio = nzo.check_quality()
req_ratio = float(cfg.req_completion_rate()) / 100.0
# Make sure that rounded ratio doesn't equal required ratio
# when it is actually below required
if (ratio < req_ratio) and (req_ratio - ratio) < 0.001:
ratio = req_ratio - 0.001
emsg = '%.1f%%' % (ratio * 100.0)
emsg2 = '%.1f%%' % float(cfg.req_completion_rate())
emsg = T('Download might fail, only %s of required %s available') % (emsg, emsg2)
else:
emsg = T('Download failed - Not on your server(s)')
empty = True
nzo.fail_msg = emsg
nzo.set_unpack_info('Fail', emsg)
nzo.status = Status.FAILED
# do not run unpacking or parity verification
flag_repair = flag_unpack = False
all_ok = cfg.empty_postproc() and empty
if not all_ok:
par_error = True
unpack_error = 1
script = nzo.script
cat = nzo.cat
logging.info('Starting Post-Processing on %s' +
' => Repair:%s, Unpack:%s, Delete:%s, Script:%s, Cat:%s',
filename, flag_repair, flag_unpack, flag_delete, script, cat)
# Set complete dir to workdir in case we need to abort
workdir_complete = workdir
dirname = nzo.final_name
marker_file = None
# Par processing, if enabled
if all_ok and flag_repair:
if not check_win_maxpath(workdir):
crash_msg = T('Path exceeds 260, repair by "par2" is not possible')
raise WindowsError
par_error, re_add = parring(nzo, workdir)
if re_add:
# Try to get more par files
return False
# Check if user allows unsafe post-processing
if flag_repair and cfg.safe_postproc():
all_ok = all_ok and not par_error
if all_ok:
fix_unix_encoding(workdir)
one_folder = False
# Determine class directory
if cfg.create_group_folders():
complete_dir = addPrefixes(cfg.complete_dir.get_path(), nzo.dirprefix)
complete_dir = create_dirs(complete_dir)
else:
catdir = config.get_categories(cat).dir()
if catdir.endswith('*'):
catdir = catdir.strip('*')
one_folder = True
complete_dir = real_path(cfg.complete_dir.get_path(), catdir)
complete_dir = long_path(complete_dir)
# TV/Movie/Date Renaming code part 1 - detect and construct paths
if cfg.enable_meta():
file_sorter = Sorter(nzo, cat)
else:
file_sorter = Sorter(None, cat)
complete_dir = file_sorter.detect(dirname, complete_dir)
if file_sorter.sort_file:
one_folder = False
complete_dir = sanitize_and_trim_path(complete_dir)
if one_folder:
workdir_complete = create_dirs(complete_dir)
else:
workdir_complete = get_unique_path(os.path.join(complete_dir, dirname), create_dir=True)
marker_file = set_marker(workdir_complete)
if not workdir_complete or not os.path.exists(workdir_complete):
crash_msg = T('Cannot create final folder %s') % unicoder(os.path.join(complete_dir, dirname))
raise IOError
if cfg.folder_rename() and not one_folder:
tmp_workdir_complete = prefix(workdir_complete, '_UNPACK_')
try:
renamer(workdir_complete, tmp_workdir_complete)
except:
pass # On failure, just use the original name
else:
tmp_workdir_complete = workdir_complete
newfiles = []
# Run Stage 2: Unpack
if flag_unpack:
if all_ok:
# set the current nzo status to "Extracting...". Used in History
nzo.status = Status.EXTRACTING
logging.info("Running unpack_magic on %s", filename)
short_complete = short_path(tmp_workdir_complete)
unpack_error, newfiles = unpack_magic(nzo, short_path(workdir), short_complete, flag_delete, one_folder, (), (), (), (), ())
if short_complete != tmp_workdir_complete:
newfiles = [f.replace(short_complete, tmp_workdir_complete) for f in newfiles]
logging.info("unpack_magic finished on %s", filename)
else:
nzo.set_unpack_info('Unpack', T('No post-processing because of failed verification'))
if cfg.safe_postproc():
all_ok = all_ok and not unpack_error
if all_ok:
# Move any (left-over) files to destination
nzo.status = Status.MOVING
nzo.set_action_line(T('Moving'), '...')
for root, _dirs, files in os.walk(workdir):
if not root.endswith(JOB_ADMIN):
for file_ in files:
path = os.path.join(root, file_)
new_path = path.replace(workdir, tmp_workdir_complete)
ok, new_path = move_to_path(path, new_path)
newfiles.append(new_path)
if not ok:
nzo.set_unpack_info('Unpack', T('Failed moving %s to %s') % (unicoder(path), unicoder(new_path)))
all_ok = False
break
# Set permissions right
set_permissions(tmp_workdir_complete)
if all_ok and marker_file:
del_marker(os.path.join(tmp_workdir_complete, marker_file))
remove_from_list(marker_file, newfiles)
if all_ok:
# Remove files matching the cleanup list
cleanup_list(tmp_workdir_complete, True)
# Check if this is an NZB-only download, if so redirect to queue
# except when PP was Download-only
if flag_repair:
nzb_list = nzb_redirect(tmp_workdir_complete, nzo.final_name, nzo.pp, script, cat, priority=nzo.priority)
else:
nzb_list = None
if nzb_list:
nzo.set_unpack_info('Download', T('Sent %s to queue') % unicoder(nzb_list))
cleanup_empty_directories(tmp_workdir_complete)
else:
cleanup_list(tmp_workdir_complete, False)
script_output = ''
script_ret = 0
if not nzb_list:
# Give destination its final name
if cfg.folder_rename() and tmp_workdir_complete and not one_folder:
if all_ok:
try:
newfiles = rename_and_collapse_folder(tmp_workdir_complete, workdir_complete, newfiles)
except:
logging.error(T('Error renaming "%s" to "%s"'), clip_path(tmp_workdir_complete), clip_path(workdir_complete))
logging.info('Traceback: ', exc_info=True)
# Better disable sorting because filenames are all off now
file_sorter.sort_file = None
else:
workdir_complete = tmp_workdir_complete.replace('_UNPACK_', '_FAILED_')
workdir_complete = get_unique_path(workdir_complete, n=0, create_dir=False)
workdir_complete = workdir_complete
if empty:
job_result = -1
else:
job_result = int(par_error) + int(bool(unpack_error)) * 2
if cfg.ignore_samples():
remove_samples(workdir_complete)
# TV/Movie/Date Renaming code part 2 - rename and move files to parent folder
if all_ok and file_sorter.sort_file:
if newfiles:
file_sorter.rename(newfiles, workdir_complete)
workdir_complete, ok = file_sorter.move(workdir_complete)
else:
workdir_complete, ok = file_sorter.rename_with_ext(workdir_complete)
if not ok:
nzo.set_unpack_info('Unpack', T('Failed to move files'))
all_ok = False
# Run the user script
script_path = make_script_path(script)
if (all_ok or not cfg.safe_postproc()) and (not nzb_list) and script_path:
# set the current nzo status to "Ext Script...". Used in History
nzo.status = Status.RUNNING
nzo.set_action_line(T('Running script'), unicoder(script))
nzo.set_unpack_info('Script', T('Running user script %s') % unicoder(script), unique=True)
script_log, script_ret = external_processing(short_path(script_path, False), short_path(workdir_complete, False), nzo.filename,
dirname, cat, nzo.group, job_result,
nzo.nzo_info.get('failure', ''))
script_line = get_last_line(script_log)
if script_log:
script_output = nzo.nzo_id
if script_line:
nzo.set_unpack_info('Script', unicoder(script_line), unique=True)
else:
nzo.set_unpack_info('Script', T('Ran %s') % unicoder(script), unique=True)
else:
script = ""
script_line = ""
script_ret = 0
# Maybe bad script result should fail job
if script_ret and cfg.script_can_fail():
script_error = True
all_ok = False
nzo.fail_msg = T('Script exit code is %s') % script_ret
else:
script_error = False
# Email the results
if (not nzb_list) and cfg.email_endjob():
if (cfg.email_endjob() == 1) or (cfg.email_endjob() == 2 and (unpack_error or par_error or script_error)):
emailer.endjob(dirname, cat, all_ok, workdir_complete, nzo.bytes_downloaded,
nzo.fail_msg, nzo.unpack_info, script, TRANS(script_log), script_ret)
if script_output:
# Can do this only now, otherwise it would show up in the email
if script_ret:
script_ret = 'Exit(%s) ' % script_ret
else:
script_ret = ''
if len(script_log.rstrip().split('\n')) > 1:
nzo.set_unpack_info('Script',
u'%s%s <a href="./scriptlog?name=%s">(%s)</a>' % (script_ret, xml.sax.saxutils.escape(script_line),
xml.sax.saxutils.escape(script_output), T('More')), unique=True)
else:
# No '(more)' button needed
nzo.set_unpack_info('Script', u'%s%s ' % (script_ret, xml.sax.saxutils.escape(script_line)), unique=True)
# Cleanup again, including NZB files
if all_ok:
cleanup_list(workdir_complete, False)
# Force error for empty result
all_ok = all_ok and not empty
# Update indexer with results
if cfg.rating_enable():
if nzo.encrypted > 0:
Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_ENCRYPTED)
if empty:
hosts = map(lambda s: s.host, sabnzbd.downloader.Downloader.do.nzo_servers(nzo))
if not hosts:
hosts = [None]
for host in hosts:
Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_EXPIRED, host)
# Show final status in history
if all_ok:
notifier.send_notification(T('Download Completed'), filename, 'complete')
nzo.status = Status.COMPLETED
else:
notifier.send_notification(T('Download Failed'), filename, 'failed')
nzo.status = Status.FAILED
except:
logging.error(T('Post Processing Failed for %s (%s)'), filename, crash_msg)
if not crash_msg:
logging.info("Traceback: ", exc_info=True)
crash_msg = T('see logfile')
nzo.fail_msg = T('PostProcessing was aborted (%s)') % unicoder(crash_msg)
notifier.send_notification(T('Download Failed'), filename, 'failed')
nzo.status = Status.FAILED
par_error = True
all_ok = False
if cfg.email_endjob():
emailer.endjob(dirname, cat, all_ok, clip_path(workdir_complete), nzo.bytes_downloaded,
nzo.fail_msg, nzo.unpack_info, '', '', 0)
if all_ok:
# If the folder only contains one file OR folder, have that as the path
# Be aware that series/generic/date sorting may move a single file into a folder containing other files
workdir_complete = one_file_or_folder(workdir_complete)
workdir_complete = os.path.normpath(workdir_complete)
# Log the overall time taken for postprocessing
postproc_time = int(time.time() - start)
# Create the history DB instance
history_db = database.HistoryDB()
# Add the nzo to the database. Only the path, script and time taken is passed
# Other information is obtained from the nzo
history_db.add_history_db(nzo, clip_path(workdir_complete), nzo.downpath, postproc_time, script_log, script_line)
# The connection is only used once, so close it here
history_db.close()
# Clean up the NZO
try:
logging.info('Cleaning up %s (keep_basic=%s)', filename, str(not all_ok))
sabnzbd.nzbqueue.NzbQueue.do.cleanup_nzo(nzo, keep_basic=not all_ok)
except:
logging.error(T('Cleanup of %s failed.'), nzo.final_name)
logging.info("Traceback: ", exc_info=True)
# Remove download folder
if all_ok:
try:
if os.path.exists(workdir):
logging.debug('Removing workdir %s', workdir)
remove_all(workdir, recursive=True)
except:
logging.error(T('Error removing workdir (%s)'), clip_path(workdir))
logging.info("Traceback: ", exc_info=True)
# Use automatic retry link on par2 errors and encrypted/bad RARs
if par_error or unpack_error in (2, 3):
try_alt_nzb(nzo)
# Update the last check time
sabnzbd.LAST_HISTORY_UPDATE = time.time()
return True
def parring(nzo, workdir):
""" Perform par processing. Returns: (par_error, re_add) """
filename = nzo.final_name
notifier.send_notification(T('Post-processing'), filename, 'pp', nzo.cat)
logging.info('Starting verification and repair of %s', filename)
# Get verification status of sets
verified = sabnzbd.load_data(VERIFIED_FILE, nzo.workpath, remove=False) or {}
repair_sets = nzo.extrapars.keys()
re_add = False
par_error = False
single = len(repair_sets) == 1
if repair_sets:
for setname in repair_sets:
if cfg.ignore_samples() and RE_SAMPLE.search(setname.lower()):
continue
if not verified.get(setname, False):
logging.info("Running verification and repair on set %s", setname)
parfile_nzf = nzo.partable[setname]
# Check if file maybe wasn't deleted and if we maybe have more files in the parset
if os.path.exists(os.path.join(nzo.downpath, parfile_nzf.filename)) or nzo.extrapars[setname]:
need_re_add, res = par2_repair(parfile_nzf, nzo, workdir, setname, single=single)
# Was it aborted?
if not nzo.pp_active:
re_add = False
par_error = True
break
re_add = re_add or need_re_add
verified[setname] = res
else:
continue
par_error = par_error or not res
else:
# We must not have found any par2..
logging.info("No par2 sets for %s", filename)
nzo.set_unpack_info('Repair', T('[%s] No par2 sets') % unicoder(filename))
if cfg.sfv_check() and not verified.get('', False):
par_error = not try_sfv_check(nzo, workdir, '')
verified[''] = not par_error
# If still no success, do RAR-check
if not par_error and cfg.enable_unrar():
par_error = not try_rar_check(nzo, workdir, '')
verified[''] = not par_error
if re_add:
logging.info('Re-added %s to queue', filename)
if nzo.priority != TOP_PRIORITY:
nzo.priority = REPAIR_PRIORITY
nzo.status = Status.FETCHING
sabnzbd.nzbqueue.NzbQueue.do.add(nzo)
sabnzbd.downloader.Downloader.do.resume_from_postproc()
sabnzbd.save_data(verified, VERIFIED_FILE, nzo.workpath)
logging.info('Verification and repair finished for %s', filename)
return par_error, re_add
def process_job(nzo):
""" Process one job """
start = time.time()
# keep track of whether we can continue
all_ok = True
# keep track of par problems
par_error = False
# keep track of any unpacking errors
unpack_error = False
# Signal empty download, for when 'empty_postproc' is enabled
empty = False
nzb_list = []
# These need to be initialized in case of a crash
workdir_complete = ""
script_log = ""
script_line = ""
# Get the job flags
nzo.save_attribs()
flag_repair, flag_unpack, flag_delete = nzo.repair_opts
# Normalize PP
if flag_delete:
flag_unpack = True
if flag_unpack:
flag_repair = True
# Get the NZB name
filename = nzo.final_name
# Download-processes can mark job as failed
if nzo.fail_msg:
nzo.status = Status.FAILED
nzo.save_attribs()
all_ok = False
par_error = True
unpack_error = 1
try:
# Get the folder containing the download result
workdir = nzo.downpath
tmp_workdir_complete = None
# if no files are present (except __admin__), fail the job
if all_ok and len(globber(workdir)) < 2:
if nzo.precheck:
_, ratio = nzo.check_availability_ratio()
emsg = T(
"Download might fail, only %s of required %s available"
) % (ratio, cfg.req_completion_rate())
else:
emsg = T("Download failed - Not on your server(s)")
empty = True
emsg += " - https://sabnzbd.org/not-complete"
nzo.fail_msg = emsg
nzo.set_unpack_info("Fail", emsg)
nzo.status = Status.FAILED
# do not run unpacking or parity verification
flag_repair = flag_unpack = False
all_ok = cfg.empty_postproc() and empty
if not all_ok:
par_error = True
unpack_error = 1
script = nzo.script
logging.info(
"Starting Post-Processing on %s => Repair:%s, Unpack:%s, Delete:%s, Script:%s, Cat:%s",
filename,
flag_repair,
flag_unpack,
flag_delete,
script,
nzo.cat,
)
# Set complete dir to workdir in case we need to abort
workdir_complete = workdir
# Send post-processing notification
notifier.send_notification(T("Post-processing"), nzo.final_name, "pp",
nzo.cat)
# Par processing, if enabled
if all_ok and flag_repair:
par_error, re_add = parring(nzo, workdir)
if re_add:
# Try to get more par files
return False
# If we don't need extra par2, we can disconnect
if sabnzbd.nzbqueue.NzbQueue.do.actives(
grabs=False) == 0 and cfg.autodisconnect():
# This was the last job, close server connections
sabnzbd.downloader.Downloader.do.disconnect()
# Sanitize the resulting files
if sabnzbd.WIN32:
sanitize_files_in_folder(workdir)
# Check if user allows unsafe post-processing
if flag_repair and cfg.safe_postproc():
all_ok = all_ok and not par_error
if all_ok:
# Fix encodings
fix_unix_encoding(workdir)
# Use dirs generated by direct-unpacker
if nzo.direct_unpacker and nzo.direct_unpacker.unpack_dir_info:
(
tmp_workdir_complete,
workdir_complete,
file_sorter,
one_folder,
marker_file,
) = nzo.direct_unpacker.unpack_dir_info
else:
# Generate extraction path
tmp_workdir_complete, workdir_complete, file_sorter, one_folder, marker_file = prepare_extraction_path(
nzo)
newfiles = []
# Run Stage 2: Unpack
if flag_unpack:
# Set the current nzo status to "Extracting...". Used in History
nzo.status = Status.EXTRACTING
logging.info("Running unpack_magic on %s", filename)
unpack_error, newfiles = unpack_magic(nzo, workdir,
tmp_workdir_complete,
flag_delete, one_folder,
(), (), (), (), ())
logging.info("Unpacked files %s", newfiles)
if sabnzbd.WIN32:
# Sanitize the resulting files
newfiles = sanitize_files_in_folder(tmp_workdir_complete)
logging.info("Finished unpack_magic on %s", filename)
if cfg.safe_postproc():
all_ok = all_ok and not unpack_error
if all_ok:
# Move any (left-over) files to destination
nzo.status = Status.MOVING
nzo.set_action_line(T("Moving"), "...")
for root, _dirs, files in os.walk(workdir):
if not root.endswith(JOB_ADMIN):
for file_ in files:
path = os.path.join(root, file_)
new_path = path.replace(workdir,
tmp_workdir_complete)
ok, new_path = move_to_path(path, new_path)
if new_path:
newfiles.append(new_path)
if not ok:
nzo.set_unpack_info(
"Unpack",
T("Failed moving %s to %s") %
(path, new_path))
all_ok = False
break
# Set permissions right
set_permissions(tmp_workdir_complete)
if all_ok and marker_file:
del_marker(os.path.join(tmp_workdir_complete, marker_file))
remove_from_list(marker_file, newfiles)
if all_ok:
# Remove files matching the cleanup list
cleanup_list(tmp_workdir_complete, skip_nzb=True)
# Check if this is an NZB-only download, if so redirect to queue
# except when PP was Download-only
if flag_repair:
nzb_list = nzb_redirect(tmp_workdir_complete,
nzo.final_name, nzo.pp, script,
nzo.cat, nzo.priority)
else:
nzb_list = None
if nzb_list:
nzo.set_unpack_info("Download",
T("Sent %s to queue") % nzb_list)
cleanup_empty_directories(tmp_workdir_complete)
else:
# Full cleanup including nzb's
cleanup_list(tmp_workdir_complete, skip_nzb=False)
script_output = ""
script_ret = 0
if not nzb_list:
# Give destination its final name
if cfg.folder_rename() and tmp_workdir_complete and not one_folder:
if not all_ok:
# Rename failed folders so they are easy to recognize
workdir_complete = tmp_workdir_complete.replace(
"_UNPACK_", "_FAILED_")
workdir_complete = get_unique_path(workdir_complete,
create_dir=False)
try:
newfiles = rename_and_collapse_folder(
tmp_workdir_complete, workdir_complete, newfiles)
except:
logging.error(
T('Error renaming "%s" to "%s"'),
clip_path(tmp_workdir_complete),
clip_path(workdir_complete),
)
logging.info("Traceback: ", exc_info=True)
# Better disable sorting because filenames are all off now
file_sorter.sort_file = None
if empty:
job_result = -1
else:
job_result = int(par_error) + int(bool(unpack_error)) * 2
if cfg.ignore_samples():
remove_samples(workdir_complete)
# TV/Movie/Date Renaming code part 2 - rename and move files to parent folder
if all_ok and file_sorter.sort_file:
if newfiles:
file_sorter.rename(newfiles, workdir_complete)
workdir_complete, ok = file_sorter.move(workdir_complete)
else:
workdir_complete, ok = file_sorter.rename_with_ext(
workdir_complete)
if not ok:
nzo.set_unpack_info("Unpack", T("Failed to move files"))
all_ok = False
if cfg.deobfuscate_final_filenames() and all_ok and not nzb_list:
# deobfuscate the filenames
logging.info("Running deobfuscate")
deobfuscate.deobfuscate_list(newfiles, nzo.final_name)
# Run the user script
script_path = make_script_path(script)
if (all_ok or not cfg.safe_postproc()) and (
not nzb_list) and script_path:
# Set the current nzo status to "Ext Script...". Used in History
nzo.status = Status.RUNNING
nzo.set_action_line(T("Running script"), script)
nzo.set_unpack_info("Script",
T("Running user script %s") % script,
unique=True)
script_log, script_ret = external_processing(
script_path, nzo, clip_path(workdir_complete),
nzo.final_name, job_result)
script_line = get_last_line(script_log)
if script_log:
script_output = nzo.nzo_id
if script_line:
nzo.set_unpack_info("Script", script_line, unique=True)
else:
nzo.set_unpack_info("Script",
T("Ran %s") % script,
unique=True)
else:
script = ""
script_line = ""
script_ret = 0
# Maybe bad script result should fail job
if script_ret and cfg.script_can_fail():
script_error = True
all_ok = False
nzo.fail_msg = T("Script exit code is %s") % script_ret
else:
script_error = False
# Email the results
if (not nzb_list) and cfg.email_endjob():
if (cfg.email_endjob()
== 1) or (cfg.email_endjob() == 2 and
(unpack_error or par_error or script_error)):
emailer.endjob(
nzo.final_name,
nzo.cat,
all_ok,
workdir_complete,
nzo.bytes_downloaded,
nzo.fail_msg,
nzo.unpack_info,
script,
script_log,
script_ret,
)
if script_output:
# Can do this only now, otherwise it would show up in the email
if script_ret:
script_ret = "Exit(%s) " % script_ret
else:
script_ret = ""
if len(script_log.rstrip().split("\n")) > 1:
nzo.set_unpack_info(
"Script",
'%s%s <a href="./scriptlog?name=%s">(%s)</a>' %
(script_ret, script_line, encoding.xml_name(script_output),
T("More")),
unique=True,
)
else:
# No '(more)' button needed
nzo.set_unpack_info("Script",
"%s%s " % (script_ret, script_line),
unique=True)
# Cleanup again, including NZB files
if all_ok:
cleanup_list(workdir_complete, False)
# Force error for empty result
all_ok = all_ok and not empty
# Update indexer with results
if cfg.rating_enable():
if nzo.encrypted > 0:
Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_ENCRYPTED)
if empty:
hosts = [
s.host
for s in sabnzbd.downloader.Downloader.do.nzo_servers(nzo)
]
if not hosts:
hosts = [None]
for host in hosts:
Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_EXPIRED,
host)
except:
logging.error(T("Post Processing Failed for %s (%s)"), filename,
T("see logfile"))
logging.info("Traceback: ", exc_info=True)
nzo.fail_msg = T("PostProcessing was aborted (%s)") % T("see logfile")
notifier.send_notification(T("Download Failed"), filename, "failed",
nzo.cat)
nzo.status = Status.FAILED
par_error = True
all_ok = False
if cfg.email_endjob():
emailer.endjob(
nzo.final_name,
nzo.cat,
all_ok,
clip_path(workdir_complete),
nzo.bytes_downloaded,
nzo.fail_msg,
nzo.unpack_info,
"",
"",
0,
)
if all_ok:
# If the folder only contains one file OR folder, have that as the path
# Be aware that series/generic/date sorting may move a single file into a folder containing other files
workdir_complete = one_file_or_folder(workdir_complete)
workdir_complete = os.path.normpath(workdir_complete)
# Clean up the NZO data
try:
nzo.purge_data(delete_all_data=all_ok)
except:
logging.error(T("Cleanup of %s failed."), nzo.final_name)
logging.info("Traceback: ", exc_info=True)
# Use automatic retry link on par2 errors and encrypted/bad RARs
if par_error or unpack_error in (2, 3):
try_alt_nzb(nzo)
# Show final status in history
if all_ok:
notifier.send_notification(T("Download Completed"), filename,
"complete", nzo.cat)
nzo.status = Status.COMPLETED
else:
notifier.send_notification(T("Download Failed"), filename, "failed",
nzo.cat)
nzo.status = Status.FAILED
# Log the overall time taken for postprocessing
postproc_time = int(time.time() - start)
# Create the history DB instance
history_db = database.HistoryDB()
# Add the nzo to the database. Only the path, script and time taken is passed
# Other information is obtained from the nzo
history_db.add_history_db(nzo, workdir_complete, postproc_time, script_log,
script_line)
# Purge items
history_db.auto_history_purge()
# The connection is only used once, so close it here
history_db.close()
sabnzbd.history_updated()
return True
def parring(nzo, workdir):
""" Perform par processing. Returns: (par_error, re_add) """
if 0:
assert isinstance(
nzo, sabnzbd.nzbstuff.NzbObject) # Assert only for debug purposes
filename = nzo.final_name
notifier.send_notification(T('Post-processing'), filename, 'pp')
logging.info('Starting verification and repair of %s', filename)
# Get verification status of sets
verified = sabnzbd.load_data(VERIFIED_FILE, nzo.workpath,
remove=False) or {}
# Collect the par files
if nzo.partable:
par_table = nzo.partable.copy()
else:
par_table = {}
repair_sets = par_table.keys()
re_add = False
par_error = False
single = len(repair_sets) == 1
if repair_sets:
for setname in repair_sets:
if cfg.ignore_samples() and 'sample' in setname.lower():
continue
if not verified.get(setname, False):
logging.info("Running verification and repair on set %s",
setname)
parfile_nzf = par_table[setname]
if os.path.exists(
os.path.join(
nzo.downpath,
parfile_nzf.filename)) or parfile_nzf.extrapars:
need_re_add, res = par2_repair(parfile_nzf,
nzo,
workdir,
setname,
single=single)
# Was it aborted?
if not nzo.pp_active:
re_add = False
par_error = True
break
re_add = re_add or need_re_add
if not res and not need_re_add and cfg.sfv_check():
res = try_sfv_check(nzo, workdir, setname)
if not res and not need_re_add and cfg.enable_unrar():
res = try_rar_check(nzo, workdir, setname)
verified[setname] = res
else:
continue
par_error = par_error or not res
else:
# Obfuscated par2 check
logging.info('No par2 sets found, running obfuscated check on %s',
filename)
# Get the NZF's and sort them based on size
nzfs_sorted = sorted(nzo.finished_files, key=lambda x: x.bytes)
# We will have to make 'fake' par files that are recognized
par2_vol = 0
par2_filename = None
for nzf_try in nzfs_sorted:
# run through list of files, looking for par2 signature..
logging.debug("Checking par2 signature of %s", nzf_try.filename)
try:
nzf_path = os.path.join(workdir, nzf_try.filename)
if (is_parfile(nzf_path)):
# We need 1 base-name so they are recognized as 1 set
if not par2_filename:
par2_filename = nzf_path
# Rename so handle_par2() picks it up
newpath = '%s.vol%d+%d.par2' % (par2_filename, par2_vol,
par2_vol + 1)
renamer(nzf_path, newpath)
nzf_try.filename = os.path.split(newpath)[1]
# Let the magic happen
nzo.handle_par2(nzf_try, file_done=True)
par2_vol += 1
except:
pass
if par2_vol > 0:
# Pars found, we do it again
par_error, re_add = parring(nzo, workdir)
else:
# We must not have found any par2..
logging.info("No par2 sets for %s", filename)
nzo.set_unpack_info('Repair',
T('[%s] No par2 sets') % unicoder(filename))
if cfg.sfv_check() and not verified.get('', False):
par_error = not try_sfv_check(nzo, workdir, '')
verified[''] = not par_error
# If still no success, do RAR-check
if not par_error and cfg.enable_unrar():
par_error = not try_rar_check(nzo, workdir, '')
verified[''] = not par_error
if re_add:
logging.info('Re-added %s to queue', filename)
if nzo.priority != TOP_PRIORITY:
nzo.priority = REPAIR_PRIORITY
sabnzbd.nzbqueue.add_nzo(nzo)
sabnzbd.downloader.Downloader.do.resume_from_postproc()
sabnzbd.save_data(verified, VERIFIED_FILE, nzo.workpath)
logging.info('Verification and repair finished for %s', filename)
return par_error, re_add
def stop(self):
self.shutdown = True
notifier.send_notification("SABnzbd", T('Shutting down'), 'startup')
def parring(nzo, workdir):
""" Perform par processing. Returns: (par_error, re_add) """
filename = nzo.final_name
notifier.send_notification(T('Post-processing'), filename, 'pp')
logging.info('Starting verification and repair of %s', filename)
# Get verification status of sets
verified = sabnzbd.load_data(VERIFIED_FILE, nzo.workpath, remove=False) or {}
repair_sets = nzo.partable.keys()
re_add = False
par_error = False
single = len(repair_sets) == 1
if repair_sets:
for setname in repair_sets:
if cfg.ignore_samples() and RE_SAMPLE.search(setname.lower()):
continue
if not verified.get(setname, False):
logging.info("Running verification and repair on set %s", setname)
parfile_nzf = nzo.partable[setname]
if os.path.exists(os.path.join(nzo.downpath, parfile_nzf.filename)) or parfile_nzf.extrapars:
need_re_add, res = par2_repair(parfile_nzf, nzo, workdir, setname, single=single)
# Was it aborted?
if not nzo.pp_active:
re_add = False
par_error = True
break
re_add = re_add or need_re_add
verified[setname] = res
else:
continue
par_error = par_error or not res
else:
# Obfuscated par2 check
logging.info('No par2 sets found, running obfuscated check on %s', filename)
# Get the NZF's and sort them based on size
nzfs_sorted = sorted(nzo.finished_files, key=lambda x: x.bytes)
# We will have to make 'fake' par files that are recognized
par2_vol = 0
par2_filename = None
for nzf_try in nzfs_sorted:
# run through list of files, looking for par2 signature..
logging.debug("Checking par2 signature of %s", nzf_try.filename)
try:
nzf_path = os.path.join(workdir, nzf_try.filename)
if(is_parfile(nzf_path)):
# We need 1 base-name so they are recognized as 1 set
if not par2_filename:
par2_filename = nzf_path
# Rename so handle_par2() picks it up
newpath = '%s.vol%d+%d.par2' % (par2_filename, par2_vol, par2_vol + 1)
renamer(nzf_path, newpath)
nzf_try.filename = os.path.split(newpath)[1]
# Let the magic happen
nzo.handle_par2(nzf_try, file_done=True)
par2_vol += 1
except:
pass
if par2_vol > 0:
# Pars found, we do it again
par_error, re_add = parring(nzo, workdir)
else:
# We must not have found any par2..
logging.info("No par2 sets for %s", filename)
nzo.set_unpack_info('Repair', T('[%s] No par2 sets') % unicoder(filename))
if cfg.sfv_check() and not verified.get('', False):
par_error = not try_sfv_check(nzo, workdir, '')
verified[''] = not par_error
# If still no success, do RAR-check
if not par_error and cfg.enable_unrar():
par_error = not try_rar_check(nzo, workdir, '')
verified[''] = not par_error
if re_add:
logging.info('Re-added %s to queue', filename)
if nzo.priority != TOP_PRIORITY:
nzo.priority = REPAIR_PRIORITY
sabnzbd.nzbqueue.add_nzo(nzo)
sabnzbd.downloader.Downloader.do.resume_from_postproc()
sabnzbd.save_data(verified, VERIFIED_FILE, nzo.workpath)
logging.info('Verification and repair finished for %s', filename)
return par_error, re_add
