def try_sfv_check(nzo, workdir, setname):
""" Attempt to verify set using SFV file
Return True if verified, False when failed
When setname is '', all SFV files will be used, otherwise only the matching one
When setname is '' and no SFV files are found, True is returned
"""
# Get list of SFV names; shortest name first, minimizes the chance on a mismatch
sfvs = globber(workdir, '*.sfv')
sfvs.sort(lambda x, y: len(x) - len(y))
par_error = False
found = False
for sfv in sfvs:
if setname in os.path.basename(sfv):
found = True
nzo.set_unpack_info('Repair', T('Trying SFV verification'))
failed = sfv_check(sfv)
if failed:
msg = T('Some files failed to verify against "%s"') % unicoder(
os.path.basename(sfv))
msg += '; '
msg += '; '.join(failed)
nzo.set_unpack_info('Repair', msg)
par_error = True
else:
nzo.set_unpack_info('Repair',
T('Verified successfully using SFV files'))
if setname:
break
return (found or not setname) and not par_error
def try_sfv_check(nzo, workdir, setname):
""" Attempt to verify set using SFV file
Return True if verified, False when failed
When setname is '', all SFV files will be used, otherwise only the matching one
When setname is '' and no SFV files are found, True is returned
"""
# Get list of SFV names; shortest name first, minimizes the chance on a mismatch
sfvs = globber(workdir, '*.sfv')
sfvs.sort(lambda x, y: len(x) - len(y))
par_error = False
found = False
for sfv in sfvs:
if setname in os.path.basename(sfv):
found = True
nzo.set_unpack_info('Repair', T('Trying SFV verification'))
failed = sfv_check(sfv)
if failed:
msg = T('Some files failed to verify against "%s"') % unicoder(os.path.basename(sfv))
msg += '; '
msg += '; '.join(failed)
nzo.set_unpack_info('Repair', msg)
par_error = True
else:
nzo.set_unpack_info('Repair', T('Verified successfully using SFV files'))
if setname:
break
return (found or not setname) and not par_error
def rename_and_collapse_folder(oldpath, newpath, files):
""" Rename folder, collapsing when there's just a single subfolder
oldpath --> newpath OR oldpath/subfolder --> newpath
Modify list of filenames accordingly
"""
orgpath = oldpath
items = globber(oldpath)
if len(items) == 1:
folder = items[0]
folder_path = os.path.join(oldpath, folder)
if os.path.isdir(folder_path) and folder not in ('VIDEO_TS',
'AUDIO_TS'):
logging.info('Collapsing %s', os.path.join(newpath, folder))
oldpath = folder_path
oldpath = os.path.normpath(oldpath)
newpath = os.path.normpath(newpath)
files = [os.path.normpath(f).replace(oldpath, newpath) for f in files]
renamer(oldpath, newpath)
try:
remove_dir(orgpath)
except:
pass
return files
def repair_job(self, folder, new_nzb=None):
""" Reconstruct admin for a single job folder, optionally with new NZB
"""
def all_verified(path):
""" Return True when all sets have been successfully verified """
verified = sabnzbd.load_data(VERIFIED_FILE, path, remove=False) or {'x':False}
return not bool([True for x in verified if not verified[x]])
name = os.path.basename(folder)
path = os.path.join(folder, JOB_ADMIN)
if hasattr(new_nzb, 'filename'):
filename = new_nzb.filename
else:
filename = ''
if not filename:
if not all_verified(path):
filename = globber(path, '*.gz')
if len(filename) > 0:
logging.debug('Repair job %s by reparsing stored NZB', latin1(name))
sabnzbd.add_nzbfile(filename[0], pp=None, script=None, cat=None, priority=None, nzbname=name, reuse=True)
else:
logging.debug('Repair job %s without stored NZB', latin1(name))
nzo = NzbObject(name, 0, pp=None, script=None, nzb='', cat=None, priority=None, nzbname=name, reuse=True)
self.add(nzo)
else:
remove_all(path, '*.gz')
logging.debug('Repair job %s with new NZB (%s)', latin1(name), latin1(filename))
sabnzbd.add_nzbfile(new_nzb, pp=None, script=None, cat=None, priority=None, nzbname=name, reuse=True)
def scan_jobs(self, all=False, action=True):
""" Scan "incomplete" for mssing folders,
'all' is True: Include active folders
'action' is True, do the recovery action
returns list of orphaned folders
"""
result = []
# Folders from the download queue
if all:
registered = []
else:
registered = [nzo.work_name for nzo in self.__nzo_list]
# Retryable folders from History
items = sabnzbd.proxy_build_history()[0]
# Anything waiting or active or retryable is a known item
registered.extend([platform_encode(os.path.basename(item['path'])) \
for item in items if item['retry'] or item['loaded'] or item['status'] == 'Queued'])
# Repair unregistered folders
for folder in globber(cfg.download_dir.get_path()):
if os.path.isdir(folder) and os.path.basename(folder) not in registered:
if action:
logging.info('Repairing job %s', folder)
self.repair_job(folder)
result.append(os.path.basename(folder))
else:
if action:
logging.info('Skipping repair for job %s', folder)
return result
def parring(nzo, workdir):
""" Perform par processing. Returns: (par_error, re_add)
"""
filename = nzo.final_name
growler.send_notification(T('Post-processing'), nzo.final_name, 'pp')
logging.info('Par2 check starting on %s', filename)
## Collect the par files
if nzo.partable:
par_table = nzo.partable.copy()
else:
par_table = {}
repair_sets = par_table.keys()
re_add = False
par_error = False
if repair_sets:
for set_ in repair_sets:
logging.info("Running repair on set %s", set_)
parfile_nzf = par_table[set_]
need_re_add, res = par2_repair(parfile_nzf, nzo, workdir, set_)
if need_re_add:
re_add = True
else:
par_error = par_error or not res
if re_add:
logging.info('Readded %s to queue', filename)
nzo.priority = REPAIR_PRIORITY
sabnzbd.nzbqueue.add_nzo(nzo)
sabnzbd.downloader.Downloader.do.resume_from_postproc()
logging.info('Par2 check finished on %s', filename)
if (par_error and not re_add) or not repair_sets:
# See if alternative SFV check is possible
if cfg.sfv_check():
sfvs = globber(workdir, '*.sfv')
else:
sfvs = None
if sfvs:
par_error = False
nzo.set_unpack_info('Repair', T('Trying SFV verification'))
for sfv in sfvs:
if not sfv_check(sfv):
nzo.set_unpack_info('Repair', T('Some files failed to verify against "%s"') % unicoder(os.path.basename(sfv)))
par_error = True
if not par_error:
nzo.set_unpack_info('Repair', T('Verified successfully using SFV files'))
elif not repair_sets:
logging.info("No par2 sets for %s", filename)
nzo.set_unpack_info('Repair', T('[%s] No par2 sets') % unicoder(filename))
if not par_error:
verified_flag_file(workdir, create=True)
return par_error, re_add
def read_queue(self, repair):
""" Read queue from disk, supporting repair modes
0 = no repairs
1 = use existing queue, add missing "incomplete" folders
2 = Discard all queue admin, reconstruct from "incomplete" folders
"""
nzo_ids = []
if repair < 2:
# Read the queue from the saved files
data = sabnzbd.load_admin(QUEUE_FILE_NAME)
if data:
try:
queue_vers, nzo_ids, dummy = data
if not queue_vers == QUEUE_VERSION:
nzo_ids = []
logging.error(
Ta('Incompatible queuefile found, cannot proceed'))
if not repair:
panic_queue(
os.path.join(cfg.cache_dir.get_path(),
QUEUE_FILE_NAME))
exit_sab(2)
except ValueError:
nzo_ids = []
logging.error(
Ta('Error loading %s, corrupt file detected'),
os.path.join(cfg.cache_dir.get_path(),
QUEUE_FILE_NAME))
if not repair:
return
# First handle jobs in the queue file
folders = []
for nzo_id in nzo_ids:
folder, _id = os.path.split(nzo_id)
# Try as normal job
path = get_admin_path(bool(folder), folder, False)
nzo = sabnzbd.load_data(_id, path, remove=False)
if not nzo:
# Try as future job
path = get_admin_path(bool(folder), folder, True)
nzo = sabnzbd.load_data(_id, path)
if nzo:
self.add(nzo, save=False, quiet=True)
folders.append(folder)
# Scan for any folders in "incomplete" that are not yet in the queue
if repair:
self.scan_jobs(not folders)
# Handle any lost future jobs
for path in globber(
os.path.join(cfg.admin_dir.get_path(), FUTURE_Q_FOLDER)):
path, nzo_id = os.path.split(path)
if nzo_id not in self.__nzo_table:
nzo = sabnzbd.load_data(nzo_id, path, remove=True)
if nzo:
self.add(nzo, save=True)
def repair_job(self, folder, new_nzb=None):
""" Reconstruct admin for a single job folder, optionally with new NZB
"""
def all_verified(path):
""" Return True when all sets have been successfully verified """
verified = sabnzbd.load_data(VERIFIED_FILE, path,
remove=False) or {
'x': False
}
return not bool([True for x in verified if not verified[x]])
nzo_id = None
name = os.path.basename(folder)
path = os.path.join(folder, JOB_ADMIN)
if hasattr(new_nzb, 'filename'):
filename = new_nzb.filename
else:
filename = ''
if not filename:
if not all_verified(path):
filename = globber(path, '*.gz')
if len(filename) > 0:
logging.debug('Repair job %s by reparsing stored NZB',
latin1(name))
nzo_id = sabnzbd.add_nzbfile(filename[0],
pp=None,
script=None,
cat=None,
priority=None,
nzbname=name,
reuse=True)[1]
else:
logging.debug('Repair job %s without stored NZB', latin1(name))
nzo = NzbObject(name,
0,
pp=None,
script=None,
nzb='',
cat=None,
priority=None,
nzbname=name,
reuse=True)
self.add(nzo)
nzo_id = nzo.nzo_id
else:
remove_all(path, '*.gz')
logging.debug('Repair job %s with new NZB (%s)', latin1(name),
latin1(filename))
nzo_id = sabnzbd.add_nzbfile(new_nzb,
pp=None,
script=None,
cat=None,
priority=None,
nzbname=name,
reuse=True)[1]
return nzo_id
def send_back(self, nzo):
""" Send back job to queue after successful pre-check """
try:
nzb_path = globber(nzo.workpath, '*.gz')[0]
except:
logging.debug('Failed to find NZB file after pre-check (%s)', nzo.nzo_id)
return
from sabnzbd.dirscanner import ProcessSingleFile
nzo_id = ProcessSingleFile(os.path.split(nzb_path)[1], nzb_path, reuse=True)[1][0]
self.replace_in_q(nzo, nzo_id)
def parring(nzo, workdir):
""" Perform par processing. Returns: (par_error, re_add)
"""
filename = nzo.final_name
osx.sendGrowlMsg(T('Post-processing'), nzo.final_name,
osx.NOTIFICATION['pp'])
logging.info('Par2 check starting on %s', filename)
## Collect the par files
if nzo.partable:
par_table = nzo.partable.copy()
else:
par_table = {}
repair_sets = par_table.keys()
re_add = False
par_error = False
if repair_sets:
for set_ in repair_sets:
logging.info("Running repair on set %s", set_)
parfile_nzf = par_table[set_]
need_re_add, res = par2_repair(parfile_nzf, nzo, workdir, set_)
if need_re_add:
re_add = True
else:
par_error = par_error or not res
if re_add:
logging.info('Readded %s to queue', filename)
nzo.priority = REPAIR_PRIORITY
sabnzbd.nzbqueue.add_nzo(nzo)
sabnzbd.downloader.Downloader.do.resume_from_postproc()
logging.info('Par2 check finished on %s', filename)
else:
# See if alternative SFV check is possible
sfv = None
if cfg.sfv_check():
for sfv in globber(workdir, '*.sfv'):
par_error = par_error or not sfv_check(sfv)
if par_error:
nzo.set_unpack_info(
'Repair',
T('Some files failed to verify against "%s"') %
unicoder(os.path.basename(sfv)))
if not sfv:
logging.info("No par2 sets for %s", filename)
nzo.set_unpack_info('Repair',
T('[%s] No par2 sets') % unicoder(filename))
return par_error, re_add
def send_back(self, nzo):
""" Send back job to queue after successful pre-check """
try:
nzb_path = globber(nzo.workpath, '*.gz')[0]
except:
logging.debug('Failed to find NZB file after pre-check (%s)', nzo.nzo_id)
return
from sabnzbd.dirscanner import ProcessSingleFile
res, nzo_ids = ProcessSingleFile(nzo.work_name + '.nzb', nzb_path, reuse=True)
if res == 0 and nzo_ids:
self.replace_in_q(nzo, nzo_ids[0])
def read_queue(self, repair):
""" Read queue from disk, supporting repair modes
0 = no repairs
1 = use existing queue, add missing "incomplete" folders
2 = Discard all queue admin, reconstruct from "incomplete" folders
"""
nzo_ids = []
if repair < 2:
# Read the queue from the saved files
data = sabnzbd.load_admin(QUEUE_FILE_NAME)
if data:
try:
queue_vers, nzo_ids, dummy = data
if not queue_vers == QUEUE_VERSION:
nzo_ids = []
logging.error(Ta("Incompatible queuefile found, cannot proceed"))
if not repair:
panic_queue(os.path.join(cfg.cache_dir.get_path(), QUEUE_FILE_NAME))
exit_sab(2)
except ValueError:
nzo_ids = []
logging.error(
Ta("Error loading %s, corrupt file detected"),
os.path.join(cfg.cache_dir.get_path(), QUEUE_FILE_NAME),
)
if not repair:
return
# First handle jobs in the queue file
folders = []
for nzo_id in nzo_ids:
folder, _id = os.path.split(nzo_id)
# Try as normal job
path = get_admin_path(bool(folder), folder, False)
nzo = sabnzbd.load_data(_id, path, remove=False)
if not nzo:
# Try as future job
path = get_admin_path(bool(folder), folder, True)
nzo = sabnzbd.load_data(_id, path)
if nzo:
self.add(nzo, save=False, quiet=True)
folders.append(folder)
# Scan for any folders in "incomplete" that are not yet in the queue
if repair:
self.scan_jobs(not folders)
# Handle any lost future jobs
for path in globber(os.path.join(cfg.admin_dir.get_path(), FUTURE_Q_FOLDER)):
path, nzo_id = os.path.split(path)
if nzo_id not in self.__nzo_table:
nzo = sabnzbd.load_data(nzo_id, path, remove=True)
if nzo:
self.add(nzo, save=True)
def parring(nzo, workdir):
""" Perform par processing. Returns: (par_error, re_add)
"""
filename = nzo.final_name
osx.sendGrowlMsg(T('Post-processing'), nzo.final_name, osx.NOTIFICATION['pp'])
logging.info('Par2 check starting on %s', filename)
## Collect the par files
if nzo.partable:
par_table = nzo.partable.copy()
else:
par_table = {}
repair_sets = par_table.keys()
re_add = False
par_error = False
if repair_sets:
for set_ in repair_sets:
logging.info("Running repair on set %s", set_)
parfile_nzf = par_table[set_]
need_re_add, res = par2_repair(parfile_nzf, nzo, workdir, set_)
if need_re_add:
re_add = True
else:
par_error = par_error or not res
if re_add:
logging.info('Readded %s to queue', filename)
nzo.priority = REPAIR_PRIORITY
sabnzbd.nzbqueue.add_nzo(nzo)
sabnzbd.downloader.Downloader.do.resume_from_postproc()
logging.info('Par2 check finished on %s', filename)
else:
# See if alternative SFV check is possible
sfv = None
if cfg.sfv_check():
for sfv in globber(workdir, '*.sfv'):
par_error = par_error or not sfv_check(sfv)
if par_error:
nzo.set_unpack_info('Repair', T('Some files failed to verify against "%s"') % unicoder(os.path.basename(sfv)))
if not sfv:
logging.info("No par2 sets for %s", filename)
nzo.set_unpack_info('Repair', T('[%s] No par2 sets') % unicoder(filename))
return par_error, re_add
def send_back(self, nzo):
""" Send back job to queue after successful pre-check """
try:
nzb_path = globber(nzo.workpath, '*.gz')[0]
except:
logging.debug('Failed to find NZB file after pre-check (%s)',
nzo.nzo_id)
return
from sabnzbd.dirscanner import ProcessSingleFile
res, nzo_ids = ProcessSingleFile(nzo.work_name + '.nzb',
nzb_path,
reuse=True)
if res == 0 and nzo_ids:
self.replace_in_q(nzo, nzo_ids[0])
def repair_job(self, folder, new_nzb=None):
""" Reconstruct admin for a single job folder, optionally with new NZB """
name = os.path.basename(folder)
path = os.path.join(folder, JOB_ADMIN)
if new_nzb is None or not new_nzb.filename:
filename = globber(path, '*.gz')
if len(filename) > 0:
logging.debug('Repair job %s by reparsing stored NZB', latin1(name))
sabnzbd.add_nzbfile(filename[0], pp=None, script=None, cat=None, priority=None, nzbname=name, reuse=True)
else:
logging.debug('Repair job %s without stored NZB', latin1(name))
nzo = NzbObject(name, 0, pp=None, script=None, nzb='', cat=None, priority=None, nzbname=name, reuse=True)
self.add(nzo)
else:
remove_all(path, '*.gz')
logging.debug('Repair job %s without new NZB (%s)', latin1(name), latin1(new_nzb.filename))
sabnzbd.add_nzbfile(new_nzb, pp=None, script=None, cat=None, priority=None, nzbname=name, reuse=True)
def collapse_folder(oldpath, newpath):
""" Rename folder, collapsing when there's just a single subfolder
oldpath --> newpath OR oldpath/subfolder --> newpath
"""
orgpath = oldpath
items = globber(oldpath)
if len(items) == 1:
folder_path = items[0]
folder = os.path.split(folder_path)[1]
if os.path.isdir(folder_path) and folder not in ('VIDEO_TS', 'AUDIO_TS'):
logging.info('Collapsing %s', os.path.join(newpath, folder))
oldpath = folder_path
renamer(oldpath, newpath)
try:
remove_dir(orgpath)
except:
pass
def collapse_folder(oldpath, newpath):
""" Rename folder, collapsing when there's just a single subfolder
oldpath --> newpath OR oldpath/subfolder --> newpath
"""
orgpath = oldpath
items = globber(oldpath)
if len(items) == 1:
folder_path = items[0]
folder = os.path.split(folder_path)[1]
if os.path.isdir(folder_path) and folder not in ('VIDEO_TS',
'AUDIO_TS'):
logging.info('Collapsing %s', os.path.join(newpath, folder))
oldpath = folder_path
renamer(oldpath, newpath)
try:
remove_dir(orgpath)
except:
pass
def rename_and_collapse_folder(oldpath, newpath, files):
""" Rename folder, collapsing when there's just a single subfolder
oldpath --> newpath OR oldpath/subfolder --> newpath
Modify list of filenames accordingly
"""
orgpath = oldpath
items = globber(oldpath)
if len(items) == 1:
folder_path = items[0]
folder = os.path.split(folder_path)[1]
if os.path.isdir(folder_path) and folder not in ('VIDEO_TS', 'AUDIO_TS'):
logging.info('Collapsing %s', os.path.join(newpath, folder))
oldpath = folder_path
oldpath = os.path.normpath(oldpath)
newpath = os.path.normpath(newpath)
files = [os.path.normpath(f).replace(oldpath, newpath) for f in files]
renamer(oldpath, newpath)
try:
remove_dir(orgpath)
except:
pass
return files
def process_job(nzo):
""" Process one job """
assert isinstance(nzo, sabnzbd.nzbstuff.NzbObject)
start = time.time()
# keep track of whether we can continue
all_ok = True
# keep track of par problems
par_error = False
# keep track of any unpacking errors
unpack_error = False
nzb_list = []
# These need to be initialised incase of a crash
workdir_complete = ''
postproc_time = 0
script_log = ''
script_line = ''
crash_msg = ''
## Get the job flags
nzo.save_attribs()
flag_repair, flag_unpack, flag_delete = nzo.repair_opts
# Normalize PP
if flag_delete: flag_unpack = True
if flag_unpack: flag_repair = True
# Get the NZB name
filename = nzo.final_name
msgid = nzo.msgid
if cfg.allow_streaming() and not (flag_repair or flag_unpack
or flag_delete):
# After streaming, force +D
nzo.set_pp(3)
nzo.status = 'Failed'
nzo.save_attribs()
all_ok = False
try:
# Get the folder containing the download result
workdir = nzo.downpath
tmp_workdir_complete = None
# if no files are present (except __admin__), fail the job
if len(globber(workdir)) < 2:
emsg = T('Download failed - Out of your server\'s retention?')
nzo.fail_msg = emsg
nzo.status = 'Failed'
# do not run unpacking or parity verification
flag_repair = flag_unpack = False
par_error = unpack_error = True
all_ok = False
script = nzo.script
cat = nzo.cat
logging.info('Starting PostProcessing on %s' + \
' => Repair:%s, Unpack:%s, Delete:%s, Script:%s, Cat:%s',
filename, flag_repair, flag_unpack, flag_delete, script, cat)
## Par processing, if enabled
if flag_repair:
par_error, re_add = parring(nzo, workdir)
if re_add:
# Try to get more par files
return False
## Check if user allows unsafe post-processing
if flag_repair and cfg.safe_postproc():
all_ok = all_ok and not par_error
# Set complete dir to workdir in case we need to abort
workdir_complete = workdir
dirname = nzo.final_name
if all_ok:
one_folder = False
## Determine class directory
if cfg.create_group_folders():
complete_dir = addPrefixes(cfg.complete_dir.get_path(),
nzo.dirprefix)
complete_dir = create_dirs(complete_dir)
else:
catdir = config.get_categories(cat).dir()
if catdir.endswith('*'):
catdir = catdir.strip('*')
one_folder = True
complete_dir = real_path(cfg.complete_dir.get_path(), catdir)
## TV/Movie/Date Renaming code part 1 - detect and construct paths
file_sorter = Sorter(cat)
complete_dir = file_sorter.detect(dirname, complete_dir)
if file_sorter.is_sortfile():
one_folder = False
if one_folder:
workdir_complete = create_dirs(complete_dir)
else:
workdir_complete = get_unique_path(os.path.join(
complete_dir, dirname),
create_dir=True)
if not workdir_complete or not os.path.exists(workdir_complete):
crash_msg = T('Cannot create final folder %s') % unicoder(
os.path.join(complete_dir, dirname))
raise IOError
if cfg.folder_rename() and not one_folder:
tmp_workdir_complete = prefix(workdir_complete, '_UNPACK_')
try:
renamer(workdir_complete, tmp_workdir_complete)
except:
pass # On failure, just use the original name
else:
tmp_workdir_complete = workdir_complete
newfiles = []
## Run Stage 2: Unpack
if flag_unpack:
if all_ok:
#set the current nzo status to "Extracting...". Used in History
nzo.status = 'Extracting'
logging.info("Running unpack_magic on %s", filename)
unpack_error, newfiles = unpack_magic(
nzo, workdir, tmp_workdir_complete, flag_delete,
one_folder, (), (), (), ())
logging.info("unpack_magic finished on %s", filename)
else:
nzo.set_unpack_info(
'Unpack',
T('No post-processing because of failed verification'))
if cfg.safe_postproc():
all_ok = all_ok and not unpack_error
if all_ok:
## Move any (left-over) files to destination
nzo.status = 'Moving'
nzo.set_action_line(T('Moving'), '...')
for root, dirs, files in os.walk(workdir):
if not root.endswith(JOB_ADMIN):
for file_ in files:
path = os.path.join(root, file_)
new_path = path.replace(workdir,
tmp_workdir_complete)
new_path = get_unique_filename(new_path)
move_to_path(path, new_path, unique=False)
## Set permissions right
if not sabnzbd.WIN32:
perm_script(tmp_workdir_complete, cfg.umask())
if all_ok:
## Remove files matching the cleanup list
cleanup_list(tmp_workdir_complete, True)
## Check if this is an NZB-only download, if so redirect to queue
## except when PP was Download-only
if flag_repair:
nzb_list = nzb_redirect(tmp_workdir_complete,
nzo.final_name,
nzo.pp,
script,
cat,
priority=nzo.priority)
else:
nzb_list = None
if nzb_list:
nzo.set_unpack_info(
'Download',
T('Sent %s to queue') % unicoder(nzb_list))
try:
remove_dir(tmp_workdir_complete)
except:
pass
else:
cleanup_list(tmp_workdir_complete, False)
script_output = ''
script_ret = 0
if not nzb_list:
## Give destination its final name
if cfg.folder_rename() and tmp_workdir_complete and not one_folder:
if not all_ok:
workdir_complete = tmp_workdir_complete.replace(
'_UNPACK_', '_FAILED_')
workdir_complete = get_unique_path(workdir_complete,
n=0,
create_dir=False)
try:
collapse_folder(tmp_workdir_complete, workdir_complete)
except:
logging.error(Ta('Error renaming "%s" to "%s"'),
tmp_workdir_complete, workdir_complete)
logging.info("Traceback: ", exc_info=True)
job_result = int(par_error) + int(unpack_error) * 2
if cfg.ignore_samples() > 0:
remove_samples(workdir_complete)
## TV/Movie/Date Renaming code part 2 - rename and move files to parent folder
if all_ok:
if newfiles and file_sorter.is_sortfile():
file_sorter.rename(newfiles, workdir_complete)
workdir_complete = file_sorter.move(workdir_complete)
## Run the user script
script_path = make_script_path(script)
if all_ok and (not nzb_list) and script_path:
#set the current nzo status to "Ext Script...". Used in History
nzo.status = 'Running'
nzo.set_action_line(T('Running script'), unicoder(script))
nzo.set_unpack_info('Script',
T('Running user script %s') %
unicoder(script),
unique=True)
script_log, script_ret = external_processing(
script_path, workdir_complete, nzo.filename, msgid,
dirname, cat, nzo.group, job_result)
script_line = get_last_line(script_log)
if script_log:
script_output = nzo.nzo_id
if script_line:
nzo.set_unpack_info('Script',
unicoder(script_line),
unique=True)
else:
nzo.set_unpack_info('Script',
T('Ran %s') % unicoder(script),
unique=True)
else:
script = ""
script_line = ""
script_ret = 0
## Email the results
if (not nzb_list) and cfg.email_endjob():
if (cfg.email_endjob() == 1) or (cfg.email_endjob() == 2 and
(unpack_error or par_error)):
emailer.endjob(dirname, msgid, cat, all_ok, workdir_complete,
nzo.bytes_downloaded, nzo.unpack_info, script,
TRANS(script_log), script_ret)
if script_output:
# Can do this only now, otherwise it would show up in the email
if script_ret:
script_ret = 'Exit(%s) ' % script_ret
else:
script_ret = ''
if script_line:
nzo.set_unpack_info(
'Script',
u'%s%s <a href="./scriptlog?name=%s">(%s)</a>' %
(script_ret, unicoder(script_line),
urllib.quote(script_output), T('More')),
unique=True)
else:
nzo.set_unpack_info('Script',
u'%s<a href="./scriptlog?name=%s">%s</a>' %
(script_ret, urllib.quote(script_output),
T('View script output')),
unique=True)
## Cleanup again, including NZB files
cleanup_list(workdir_complete, False)
## Remove newzbin bookmark, if any
if msgid and all_ok:
Bookmarks.do.del_bookmark(msgid)
elif all_ok:
sabnzbd.proxy_rm_bookmark(nzo.url)
## Show final status in history
if all_ok:
osx.sendGrowlMsg(T('Download Completed'), filename,
osx.NOTIFICATION['complete'])
nzo.status = 'Completed'
else:
osx.sendGrowlMsg(T('Download Failed'), filename,
osx.NOTIFICATION['complete'])
nzo.status = 'Failed'
except:
logging.error(Ta('Post Processing Failed for %s (%s)'), filename,
crash_msg)
if not crash_msg:
logging.info("Traceback: ", exc_info=True)
crash_msg = T('see logfile')
nzo.fail_msg = T('PostProcessing was aborted (%s)') % unicoder(
crash_msg)
osx.sendGrowlMsg(T('Download Failed'), filename,
osx.NOTIFICATION['complete'])
nzo.status = 'Failed'
par_error = True
all_ok = False
if all_ok:
# If the folder only contains one file OR folder, have that as the path
# Be aware that series/generic/date sorting may move a single file into a folder containing other files
workdir_complete = one_file_or_folder(workdir_complete)
workdir_complete = os.path.normpath(workdir_complete)
# Log the overall time taken for postprocessing
postproc_time = int(time.time() - start)
# Create the history DB instance
history_db = database.get_history_handle()
# Add the nzo to the database. Only the path, script and time taken is passed
# Other information is obtained from the nzo
history_db.add_history_db(nzo, workdir_complete, nzo.downpath,
postproc_time, script_log, script_line)
# The connection is only used once, so close it here
history_db.close()
## Clean up the NZO
try:
logging.info('Cleaning up %s (keep_basic=%s)', filename,
str(not all_ok))
sabnzbd.nzbqueue.NzbQueue.do.cleanup_nzo(nzo, keep_basic=not all_ok)
except:
logging.error(Ta('Cleanup of %s failed.'), nzo.final_name)
logging.info("Traceback: ", exc_info=True)
## Remove download folder
if all_ok:
try:
if os.path.exists(workdir):
logging.debug('Removing workdir %s', workdir)
remove_all(workdir, recursive=True)
except:
logging.error(Ta('Error removing workdir (%s)'), workdir)
logging.info("Traceback: ", exc_info=True)
return True
def parring(nzo, workdir):
""" Perform par processing. Returns: (par_error, re_add)
"""
assert isinstance(nzo, sabnzbd.nzbstuff.NzbObject)
filename = nzo.final_name
growler.send_notification(T('Post-processing'), nzo.final_name, 'pp')
logging.info('Par2 check starting on %s', filename)
## Collect the par files
if nzo.partable:
par_table = nzo.partable.copy()
else:
par_table = {}
repair_sets = par_table.keys()
re_add = False
par_error = False
if repair_sets:
for set_ in repair_sets:
logging.info("Running repair on set %s", set_)
parfile_nzf = par_table[set_]
need_re_add, res = par2_repair(parfile_nzf, nzo, workdir, set_)
if need_re_add:
re_add = True
par_error = par_error or not res
if re_add:
logging.info('Readded %s to queue', filename)
nzo.priority = REPAIR_PRIORITY
sabnzbd.nzbqueue.add_nzo(nzo)
sabnzbd.downloader.Downloader.do.resume_from_postproc()
logging.info('Par2 check finished on %s', filename)
if (par_error and not re_add) or not repair_sets:
# See if alternative SFV check is possible
if cfg.sfv_check():
sfvs = globber(workdir, '*.sfv')
else:
sfvs = None
if sfvs:
par_error = False
nzo.set_unpack_info('Repair', T('Trying SFV verification'))
for sfv in sfvs:
failed = sfv_check(sfv)
if failed:
msg = T('Some files failed to verify against "%s"'
) % unicoder(os.path.basename(sfv))
msg += '; '
msg += '; '.join(failed)
nzo.set_unpack_info('Repair', msg)
par_error = True
if not par_error:
nzo.set_unpack_info('Repair',
T('Verified successfully using SFV files'))
elif not repair_sets:
logging.info("No par2 sets for %s", filename)
nzo.set_unpack_info('Repair',
T('[%s] No par2 sets') % unicoder(filename))
if not par_error:
flag_file(workdir, VERIFIED_FILE, create=True)
return par_error, re_add
def process_job(nzo):
""" Process one job """
assert isinstance(nzo, sabnzbd.nzbstuff.NzbObject)
start = time.time()
# keep track of whether we can continue
all_ok = True
# keep track of par problems
par_error = False
# keep track of any unpacking errors
unpack_error = False
# Signal empty download, for when 'empty_postproc' is enabled
empty = False
nzb_list = []
# These need to be initialised incase of a crash
workdir_complete = ''
postproc_time = 0
script_log = ''
script_line = ''
crash_msg = ''
## Get the job flags
nzo.save_attribs()
flag_repair, flag_unpack, flag_delete = nzo.repair_opts
# Normalize PP
if flag_delete: flag_unpack = True
if flag_unpack: flag_repair = True
# Get the NZB name
filename = nzo.final_name
msgid = nzo.msgid
if cfg.allow_streaming() and not (flag_repair or flag_unpack or flag_delete):
# After streaming, force +D
nzo.set_pp(3)
nzo.status = Status.FAILED
nzo.save_attribs()
all_ok = False
if nzo.fail_msg: # Special case: aborted due to too many missing data
nzo.status = Status.FAILED
nzo.save_attribs()
all_ok = False
par_error = unpack_error = True
try:
# Get the folder containing the download result
workdir = nzo.downpath
tmp_workdir_complete = None
# if no files are present (except __admin__), fail the job
if all_ok and len(globber(workdir)) < 2:
if nzo.precheck:
enough, ratio = nzo.check_quality()
req_ratio = float(cfg.req_completion_rate()) / 100.0
# Make sure that rounded ratio doesn't equal required ratio
# when it is actually below required
if (ratio < req_ratio) and (req_ratio - ratio) < 0.001:
ratio = req_ratio - 0.001
emsg = '%.1f%%' % (ratio * 100.0)
emsg2 = '%.1f%%' % float(cfg.req_completion_rate())
emsg = T('Download might fail, only %s of required %s available') % (emsg, emsg2)
else:
emsg = T('Download failed - Out of your server\'s retention?')
empty = True
nzo.fail_msg = emsg
nzo.set_unpack_info('Fail', emsg)
nzo.status = Status.FAILED
# do not run unpacking or parity verification
flag_repair = flag_unpack = False
all_ok = cfg.empty_postproc() and empty
if not all_ok:
par_error = unpack_error = True
script = nzo.script
cat = nzo.cat
logging.info('Starting PostProcessing on %s' + \
' => Repair:%s, Unpack:%s, Delete:%s, Script:%s, Cat:%s',
filename, flag_repair, flag_unpack, flag_delete, script, cat)
## Par processing, if enabled
if all_ok and flag_repair:
par_error, re_add = parring(nzo, workdir)
if re_add:
# Try to get more par files
return False
## Check if user allows unsafe post-processing
if flag_repair and cfg.safe_postproc():
all_ok = all_ok and not par_error
# Set complete dir to workdir in case we need to abort
workdir_complete = workdir
dirname = nzo.final_name
marker_file = None
if all_ok:
one_folder = False
## Determine class directory
if cfg.create_group_folders():
complete_dir = addPrefixes(cfg.complete_dir.get_path(), nzo.dirprefix)
complete_dir = create_dirs(complete_dir)
else:
catdir = config.get_categories(cat).dir()
if catdir.endswith('*'):
catdir = catdir.strip('*')
one_folder = True
complete_dir = real_path(cfg.complete_dir.get_path(), catdir)
## TV/Movie/Date Renaming code part 1 - detect and construct paths
file_sorter = Sorter(cat)
complete_dir = file_sorter.detect(dirname, complete_dir)
if file_sorter.sort_file:
one_folder = False
if one_folder:
workdir_complete = create_dirs(complete_dir)
else:
workdir_complete = get_unique_path(os.path.join(complete_dir, dirname), create_dir=True)
marker_file = set_marker(workdir_complete)
if not workdir_complete or not os.path.exists(workdir_complete):
crash_msg = T('Cannot create final folder %s') % unicoder(os.path.join(complete_dir, dirname))
raise IOError
if cfg.folder_rename() and not one_folder:
tmp_workdir_complete = prefix(workdir_complete, '_UNPACK_')
try:
renamer(workdir_complete, tmp_workdir_complete)
except:
pass # On failure, just use the original name
else:
tmp_workdir_complete = workdir_complete
newfiles = []
## Run Stage 2: Unpack
if flag_unpack:
if all_ok:
#set the current nzo status to "Extracting...". Used in History
nzo.status = Status.EXTRACTING
logging.info("Running unpack_magic on %s", filename)
unpack_error, newfiles = unpack_magic(nzo, workdir, tmp_workdir_complete, flag_delete, one_folder, (), (), (), ())
logging.info("unpack_magic finished on %s", filename)
else:
nzo.set_unpack_info('Unpack', T('No post-processing because of failed verification'))
if cfg.safe_postproc():
all_ok = all_ok and not unpack_error
if all_ok:
## Move any (left-over) files to destination
nzo.status = Status.MOVING
nzo.set_action_line(T('Moving'), '...')
for root, dirs, files in os.walk(workdir):
if not root.endswith(JOB_ADMIN):
for file_ in files:
path = os.path.join(root, file_)
new_path = path.replace(workdir, tmp_workdir_complete)
ok, new_path = move_to_path(path, new_path)
newfiles.append(new_path)
if not ok:
nzo.set_unpack_info('Unpack', T('Failed moving %s to %s') % (unicoder(path), unicoder(new_path)))
all_ok = False
break
## Set permissions right
set_permissions(tmp_workdir_complete)
if all_ok and marker_file:
del_marker(os.path.join(tmp_workdir_complete, marker_file))
remove_from_list(marker_file, newfiles)
if all_ok:
## Remove files matching the cleanup list
cleanup_list(tmp_workdir_complete, True)
## Check if this is an NZB-only download, if so redirect to queue
## except when PP was Download-only
if flag_repair:
nzb_list = nzb_redirect(tmp_workdir_complete, nzo.final_name, nzo.pp, script, cat, priority=nzo.priority)
else:
nzb_list = None
if nzb_list:
nzo.set_unpack_info('Download', T('Sent %s to queue') % unicoder(nzb_list))
cleanup_empty_directories(tmp_workdir_complete)
else:
cleanup_list(tmp_workdir_complete, False)
script_output = ''
script_ret = 0
if not nzb_list:
## Give destination its final name
if cfg.folder_rename() and tmp_workdir_complete and not one_folder:
if all_ok:
try:
newfiles = rename_and_collapse_folder(tmp_workdir_complete, workdir_complete, newfiles)
except:
logging.error(Ta('Error renaming "%s" to "%s"'), tmp_workdir_complete, workdir_complete)
logging.info('Traceback: ', exc_info = True)
# Better disable sorting because filenames are all off now
file_sorter.sort_file = None
else:
workdir_complete = tmp_workdir_complete.replace('_UNPACK_', '_FAILED_')
workdir_complete = get_unique_path(workdir_complete, n=0, create_dir=False)
if empty:
job_result = -1
else:
job_result = int(par_error) + int(unpack_error)*2
if cfg.ignore_samples() > 0:
remove_samples(workdir_complete)
## TV/Movie/Date Renaming code part 2 - rename and move files to parent folder
if all_ok and file_sorter.sort_file:
if newfiles:
file_sorter.rename(newfiles, workdir_complete)
workdir_complete, ok = file_sorter.move(workdir_complete)
else:
workdir_complete, ok = file_sorter.rename_with_ext(workdir_complete)
if not ok:
nzo.set_unpack_info('Unpack', T('Failed to move files'))
all_ok = False
## Run the user script
script_path = make_script_path(script)
if (all_ok or not cfg.safe_postproc()) and (not nzb_list) and script_path:
#set the current nzo status to "Ext Script...". Used in History
nzo.status = Status.RUNNING
nzo.set_action_line(T('Running script'), unicoder(script))
nzo.set_unpack_info('Script', T('Running user script %s') % unicoder(script), unique=True)
script_log, script_ret = external_processing(script_path, workdir_complete, nzo.filename,
msgid, dirname, cat, nzo.group, job_result)
script_line = get_last_line(script_log)
if script_log:
script_output = nzo.nzo_id
if script_line:
nzo.set_unpack_info('Script', unicoder(script_line), unique=True)
else:
nzo.set_unpack_info('Script', T('Ran %s') % unicoder(script), unique=True)
else:
script = ""
script_line = ""
script_ret = 0
## Email the results
if (not nzb_list) and cfg.email_endjob():
if (cfg.email_endjob() == 1) or (cfg.email_endjob() == 2 and (unpack_error or par_error)):
emailer.endjob(dirname, msgid, cat, all_ok, workdir_complete, nzo.bytes_downloaded,
nzo.fail_msg, nzo.unpack_info, script, TRANS(script_log), script_ret)
if script_output:
# Can do this only now, otherwise it would show up in the email
if script_ret:
script_ret = 'Exit(%s) ' % script_ret
else:
script_ret = ''
if script_line:
nzo.set_unpack_info('Script',
u'%s%s <a href="./scriptlog?name=%s">(%s)</a>' % (script_ret, unicoder(script_line), urllib.quote(script_output),
T('More')), unique=True)
else:
nzo.set_unpack_info('Script',
u'%s<a href="./scriptlog?name=%s">%s</a>' % (script_ret, urllib.quote(script_output),
T('View script output')), unique=True)
## Cleanup again, including NZB files
if all_ok:
cleanup_list(workdir_complete, False)
## Remove newzbin bookmark, if any
if msgid and all_ok:
Bookmarks.do.del_bookmark(msgid)
elif all_ok and isinstance(nzo.url, str):
sabnzbd.proxy_rm_bookmark(nzo.url)
## Force error for empty result
all_ok = all_ok and not empty
## Show final status in history
if all_ok:
growler.send_notification(T('Download Completed'), filename, 'complete')
nzo.status = Status.COMPLETED
else:
growler.send_notification(T('Download Failed'), filename, 'complete')
nzo.status = Status.FAILED
except:
logging.error(Ta('Post Processing Failed for %s (%s)'), filename, crash_msg)
if not crash_msg:
logging.info("Traceback: ", exc_info = True)
crash_msg = T('see logfile')
nzo.fail_msg = T('PostProcessing was aborted (%s)') % unicoder(crash_msg)
growler.send_notification(T('Download Failed'), filename, 'complete')
nzo.status = Status.FAILED
par_error = True
all_ok = False
if cfg.email_endjob():
emailer.endjob(dirname, msgid, cat, all_ok, workdir_complete, nzo.bytes_downloaded,
nzo.fail_msg, nzo.unpack_info, '', '', 0)
if all_ok:
# If the folder only contains one file OR folder, have that as the path
# Be aware that series/generic/date sorting may move a single file into a folder containing other files
workdir_complete = one_file_or_folder(workdir_complete)
workdir_complete = os.path.normpath(workdir_complete)
# Log the overall time taken for postprocessing
postproc_time = int(time.time() - start)
# Create the history DB instance
history_db = database.get_history_handle()
# Add the nzo to the database. Only the path, script and time taken is passed
# Other information is obtained from the nzo
history_db.add_history_db(nzo, workdir_complete, nzo.downpath, postproc_time, script_log, script_line)
# The connection is only used once, so close it here
history_db.close()
## Clean up the NZO
try:
logging.info('Cleaning up %s (keep_basic=%s)', filename, str(not all_ok))
sabnzbd.nzbqueue.NzbQueue.do.cleanup_nzo(nzo, keep_basic=not all_ok)
except:
logging.error(Ta('Cleanup of %s failed.'), nzo.final_name)
logging.info("Traceback: ", exc_info = True)
## Remove download folder
if all_ok:
try:
if os.path.exists(workdir):
logging.debug('Removing workdir %s', workdir)
remove_all(workdir, recursive=True)
except:
logging.error(Ta('Error removing workdir (%s)'), workdir)
logging.info("Traceback: ", exc_info = True)
return True
def initialize(pause_downloader=False,
clean_up=False,
evalSched=False,
repair=0):
global __INITIALIZED__, __SHUTTING_DOWN__,\
LOGFILE, WEBLOGFILE, LOGHANDLER, GUIHANDLER, AMBI_LOCALHOST, WAITEXIT, \
DAEMON, MY_NAME, MY_FULLNAME, NEW_VERSION, \
DIR_HOME, DIR_APPDATA, DIR_LCLDATA, DIR_PROG , DIR_INTERFACES, \
DARWIN, RESTART_REQ, OLD_QUEUE
if __INITIALIZED__:
return False
__SHUTTING_DOWN__ = False
### Set global database connection for Web-UI threads
cherrypy.engine.subscribe('start_thread', connect_db)
### Paused?
pause_downloader = pause_downloader or cfg.start_paused()
### Clean-up, if requested
if clean_up:
# Old cache folder
misc.remove_all(cfg.cache_dir.get_path(), '*.sab')
misc.remove_all(cfg.cache_dir.get_path(), 'SABnzbd_*')
# New admin folder
misc.remove_all(cfg.admin_dir.get_path(), '*.sab')
### Optionally wait for "incomplete" to become online
if cfg.wait_for_dfolder():
wait_for_download_folder()
else:
cfg.download_dir.set(cfg.download_dir(), create=True)
cfg.download_dir.set_create(True)
### Set access rights for "incomplete" base folder
misc.set_permissions(cfg.download_dir.get_path(), recursive=False)
### If dirscan_dir cannot be created, set a proper value anyway.
### Maybe it's a network path that's temporarily missing.
path = cfg.dirscan_dir.get_path()
if not os.path.exists(path):
sabnzbd.misc.create_real_path(cfg.dirscan_dir.ident(), '', path, False)
### Set call backs for Config items
cfg.cache_limit.callback(new_limit)
cfg.cherryhost.callback(guard_restart)
cfg.cherryport.callback(guard_restart)
cfg.web_dir.callback(guard_restart)
cfg.web_dir2.callback(guard_restart)
cfg.web_color.callback(guard_restart)
cfg.web_color2.callback(guard_restart)
cfg.log_dir.callback(guard_restart)
cfg.cache_dir.callback(guard_restart)
cfg.https_port.callback(guard_restart)
cfg.https_cert.callback(guard_restart)
cfg.https_key.callback(guard_restart)
cfg.enable_https.callback(guard_restart)
cfg.bandwidth_limit.callback(guard_speedlimit)
cfg.top_only.callback(guard_top_only)
cfg.pause_on_post_processing.callback(guard_pause_on_pp)
cfg.growl_server.callback(sabnzbd.growler.change_value)
cfg.growl_password.callback(sabnzbd.growler.change_value)
cfg.quota_size.callback(guard_quota_size)
cfg.quota_day.callback(guard_quota_dp)
cfg.quota_period.callback(guard_quota_dp)
cfg.fsys_type.callback(guard_fsys_type)
cfg.language.callback(sabnzbd.growler.reset_growl)
### Set Posix filesystem encoding
sabnzbd.encoding.change_fsys(cfg.fsys_type())
### Set cache limit
if (sabnzbd.WIN32 or sabnzbd.DARWIN) and not cfg.cache_limit():
cfg.cache_limit.set('200M')
ArticleCache.do.new_limit(cfg.cache_limit.get_int())
check_incomplete_vs_complete()
### Handle language upgrade from 0.5.x to 0.6.x
cfg.language.set(LANG_MAP.get(cfg.language(), cfg.language()))
### Set language files
lang.set_locale_info('SABnzbd', DIR_LANGUAGE)
lang.set_language(cfg.language())
sabnzbd.api.clear_trans_cache()
### Check for old queue (when a new queue is not present)
if not os.path.exists(
os.path.join(cfg.cache_dir.get_path(), QUEUE_FILE_NAME)):
OLD_QUEUE = bool(
misc.globber(cfg.cache_dir.get_path(), QUEUE_FILE_TMPL % '?'))
sabnzbd.change_queue_complete_action(cfg.queue_complete(), new=False)
if check_repair_request():
repair = 2
pause_downloader = True
else:
# Check crash detection file
#if load_admin(TERM_FLAG_FILE, remove=True):
# Repair mode 2 is a bit over an over-reaction!
pass # repair = 2
# Set crash detection file
#save_admin(1, TERM_FLAG_FILE)
###
### Initialize threads
###
Bookmarks()
rss.init()
paused = BPSMeter.do.read()
PostProcessor()
NzbQueue()
Assembler()
NzbQueue.do.read_queue(repair)
Downloader(pause_downloader or paused)
DirScanner()
MSGIDGrabber()
Rating()
URLGrabber()
scheduler.init()
if evalSched:
scheduler.analyse(pause_downloader)
logging.info('All processes started')
RESTART_REQ = False
__INITIALIZED__ = True
return True
def process_job(nzo):
""" Process one job """
start = time.time()
# keep track of whether we can continue
all_ok = True
# keep track of par problems
par_error = False
# keep track of any unpacking errors
unpack_error = False
# Signal empty download, for when 'empty_postproc' is enabled
empty = False
nzb_list = []
# These need to be initialized in case of a crash
workdir_complete = ''
script_log = ''
script_line = ''
# Get the job flags
nzo.save_attribs()
flag_repair, flag_unpack, flag_delete = nzo.repair_opts
# Normalize PP
if flag_delete:
flag_unpack = True
if flag_unpack:
flag_repair = True
# Get the NZB name
filename = nzo.final_name
if nzo.fail_msg: # Special case: aborted due to too many missing data
nzo.status = Status.FAILED
nzo.save_attribs()
all_ok = False
par_error = True
unpack_error = 1
try:
# Get the folder containing the download result
workdir = nzo.downpath
tmp_workdir_complete = None
# if no files are present (except __admin__), fail the job
if all_ok and len(globber(workdir)) < 2:
if nzo.precheck:
_enough, ratio = nzo.check_quality()
req_ratio = float(cfg.req_completion_rate()) / 100.0
# Make sure that rounded ratio doesn't equal required ratio
# when it is actually below required
if (ratio < req_ratio) and (req_ratio - ratio) < 0.001:
ratio = req_ratio - 0.001
emsg = '%.1f%%' % (ratio * 100.0)
emsg2 = '%.1f%%' % float(cfg.req_completion_rate())
emsg = T(
'Download might fail, only %s of required %s available'
) % (emsg, emsg2)
else:
emsg = T('Download failed - Not on your server(s)')
empty = True
emsg += ' - https://sabnzbd.org/not-complete'
nzo.fail_msg = emsg
nzo.set_unpack_info('Fail', emsg)
nzo.status = Status.FAILED
# do not run unpacking or parity verification
flag_repair = flag_unpack = False
all_ok = cfg.empty_postproc() and empty
if not all_ok:
par_error = True
unpack_error = 1
script = nzo.script
logging.info(
'Starting Post-Processing on %s' +
' => Repair:%s, Unpack:%s, Delete:%s, Script:%s, Cat:%s', filename,
flag_repair, flag_unpack, flag_delete, script, nzo.cat)
# Set complete dir to workdir in case we need to abort
workdir_complete = workdir
# Par processing, if enabled
if all_ok and flag_repair:
par_error, re_add = parring(nzo, workdir)
if re_add:
# Try to get more par files
return False
# If we don't need extra par2, we can disconnect
if sabnzbd.nzbqueue.NzbQueue.do.actives(
grabs=False) == 0 and cfg.autodisconnect():
# This was the last job, close server connections
sabnzbd.downloader.Downloader.do.disconnect()
# Sanitize the resulting files
if sabnzbd.WIN32:
sanitize_files_in_folder(workdir)
# Check if user allows unsafe post-processing
if flag_repair and cfg.safe_postproc():
all_ok = all_ok and not par_error
if all_ok:
# Fix encodings
fix_unix_encoding(workdir)
# Use dirs generated by direct-unpacker
if nzo.direct_unpacker and nzo.direct_unpacker.unpack_dir_info:
tmp_workdir_complete, workdir_complete, file_sorter, one_folder, marker_file = nzo.direct_unpacker.unpack_dir_info
else:
# Generate extraction path
tmp_workdir_complete, workdir_complete, file_sorter, one_folder, marker_file = prepare_extraction_path(
nzo)
newfiles = []
# Run Stage 2: Unpack
if flag_unpack:
# set the current nzo status to "Extracting...". Used in History
nzo.status = Status.EXTRACTING
logging.info("Running unpack_magic on %s", filename)
unpack_error, newfiles = unpack_magic(nzo, workdir,
tmp_workdir_complete,
flag_delete, one_folder,
(), (), (), (), ())
logging.info("Unpacked files %s", newfiles)
if sabnzbd.WIN32:
# Sanitize the resulting files
newfiles = sanitize_files_in_folder(tmp_workdir_complete)
logging.info("Finished unpack_magic on %s", filename)
if cfg.safe_postproc():
all_ok = all_ok and not unpack_error
if all_ok:
# Move any (left-over) files to destination
nzo.status = Status.MOVING
nzo.set_action_line(T('Moving'), '...')
for root, _dirs, files in os.walk(workdir):
if not root.endswith(JOB_ADMIN):
for file_ in files:
path = os.path.join(root, file_)
new_path = path.replace(workdir,
tmp_workdir_complete)
ok, new_path = move_to_path(path, new_path)
if new_path:
newfiles.append(new_path)
if not ok:
nzo.set_unpack_info(
'Unpack',
T('Failed moving %s to %s') %
(unicoder(path), unicoder(new_path)))
all_ok = False
break
# Set permissions right
set_permissions(tmp_workdir_complete)
if all_ok and marker_file:
del_marker(os.path.join(tmp_workdir_complete, marker_file))
remove_from_list(marker_file, newfiles)
if all_ok:
# Remove files matching the cleanup list
cleanup_list(tmp_workdir_complete, True)
# Check if this is an NZB-only download, if so redirect to queue
# except when PP was Download-only
if flag_repair:
nzb_list = nzb_redirect(tmp_workdir_complete,
nzo.final_name,
nzo.pp,
script,
nzo.cat,
priority=nzo.priority)
else:
nzb_list = None
if nzb_list:
nzo.set_unpack_info(
'Download',
T('Sent %s to queue') % unicoder(nzb_list))
cleanup_empty_directories(tmp_workdir_complete)
else:
cleanup_list(tmp_workdir_complete, False)
script_output = ''
script_ret = 0
if not nzb_list:
# Give destination its final name
if cfg.folder_rename() and tmp_workdir_complete and not one_folder:
if all_ok:
try:
newfiles = rename_and_collapse_folder(
tmp_workdir_complete, workdir_complete, newfiles)
except:
logging.error(T('Error renaming "%s" to "%s"'),
clip_path(tmp_workdir_complete),
clip_path(workdir_complete))
logging.info('Traceback: ', exc_info=True)
# Better disable sorting because filenames are all off now
file_sorter.sort_file = None
else:
workdir_complete = tmp_workdir_complete.replace(
'_UNPACK_', '_FAILED_')
workdir_complete = get_unique_path(workdir_complete,
n=0,
create_dir=False)
if empty:
job_result = -1
else:
job_result = int(par_error) + int(bool(unpack_error)) * 2
if cfg.ignore_samples():
remove_samples(workdir_complete)
# TV/Movie/Date Renaming code part 2 - rename and move files to parent folder
if all_ok and file_sorter.sort_file:
if newfiles:
file_sorter.rename(newfiles, workdir_complete)
workdir_complete, ok = file_sorter.move(workdir_complete)
else:
workdir_complete, ok = file_sorter.rename_with_ext(
workdir_complete)
if not ok:
nzo.set_unpack_info('Unpack', T('Failed to move files'))
all_ok = False
# Run the user script
script_path = make_script_path(script)
if (all_ok or not cfg.safe_postproc()) and (
not nzb_list) and script_path:
# Set the current nzo status to "Ext Script...". Used in History
nzo.status = Status.RUNNING
nzo.set_action_line(T('Running script'), unicoder(script))
nzo.set_unpack_info('Script',
T('Running user script %s') %
unicoder(script),
unique=True)
script_log, script_ret = external_processing(
script_path, nzo, clip_path(workdir_complete),
nzo.final_name, job_result)
script_line = get_last_line(script_log)
if script_log:
script_output = nzo.nzo_id
if script_line:
nzo.set_unpack_info('Script',
unicoder(script_line),
unique=True)
else:
nzo.set_unpack_info('Script',
T('Ran %s') % unicoder(script),
unique=True)
else:
script = ""
script_line = ""
script_ret = 0
# Maybe bad script result should fail job
if script_ret and cfg.script_can_fail():
script_error = True
all_ok = False
nzo.fail_msg = T('Script exit code is %s') % script_ret
else:
script_error = False
# Email the results
if (not nzb_list) and cfg.email_endjob():
if (cfg.email_endjob()
== 1) or (cfg.email_endjob() == 2 and
(unpack_error or par_error or script_error)):
emailer.endjob(nzo.final_name, nzo.cat, all_ok,
workdir_complete, nzo.bytes_downloaded,
nzo.fail_msg, nzo.unpack_info, script,
TRANS(script_log), script_ret)
if script_output:
# Can do this only now, otherwise it would show up in the email
if script_ret:
script_ret = 'Exit(%s) ' % script_ret
else:
script_ret = ''
if len(script_log.rstrip().split('\n')) > 1:
nzo.set_unpack_info(
'Script',
u'%s%s <a href="./scriptlog?name=%s">(%s)</a>' %
(script_ret, script_line,
xml.sax.saxutils.escape(script_output), T('More')),
unique=True)
else:
# No '(more)' button needed
nzo.set_unpack_info('Script',
u'%s%s ' % (script_ret, script_line),
unique=True)
# Cleanup again, including NZB files
if all_ok:
cleanup_list(workdir_complete, False)
# Force error for empty result
all_ok = all_ok and not empty
# Update indexer with results
if cfg.rating_enable():
if nzo.encrypted > 0:
Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_ENCRYPTED)
if empty:
hosts = map(lambda s: s.host,
sabnzbd.downloader.Downloader.do.nzo_servers(nzo))
if not hosts:
hosts = [None]
for host in hosts:
Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_EXPIRED,
host)
except:
logging.error(T('Post Processing Failed for %s (%s)'), filename,
T('see logfile'))
logging.info("Traceback: ", exc_info=True)
nzo.fail_msg = T('PostProcessing was aborted (%s)') % T('see logfile')
notifier.send_notification(T('Download Failed'), filename, 'failed',
nzo.cat)
nzo.status = Status.FAILED
par_error = True
all_ok = False
if cfg.email_endjob():
emailer.endjob(nzo.final_name, nzo.cat, all_ok,
clip_path(workdir_complete), nzo.bytes_downloaded,
nzo.fail_msg, nzo.unpack_info, '', '', 0)
if all_ok:
# If the folder only contains one file OR folder, have that as the path
# Be aware that series/generic/date sorting may move a single file into a folder containing other files
workdir_complete = one_file_or_folder(workdir_complete)
workdir_complete = os.path.normpath(workdir_complete)
# Clean up the NZO
try:
logging.info('Cleaning up %s (keep_basic=%s)', filename,
str(not all_ok))
sabnzbd.nzbqueue.NzbQueue.do.cleanup_nzo(nzo, keep_basic=not all_ok)
except:
logging.error(T('Cleanup of %s failed.'), nzo.final_name)
logging.info("Traceback: ", exc_info=True)
# Remove download folder
if all_ok:
try:
if os.path.exists(workdir):
logging.debug('Removing workdir %s', workdir)
remove_all(workdir, recursive=True)
except:
logging.error(T('Error removing workdir (%s)'), clip_path(workdir))
logging.info("Traceback: ", exc_info=True)
# Use automatic retry link on par2 errors and encrypted/bad RARs
if par_error or unpack_error in (2, 3):
try_alt_nzb(nzo)
# Show final status in history
if all_ok:
notifier.send_notification(T('Download Completed'), filename,
'complete', nzo.cat)
nzo.status = Status.COMPLETED
else:
notifier.send_notification(T('Download Failed'), filename, 'failed',
nzo.cat)
nzo.status = Status.FAILED
# Log the overall time taken for postprocessing
postproc_time = int(time.time() - start)
# Create the history DB instance
history_db = database.HistoryDB()
# Add the nzo to the database. Only the path, script and time taken is passed
# Other information is obtained from the nzo
history_db.add_history_db(nzo, clip_path(workdir_complete), nzo.downpath,
postproc_time, script_log, script_line)
# Purge items
history_db.auto_history_purge()
# The connection is only used once, so close it here
history_db.close()
sabnzbd.history_updated()
return True
def process_job(nzo):
""" Process one job """
start = time.time()
# keep track of whether we can continue
all_ok = True
# keep track of par problems
par_error = False
# keep track of any unpacking errors
unpack_error = False
# Signal empty download, for when 'empty_postproc' is enabled
empty = False
nzb_list = []
# These need to be initialized in case of a crash
workdir_complete = ''
script_log = ''
script_line = ''
# Get the job flags
nzo.save_attribs()
flag_repair, flag_unpack, flag_delete = nzo.repair_opts
# Normalize PP
if flag_delete:
flag_unpack = True
if flag_unpack:
flag_repair = True
# Get the NZB name
filename = nzo.final_name
if nzo.fail_msg: # Special case: aborted due to too many missing data
nzo.status = Status.FAILED
nzo.save_attribs()
all_ok = False
par_error = True
unpack_error = 1
try:
# Get the folder containing the download result
workdir = nzo.downpath
tmp_workdir_complete = None
# if no files are present (except __admin__), fail the job
if all_ok and len(globber(workdir)) < 2:
if nzo.precheck:
_enough, ratio = nzo.check_quality()
req_ratio = float(cfg.req_completion_rate()) / 100.0
# Make sure that rounded ratio doesn't equal required ratio
# when it is actually below required
if (ratio < req_ratio) and (req_ratio - ratio) < 0.001:
ratio = req_ratio - 0.001
emsg = '%.1f%%' % (ratio * 100.0)
emsg2 = '%.1f%%' % float(cfg.req_completion_rate())
emsg = T('Download might fail, only %s of required %s available') % (emsg, emsg2)
else:
emsg = T('Download failed - Not on your server(s)')
empty = True
emsg += ' - https://sabnzbd.org/not-complete'
nzo.fail_msg = emsg
nzo.set_unpack_info('Fail', emsg)
nzo.status = Status.FAILED
# do not run unpacking or parity verification
flag_repair = flag_unpack = False
all_ok = cfg.empty_postproc() and empty
if not all_ok:
par_error = True
unpack_error = 1
script = nzo.script
logging.info('Starting Post-Processing on %s' +
' => Repair:%s, Unpack:%s, Delete:%s, Script:%s, Cat:%s',
filename, flag_repair, flag_unpack, flag_delete, script, nzo.cat)
# Set complete dir to workdir in case we need to abort
workdir_complete = workdir
# Par processing, if enabled
if all_ok and flag_repair:
par_error, re_add = parring(nzo, workdir)
if re_add:
# Try to get more par files
return False
# If we don't need extra par2, we can disconnect
if sabnzbd.nzbqueue.NzbQueue.do.actives(grabs=False) == 0 and cfg.autodisconnect():
# This was the last job, close server connections
sabnzbd.downloader.Downloader.do.disconnect()
# Sanitize the resulting files
if sabnzbd.WIN32:
sanitize_files_in_folder(workdir)
# Check if user allows unsafe post-processing
if flag_repair and cfg.safe_postproc():
all_ok = all_ok and not par_error
if all_ok:
# Fix encodings
fix_unix_encoding(workdir)
# Use dirs generated by direct-unpacker
if nzo.direct_unpacker and nzo.direct_unpacker.unpack_dir_info:
tmp_workdir_complete, workdir_complete, file_sorter, one_folder, marker_file = nzo.direct_unpacker.unpack_dir_info
else:
# Generate extraction path
tmp_workdir_complete, workdir_complete, file_sorter, one_folder, marker_file = prepare_extraction_path(nzo)
newfiles = []
# Run Stage 2: Unpack
if flag_unpack:
if all_ok:
# set the current nzo status to "Extracting...". Used in History
nzo.status = Status.EXTRACTING
logging.info("Running unpack_magic on %s", filename)
unpack_error, newfiles = unpack_magic(nzo, workdir, tmp_workdir_complete, flag_delete, one_folder, (), (), (), (), ())
logging.info("Unpacked files %s", newfiles)
if sabnzbd.WIN32:
# Sanitize the resulting files
newfiles = sanitize_files_in_folder(tmp_workdir_complete)
logging.info("Finished unpack_magic on %s", filename)
else:
nzo.set_unpack_info('Unpack', T('No post-processing because of failed verification'))
if cfg.safe_postproc():
all_ok = all_ok and not unpack_error
if all_ok:
# Move any (left-over) files to destination
nzo.status = Status.MOVING
nzo.set_action_line(T('Moving'), '...')
for root, _dirs, files in os.walk(workdir):
if not root.endswith(JOB_ADMIN):
for file_ in files:
path = os.path.join(root, file_)
new_path = path.replace(workdir, tmp_workdir_complete)
ok, new_path = move_to_path(path, new_path)
if new_path:
newfiles.append(new_path)
if not ok:
nzo.set_unpack_info('Unpack', T('Failed moving %s to %s') % (unicoder(path), unicoder(new_path)))
all_ok = False
break
# Set permissions right
set_permissions(tmp_workdir_complete)
if all_ok and marker_file:
del_marker(os.path.join(tmp_workdir_complete, marker_file))
remove_from_list(marker_file, newfiles)
if all_ok:
# Remove files matching the cleanup list
cleanup_list(tmp_workdir_complete, True)
# Check if this is an NZB-only download, if so redirect to queue
# except when PP was Download-only
if flag_repair:
nzb_list = nzb_redirect(tmp_workdir_complete, nzo.final_name, nzo.pp, script, nzo.cat, priority=nzo.priority)
else:
nzb_list = None
if nzb_list:
nzo.set_unpack_info('Download', T('Sent %s to queue') % unicoder(nzb_list))
cleanup_empty_directories(tmp_workdir_complete)
else:
cleanup_list(tmp_workdir_complete, False)
script_output = ''
script_ret = 0
if not nzb_list:
# Give destination its final name
if cfg.folder_rename() and tmp_workdir_complete and not one_folder:
if all_ok:
try:
newfiles = rename_and_collapse_folder(tmp_workdir_complete, workdir_complete, newfiles)
except:
logging.error(T('Error renaming "%s" to "%s"'), clip_path(tmp_workdir_complete), clip_path(workdir_complete))
logging.info('Traceback: ', exc_info=True)
# Better disable sorting because filenames are all off now
file_sorter.sort_file = None
else:
workdir_complete = tmp_workdir_complete.replace('_UNPACK_', '_FAILED_')
workdir_complete = get_unique_path(workdir_complete, n=0, create_dir=False)
workdir_complete = workdir_complete
if empty:
job_result = -1
else:
job_result = int(par_error) + int(bool(unpack_error)) * 2
if cfg.ignore_samples():
remove_samples(workdir_complete)
# TV/Movie/Date Renaming code part 2 - rename and move files to parent folder
if all_ok and file_sorter.sort_file:
if newfiles:
file_sorter.rename(newfiles, workdir_complete)
workdir_complete, ok = file_sorter.move(workdir_complete)
else:
workdir_complete, ok = file_sorter.rename_with_ext(workdir_complete)
if not ok:
nzo.set_unpack_info('Unpack', T('Failed to move files'))
all_ok = False
# Run the user script
script_path = make_script_path(script)
if (all_ok or not cfg.safe_postproc()) and (not nzb_list) and script_path:
# Set the current nzo status to "Ext Script...". Used in History
nzo.status = Status.RUNNING
nzo.set_action_line(T('Running script'), unicoder(script))
nzo.set_unpack_info('Script', T('Running user script %s') % unicoder(script), unique=True)
script_log, script_ret = external_processing(script_path, nzo, clip_path(workdir_complete),
nzo.final_name, job_result)
script_line = get_last_line(script_log)
if script_log:
script_output = nzo.nzo_id
if script_line:
nzo.set_unpack_info('Script', unicoder(script_line), unique=True)
else:
nzo.set_unpack_info('Script', T('Ran %s') % unicoder(script), unique=True)
else:
script = ""
script_line = ""
script_ret = 0
# Maybe bad script result should fail job
if script_ret and cfg.script_can_fail():
script_error = True
all_ok = False
nzo.fail_msg = T('Script exit code is %s') % script_ret
else:
script_error = False
# Email the results
if (not nzb_list) and cfg.email_endjob():
if (cfg.email_endjob() == 1) or (cfg.email_endjob() == 2 and (unpack_error or par_error or script_error)):
emailer.endjob(nzo.final_name, nzo.cat, all_ok, workdir_complete, nzo.bytes_downloaded,
nzo.fail_msg, nzo.unpack_info, script, TRANS(script_log), script_ret)
if script_output:
# Can do this only now, otherwise it would show up in the email
if script_ret:
script_ret = 'Exit(%s) ' % script_ret
else:
script_ret = ''
if len(script_log.rstrip().split('\n')) > 1:
nzo.set_unpack_info('Script',
u'%s%s <a href="./scriptlog?name=%s">(%s)</a>' % (script_ret, script_line,
xml.sax.saxutils.escape(script_output), T('More')), unique=True)
else:
# No '(more)' button needed
nzo.set_unpack_info('Script', u'%s%s ' % (script_ret, script_line), unique=True)
# Cleanup again, including NZB files
if all_ok:
cleanup_list(workdir_complete, False)
# Force error for empty result
all_ok = all_ok and not empty
# Update indexer with results
if cfg.rating_enable():
if nzo.encrypted > 0:
Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_ENCRYPTED)
if empty:
hosts = map(lambda s: s.host, sabnzbd.downloader.Downloader.do.nzo_servers(nzo))
if not hosts:
hosts = [None]
for host in hosts:
Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_EXPIRED, host)
except:
logging.error(T('Post Processing Failed for %s (%s)'), filename, T('see logfile'))
logging.info("Traceback: ", exc_info=True)
nzo.fail_msg = T('PostProcessing was aborted (%s)') % T('see logfile')
notifier.send_notification(T('Download Failed'), filename, 'failed', nzo.cat)
nzo.status = Status.FAILED
par_error = True
all_ok = False
if cfg.email_endjob():
emailer.endjob(nzo.final_name, nzo.cat, all_ok, clip_path(workdir_complete), nzo.bytes_downloaded,
nzo.fail_msg, nzo.unpack_info, '', '', 0)
if all_ok:
# If the folder only contains one file OR folder, have that as the path
# Be aware that series/generic/date sorting may move a single file into a folder containing other files
workdir_complete = one_file_or_folder(workdir_complete)
workdir_complete = os.path.normpath(workdir_complete)
# Clean up the NZO
try:
logging.info('Cleaning up %s (keep_basic=%s)', filename, str(not all_ok))
sabnzbd.nzbqueue.NzbQueue.do.cleanup_nzo(nzo, keep_basic=not all_ok)
except:
logging.error(T('Cleanup of %s failed.'), nzo.final_name)
logging.info("Traceback: ", exc_info=True)
# Remove download folder
if all_ok:
try:
if os.path.exists(workdir):
logging.debug('Removing workdir %s', workdir)
remove_all(workdir, recursive=True)
except:
logging.error(T('Error removing workdir (%s)'), clip_path(workdir))
logging.info("Traceback: ", exc_info=True)
# Use automatic retry link on par2 errors and encrypted/bad RARs
if par_error or unpack_error in (2, 3):
try_alt_nzb(nzo)
# Show final status in history
if all_ok:
notifier.send_notification(T('Download Completed'), filename, 'complete', nzo.cat)
nzo.status = Status.COMPLETED
else:
notifier.send_notification(T('Download Failed'), filename, 'failed', nzo.cat)
nzo.status = Status.FAILED
# Log the overall time taken for postprocessing
postproc_time = int(time.time() - start)
# Create the history DB instance
history_db = database.HistoryDB()
# Add the nzo to the database. Only the path, script and time taken is passed
# Other information is obtained from the nzo
history_db.add_history_db(nzo, clip_path(workdir_complete), nzo.downpath, postproc_time, script_log, script_line)
# Purge items
history_db.auto_history_purge()
# The connection is only used once, so close it here
history_db.close()
sabnzbd.history_updated()
return True
def initialize(pause_downloader = False, clean_up = False, evalSched=False, repair=0):
global __INITIALIZED__, __SHUTTING_DOWN__,\
LOGFILE, WEBLOGFILE, LOGHANDLER, GUIHANDLER, AMBI_LOCALHOST, WAITEXIT, \
DAEMON, MY_NAME, MY_FULLNAME, NEW_VERSION, \
DIR_HOME, DIR_APPDATA, DIR_LCLDATA, DIR_PROG , DIR_INTERFACES, \
DARWIN, RESTART_REQ, OSX_ICON, OLD_QUEUE
if __INITIALIZED__:
return False
__SHUTTING_DOWN__ = False
### Set global database connection for Web-UI threads
cherrypy.engine.subscribe('start_thread', connect_db)
### Clean-up, if requested
if clean_up:
# Old cache folder
misc.remove_all(cfg.cache_dir.get_path(), '*.sab')
misc.remove_all(cfg.cache_dir.get_path(), 'SABnzbd_*')
# New admin folder
misc.remove_all(cfg.admin_dir.get_path(), '*.sab')
### If dirscan_dir cannot be created, set a proper value anyway.
### Maybe it's a network path that's temporarily missing.
path = cfg.dirscan_dir.get_path()
if not os.path.exists(path):
sabnzbd.misc.create_real_path(cfg.dirscan_dir.ident(), '', path, False)
### Set call backs for Config items
cfg.cache_limit.callback(new_limit)
cfg.cherryhost.callback(guard_restart)
cfg.cherryport.callback(guard_restart)
cfg.web_dir.callback(guard_restart)
cfg.web_dir2.callback(guard_restart)
cfg.web_color.callback(guard_restart)
cfg.web_color2.callback(guard_restart)
cfg.log_dir.callback(guard_restart)
cfg.cache_dir.callback(guard_restart)
cfg.https_port.callback(guard_restart)
cfg.https_cert.callback(guard_restart)
cfg.https_key.callback(guard_restart)
cfg.enable_https.callback(guard_restart)
cfg.bandwidth_limit.callback(guard_speedlimit)
cfg.top_only.callback(guard_top_only)
cfg.pause_on_post_processing.callback(guard_pause_on_pp)
### Set cache limit
ArticleCache.do.new_limit(cfg.cache_limit.get_int())
### Handle language upgrade from 0.5.x to 0.6.x
cfg.language.set(LANG_MAP.get(cfg.language(), cfg.language()))
### Set language files
lang.set_locale_info('SABnzbd', DIR_LANGUAGE)
lang.set_language(cfg.language())
sabnzbd.api.cache_skin_trans()
### Check for old queue (when a new queue is not present)
if not os.path.exists(os.path.join(cfg.cache_dir.get_path(), QUEUE_FILE_NAME)):
OLD_QUEUE = bool(misc.globber(cfg.cache_dir.get_path(), QUEUE_FILE_TMPL % '?'))
sabnzbd.change_queue_complete_action(cfg.queue_complete(), new=False)
if check_repair_request():
repair = 2
pause_downloader = True
else:
# Check crash detection file
#if load_admin(TERM_FLAG_FILE, remove=True):
# Repair mode 2 is a bit over an over-reaction!
pass # repair = 2
# Set crash detection file
#save_admin(1, TERM_FLAG_FILE)
###
### Initialize threads
###
Bookmarks()
rss.init()
BPSMeter.do.read()
PostProcessor()
NzbQueue()
NzbQueue.do.read_queue(repair)
Assembler()
Downloader(pause_downloader)
DirScanner()
MSGIDGrabber()
URLGrabber()
scheduler.init()
if evalSched:
scheduler.analyse(pause_downloader)
logging.info('All processes started')
RESTART_REQ = False
__INITIALIZED__ = True
return True
def process_job(nzo):
""" Process one job """
assert isinstance(nzo, sabnzbd.nzbstuff.NzbObject)
start = time.time()
# keep track of whether we can continue
all_ok = True
# keep track of par problems
par_error = False
# keep track of any unpacking errors
unpack_error = False
# Signal empty download, for when 'empty_postproc' is enabled
empty = False
nzb_list = []
# These need to be initialised incase of a crash
workdir_complete = ''
postproc_time = 0
script_log = ''
script_line = ''
crash_msg = ''
## Get the job flags
nzo.save_attribs()
flag_repair, flag_unpack, flag_delete = nzo.repair_opts
# Normalize PP
if flag_delete: flag_unpack = True
if flag_unpack: flag_repair = True
# Get the NZB name
filename = nzo.final_name
msgid = nzo.msgid
if cfg.allow_streaming() and not (flag_repair or flag_unpack
or flag_delete):
# After streaming, force +D
nzo.set_pp(3)
nzo.status = Status.FAILED
nzo.save_attribs()
all_ok = False
if nzo.fail_msg: # Special case: aborted due to too many missing data
nzo.status = Status.FAILED
nzo.save_attribs()
all_ok = False
par_error = unpack_error = True
try:
# Get the folder containing the download result
workdir = nzo.downpath
tmp_workdir_complete = None
# if no files are present (except __admin__), fail the job
if all_ok and len(globber(workdir)) < 2:
if nzo.precheck:
enough, ratio = nzo.check_quality()
req_ratio = float(cfg.req_completion_rate()) / 100.0
# Make sure that rounded ratio doesn't equal required ratio
# when it is actually below required
if (ratio < req_ratio) and (req_ratio - ratio) < 0.001:
ratio = req_ratio - 0.001
emsg = '%.1f%%' % (ratio * 100.0)
emsg2 = '%.1f%%' % float(cfg.req_completion_rate())
emsg = T(
'Download might fail, only %s of required %s available'
) % (emsg, emsg2)
else:
emsg = T('Download failed - Out of your server\'s retention?')
empty = True
nzo.fail_msg = emsg
nzo.set_unpack_info('Fail', emsg)
nzo.status = Status.FAILED
# do not run unpacking or parity verification
flag_repair = flag_unpack = False
all_ok = cfg.empty_postproc() and empty
if not all_ok:
par_error = unpack_error = True
script = nzo.script
cat = nzo.cat
logging.info('Starting PostProcessing on %s' + \
' => Repair:%s, Unpack:%s, Delete:%s, Script:%s, Cat:%s',
filename, flag_repair, flag_unpack, flag_delete, script, cat)
## Par processing, if enabled
if all_ok and flag_repair:
par_error, re_add = parring(nzo, workdir)
if re_add:
# Try to get more par files
return False
## Check if user allows unsafe post-processing
if flag_repair and cfg.safe_postproc():
all_ok = all_ok and not par_error
# Set complete dir to workdir in case we need to abort
workdir_complete = workdir
dirname = nzo.final_name
marker_file = None
if all_ok:
one_folder = False
## Determine class directory
if cfg.create_group_folders():
complete_dir = addPrefixes(cfg.complete_dir.get_path(),
nzo.dirprefix)
complete_dir = create_dirs(complete_dir)
else:
catdir = config.get_categories(cat).dir()
if catdir.endswith('*'):
catdir = catdir.strip('*')
one_folder = True
complete_dir = real_path(cfg.complete_dir.get_path(), catdir)
## TV/Movie/Date Renaming code part 1 - detect and construct paths
if cfg.enable_meta():
file_sorter = Sorter(nzo, cat)
else:
file_sorter = Sorter(None, cat)
complete_dir = file_sorter.detect(dirname, complete_dir)
if file_sorter.sort_file:
one_folder = False
if one_folder:
workdir_complete = create_dirs(complete_dir)
else:
workdir_complete = get_unique_path(os.path.join(
complete_dir, dirname),
create_dir=True)
marker_file = set_marker(workdir_complete)
if not workdir_complete or not os.path.exists(workdir_complete):
crash_msg = T('Cannot create final folder %s') % unicoder(
os.path.join(complete_dir, dirname))
raise IOError
if cfg.folder_rename() and not one_folder:
tmp_workdir_complete = prefix(workdir_complete, '_UNPACK_')
try:
renamer(workdir_complete, tmp_workdir_complete)
except:
pass # On failure, just use the original name
else:
tmp_workdir_complete = workdir_complete
newfiles = []
## Run Stage 2: Unpack
if flag_unpack:
if all_ok:
#set the current nzo status to "Extracting...". Used in History
nzo.status = Status.EXTRACTING
logging.info("Running unpack_magic on %s", filename)
unpack_error, newfiles = unpack_magic(
nzo, workdir, tmp_workdir_complete, flag_delete,
one_folder, (), (), (), ())
logging.info("unpack_magic finished on %s", filename)
else:
nzo.set_unpack_info(
'Unpack',
T('No post-processing because of failed verification'))
if cfg.safe_postproc():
all_ok = all_ok and not unpack_error
if all_ok:
## Move any (left-over) files to destination
nzo.status = Status.MOVING
nzo.set_action_line(T('Moving'), '...')
for root, dirs, files in os.walk(workdir):
if not root.endswith(JOB_ADMIN):
for file_ in files:
path = os.path.join(root, file_)
new_path = path.replace(workdir,
tmp_workdir_complete)
ok, new_path = move_to_path(path, new_path)
newfiles.append(new_path)
if not ok:
nzo.set_unpack_info(
'Unpack',
T('Failed moving %s to %s') %
(unicoder(path), unicoder(new_path)))
all_ok = False
break
## Set permissions right
set_permissions(tmp_workdir_complete)
if all_ok and marker_file:
del_marker(os.path.join(tmp_workdir_complete, marker_file))
remove_from_list(marker_file, newfiles)
if all_ok:
## Remove files matching the cleanup list
cleanup_list(tmp_workdir_complete, True)
## Check if this is an NZB-only download, if so redirect to queue
## except when PP was Download-only
if flag_repair:
nzb_list = nzb_redirect(tmp_workdir_complete,
nzo.final_name,
nzo.pp,
script,
cat,
priority=nzo.priority)
else:
nzb_list = None
if nzb_list:
nzo.set_unpack_info(
'Download',
T('Sent %s to queue') % unicoder(nzb_list))
cleanup_empty_directories(tmp_workdir_complete)
else:
cleanup_list(tmp_workdir_complete, False)
script_output = ''
script_ret = 0
if not nzb_list:
## Give destination its final name
if cfg.folder_rename() and tmp_workdir_complete and not one_folder:
if all_ok:
try:
newfiles = rename_and_collapse_folder(
tmp_workdir_complete, workdir_complete, newfiles)
except:
logging.error(Ta('Error renaming "%s" to "%s"'),
tmp_workdir_complete, workdir_complete)
logging.info('Traceback: ', exc_info=True)
# Better disable sorting because filenames are all off now
file_sorter.sort_file = None
else:
workdir_complete = tmp_workdir_complete.replace(
'_UNPACK_', '_FAILED_')
workdir_complete = get_unique_path(workdir_complete,
n=0,
create_dir=False)
if empty:
job_result = -1
else:
job_result = int(par_error) + int(unpack_error) * 2
if cfg.ignore_samples() > 0:
remove_samples(workdir_complete)
## TV/Movie/Date Renaming code part 2 - rename and move files to parent folder
if all_ok and file_sorter.sort_file:
if newfiles:
file_sorter.rename(newfiles, workdir_complete)
workdir_complete, ok = file_sorter.move(workdir_complete)
else:
workdir_complete, ok = file_sorter.rename_with_ext(
workdir_complete)
if not ok:
nzo.set_unpack_info('Unpack', T('Failed to move files'))
all_ok = False
## Run the user script
script_path = make_script_path(script)
if (all_ok or not cfg.safe_postproc()) and (
not nzb_list) and script_path:
#set the current nzo status to "Ext Script...". Used in History
nzo.status = Status.RUNNING
nzo.set_action_line(T('Running script'), unicoder(script))
nzo.set_unpack_info('Script',
T('Running user script %s') %
unicoder(script),
unique=True)
script_log, script_ret = external_processing(
script_path, workdir_complete, nzo.filename, msgid,
dirname, cat, nzo.group, job_result,
nzo.nzo_info.get('failure', ''))
script_line = get_last_line(script_log)
if script_log:
script_output = nzo.nzo_id
if script_line:
nzo.set_unpack_info('Script',
unicoder(script_line),
unique=True)
else:
nzo.set_unpack_info('Script',
T('Ran %s') % unicoder(script),
unique=True)
else:
script = ""
script_line = ""
script_ret = 0
## Email the results
if (not nzb_list) and cfg.email_endjob():
if (cfg.email_endjob() == 1) or (cfg.email_endjob() == 2 and
(unpack_error or par_error)):
emailer.endjob(dirname, msgid, cat, all_ok, workdir_complete,
nzo.bytes_downloaded,
nzo.fail_msg, nzo.unpack_info, script,
TRANS(script_log), script_ret)
if script_output:
# Can do this only now, otherwise it would show up in the email
if script_ret:
script_ret = 'Exit(%s) ' % script_ret
else:
script_ret = ''
if script_line:
nzo.set_unpack_info(
'Script',
u'%s%s <a href="./scriptlog?name=%s">(%s)</a>' %
(script_ret, unicoder(script_line),
urllib.quote(script_output), T('More')),
unique=True)
else:
nzo.set_unpack_info('Script',
u'%s<a href="./scriptlog?name=%s">%s</a>' %
(script_ret, urllib.quote(script_output),
T('View script output')),
unique=True)
## Cleanup again, including NZB files
if all_ok:
cleanup_list(workdir_complete, False)
## Remove newzbin bookmark, if any
if msgid and all_ok:
Bookmarks.do.del_bookmark(msgid)
elif all_ok and isinstance(nzo.url, str):
sabnzbd.proxy_rm_bookmark(nzo.url)
## Force error for empty result
all_ok = all_ok and not empty
## Update indexer with results
if nzo.encrypted > 0:
Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_ENCRYPTED)
if empty:
hosts = map(lambda s: s.host,
sabnzbd.downloader.Downloader.do.nzo_servers(nzo))
if not hosts: hosts = [None]
for host in hosts:
Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_EXPIRED,
host)
## Show final status in history
if all_ok:
growler.send_notification(T('Download Completed'), filename,
'complete')
nzo.status = Status.COMPLETED
else:
growler.send_notification(T('Download Failed'), filename,
'complete')
nzo.status = Status.FAILED
except:
logging.error(Ta('Post Processing Failed for %s (%s)'), filename,
crash_msg)
if not crash_msg:
logging.info("Traceback: ", exc_info=True)
crash_msg = T('see logfile')
nzo.fail_msg = T('PostProcessing was aborted (%s)') % unicoder(
crash_msg)
growler.send_notification(T('Download Failed'), filename, 'complete')
nzo.status = Status.FAILED
par_error = True
all_ok = False
if cfg.email_endjob():
emailer.endjob(dirname, msgid, cat, all_ok, workdir_complete,
nzo.bytes_downloaded, nzo.fail_msg, nzo.unpack_info,
'', '', 0)
if all_ok:
# If the folder only contains one file OR folder, have that as the path
# Be aware that series/generic/date sorting may move a single file into a folder containing other files
workdir_complete = one_file_or_folder(workdir_complete)
workdir_complete = os.path.normpath(workdir_complete)
# Log the overall time taken for postprocessing
postproc_time = int(time.time() - start)
# Create the history DB instance
history_db = database.get_history_handle()
# Add the nzo to the database. Only the path, script and time taken is passed
# Other information is obtained from the nzo
history_db.add_history_db(nzo, workdir_complete, nzo.downpath,
postproc_time, script_log, script_line)
# The connection is only used once, so close it here
history_db.close()
## Clean up the NZO
try:
logging.info('Cleaning up %s (keep_basic=%s)', filename,
str(not all_ok))
sabnzbd.nzbqueue.NzbQueue.do.cleanup_nzo(nzo, keep_basic=not all_ok)
except:
logging.error(Ta('Cleanup of %s failed.'), nzo.final_name)
logging.info("Traceback: ", exc_info=True)
## Remove download folder
if all_ok:
try:
if os.path.exists(workdir):
logging.debug('Removing workdir %s', workdir)
remove_all(workdir, recursive=True)
except:
logging.error(Ta('Error removing workdir (%s)'), workdir)
logging.info("Traceback: ", exc_info=True)
return True
def run(self):
# Input and output
linebuf = ''
last_volume_linebuf = ''
unrar_log = []
rarfiles = []
start_time = time.time()
# Need to read char-by-char because there's no newline after new-disk message
while 1:
if not self.active_instance:
break
char = self.active_instance.stdout.read(1)
linebuf += char
if not char:
# End of program
break
# Error? Let PP-handle it
if linebuf.endswith(('ERROR: ', 'Cannot create', 'in the encrypted file', 'CRC failed', \
'checksum failed', 'You need to start extraction from a previous volume', \
'password is incorrect', 'Write error', 'checksum error', \
'start extraction from a previous volume')):
logging.info('Error in DirectUnpack of %s', self.cur_setname)
self.abort()
if linebuf.startswith('Extracting from') and linebuf.endswith(
'\n'):
filename = TRANS((re.search(EXTRACTFROM_RE,
linebuf.strip()).group(1)))
if filename not in rarfiles:
rarfiles.append(filename)
# Did we reach the end?
if linebuf.endswith('All OK'):
# Stop timer and finish
self.unpack_time += time.time() - start_time
ACTIVE_UNPACKERS.remove(self)
# Add to success
rarfile_path = os.path.join(self.nzo.downpath,
self.rarfile_nzf.filename)
self.success_sets[self.cur_setname] = rar_volumelist(
rarfile_path, self.nzo.password, rarfiles)
logging.info('DirectUnpack completed for %s', self.cur_setname)
self.nzo.set_action_line(T('Direct Unpack'), T('Completed'))
# Write current log and clear
unrar_log.append(linebuf.strip())
linebuf = ''
logging.debug('DirectUnpack Unrar output %s',
'\n'.join(unrar_log))
unrar_log = []
rarfiles = []
# Are there more files left?
while self.nzo.files and not self.next_sets:
with self.next_file_lock:
self.next_file_lock.wait()
# Is there another set to do?
if self.next_sets:
# Start new instance
nzf = self.next_sets.pop(0)
self.reset_active()
self.cur_setname = nzf.setname
# Wait for the 1st volume to appear
self.wait_for_next_volume()
self.create_unrar_instance()
start_time = time.time()
else:
self.killed = True
break
if linebuf.endswith('[C]ontinue, [Q]uit '):
# Stop timer
self.unpack_time += time.time() - start_time
# Wait for the next one..
self.wait_for_next_volume()
# Possible that the instance was deleted while locked
if not self.killed:
# Give unrar some time to do it's thing
self.active_instance.stdin.write('\n')
start_time = time.time()
time.sleep(0.1)
# Did we unpack a new volume? Sometimes UnRar hangs on 1 volume
if not last_volume_linebuf or last_volume_linebuf != linebuf:
# Next volume
self.cur_volume += 1
self.nzo.set_action_line(T('Direct Unpack'),
self.get_formatted_stats())
logging.info('DirectUnpacked volume %s for %s',
self.cur_volume, self.cur_setname)
# If lines did not change and we don't have the next volume, this download is missing files!
if last_volume_linebuf == linebuf and not self.have_next_volume(
):
logging.info(
'DirectUnpack failed due to missing files %s',
self.cur_setname)
self.abort()
last_volume_linebuf = linebuf
# Show the log
if linebuf.endswith('\n'):
unrar_log.append(linebuf.strip())
linebuf = ''
# Add last line
unrar_log.append(linebuf.strip())
logging.debug('DirectUnpack Unrar output %s', '\n'.join(unrar_log))
# Save information if success
if self.success_sets:
# The number is wrong if one_folder, just leave empty
nr_files = '' if self.unpack_dir_info[3] else len(
globber(self.unpack_dir_info[0]))
msg = T('Unpacked %s files/folders in %s') % (
nr_files, format_time_string(self.unpack_time))
msg = '%s - %s' % (T('Direct Unpack'), msg)
self.nzo.set_unpack_info(
'Unpack', '[%s] %s' % (unicoder(self.cur_setname), msg))
# Make more space
self.reset_active()
if self in ACTIVE_UNPACKERS:
ACTIVE_UNPACKERS.remove(self)
# Set the thread to killed so it never gets restarted by accident
self.killed = True
def process_job(nzo):
""" Process one job """
assert isinstance(nzo, sabnzbd.nzbstuff.NzbObject)
start = time.time()
# keep track of whether we can continue
all_ok = True
# keep track of par problems
par_error = False
# keep track of any unpacking errors
unpack_error = False
# Signal empty download, for when 'empty_postproc' is enabled
empty = False
nzb_list = []
# These need to be initialized in case of a crash
workdir_complete = ""
postproc_time = 0 # @UnusedVariable -- pep8 bug?
script_log = ""
script_line = ""
crash_msg = ""
# Get the job flags
nzo.save_attribs()
flag_repair, flag_unpack, flag_delete = nzo.repair_opts
# Normalize PP
if flag_delete:
flag_unpack = True
if flag_unpack:
flag_repair = True
# Get the NZB name
filename = nzo.final_name
if cfg.allow_streaming() and not (flag_repair or flag_unpack or flag_delete):
# After streaming, force +D
nzo.set_pp(3)
nzo.status = Status.FAILED
nzo.save_attribs()
all_ok = False
if nzo.fail_msg: # Special case: aborted due to too many missing data
nzo.status = Status.FAILED
nzo.save_attribs()
all_ok = False
par_error = True
unpack_error = 1
try:
# Get the folder containing the download result
workdir = nzo.downpath
tmp_workdir_complete = None
# if no files are present (except __admin__), fail the job
if all_ok and len(globber(workdir)) < 2:
if nzo.precheck:
_enough, ratio = nzo.check_quality()
req_ratio = float(cfg.req_completion_rate()) / 100.0
# Make sure that rounded ratio doesn't equal required ratio
# when it is actually below required
if (ratio < req_ratio) and (req_ratio - ratio) < 0.001:
ratio = req_ratio - 0.001
emsg = "%.1f%%" % (ratio * 100.0)
emsg2 = "%.1f%%" % float(cfg.req_completion_rate())
emsg = T("Download might fail, only %s of required %s available") % (emsg, emsg2)
else:
emsg = T("Download failed - Not on your server(s)")
empty = True
nzo.fail_msg = emsg
nzo.set_unpack_info("Fail", emsg)
nzo.status = Status.FAILED
# do not run unpacking or parity verification
flag_repair = flag_unpack = False
all_ok = cfg.empty_postproc() and empty
if not all_ok:
par_error = True
unpack_error = 1
script = nzo.script
cat = nzo.cat
logging.info(
"Starting PostProcessing on %s" + " => Repair:%s, Unpack:%s, Delete:%s, Script:%s, Cat:%s",
filename,
flag_repair,
flag_unpack,
flag_delete,
script,
cat,
)
# Set complete dir to workdir in case we need to abort
workdir_complete = workdir
dirname = nzo.final_name
marker_file = None
# Par processing, if enabled
if all_ok and flag_repair:
if not check_win_maxpath(workdir):
crash_msg = T('Path exceeds 260, repair by "par2" is not possible')
raise WindowsError
par_error, re_add = parring(nzo, workdir)
if re_add:
# Try to get more par files
return False
# Check if user allows unsafe post-processing
if flag_repair and cfg.safe_postproc():
all_ok = all_ok and not par_error
if all_ok:
fix_unix_encoding(workdir)
one_folder = False
# Determine class directory
if cfg.create_group_folders():
complete_dir = addPrefixes(cfg.complete_dir.get_path(), nzo.dirprefix)
complete_dir = create_dirs(complete_dir)
else:
catdir = config.get_categories(cat).dir()
if catdir.endswith("*"):
catdir = catdir.strip("*")
one_folder = True
complete_dir = real_path(cfg.complete_dir.get_path(), catdir)
complete_dir = long_path(complete_dir)
# TV/Movie/Date Renaming code part 1 - detect and construct paths
if cfg.enable_meta():
file_sorter = Sorter(nzo, cat)
else:
file_sorter = Sorter(None, cat)
complete_dir = file_sorter.detect(dirname, complete_dir)
if file_sorter.sort_file:
one_folder = False
complete_dir = sanitize_and_trim_path(complete_dir)
if one_folder:
workdir_complete = create_dirs(complete_dir)
else:
workdir_complete = get_unique_path(os.path.join(complete_dir, dirname), create_dir=True)
marker_file = set_marker(workdir_complete)
if not workdir_complete or not os.path.exists(workdir_complete):
crash_msg = T("Cannot create final folder %s") % unicoder(os.path.join(complete_dir, dirname))
raise IOError
if cfg.folder_rename() and not one_folder:
tmp_workdir_complete = prefix(workdir_complete, "_UNPACK_")
try:
renamer(workdir_complete, tmp_workdir_complete)
except:
pass # On failure, just use the original name
else:
tmp_workdir_complete = workdir_complete
newfiles = []
# Run Stage 2: Unpack
if flag_unpack:
if all_ok:
# set the current nzo status to "Extracting...". Used in History
nzo.status = Status.EXTRACTING
logging.info("Running unpack_magic on %s", filename)
short_complete = short_path(tmp_workdir_complete)
unpack_error, newfiles = unpack_magic(
nzo, short_path(workdir), short_complete, flag_delete, one_folder, (), (), (), (), ()
)
if short_complete != tmp_workdir_complete:
newfiles = [f.replace(short_complete, tmp_workdir_complete) for f in newfiles]
logging.info("unpack_magic finished on %s", filename)
else:
nzo.set_unpack_info("Unpack", T("No post-processing because of failed verification"))
if cfg.safe_postproc():
all_ok = all_ok and not unpack_error
if all_ok:
# Move any (left-over) files to destination
nzo.status = Status.MOVING
nzo.set_action_line(T("Moving"), "...")
for root, _dirs, files in os.walk(workdir):
if not root.endswith(JOB_ADMIN):
for file_ in files:
path = os.path.join(root, file_)
new_path = path.replace(workdir, tmp_workdir_complete)
ok, new_path = move_to_path(path, new_path)
newfiles.append(new_path)
if not ok:
nzo.set_unpack_info(
"Unpack", T("Failed moving %s to %s") % (unicoder(path), unicoder(new_path))
)
all_ok = False
break
# Set permissions right
set_permissions(tmp_workdir_complete)
if all_ok and marker_file:
del_marker(os.path.join(tmp_workdir_complete, marker_file))
remove_from_list(marker_file, newfiles)
if all_ok:
# Remove files matching the cleanup list
cleanup_list(tmp_workdir_complete, True)
# Check if this is an NZB-only download, if so redirect to queue
# except when PP was Download-only
if flag_repair:
nzb_list = nzb_redirect(
tmp_workdir_complete, nzo.final_name, nzo.pp, script, cat, priority=nzo.priority
)
else:
nzb_list = None
if nzb_list:
nzo.set_unpack_info("Download", T("Sent %s to queue") % unicoder(nzb_list))
cleanup_empty_directories(tmp_workdir_complete)
else:
cleanup_list(tmp_workdir_complete, False)
script_output = ""
script_ret = 0
if not nzb_list:
# Give destination its final name
if cfg.folder_rename() and tmp_workdir_complete and not one_folder:
if all_ok:
try:
newfiles = rename_and_collapse_folder(tmp_workdir_complete, workdir_complete, newfiles)
except:
logging.error(
T('Error renaming "%s" to "%s"'),
clip_path(tmp_workdir_complete),
clip_path(workdir_complete),
)
logging.info("Traceback: ", exc_info=True)
# Better disable sorting because filenames are all off now
file_sorter.sort_file = None
else:
workdir_complete = tmp_workdir_complete.replace("_UNPACK_", "_FAILED_")
workdir_complete = get_unique_path(workdir_complete, n=0, create_dir=False)
workdir_complete = workdir_complete
if empty:
job_result = -1
else:
job_result = int(par_error) + int(bool(unpack_error)) * 2
if cfg.ignore_samples():
remove_samples(workdir_complete)
# TV/Movie/Date Renaming code part 2 - rename and move files to parent folder
if all_ok and file_sorter.sort_file:
if newfiles:
file_sorter.rename(newfiles, workdir_complete)
workdir_complete, ok = file_sorter.move(workdir_complete)
else:
workdir_complete, ok = file_sorter.rename_with_ext(workdir_complete)
if not ok:
nzo.set_unpack_info("Unpack", T("Failed to move files"))
all_ok = False
# Run the user script
script_path = make_script_path(script)
if (all_ok or not cfg.safe_postproc()) and (not nzb_list) and script_path:
# set the current nzo status to "Ext Script...". Used in History
nzo.status = Status.RUNNING
nzo.set_action_line(T("Running script"), unicoder(script))
nzo.set_unpack_info("Script", T("Running user script %s") % unicoder(script), unique=True)
script_log, script_ret = external_processing(
short_path(script_path, False),
short_path(workdir_complete, False),
nzo.filename,
dirname,
cat,
nzo.group,
job_result,
nzo.nzo_info.get("failure", ""),
)
script_line = get_last_line(script_log)
if script_log:
script_output = nzo.nzo_id
if script_line:
nzo.set_unpack_info("Script", unicoder(script_line), unique=True)
else:
nzo.set_unpack_info("Script", T("Ran %s") % unicoder(script), unique=True)
else:
script = ""
script_line = ""
script_ret = 0
# Maybe bad script result should fail job
if script_ret and cfg.script_can_fail():
script_error = True
all_ok = False
nzo.fail_msg = T("Script exit code is %s") % script_ret
else:
script_error = False
# Email the results
if (not nzb_list) and cfg.email_endjob():
if (cfg.email_endjob() == 1) or (cfg.email_endjob() == 2 and (unpack_error or par_error or script_error)):
emailer.endjob(
dirname,
cat,
all_ok,
workdir_complete,
nzo.bytes_downloaded,
nzo.fail_msg,
nzo.unpack_info,
script,
TRANS(script_log),
script_ret,
)
if script_output:
# Can do this only now, otherwise it would show up in the email
if script_ret:
script_ret = "Exit(%s) " % script_ret
else:
script_ret = ""
if script_line:
nzo.set_unpack_info(
"Script",
u'%s%s <a href="./scriptlog?name=%s">(%s)</a>'
% (script_ret, unicoder(script_line), urllib.quote(script_output), T("More")),
unique=True,
)
else:
nzo.set_unpack_info(
"Script",
u'%s<a href="./scriptlog?name=%s">%s</a>'
% (script_ret, urllib.quote(script_output), T("View script output")),
unique=True,
)
# Cleanup again, including NZB files
if all_ok:
cleanup_list(workdir_complete, False)
# Force error for empty result
all_ok = all_ok and not empty
# Update indexer with results
if cfg.rating_enable():
if nzo.encrypted > 0:
Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_ENCRYPTED)
if empty:
hosts = map(lambda s: s.host, sabnzbd.downloader.Downloader.do.nzo_servers(nzo))
if not hosts:
hosts = [None]
for host in hosts:
Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_EXPIRED, host)
# Show final status in history
if all_ok:
growler.send_notification(T("Download Completed"), filename, "complete")
nzo.status = Status.COMPLETED
else:
growler.send_notification(T("Download Failed"), filename, "failed")
nzo.status = Status.FAILED
except:
logging.error(T("Post Processing Failed for %s (%s)"), filename, crash_msg)
if not crash_msg:
logging.info("Traceback: ", exc_info=True)
crash_msg = T("see logfile")
nzo.fail_msg = T("PostProcessing was aborted (%s)") % unicoder(crash_msg)
growler.send_notification(T("Download Failed"), filename, "failed")
nzo.status = Status.FAILED
par_error = True
all_ok = False
if cfg.email_endjob():
emailer.endjob(
dirname,
cat,
all_ok,
clip_path(workdir_complete),
nzo.bytes_downloaded,
nzo.fail_msg,
nzo.unpack_info,
"",
"",
0,
)
if all_ok:
# If the folder only contains one file OR folder, have that as the path
# Be aware that series/generic/date sorting may move a single file into a folder containing other files
workdir_complete = one_file_or_folder(workdir_complete)
workdir_complete = os.path.normpath(workdir_complete)
# Log the overall time taken for postprocessing
postproc_time = int(time.time() - start)
# Create the history DB instance
history_db = database.get_history_handle()
# Add the nzo to the database. Only the path, script and time taken is passed
# Other information is obtained from the nzo
history_db.add_history_db(nzo, clip_path(workdir_complete), nzo.downpath, postproc_time, script_log, script_line)
# The connection is only used once, so close it here
history_db.close()
# Clean up the NZO
try:
logging.info("Cleaning up %s (keep_basic=%s)", filename, str(not all_ok))
sabnzbd.nzbqueue.NzbQueue.do.cleanup_nzo(nzo, keep_basic=not all_ok)
except:
logging.error(T("Cleanup of %s failed."), nzo.final_name)
logging.info("Traceback: ", exc_info=True)
# Remove download folder
if all_ok:
try:
if os.path.exists(workdir):
logging.debug("Removing workdir %s", workdir)
remove_all(workdir, recursive=True)
except:
logging.error(T("Error removing workdir (%s)"), clip_path(workdir))
logging.info("Traceback: ", exc_info=True)
# Use automatic retry link on par2 errors and encrypted/bad RARs
if par_error or unpack_error in (2, 3):
try_alt_nzb(nzo)
return True
