def rename(self, files, current_path):
logging.debug("Renaming Date file")
renamed = None
#find the master file to rename
for file in files:
if is_full_path(file):
filepath = file.replace('_UNPACK_', '')
else:
filepath = os.path.join(current_path, file)
if os.path.exists(filepath):
size = os.stat(filepath).st_size
if size > 130000000:
if 'sample' not in file:
tmp, ext = os.path.splitext(file)
self.fname = tmp
newname = "%s%s" % (self.filename_set, ext)
newname = newname.replace('%fn', tmp)
newpath = os.path.join(current_path, newname)
if not os.path.exists(newpath):
try:
logging.debug("Rename: %s to %s", filepath,
newpath)
renamer(filepath, newpath)
except:
logging.error(Ta('Failed to rename: %s to %s'),
current_path, newpath)
logging.info("Traceback: ", exc_info=True)
rename_similar(current_path, file,
self.filename_set)
break
def rename_similar(folder, skip_ext, name, skipped_files):
""" Rename all other files in the 'folder' hierarchy after 'name'
and move them to the root of 'folder'.
Files having extension 'skip_ext' will be moved, but not renamed.
Don't touch files in list `skipped_files`
"""
logging.debug('Give files in set "%s" matching names.', name)
folder = os.path.normpath(folder)
skip_ext = skip_ext.lower()
for root, dirs, files in os.walk(folder):
for f in files:
path = os.path.join(root, f)
if path in skipped_files:
continue
org, ext = os.path.splitext(f)
if ext.lower() == skip_ext:
# Move file, but do not rename
newpath = os.path.join(folder, f)
else:
# Move file and rename
newname = "%s%s" % (name, ext)
newname = newname.replace('%fn', org)
newpath = os.path.join(folder, newname)
if path != newpath:
newpath = get_unique_filename(newpath)
try:
logging.debug("Rename: %s to %s", path, newpath)
renamer(path, newpath)
except:
logging.error(T('Failed to rename similar file: %s to %s'),
clip_path(path), clip_path(newpath))
logging.info("Traceback: ", exc_info=True)
cleanup_empty_directories(folder)
def rename_and_collapse_folder(oldpath, newpath, files):
""" Rename folder, collapsing when there's just a single subfolder
oldpath --> newpath OR oldpath/subfolder --> newpath
Modify list of filenames accordingly
"""
orgpath = oldpath
items = globber(oldpath)
if len(items) == 1:
folder = items[0]
folder_path = os.path.join(oldpath, folder)
if os.path.isdir(folder_path) and folder not in ('VIDEO_TS',
'AUDIO_TS'):
logging.info('Collapsing %s', os.path.join(newpath, folder))
oldpath = folder_path
oldpath = os.path.normpath(oldpath)
newpath = os.path.normpath(newpath)
files = [os.path.normpath(f).replace(oldpath, newpath) for f in files]
renamer(oldpath, newpath)
try:
remove_dir(orgpath)
except:
pass
return files
def move(self, workdir_complete):
ok = True
if self.type == 'movie':
move_to_parent = True
# check if we should leave the files inside an extra folder
if cfg.movie_extra_folders():
# if there is a folder in the download, leave it in an extra folder
move_to_parent = not check_for_folder(workdir_complete)
if move_to_parent:
workdir_complete, ok = move_to_parent_folder(workdir_complete)
else:
workdir_complete, ok = move_to_parent_folder(workdir_complete)
if not ok:
return workdir_complete, False
path, part = os.path.split(workdir_complete)
if '%fn' in part and self.sorter.fname:
old = workdir_complete
workdir_complete = os.path.join(path, part.replace('%fn', self.sorter.fname))
workdir_complete = get_unique_path(workdir_complete, create_dir=False)
try:
renamer(old, workdir_complete)
except:
logging.error(T('Cannot create directory %s'), clip_path(workdir_complete))
workdir_complete = old
ok = False
return workdir_complete, ok
def rename_similar(folder, skip_ext, name, skipped_files):
""" Rename all other files in the 'folder' hierarchy after 'name'
and move them to the root of 'folder'.
Files having extension 'skip_ext' will be moved, but not renamed.
Don't touch files in list `skipped_files`
"""
logging.debug('Give files in set "%s" matching names.', name)
folder = os.path.normpath(folder)
skip_ext = skip_ext.lower()
for root, dirs, files in os.walk(folder):
for f in files:
path = os.path.join(root, f)
if path in skipped_files:
continue
org, ext = os.path.splitext(f)
if ext.lower() == skip_ext:
# Move file, but do not rename
newpath = os.path.join(folder, f)
else:
# Move file and rename
newname = "%s%s" % (name, ext)
newname = newname.replace('%fn', org)
newpath = os.path.join(folder, newname)
if path != newpath:
newpath = get_unique_filename(newpath)
try:
logging.debug("Rename: %s to %s", path, newpath)
renamer(path, newpath)
except:
logging.error(T('Failed to rename similar file: %s to %s'), clip_path(path), clip_path(newpath))
logging.info("Traceback: ", exc_info=True)
cleanup_empty_directories(folder)
def move(self, workdir_complete):
ok = True
if self.type == 'movie':
move_to_parent = True
# check if we should leave the files inside an extra folder
if cfg.movie_extra_folders():
# if there is a folder in the download, leave it in an extra folder
move_to_parent = not check_for_folder(workdir_complete)
if move_to_parent:
workdir_complete, ok = move_to_parent_folder(workdir_complete)
else:
workdir_complete, ok = move_to_parent_folder(workdir_complete)
if not ok:
return workdir_complete, False
path, part = os.path.split(workdir_complete)
if '%fn' in part and self.sorter.fname:
old = workdir_complete
workdir_complete = os.path.join(
path, part.replace('%fn', self.sorter.fname))
workdir_complete = get_unique_path(workdir_complete,
create_dir=False)
try:
renamer(old, workdir_complete)
except:
logging.error(T('Cannot create directory %s'),
clip_path(workdir_complete))
workdir_complete = old
ok = False
return workdir_complete, ok
def rename(self, files, current_path):
""" Renaming Date file """
logging.debug("Renaming Date file")
# find the master file to rename
for file in files:
if is_full_path(file):
filepath = os.path.normpath(file)
else:
filepath = os.path.normpath(os.path.join(current_path, file))
if os.path.exists(filepath):
size = os.stat(filepath).st_size
if size > cfg.movie_rename_limit.get_int():
if 'sample' not in file:
self.fname, ext = os.path.splitext(
os.path.split(file)[1])
newname = "%s%s" % (self.filename_set, ext)
newname = newname.replace('%fn', self.fname)
newpath = os.path.join(current_path, newname)
if not os.path.exists(newpath):
try:
logging.debug("Rename: %s to %s", filepath,
newpath)
renamer(filepath, newpath)
except:
logging.error(T('Failed to rename: %s to %s'),
clip_path(current_path),
clip_path(newpath))
logging.info("Traceback: ", exc_info=True)
rename_similar(current_path, ext,
self.filename_set, ())
break
def rename(self, files, current_path):
""" Renaming Date file """
logging.debug("Renaming Date file")
# find the master file to rename
for file in files:
if is_full_path(file):
filepath = os.path.normpath(file)
else:
filepath = os.path.normpath(os.path.join(current_path, file))
if os.path.exists(filepath):
size = os.stat(filepath).st_size
if size > cfg.movie_rename_limit.get_int():
if 'sample' not in file:
self.fname, ext = os.path.splitext(os.path.split(file)[1])
newname = "%s%s" % (self.filename_set, ext)
newname = newname.replace('%fn', self.fname)
newpath = os.path.join(current_path, newname)
if not os.path.exists(newpath):
try:
logging.debug("Rename: %s to %s", filepath, newpath)
renamer(filepath, newpath)
except:
logging.error(T('Failed to rename: %s to %s'), clip_path(current_path), clip_path(newpath))
logging.info("Traceback: ", exc_info=True)
rename_similar(current_path, ext, self.filename_set, ())
break
def _assemble(nzf, path, dupe):
if os.path.exists(path):
unique_path = get_unique_filename(path)
if dupe:
path = unique_path
else:
renamer(path, unique_path)
fout = open(path, 'ab')
if cfg.quick_check():
md5 = new_md5()
else:
md5 = None
_type = nzf.type
decodetable = nzf.decodetable
for articlenum in decodetable:
sleep(0.001)
article = decodetable[articlenum]
data = ArticleCache.do.load_article(article)
if not data:
logging.info(T('%s missing'), article)
else:
# yenc data already decoded, flush it out
if _type == 'yenc':
fout.write(data)
if md5:
md5.update(data)
# need to decode uu data now
elif _type == 'uu':
data = data.split('\r\n')
chunks = []
for line in data:
if not line:
continue
if line == '-- ' or line.startswith('Posted via '):
continue
try:
tmpdata = binascii.a2b_uu(line)
chunks.append(tmpdata)
except binascii.Error, msg:
# Workaround for broken uuencoders by
# /Fredrik Lundh
nbytes = (((ord(line[0]) - 32) & 63) * 4 + 5) / 3
try:
tmpdata = binascii.a2b_uu(line[:nbytes])
chunks.append(tmpdata)
except binascii.Error, msg:
logging.info('Decode failed in part %s: %s',
article.article, msg)
data = ''.join(chunks)
fout.write(data)
if md5:
md5.update(data)
def rename(self, files, current_path):
logging.debug("Renaming Date file")
renamed = None
#find the master file to rename
for file in files:
if is_full_path(file):
filepath = file.replace('_UNPACK_', '')
else:
filepath = os.path.join(current_path, file)
if os.path.exists(filepath):
size = os.stat(filepath).st_size
if size > 130000000:
if 'sample' not in file:
tmp, ext = os.path.splitext(file)
newname = "%s%s" % (self.filename_set,ext)
newname = newname.replace('%fn',tmp)
newpath = os.path.join(current_path, newname)
if not os.path.exists(newpath):
try:
logging.debug("Rename: %s to %s", filepath,newpath)
renamer(filepath,newpath)
except:
logging.error(Ta('error-tvRename@2'), current_path, newpath)
logging.debug("Traceback: ", exc_info = True)
rename_similar(current_path, file, self.filename_set)
break
def prepare_extraction_path(nzo):
""" Based on the information that we have, generate
the extraction path and create the directory.
Separated so it can be called from DirectUnpacker
"""
one_folder = False
marker_file = None
# Determine class directory
catdir = config.get_categories(nzo.cat).dir()
if catdir.endswith('*'):
catdir = catdir.strip('*')
one_folder = True
complete_dir = real_path(cfg.complete_dir.get_path(), catdir)
complete_dir = long_path(complete_dir)
# TV/Movie/Date Renaming code part 1 - detect and construct paths
if cfg.enable_meta():
file_sorter = Sorter(nzo, nzo.cat)
else:
file_sorter = Sorter(None, nzo.cat)
complete_dir = file_sorter.detect(nzo.final_name, complete_dir)
if file_sorter.sort_file:
one_folder = False
complete_dir = sanitize_and_trim_path(complete_dir)
if one_folder:
workdir_complete = create_dirs(complete_dir)
else:
workdir_complete = get_unique_path(os.path.join(
complete_dir, nzo.final_name),
create_dir=True)
marker_file = set_marker(workdir_complete)
if not workdir_complete or not os.path.exists(workdir_complete):
logging.error(
T('Cannot create final folder %s') %
unicoder(os.path.join(complete_dir, nzo.final_name)))
raise IOError
if cfg.folder_rename() and not one_folder:
prefixed_path = prefix(workdir_complete, '_UNPACK_')
tmp_workdir_complete = get_unique_path(prefix(workdir_complete,
'_UNPACK_'),
create_dir=False)
try:
renamer(workdir_complete, tmp_workdir_complete)
except:
pass # On failure, just use the original name
# Is the unique path different? Then we also need to modify the final path
if prefixed_path != tmp_workdir_complete:
workdir_complete = workdir_complete + os.path.splitext(
tmp_workdir_complete)[1]
else:
tmp_workdir_complete = workdir_complete
return tmp_workdir_complete, workdir_complete, file_sorter, one_folder, marker_file
def _assemble(nzf, path, dupe):
if os.path.exists(path):
unique_path = get_unique_filename(path)
if dupe:
path = unique_path
else:
renamer(path, unique_path)
fout = open(path, 'ab')
if cfg.quick_check():
md5 = new_md5()
else:
md5 = None
_type = nzf.type
decodetable = nzf.decodetable
for articlenum in decodetable:
sleep(0.001)
article = decodetable[articlenum]
data = ArticleCache.do.load_article(article)
if not data:
logging.info(T('%s missing'), article)
else:
# yenc data already decoded, flush it out
if _type == 'yenc':
fout.write(data)
if md5:
md5.update(data)
# need to decode uu data now
elif _type == 'uu':
data = data.split('\r\n')
chunks = []
for line in data:
if not line:
continue
if line == '-- ' or line.startswith('Posted via '):
continue
try:
tmpdata = binascii.a2b_uu(line)
chunks.append(tmpdata)
except binascii.Error, msg:
# Workaround for broken uuencoders by
# /Fredrik Lundh
nbytes = (((ord(line[0]) - 32) & 63) * 4 + 5) / 3
try:
tmpdata = binascii.a2b_uu(line[:nbytes])
chunks.append(tmpdata)
except binascii.Error, msg:
logging.info('Decode failed in part %s: %s', article.article, msg)
data = ''.join(chunks)
fout.write(data)
if md5:
md5.update(data)
def rename(self, files, current_path):
""" Rename for Series
"""
logging.debug("Renaming Series")
largest = (None, None, 0)
def to_filepath(f, current_path):
if is_full_path(f):
filepath = f.replace('_UNPACK_', '')
else:
filepath = os.path.join(current_path, f)
return filepath
# Create a generator of filepaths, ignore sample files and excluded files (vobs ect)
filepaths = ((file, to_filepath(file, current_path)) for file in files if not RE_SAMPLE.search(file) \
and get_ext(file) not in EXCLUDED_FILE_EXTS)
# Find the largest existing file
for file, fp in filepaths:
# If for some reason the file no longer exists, skip
if not os.path.exists(fp):
continue
size = os.stat(fp).st_size
f_file, f_fp, f_size = largest
if size > f_size:
largest = (file, fp, size)
file, filepath, size = largest
# >20MB
if filepath and size > 20971520:
tmp, self.ext = os.path.splitext(file)
self.fname = tmp
newname = "%s%s" % (self.filename_set, self.ext)
# Replace %fn with the original filename
newname = newname.replace('%fn', tmp)
newpath = os.path.join(current_path, newname)
# Replace %ext with extension
newpath = newpath.replace('%ext', self.ext)
if not os.path.exists(newpath):
try:
logging.debug("Rename: %s to %s", filepath, newpath)
renamer(filepath, newpath)
except:
logging.error("Failed to rename: %s to %s", current_path,
newpath)
logging.info("Traceback: ", exc_info=True)
rename_similar(current_path, self.ext, self.filename_set)
else:
logging.debug(
'Current path already exists, skipping rename, %s',
newpath)
else:
logging.debug('Nothing to rename, %s', files)
def rename_with_ext(self, workdir_complete):
""" Special renamer for %ext """
if self.sorter.rename_or_not and '%ext' in workdir_complete and self.ext:
# Replace %ext with extension
newpath = workdir_complete.replace('%ext', self.ext)
try:
renamer(workdir_complete, newpath)
except:
return newpath, False
return newpath, True
else:
return workdir_complete, True
def rename_with_ext(self, workdir_complete):
""" Special renamer for %ext """
if self.sorter.rename_or_not and '%ext' in workdir_complete and self.ext:
# Replace %ext with extension
newpath = workdir_complete.replace('%ext', self.ext)
try:
renamer(workdir_complete, newpath)
except:
return newpath, False
return newpath, True
else:
return workdir_complete, True
def prepare_extraction_path(nzo):
""" Based on the information that we have, generate
the extraction path and create the directory.
Separated so it can be called from DirectUnpacker
"""
one_folder = False
marker_file = None
# Determine class directory
catdir = config.get_categories(nzo.cat).dir()
if catdir.endswith('*'):
catdir = catdir.strip('*')
one_folder = True
complete_dir = real_path(cfg.complete_dir.get_path(), catdir)
complete_dir = long_path(complete_dir)
# TV/Movie/Date Renaming code part 1 - detect and construct paths
if cfg.enable_meta():
file_sorter = Sorter(nzo, nzo.cat)
else:
file_sorter = Sorter(None, nzo.cat)
complete_dir = file_sorter.detect(nzo.final_name, complete_dir)
if file_sorter.sort_file:
one_folder = False
complete_dir = sanitize_and_trim_path(complete_dir)
if one_folder:
workdir_complete = create_dirs(complete_dir)
else:
workdir_complete = get_unique_path(os.path.join(complete_dir, nzo.final_name), create_dir=True)
marker_file = set_marker(workdir_complete)
if not workdir_complete or not os.path.exists(workdir_complete):
logging.error(T('Cannot create final folder %s') % unicoder(os.path.join(complete_dir, nzo.final_name)))
raise IOError
if cfg.folder_rename() and not one_folder:
prefixed_path = prefix(workdir_complete, '_UNPACK_')
tmp_workdir_complete = get_unique_path(prefix(workdir_complete, '_UNPACK_'), create_dir=False)
try:
renamer(workdir_complete, tmp_workdir_complete)
except:
pass # On failure, just use the original name
# Is the unique path different? Then we also need to modify the final path
if prefixed_path != tmp_workdir_complete:
workdir_complete = workdir_complete + os.path.splitext(tmp_workdir_complete)[1]
else:
tmp_workdir_complete = workdir_complete
return tmp_workdir_complete, workdir_complete, file_sorter, one_folder, marker_file
def rename(self, files, current_path):
""" Rename for Series
"""
logging.debug("Renaming Series")
largest = (None, None, 0)
def to_filepath(f, current_path):
if is_full_path(f):
filepath = f.replace('_UNPACK_', '')
else:
filepath = os.path.join(current_path, f)
return filepath
# Create a generator of filepaths, ignore sample files and excluded files (vobs ect)
filepaths = ((file, to_filepath(file, current_path)) for file in files if not RE_SAMPLE.search(file) \
and get_ext(file) not in EXCLUDED_FILE_EXTS)
# Find the largest existing file
for file, fp in filepaths:
# If for some reason the file no longer exists, skip
if not os.path.exists(fp):
continue
size = os.stat(fp).st_size
f_file, f_fp, f_size = largest
if size > f_size:
largest = (file, fp, size)
file, filepath, size = largest
# >20MB
if filepath and size > 20971520:
tmp, self.ext = os.path.splitext(file)
self.fname = tmp
newname = "%s%s" % (self.filename_set, self.ext)
# Replace %fn with the original filename
newname = newname.replace('%fn', tmp)
newpath = os.path.join(current_path, newname)
# Replace %ext with extension
newpath = newpath.replace('%ext', self.ext)
if not os.path.exists(newpath):
try:
logging.debug("Rename: %s to %s", filepath, newpath)
renamer(filepath, newpath)
except:
logging.error("Failed to rename: %s to %s", current_path, newpath)
logging.info("Traceback: ", exc_info = True)
rename_similar(current_path, self.ext, self.filename_set)
else:
logging.debug('Current path already exists, skipping rename, %s', newpath)
else:
logging.debug('Nothing to rename, %s', files)
def rename_similar(path, file, name):
logging.debug('Renaming files similar to: %s to %s', file, name)
file_prefix, ext = os.path.splitext(file)
for root, dirs, files in os.walk(path):
for _file in files:
fpath = os.path.join(root, _file)
tmp, ext = os.path.splitext(_file)
if tmp == file_prefix:
newname = "%s%s" % (name,ext)
newname = newname.replace('%fn',tmp)
newpath = os.path.join(path, newname)
if not os.path.exists(newpath):
try:
logging.debug("Rename: %s to %s", fpath,newpath)
renamer(fpath,newpath)
except:
logging.error(Ta('error-tvSimRename@2'), path, newpath)
logging.debug("Traceback: ", exc_info = True)
def collapse_folder(oldpath, newpath):
""" Rename folder, collapsing when there's just a single subfolder
oldpath --> newpath OR oldpath/subfolder --> newpath
"""
orgpath = oldpath
items = globber(oldpath)
if len(items) == 1:
folder_path = items[0]
folder = os.path.split(folder_path)[1]
if os.path.isdir(folder_path) and folder not in ('VIDEO_TS', 'AUDIO_TS'):
logging.info('Collapsing %s', os.path.join(newpath, folder))
oldpath = folder_path
renamer(oldpath, newpath)
try:
remove_dir(orgpath)
except:
pass
def _assemble(nzf, path, dupe):
if os.path.exists(path):
unique_path = get_unique_filename(path)
if dupe:
path = unique_path
else:
renamer(path, unique_path)
fout = open(path, "ab")
if cfg.quick_check():
md5 = new_md5()
else:
md5 = None
decodetable = nzf.decodetable
for articlenum in decodetable:
# Break if deleted during writing
if nzf.nzo.status is Status.DELETED:
break
# Sleep to allow decoder/assembler switching
sleep(0.001)
article = decodetable[articlenum]
data = ArticleCache.do.load_article(article)
if not data:
logging.info(T("%s missing"), article)
else:
# yenc data already decoded, flush it out
fout.write(data)
if md5:
md5.update(data)
fout.flush()
fout.close()
set_permissions(path)
if md5:
nzf.md5sum = md5.digest()
del md5
return path
def _assemble(nzf, path, dupe):
if os.path.exists(path):
unique_path = get_unique_filename(path)
if dupe:
path = unique_path
else:
renamer(path, unique_path)
fout = open(path, 'ab')
if cfg.quick_check():
md5 = hashlib.md5()
else:
md5 = None
decodetable = nzf.decodetable
for articlenum in decodetable:
# Break if deleted during writing
if nzf.nzo.status is Status.DELETED:
break
# Sleep to allow decoder/assembler switching
sleep(0.0001)
article = decodetable[articlenum]
data = ArticleCache.do.load_article(article)
if not data:
logging.info(T('%s missing'), article)
else:
# yenc data already decoded, flush it out
fout.write(data)
if md5:
md5.update(data)
fout.flush()
fout.close()
set_permissions(path)
if md5:
nzf.md5sum = md5.digest()
del md5
return path
def collapse_folder(oldpath, newpath):
""" Rename folder, collapsing when there's just a single subfolder
oldpath --> newpath OR oldpath/subfolder --> newpath
"""
orgpath = oldpath
items = globber(oldpath)
if len(items) == 1:
folder_path = items[0]
folder = os.path.split(folder_path)[1]
if os.path.isdir(folder_path) and folder not in ('VIDEO_TS',
'AUDIO_TS'):
logging.info('Collapsing %s', os.path.join(newpath, folder))
oldpath = folder_path
renamer(oldpath, newpath)
try:
remove_dir(orgpath)
except:
pass
def rename_similar(path, file, name):
logging.debug('Renaming files similar to: %s to %s', file, name)
file_prefix, ext = os.path.splitext(file)
for root, dirs, files in os.walk(path):
for _file in files:
fpath = os.path.join(root, _file)
tmp, ext = os.path.splitext(_file)
if tmp == file_prefix:
newname = "%s%s" % (name, ext)
newname = newname.replace('%fn', tmp)
newpath = os.path.join(path, newname)
if not os.path.exists(newpath):
try:
logging.debug("Rename: %s to %s", fpath, newpath)
renamer(fpath, newpath)
except:
logging.error(
Ta('Failed to rename similar file: %s to %s'),
path, newpath)
logging.info("Traceback: ", exc_info=True)
def rename_and_collapse_folder(oldpath, newpath, files):
""" Rename folder, collapsing when there's just a single subfolder
oldpath --> newpath OR oldpath/subfolder --> newpath
Modify list of filenames accordingly
"""
orgpath = oldpath
items = globber(oldpath)
if len(items) == 1:
folder_path = items[0]
folder = os.path.split(folder_path)[1]
if os.path.isdir(folder_path) and folder not in ('VIDEO_TS', 'AUDIO_TS'):
logging.info('Collapsing %s', os.path.join(newpath, folder))
oldpath = folder_path
oldpath = os.path.normpath(oldpath)
newpath = os.path.normpath(newpath)
files = [os.path.normpath(f).replace(oldpath, newpath) for f in files]
renamer(oldpath, newpath)
try:
remove_dir(orgpath)
except:
pass
return files
def process_job(nzo):
""" Process one job """
assert isinstance(nzo, sabnzbd.nzbstuff.NzbObject)
start = time.time()
# keep track of whether we can continue
all_ok = True
# keep track of par problems
par_error = False
# keep track of any unpacking errors
unpack_error = False
# Signal empty download, for when 'empty_postproc' is enabled
empty = False
nzb_list = []
# These need to be initialized in case of a crash
workdir_complete = ""
postproc_time = 0 # @UnusedVariable -- pep8 bug?
script_log = ""
script_line = ""
crash_msg = ""
# Get the job flags
nzo.save_attribs()
flag_repair, flag_unpack, flag_delete = nzo.repair_opts
# Normalize PP
if flag_delete:
flag_unpack = True
if flag_unpack:
flag_repair = True
# Get the NZB name
filename = nzo.final_name
if cfg.allow_streaming() and not (flag_repair or flag_unpack or flag_delete):
# After streaming, force +D
nzo.set_pp(3)
nzo.status = Status.FAILED
nzo.save_attribs()
all_ok = False
if nzo.fail_msg: # Special case: aborted due to too many missing data
nzo.status = Status.FAILED
nzo.save_attribs()
all_ok = False
par_error = True
unpack_error = 1
try:
# Get the folder containing the download result
workdir = nzo.downpath
tmp_workdir_complete = None
# if no files are present (except __admin__), fail the job
if all_ok and len(globber(workdir)) < 2:
if nzo.precheck:
_enough, ratio = nzo.check_quality()
req_ratio = float(cfg.req_completion_rate()) / 100.0
# Make sure that rounded ratio doesn't equal required ratio
# when it is actually below required
if (ratio < req_ratio) and (req_ratio - ratio) < 0.001:
ratio = req_ratio - 0.001
emsg = "%.1f%%" % (ratio * 100.0)
emsg2 = "%.1f%%" % float(cfg.req_completion_rate())
emsg = T("Download might fail, only %s of required %s available") % (emsg, emsg2)
else:
emsg = T("Download failed - Not on your server(s)")
empty = True
nzo.fail_msg = emsg
nzo.set_unpack_info("Fail", emsg)
nzo.status = Status.FAILED
# do not run unpacking or parity verification
flag_repair = flag_unpack = False
all_ok = cfg.empty_postproc() and empty
if not all_ok:
par_error = True
unpack_error = 1
script = nzo.script
cat = nzo.cat
logging.info(
"Starting PostProcessing on %s" + " => Repair:%s, Unpack:%s, Delete:%s, Script:%s, Cat:%s",
filename,
flag_repair,
flag_unpack,
flag_delete,
script,
cat,
)
# Set complete dir to workdir in case we need to abort
workdir_complete = workdir
dirname = nzo.final_name
marker_file = None
# Par processing, if enabled
if all_ok and flag_repair:
if not check_win_maxpath(workdir):
crash_msg = T('Path exceeds 260, repair by "par2" is not possible')
raise WindowsError
par_error, re_add = parring(nzo, workdir)
if re_add:
# Try to get more par files
return False
# Check if user allows unsafe post-processing
if flag_repair and cfg.safe_postproc():
all_ok = all_ok and not par_error
if all_ok:
fix_unix_encoding(workdir)
one_folder = False
# Determine class directory
if cfg.create_group_folders():
complete_dir = addPrefixes(cfg.complete_dir.get_path(), nzo.dirprefix)
complete_dir = create_dirs(complete_dir)
else:
catdir = config.get_categories(cat).dir()
if catdir.endswith("*"):
catdir = catdir.strip("*")
one_folder = True
complete_dir = real_path(cfg.complete_dir.get_path(), catdir)
complete_dir = long_path(complete_dir)
# TV/Movie/Date Renaming code part 1 - detect and construct paths
if cfg.enable_meta():
file_sorter = Sorter(nzo, cat)
else:
file_sorter = Sorter(None, cat)
complete_dir = file_sorter.detect(dirname, complete_dir)
if file_sorter.sort_file:
one_folder = False
complete_dir = sanitize_and_trim_path(complete_dir)
if one_folder:
workdir_complete = create_dirs(complete_dir)
else:
workdir_complete = get_unique_path(os.path.join(complete_dir, dirname), create_dir=True)
marker_file = set_marker(workdir_complete)
if not workdir_complete or not os.path.exists(workdir_complete):
crash_msg = T("Cannot create final folder %s") % unicoder(os.path.join(complete_dir, dirname))
raise IOError
if cfg.folder_rename() and not one_folder:
tmp_workdir_complete = prefix(workdir_complete, "_UNPACK_")
try:
renamer(workdir_complete, tmp_workdir_complete)
except:
pass # On failure, just use the original name
else:
tmp_workdir_complete = workdir_complete
newfiles = []
# Run Stage 2: Unpack
if flag_unpack:
if all_ok:
# set the current nzo status to "Extracting...". Used in History
nzo.status = Status.EXTRACTING
logging.info("Running unpack_magic on %s", filename)
short_complete = short_path(tmp_workdir_complete)
unpack_error, newfiles = unpack_magic(
nzo, short_path(workdir), short_complete, flag_delete, one_folder, (), (), (), (), ()
)
if short_complete != tmp_workdir_complete:
newfiles = [f.replace(short_complete, tmp_workdir_complete) for f in newfiles]
logging.info("unpack_magic finished on %s", filename)
else:
nzo.set_unpack_info("Unpack", T("No post-processing because of failed verification"))
if cfg.safe_postproc():
all_ok = all_ok and not unpack_error
if all_ok:
# Move any (left-over) files to destination
nzo.status = Status.MOVING
nzo.set_action_line(T("Moving"), "...")
for root, _dirs, files in os.walk(workdir):
if not root.endswith(JOB_ADMIN):
for file_ in files:
path = os.path.join(root, file_)
new_path = path.replace(workdir, tmp_workdir_complete)
ok, new_path = move_to_path(path, new_path)
newfiles.append(new_path)
if not ok:
nzo.set_unpack_info(
"Unpack", T("Failed moving %s to %s") % (unicoder(path), unicoder(new_path))
)
all_ok = False
break
# Set permissions right
set_permissions(tmp_workdir_complete)
if all_ok and marker_file:
del_marker(os.path.join(tmp_workdir_complete, marker_file))
remove_from_list(marker_file, newfiles)
if all_ok:
# Remove files matching the cleanup list
cleanup_list(tmp_workdir_complete, True)
# Check if this is an NZB-only download, if so redirect to queue
# except when PP was Download-only
if flag_repair:
nzb_list = nzb_redirect(
tmp_workdir_complete, nzo.final_name, nzo.pp, script, cat, priority=nzo.priority
)
else:
nzb_list = None
if nzb_list:
nzo.set_unpack_info("Download", T("Sent %s to queue") % unicoder(nzb_list))
cleanup_empty_directories(tmp_workdir_complete)
else:
cleanup_list(tmp_workdir_complete, False)
script_output = ""
script_ret = 0
if not nzb_list:
# Give destination its final name
if cfg.folder_rename() and tmp_workdir_complete and not one_folder:
if all_ok:
try:
newfiles = rename_and_collapse_folder(tmp_workdir_complete, workdir_complete, newfiles)
except:
logging.error(
T('Error renaming "%s" to "%s"'),
clip_path(tmp_workdir_complete),
clip_path(workdir_complete),
)
logging.info("Traceback: ", exc_info=True)
# Better disable sorting because filenames are all off now
file_sorter.sort_file = None
else:
workdir_complete = tmp_workdir_complete.replace("_UNPACK_", "_FAILED_")
workdir_complete = get_unique_path(workdir_complete, n=0, create_dir=False)
workdir_complete = workdir_complete
if empty:
job_result = -1
else:
job_result = int(par_error) + int(bool(unpack_error)) * 2
if cfg.ignore_samples():
remove_samples(workdir_complete)
# TV/Movie/Date Renaming code part 2 - rename and move files to parent folder
if all_ok and file_sorter.sort_file:
if newfiles:
file_sorter.rename(newfiles, workdir_complete)
workdir_complete, ok = file_sorter.move(workdir_complete)
else:
workdir_complete, ok = file_sorter.rename_with_ext(workdir_complete)
if not ok:
nzo.set_unpack_info("Unpack", T("Failed to move files"))
all_ok = False
# Run the user script
script_path = make_script_path(script)
if (all_ok or not cfg.safe_postproc()) and (not nzb_list) and script_path:
# set the current nzo status to "Ext Script...". Used in History
nzo.status = Status.RUNNING
nzo.set_action_line(T("Running script"), unicoder(script))
nzo.set_unpack_info("Script", T("Running user script %s") % unicoder(script), unique=True)
script_log, script_ret = external_processing(
short_path(script_path, False),
short_path(workdir_complete, False),
nzo.filename,
dirname,
cat,
nzo.group,
job_result,
nzo.nzo_info.get("failure", ""),
)
script_line = get_last_line(script_log)
if script_log:
script_output = nzo.nzo_id
if script_line:
nzo.set_unpack_info("Script", unicoder(script_line), unique=True)
else:
nzo.set_unpack_info("Script", T("Ran %s") % unicoder(script), unique=True)
else:
script = ""
script_line = ""
script_ret = 0
# Maybe bad script result should fail job
if script_ret and cfg.script_can_fail():
script_error = True
all_ok = False
nzo.fail_msg = T("Script exit code is %s") % script_ret
else:
script_error = False
# Email the results
if (not nzb_list) and cfg.email_endjob():
if (cfg.email_endjob() == 1) or (cfg.email_endjob() == 2 and (unpack_error or par_error or script_error)):
emailer.endjob(
dirname,
cat,
all_ok,
workdir_complete,
nzo.bytes_downloaded,
nzo.fail_msg,
nzo.unpack_info,
script,
TRANS(script_log),
script_ret,
)
if script_output:
# Can do this only now, otherwise it would show up in the email
if script_ret:
script_ret = "Exit(%s) " % script_ret
else:
script_ret = ""
if script_line:
nzo.set_unpack_info(
"Script",
u'%s%s <a href="./scriptlog?name=%s">(%s)</a>'
% (script_ret, unicoder(script_line), urllib.quote(script_output), T("More")),
unique=True,
)
else:
nzo.set_unpack_info(
"Script",
u'%s<a href="./scriptlog?name=%s">%s</a>'
% (script_ret, urllib.quote(script_output), T("View script output")),
unique=True,
)
# Cleanup again, including NZB files
if all_ok:
cleanup_list(workdir_complete, False)
# Force error for empty result
all_ok = all_ok and not empty
# Update indexer with results
if cfg.rating_enable():
if nzo.encrypted > 0:
Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_ENCRYPTED)
if empty:
hosts = map(lambda s: s.host, sabnzbd.downloader.Downloader.do.nzo_servers(nzo))
if not hosts:
hosts = [None]
for host in hosts:
Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_EXPIRED, host)
# Show final status in history
if all_ok:
growler.send_notification(T("Download Completed"), filename, "complete")
nzo.status = Status.COMPLETED
else:
growler.send_notification(T("Download Failed"), filename, "failed")
nzo.status = Status.FAILED
except:
logging.error(T("Post Processing Failed for %s (%s)"), filename, crash_msg)
if not crash_msg:
logging.info("Traceback: ", exc_info=True)
crash_msg = T("see logfile")
nzo.fail_msg = T("PostProcessing was aborted (%s)") % unicoder(crash_msg)
growler.send_notification(T("Download Failed"), filename, "failed")
nzo.status = Status.FAILED
par_error = True
all_ok = False
if cfg.email_endjob():
emailer.endjob(
dirname,
cat,
all_ok,
clip_path(workdir_complete),
nzo.bytes_downloaded,
nzo.fail_msg,
nzo.unpack_info,
"",
"",
0,
)
if all_ok:
# If the folder only contains one file OR folder, have that as the path
# Be aware that series/generic/date sorting may move a single file into a folder containing other files
workdir_complete = one_file_or_folder(workdir_complete)
workdir_complete = os.path.normpath(workdir_complete)
# Log the overall time taken for postprocessing
postproc_time = int(time.time() - start)
# Create the history DB instance
history_db = database.get_history_handle()
# Add the nzo to the database. Only the path, script and time taken is passed
# Other information is obtained from the nzo
history_db.add_history_db(nzo, clip_path(workdir_complete), nzo.downpath, postproc_time, script_log, script_line)
# The connection is only used once, so close it here
history_db.close()
# Clean up the NZO
try:
logging.info("Cleaning up %s (keep_basic=%s)", filename, str(not all_ok))
sabnzbd.nzbqueue.NzbQueue.do.cleanup_nzo(nzo, keep_basic=not all_ok)
except:
logging.error(T("Cleanup of %s failed."), nzo.final_name)
logging.info("Traceback: ", exc_info=True)
# Remove download folder
if all_ok:
try:
if os.path.exists(workdir):
logging.debug("Removing workdir %s", workdir)
remove_all(workdir, recursive=True)
except:
logging.error(T("Error removing workdir (%s)"), clip_path(workdir))
logging.info("Traceback: ", exc_info=True)
# Use automatic retry link on par2 errors and encrypted/bad RARs
if par_error or unpack_error in (2, 3):
try_alt_nzb(nzo)
return True
def parring(nzo, workdir):
""" Perform par processing. Returns: (par_error, re_add) """
filename = nzo.final_name
notifier.send_notification(T('Post-processing'), filename, 'pp')
logging.info('Starting verification and repair of %s', filename)
# Get verification status of sets
verified = sabnzbd.load_data(VERIFIED_FILE, nzo.workpath, remove=False) or {}
repair_sets = nzo.partable.keys()
re_add = False
par_error = False
single = len(repair_sets) == 1
if repair_sets:
for setname in repair_sets:
if cfg.ignore_samples() and RE_SAMPLE.search(setname.lower()):
continue
if not verified.get(setname, False):
logging.info("Running verification and repair on set %s", setname)
parfile_nzf = nzo.partable[setname]
if os.path.exists(os.path.join(nzo.downpath, parfile_nzf.filename)) or parfile_nzf.extrapars:
need_re_add, res = par2_repair(parfile_nzf, nzo, workdir, setname, single=single)
# Was it aborted?
if not nzo.pp_active:
re_add = False
par_error = True
break
re_add = re_add or need_re_add
verified[setname] = res
else:
continue
par_error = par_error or not res
else:
# Obfuscated par2 check
logging.info('No par2 sets found, running obfuscated check on %s', filename)
# Get the NZF's and sort them based on size
nzfs_sorted = sorted(nzo.finished_files, key=lambda x: x.bytes)
# We will have to make 'fake' par files that are recognized
par2_vol = 0
par2_filename = None
for nzf_try in nzfs_sorted:
# run through list of files, looking for par2 signature..
logging.debug("Checking par2 signature of %s", nzf_try.filename)
try:
nzf_path = os.path.join(workdir, nzf_try.filename)
if(is_parfile(nzf_path)):
# We need 1 base-name so they are recognized as 1 set
if not par2_filename:
par2_filename = nzf_path
# Rename so handle_par2() picks it up
newpath = '%s.vol%d+%d.par2' % (par2_filename, par2_vol, par2_vol + 1)
renamer(nzf_path, newpath)
nzf_try.filename = os.path.split(newpath)[1]
# Let the magic happen
nzo.handle_par2(nzf_try, file_done=True)
par2_vol += 1
except:
pass
if par2_vol > 0:
# Pars found, we do it again
par_error, re_add = parring(nzo, workdir)
else:
# We must not have found any par2..
logging.info("No par2 sets for %s", filename)
nzo.set_unpack_info('Repair', T('[%s] No par2 sets') % unicoder(filename))
if cfg.sfv_check() and not verified.get('', False):
par_error = not try_sfv_check(nzo, workdir, '')
verified[''] = not par_error
# If still no success, do RAR-check
if not par_error and cfg.enable_unrar():
par_error = not try_rar_check(nzo, workdir, '')
verified[''] = not par_error
if re_add:
logging.info('Re-added %s to queue', filename)
if nzo.priority != TOP_PRIORITY:
nzo.priority = REPAIR_PRIORITY
sabnzbd.nzbqueue.add_nzo(nzo)
sabnzbd.downloader.Downloader.do.resume_from_postproc()
sabnzbd.save_data(verified, VERIFIED_FILE, nzo.workpath)
logging.info('Verification and repair finished for %s', filename)
return par_error, re_add
def process_job(nzo):
""" Process one job """
assert isinstance(nzo, sabnzbd.nzbstuff.NzbObject)
start = time.time()
# keep track of whether we can continue
all_ok = True
# keep track of par problems
par_error = False
# keep track of any unpacking errors
unpack_error = False
# Signal empty download, for when 'empty_postproc' is enabled
empty = False
nzb_list = []
# These need to be initialised incase of a crash
workdir_complete = ''
postproc_time = 0
script_log = ''
script_line = ''
crash_msg = ''
## Get the job flags
nzo.save_attribs()
flag_repair, flag_unpack, flag_delete = nzo.repair_opts
# Normalize PP
if flag_delete: flag_unpack = True
if flag_unpack: flag_repair = True
# Get the NZB name
filename = nzo.final_name
msgid = nzo.msgid
if cfg.allow_streaming() and not (flag_repair or flag_unpack or flag_delete):
# After streaming, force +D
nzo.set_pp(3)
nzo.status = Status.FAILED
nzo.save_attribs()
all_ok = False
if nzo.fail_msg: # Special case: aborted due to too many missing data
nzo.status = Status.FAILED
nzo.save_attribs()
all_ok = False
par_error = unpack_error = True
try:
# Get the folder containing the download result
workdir = nzo.downpath
tmp_workdir_complete = None
# if no files are present (except __admin__), fail the job
if all_ok and len(globber(workdir)) < 2:
if nzo.precheck:
enough, ratio = nzo.check_quality()
req_ratio = float(cfg.req_completion_rate()) / 100.0
# Make sure that rounded ratio doesn't equal required ratio
# when it is actually below required
if (ratio < req_ratio) and (req_ratio - ratio) < 0.001:
ratio = req_ratio - 0.001
emsg = '%.1f%%' % (ratio * 100.0)
emsg2 = '%.1f%%' % float(cfg.req_completion_rate())
emsg = T('Download might fail, only %s of required %s available') % (emsg, emsg2)
else:
emsg = T('Download failed - Out of your server\'s retention?')
empty = True
nzo.fail_msg = emsg
nzo.set_unpack_info('Fail', emsg)
nzo.status = Status.FAILED
# do not run unpacking or parity verification
flag_repair = flag_unpack = False
all_ok = cfg.empty_postproc() and empty
if not all_ok:
par_error = unpack_error = True
script = nzo.script
cat = nzo.cat
logging.info('Starting PostProcessing on %s' + \
' => Repair:%s, Unpack:%s, Delete:%s, Script:%s, Cat:%s',
filename, flag_repair, flag_unpack, flag_delete, script, cat)
## Par processing, if enabled
if all_ok and flag_repair:
par_error, re_add = parring(nzo, workdir)
if re_add:
# Try to get more par files
return False
## Check if user allows unsafe post-processing
if flag_repair and cfg.safe_postproc():
all_ok = all_ok and not par_error
# Set complete dir to workdir in case we need to abort
workdir_complete = workdir
dirname = nzo.final_name
marker_file = None
if all_ok:
one_folder = False
## Determine class directory
if cfg.create_group_folders():
complete_dir = addPrefixes(cfg.complete_dir.get_path(), nzo.dirprefix)
complete_dir = create_dirs(complete_dir)
else:
catdir = config.get_categories(cat).dir()
if catdir.endswith('*'):
catdir = catdir.strip('*')
one_folder = True
complete_dir = real_path(cfg.complete_dir.get_path(), catdir)
## TV/Movie/Date Renaming code part 1 - detect and construct paths
file_sorter = Sorter(cat)
complete_dir = file_sorter.detect(dirname, complete_dir)
if file_sorter.sort_file:
one_folder = False
if one_folder:
workdir_complete = create_dirs(complete_dir)
else:
workdir_complete = get_unique_path(os.path.join(complete_dir, dirname), create_dir=True)
marker_file = set_marker(workdir_complete)
if not workdir_complete or not os.path.exists(workdir_complete):
crash_msg = T('Cannot create final folder %s') % unicoder(os.path.join(complete_dir, dirname))
raise IOError
if cfg.folder_rename() and not one_folder:
tmp_workdir_complete = prefix(workdir_complete, '_UNPACK_')
try:
renamer(workdir_complete, tmp_workdir_complete)
except:
pass # On failure, just use the original name
else:
tmp_workdir_complete = workdir_complete
newfiles = []
## Run Stage 2: Unpack
if flag_unpack:
if all_ok:
#set the current nzo status to "Extracting...". Used in History
nzo.status = Status.EXTRACTING
logging.info("Running unpack_magic on %s", filename)
unpack_error, newfiles = unpack_magic(nzo, workdir, tmp_workdir_complete, flag_delete, one_folder, (), (), (), ())
logging.info("unpack_magic finished on %s", filename)
else:
nzo.set_unpack_info('Unpack', T('No post-processing because of failed verification'))
if cfg.safe_postproc():
all_ok = all_ok and not unpack_error
if all_ok:
## Move any (left-over) files to destination
nzo.status = Status.MOVING
nzo.set_action_line(T('Moving'), '...')
for root, dirs, files in os.walk(workdir):
if not root.endswith(JOB_ADMIN):
for file_ in files:
path = os.path.join(root, file_)
new_path = path.replace(workdir, tmp_workdir_complete)
ok, new_path = move_to_path(path, new_path)
newfiles.append(new_path)
if not ok:
nzo.set_unpack_info('Unpack', T('Failed moving %s to %s') % (unicoder(path), unicoder(new_path)))
all_ok = False
break
## Set permissions right
set_permissions(tmp_workdir_complete)
if all_ok and marker_file:
del_marker(os.path.join(tmp_workdir_complete, marker_file))
remove_from_list(marker_file, newfiles)
if all_ok:
## Remove files matching the cleanup list
cleanup_list(tmp_workdir_complete, True)
## Check if this is an NZB-only download, if so redirect to queue
## except when PP was Download-only
if flag_repair:
nzb_list = nzb_redirect(tmp_workdir_complete, nzo.final_name, nzo.pp, script, cat, priority=nzo.priority)
else:
nzb_list = None
if nzb_list:
nzo.set_unpack_info('Download', T('Sent %s to queue') % unicoder(nzb_list))
cleanup_empty_directories(tmp_workdir_complete)
else:
cleanup_list(tmp_workdir_complete, False)
script_output = ''
script_ret = 0
if not nzb_list:
## Give destination its final name
if cfg.folder_rename() and tmp_workdir_complete and not one_folder:
if all_ok:
try:
newfiles = rename_and_collapse_folder(tmp_workdir_complete, workdir_complete, newfiles)
except:
logging.error(Ta('Error renaming "%s" to "%s"'), tmp_workdir_complete, workdir_complete)
logging.info('Traceback: ', exc_info = True)
# Better disable sorting because filenames are all off now
file_sorter.sort_file = None
else:
workdir_complete = tmp_workdir_complete.replace('_UNPACK_', '_FAILED_')
workdir_complete = get_unique_path(workdir_complete, n=0, create_dir=False)
if empty:
job_result = -1
else:
job_result = int(par_error) + int(unpack_error)*2
if cfg.ignore_samples() > 0:
remove_samples(workdir_complete)
## TV/Movie/Date Renaming code part 2 - rename and move files to parent folder
if all_ok and file_sorter.sort_file:
if newfiles:
file_sorter.rename(newfiles, workdir_complete)
workdir_complete, ok = file_sorter.move(workdir_complete)
else:
workdir_complete, ok = file_sorter.rename_with_ext(workdir_complete)
if not ok:
nzo.set_unpack_info('Unpack', T('Failed to move files'))
all_ok = False
## Run the user script
script_path = make_script_path(script)
if (all_ok or not cfg.safe_postproc()) and (not nzb_list) and script_path:
#set the current nzo status to "Ext Script...". Used in History
nzo.status = Status.RUNNING
nzo.set_action_line(T('Running script'), unicoder(script))
nzo.set_unpack_info('Script', T('Running user script %s') % unicoder(script), unique=True)
script_log, script_ret = external_processing(script_path, workdir_complete, nzo.filename,
msgid, dirname, cat, nzo.group, job_result)
script_line = get_last_line(script_log)
if script_log:
script_output = nzo.nzo_id
if script_line:
nzo.set_unpack_info('Script', unicoder(script_line), unique=True)
else:
nzo.set_unpack_info('Script', T('Ran %s') % unicoder(script), unique=True)
else:
script = ""
script_line = ""
script_ret = 0
## Email the results
if (not nzb_list) and cfg.email_endjob():
if (cfg.email_endjob() == 1) or (cfg.email_endjob() == 2 and (unpack_error or par_error)):
emailer.endjob(dirname, msgid, cat, all_ok, workdir_complete, nzo.bytes_downloaded,
nzo.fail_msg, nzo.unpack_info, script, TRANS(script_log), script_ret)
if script_output:
# Can do this only now, otherwise it would show up in the email
if script_ret:
script_ret = 'Exit(%s) ' % script_ret
else:
script_ret = ''
if script_line:
nzo.set_unpack_info('Script',
u'%s%s <a href="./scriptlog?name=%s">(%s)</a>' % (script_ret, unicoder(script_line), urllib.quote(script_output),
T('More')), unique=True)
else:
nzo.set_unpack_info('Script',
u'%s<a href="./scriptlog?name=%s">%s</a>' % (script_ret, urllib.quote(script_output),
T('View script output')), unique=True)
## Cleanup again, including NZB files
if all_ok:
cleanup_list(workdir_complete, False)
## Remove newzbin bookmark, if any
if msgid and all_ok:
Bookmarks.do.del_bookmark(msgid)
elif all_ok and isinstance(nzo.url, str):
sabnzbd.proxy_rm_bookmark(nzo.url)
## Force error for empty result
all_ok = all_ok and not empty
## Show final status in history
if all_ok:
growler.send_notification(T('Download Completed'), filename, 'complete')
nzo.status = Status.COMPLETED
else:
growler.send_notification(T('Download Failed'), filename, 'complete')
nzo.status = Status.FAILED
except:
logging.error(Ta('Post Processing Failed for %s (%s)'), filename, crash_msg)
if not crash_msg:
logging.info("Traceback: ", exc_info = True)
crash_msg = T('see logfile')
nzo.fail_msg = T('PostProcessing was aborted (%s)') % unicoder(crash_msg)
growler.send_notification(T('Download Failed'), filename, 'complete')
nzo.status = Status.FAILED
par_error = True
all_ok = False
if cfg.email_endjob():
emailer.endjob(dirname, msgid, cat, all_ok, workdir_complete, nzo.bytes_downloaded,
nzo.fail_msg, nzo.unpack_info, '', '', 0)
if all_ok:
# If the folder only contains one file OR folder, have that as the path
# Be aware that series/generic/date sorting may move a single file into a folder containing other files
workdir_complete = one_file_or_folder(workdir_complete)
workdir_complete = os.path.normpath(workdir_complete)
# Log the overall time taken for postprocessing
postproc_time = int(time.time() - start)
# Create the history DB instance
history_db = database.get_history_handle()
# Add the nzo to the database. Only the path, script and time taken is passed
# Other information is obtained from the nzo
history_db.add_history_db(nzo, workdir_complete, nzo.downpath, postproc_time, script_log, script_line)
# The connection is only used once, so close it here
history_db.close()
## Clean up the NZO
try:
logging.info('Cleaning up %s (keep_basic=%s)', filename, str(not all_ok))
sabnzbd.nzbqueue.NzbQueue.do.cleanup_nzo(nzo, keep_basic=not all_ok)
except:
logging.error(Ta('Cleanup of %s failed.'), nzo.final_name)
logging.info("Traceback: ", exc_info = True)
## Remove download folder
if all_ok:
try:
if os.path.exists(workdir):
logging.debug('Removing workdir %s', workdir)
remove_all(workdir, recursive=True)
except:
logging.error(Ta('Error removing workdir (%s)'), workdir)
logging.info("Traceback: ", exc_info = True)
return True
def rename(self, _files, current_path):
logging.debug("Renaming Generic file")
def filter_files(_file, current_path):
if is_full_path(_file):
filepath = _file.replace('_UNPACK_', '')
else:
filepath = os.path.join(current_path, _file)
if os.path.exists(filepath):
size = os.stat(filepath).st_size
if size > 314572800 and not RE_SAMPLE.search(_file) \
and get_ext(_file) not in EXCLUDED_FILE_EXTS:
return True
return False
renamed = False
# remove any files below 300MB from this list
files = [_file for _file in _files if filter_files(_file, current_path)]
length = len(files)
## Single File Handling
if length == 1:
file = files[0]
if is_full_path(file):
filepath = file.replace('_UNPACK_', '')
else:
filepath = os.path.join(current_path, file)
if os.path.exists(filepath):
tmp, ext = os.path.splitext(file)
newname = "%s%s" % (self.filename_set,ext)
newname = newname.replace('%fn',tmp)
newpath = os.path.join(current_path, newname)
try:
logging.debug("Rename: %s to %s", filepath,newpath)
renamer(filepath,newpath)
except:
logging.error(Ta('error-tvRename@2'), filepath, newpath)
logging.debug("Traceback: ", exc_info = True)
rename_similar(current_path, file, self.filename_set)
## Sequence File Handling
# if there is more than one extracted file check for CD1/1/A in the title
elif self.extra:
matched_files = check_for_multiple(files)
# rename files marked as in a set
if matched_files:
logging.debug("Renaming a series of generic files (%s)", matched_files)
for index, file in matched_files.iteritems():
filepath = os.path.join(current_path, file)
tmp, ext = os.path.splitext(file)
name = '%s%s' % (self.filename_set, self.extra)
name = name.replace('%1', str(index)).replace('%fn',tmp)
name = name + ext
newpath = os.path.join(current_path, name)
try:
logging.debug("Rename: %s to %s", filepath,newpath)
renamer(filepath,newpath)
except:
logging.error(Ta('error-tvRename@2'), filepath, newpath)
logging.debug("Traceback: ", exc_info = True)
rename_similar(current_path, file, self.filename_set)
else:
logging.debug("Movie files not in sequence %s", _files)
def process_job(nzo):
""" Process one job """
assert isinstance(nzo, sabnzbd.nzbstuff.NzbObject)
start = time.time()
# keep track of whether we can continue
all_ok = True
# keep track of par problems
par_error = False
# keep track of any unpacking errors
unpack_error = False
nzb_list = []
# These need to be initialised incase of a crash
workdir_complete = ''
postproc_time = 0
script_log = ''
script_line = ''
crash_msg = ''
## Get the job flags
nzo.save_attribs()
flag_repair, flag_unpack, flag_delete = nzo.repair_opts
# Normalize PP
if flag_delete: flag_unpack = True
if flag_unpack: flag_repair = True
# Get the NZB name
filename = nzo.final_name
msgid = nzo.msgid
if cfg.allow_streaming() and not (flag_repair or flag_unpack
or flag_delete):
# After streaming, force +D
nzo.set_pp(3)
nzo.status = 'Failed'
nzo.save_attribs()
all_ok = False
try:
# Get the folder containing the download result
workdir = nzo.downpath
tmp_workdir_complete = None
# if no files are present (except __admin__), fail the job
if len(globber(workdir)) < 2:
emsg = T('Download failed - Out of your server\'s retention?')
nzo.fail_msg = emsg
nzo.status = 'Failed'
# do not run unpacking or parity verification
flag_repair = flag_unpack = False
par_error = unpack_error = True
all_ok = False
script = nzo.script
cat = nzo.cat
logging.info('Starting PostProcessing on %s' + \
' => Repair:%s, Unpack:%s, Delete:%s, Script:%s, Cat:%s',
filename, flag_repair, flag_unpack, flag_delete, script, cat)
## Par processing, if enabled
if flag_repair:
par_error, re_add = parring(nzo, workdir)
if re_add:
# Try to get more par files
return False
## Check if user allows unsafe post-processing
if flag_repair and cfg.safe_postproc():
all_ok = all_ok and not par_error
# Set complete dir to workdir in case we need to abort
workdir_complete = workdir
dirname = nzo.final_name
if all_ok:
one_folder = False
## Determine class directory
if cfg.create_group_folders():
complete_dir = addPrefixes(cfg.complete_dir.get_path(),
nzo.dirprefix)
complete_dir = create_dirs(complete_dir)
else:
catdir = config.get_categories(cat).dir()
if catdir.endswith('*'):
catdir = catdir.strip('*')
one_folder = True
complete_dir = real_path(cfg.complete_dir.get_path(), catdir)
## TV/Movie/Date Renaming code part 1 - detect and construct paths
file_sorter = Sorter(cat)
complete_dir = file_sorter.detect(dirname, complete_dir)
if file_sorter.is_sortfile():
one_folder = False
if one_folder:
workdir_complete = create_dirs(complete_dir)
else:
workdir_complete = get_unique_path(os.path.join(
complete_dir, dirname),
create_dir=True)
if not workdir_complete or not os.path.exists(workdir_complete):
crash_msg = T('Cannot create final folder %s') % unicoder(
os.path.join(complete_dir, dirname))
raise IOError
if cfg.folder_rename() and not one_folder:
tmp_workdir_complete = prefix(workdir_complete, '_UNPACK_')
try:
renamer(workdir_complete, tmp_workdir_complete)
except:
pass # On failure, just use the original name
else:
tmp_workdir_complete = workdir_complete
newfiles = []
## Run Stage 2: Unpack
if flag_unpack:
if all_ok:
#set the current nzo status to "Extracting...". Used in History
nzo.status = 'Extracting'
logging.info("Running unpack_magic on %s", filename)
unpack_error, newfiles = unpack_magic(
nzo, workdir, tmp_workdir_complete, flag_delete,
one_folder, (), (), (), ())
logging.info("unpack_magic finished on %s", filename)
else:
nzo.set_unpack_info(
'Unpack',
T('No post-processing because of failed verification'))
if cfg.safe_postproc():
all_ok = all_ok and not unpack_error
if all_ok:
## Move any (left-over) files to destination
nzo.status = 'Moving'
nzo.set_action_line(T('Moving'), '...')
for root, dirs, files in os.walk(workdir):
if not root.endswith(JOB_ADMIN):
for file_ in files:
path = os.path.join(root, file_)
new_path = path.replace(workdir,
tmp_workdir_complete)
new_path = get_unique_filename(new_path)
move_to_path(path, new_path, unique=False)
## Set permissions right
if not sabnzbd.WIN32:
perm_script(tmp_workdir_complete, cfg.umask())
if all_ok:
## Remove files matching the cleanup list
cleanup_list(tmp_workdir_complete, True)
## Check if this is an NZB-only download, if so redirect to queue
## except when PP was Download-only
if flag_repair:
nzb_list = nzb_redirect(tmp_workdir_complete,
nzo.final_name,
nzo.pp,
script,
cat,
priority=nzo.priority)
else:
nzb_list = None
if nzb_list:
nzo.set_unpack_info(
'Download',
T('Sent %s to queue') % unicoder(nzb_list))
try:
remove_dir(tmp_workdir_complete)
except:
pass
else:
cleanup_list(tmp_workdir_complete, False)
script_output = ''
script_ret = 0
if not nzb_list:
## Give destination its final name
if cfg.folder_rename() and tmp_workdir_complete and not one_folder:
if not all_ok:
workdir_complete = tmp_workdir_complete.replace(
'_UNPACK_', '_FAILED_')
workdir_complete = get_unique_path(workdir_complete,
n=0,
create_dir=False)
try:
collapse_folder(tmp_workdir_complete, workdir_complete)
except:
logging.error(Ta('Error renaming "%s" to "%s"'),
tmp_workdir_complete, workdir_complete)
logging.info("Traceback: ", exc_info=True)
job_result = int(par_error) + int(unpack_error) * 2
if cfg.ignore_samples() > 0:
remove_samples(workdir_complete)
## TV/Movie/Date Renaming code part 2 - rename and move files to parent folder
if all_ok:
if newfiles and file_sorter.is_sortfile():
file_sorter.rename(newfiles, workdir_complete)
workdir_complete = file_sorter.move(workdir_complete)
## Run the user script
script_path = make_script_path(script)
if all_ok and (not nzb_list) and script_path:
#set the current nzo status to "Ext Script...". Used in History
nzo.status = 'Running'
nzo.set_action_line(T('Running script'), unicoder(script))
nzo.set_unpack_info('Script',
T('Running user script %s') %
unicoder(script),
unique=True)
script_log, script_ret = external_processing(
script_path, workdir_complete, nzo.filename, msgid,
dirname, cat, nzo.group, job_result)
script_line = get_last_line(script_log)
if script_log:
script_output = nzo.nzo_id
if script_line:
nzo.set_unpack_info('Script',
unicoder(script_line),
unique=True)
else:
nzo.set_unpack_info('Script',
T('Ran %s') % unicoder(script),
unique=True)
else:
script = ""
script_line = ""
script_ret = 0
## Email the results
if (not nzb_list) and cfg.email_endjob():
if (cfg.email_endjob() == 1) or (cfg.email_endjob() == 2 and
(unpack_error or par_error)):
emailer.endjob(dirname, msgid, cat, all_ok, workdir_complete,
nzo.bytes_downloaded, nzo.unpack_info, script,
TRANS(script_log), script_ret)
if script_output:
# Can do this only now, otherwise it would show up in the email
if script_ret:
script_ret = 'Exit(%s) ' % script_ret
else:
script_ret = ''
if script_line:
nzo.set_unpack_info(
'Script',
u'%s%s <a href="./scriptlog?name=%s">(%s)</a>' %
(script_ret, unicoder(script_line),
urllib.quote(script_output), T('More')),
unique=True)
else:
nzo.set_unpack_info('Script',
u'%s<a href="./scriptlog?name=%s">%s</a>' %
(script_ret, urllib.quote(script_output),
T('View script output')),
unique=True)
## Cleanup again, including NZB files
cleanup_list(workdir_complete, False)
## Remove newzbin bookmark, if any
if msgid and all_ok:
Bookmarks.do.del_bookmark(msgid)
elif all_ok:
sabnzbd.proxy_rm_bookmark(nzo.url)
## Show final status in history
if all_ok:
osx.sendGrowlMsg(T('Download Completed'), filename,
osx.NOTIFICATION['complete'])
nzo.status = 'Completed'
else:
osx.sendGrowlMsg(T('Download Failed'), filename,
osx.NOTIFICATION['complete'])
nzo.status = 'Failed'
except:
logging.error(Ta('Post Processing Failed for %s (%s)'), filename,
crash_msg)
if not crash_msg:
logging.info("Traceback: ", exc_info=True)
crash_msg = T('see logfile')
nzo.fail_msg = T('PostProcessing was aborted (%s)') % unicoder(
crash_msg)
osx.sendGrowlMsg(T('Download Failed'), filename,
osx.NOTIFICATION['complete'])
nzo.status = 'Failed'
par_error = True
all_ok = False
if all_ok:
# If the folder only contains one file OR folder, have that as the path
# Be aware that series/generic/date sorting may move a single file into a folder containing other files
workdir_complete = one_file_or_folder(workdir_complete)
workdir_complete = os.path.normpath(workdir_complete)
# Log the overall time taken for postprocessing
postproc_time = int(time.time() - start)
# Create the history DB instance
history_db = database.get_history_handle()
# Add the nzo to the database. Only the path, script and time taken is passed
# Other information is obtained from the nzo
history_db.add_history_db(nzo, workdir_complete, nzo.downpath,
postproc_time, script_log, script_line)
# The connection is only used once, so close it here
history_db.close()
## Clean up the NZO
try:
logging.info('Cleaning up %s (keep_basic=%s)', filename,
str(not all_ok))
sabnzbd.nzbqueue.NzbQueue.do.cleanup_nzo(nzo, keep_basic=not all_ok)
except:
logging.error(Ta('Cleanup of %s failed.'), nzo.final_name)
logging.info("Traceback: ", exc_info=True)
## Remove download folder
if all_ok:
try:
if os.path.exists(workdir):
logging.debug('Removing workdir %s', workdir)
remove_all(workdir, recursive=True)
except:
logging.error(Ta('Error removing workdir (%s)'), workdir)
logging.info("Traceback: ", exc_info=True)
return True
def parring(nzo, workdir):
""" Perform par processing. Returns: (par_error, re_add) """
if 0:
assert isinstance(
nzo, sabnzbd.nzbstuff.NzbObject) # Assert only for debug purposes
filename = nzo.final_name
notifier.send_notification(T('Post-processing'), filename, 'pp')
logging.info('Starting verification and repair of %s', filename)
# Get verification status of sets
verified = sabnzbd.load_data(VERIFIED_FILE, nzo.workpath,
remove=False) or {}
# Collect the par files
if nzo.partable:
par_table = nzo.partable.copy()
else:
par_table = {}
repair_sets = par_table.keys()
re_add = False
par_error = False
single = len(repair_sets) == 1
if repair_sets:
for setname in repair_sets:
if cfg.ignore_samples() and 'sample' in setname.lower():
continue
if not verified.get(setname, False):
logging.info("Running verification and repair on set %s",
setname)
parfile_nzf = par_table[setname]
if os.path.exists(
os.path.join(
nzo.downpath,
parfile_nzf.filename)) or parfile_nzf.extrapars:
need_re_add, res = par2_repair(parfile_nzf,
nzo,
workdir,
setname,
single=single)
# Was it aborted?
if not nzo.pp_active:
re_add = False
par_error = True
break
re_add = re_add or need_re_add
if not res and not need_re_add and cfg.sfv_check():
res = try_sfv_check(nzo, workdir, setname)
if not res and not need_re_add and cfg.enable_unrar():
res = try_rar_check(nzo, workdir, setname)
verified[setname] = res
else:
continue
par_error = par_error or not res
else:
# Obfuscated par2 check
logging.info('No par2 sets found, running obfuscated check on %s',
filename)
# Get the NZF's and sort them based on size
nzfs_sorted = sorted(nzo.finished_files, key=lambda x: x.bytes)
# We will have to make 'fake' par files that are recognized
par2_vol = 0
par2_filename = None
for nzf_try in nzfs_sorted:
# run through list of files, looking for par2 signature..
logging.debug("Checking par2 signature of %s", nzf_try.filename)
try:
nzf_path = os.path.join(workdir, nzf_try.filename)
if (is_parfile(nzf_path)):
# We need 1 base-name so they are recognized as 1 set
if not par2_filename:
par2_filename = nzf_path
# Rename so handle_par2() picks it up
newpath = '%s.vol%d+%d.par2' % (par2_filename, par2_vol,
par2_vol + 1)
renamer(nzf_path, newpath)
nzf_try.filename = os.path.split(newpath)[1]
# Let the magic happen
nzo.handle_par2(nzf_try, file_done=True)
par2_vol += 1
except:
pass
if par2_vol > 0:
# Pars found, we do it again
par_error, re_add = parring(nzo, workdir)
else:
# We must not have found any par2..
logging.info("No par2 sets for %s", filename)
nzo.set_unpack_info('Repair',
T('[%s] No par2 sets') % unicoder(filename))
if cfg.sfv_check() and not verified.get('', False):
par_error = not try_sfv_check(nzo, workdir, '')
verified[''] = not par_error
# If still no success, do RAR-check
if not par_error and cfg.enable_unrar():
par_error = not try_rar_check(nzo, workdir, '')
verified[''] = not par_error
if re_add:
logging.info('Re-added %s to queue', filename)
if nzo.priority != TOP_PRIORITY:
nzo.priority = REPAIR_PRIORITY
sabnzbd.nzbqueue.add_nzo(nzo)
sabnzbd.downloader.Downloader.do.resume_from_postproc()
sabnzbd.save_data(verified, VERIFIED_FILE, nzo.workpath)
logging.info('Verification and repair finished for %s', filename)
return par_error, re_add
def rename(self, _files, current_path):
logging.debug("Renaming Generic file")
def filter_files(_file, current_path):
if is_full_path(_file):
filepath = _file.replace('_UNPACK_', '')
else:
filepath = os.path.join(current_path, _file)
if os.path.exists(filepath):
size = os.stat(filepath).st_size
if size > 314572800 and not RE_SAMPLE.search(_file) \
and get_ext(_file) not in EXCLUDED_FILE_EXTS:
return True
return False
renamed = False
# remove any files below 300MB from this list
files = [
_file for _file in _files if filter_files(_file, current_path)
]
length = len(files)
## Single File Handling
if length == 1:
file = files[0]
if is_full_path(file):
filepath = file.replace('_UNPACK_', '')
else:
filepath = os.path.join(current_path, file)
if os.path.exists(filepath):
tmp, ext = os.path.splitext(file)
self.fname = tmp
newname = "%s%s" % (self.filename_set, ext)
newname = newname.replace('%fn', tmp)
newpath = os.path.join(current_path, newname)
try:
logging.debug("Rename: %s to %s", filepath, newpath)
renamer(filepath, newpath)
except:
logging.error(Ta('Failed to rename: %s to %s'), filepath,
newpath)
logging.info("Traceback: ", exc_info=True)
rename_similar(current_path, file, self.filename_set)
## Sequence File Handling
# if there is more than one extracted file check for CD1/1/A in the title
elif self.extra:
matched_files = check_for_multiple(files)
# rename files marked as in a set
if matched_files:
logging.debug("Renaming a series of generic files (%s)",
matched_files)
for index, file in matched_files.iteritems():
filepath = os.path.join(current_path, file)
tmp, ext = os.path.splitext(file)
self.fname = tmp
name = '%s%s' % (self.filename_set, self.extra)
name = name.replace('%1', str(index)).replace('%fn', tmp)
name = name + ext
newpath = os.path.join(current_path, name)
try:
logging.debug("Rename: %s to %s", filepath, newpath)
renamer(filepath, newpath)
except:
logging.error(Ta('Failed to rename: %s to %s'),
filepath, newpath)
logging.info("Traceback: ", exc_info=True)
rename_similar(current_path, file, self.filename_set)
else:
logging.debug("Movie files not in sequence %s", _files)
def rename(self, _files, current_path):
""" Rename for Generic files """
logging.debug("Renaming Generic file")
def filter_files(_file, current_path):
if is_full_path(_file):
filepath = os.path.normpath(_file)
else:
filepath = os.path.normpath(os.path.join(current_path, _file))
if os.path.exists(filepath):
size = os.stat(filepath).st_size
if size >= cfg.movie_rename_limit.get_int() and not RE_SAMPLE.search(_file) \
and get_ext(_file) not in EXCLUDED_FILE_EXTS:
return True
return False
# remove any files below the limit from this list
files = [_file for _file in _files if filter_files(_file, current_path)]
length = len(files)
# Single File Handling
if length == 1:
file = files[0]
if is_full_path(file):
filepath = os.path.normpath(file)
else:
filepath = os.path.normpath(os.path.join(current_path, file))
if os.path.exists(filepath):
self.fname, ext = os.path.splitext(os.path.split(file)[1])
newname = "%s%s" % (self.filename_set, ext)
newname = newname.replace('%fn', self.fname)
newpath = os.path.join(current_path, newname)
try:
logging.debug("Rename: %s to %s", filepath, newpath)
renamer(filepath, newpath)
except:
logging.error(T('Failed to rename: %s to %s'), clip_path(filepath), clip_path(newpath))
logging.info("Traceback: ", exc_info=True)
rename_similar(current_path, ext, self.filename_set, ())
# Sequence File Handling
# if there is more than one extracted file check for CD1/1/A in the title
elif self.extra:
matched_files = check_for_multiple(files)
# rename files marked as in a set
if matched_files:
logging.debug("Renaming a series of generic files (%s)", matched_files)
renamed = matched_files.values()
for index, file in matched_files.iteritems():
filepath = os.path.join(current_path, file)
renamed.append(filepath)
self.fname, ext = os.path.splitext(os.path.split(file)[1])
name = '%s%s' % (self.filename_set, self.extra)
name = name.replace('%1', str(index)).replace('%fn', self.fname)
name = name + ext
newpath = os.path.join(current_path, name)
try:
logging.debug("Rename: %s to %s", filepath, newpath)
renamer(filepath, newpath)
except:
logging.error(T('Failed to rename: %s to %s'), clip_path(filepath), clip_path(newpath))
logging.info("Traceback: ", exc_info=True)
rename_similar(current_path, ext, self.filename_set, renamed)
else:
logging.debug("Movie files not in sequence %s", _files)
def process_job(nzo):
""" Process one job """
assert isinstance(nzo, sabnzbd.nzbstuff.NzbObject)
start = time.time()
# keep track of whether we can continue
all_ok = True
# keep track of par problems
par_error = False
# keep track of any unpacking errors
unpack_error = False
# Signal empty download, for when 'empty_postproc' is enabled
empty = False
nzb_list = []
# These need to be initialised incase of a crash
workdir_complete = ''
postproc_time = 0
script_log = ''
script_line = ''
crash_msg = ''
## Get the job flags
nzo.save_attribs()
flag_repair, flag_unpack, flag_delete = nzo.repair_opts
# Normalize PP
if flag_delete: flag_unpack = True
if flag_unpack: flag_repair = True
# Get the NZB name
filename = nzo.final_name
msgid = nzo.msgid
if cfg.allow_streaming() and not (flag_repair or flag_unpack
or flag_delete):
# After streaming, force +D
nzo.set_pp(3)
nzo.status = Status.FAILED
nzo.save_attribs()
all_ok = False
if nzo.fail_msg: # Special case: aborted due to too many missing data
nzo.status = Status.FAILED
nzo.save_attribs()
all_ok = False
par_error = unpack_error = True
try:
# Get the folder containing the download result
workdir = nzo.downpath
tmp_workdir_complete = None
# if no files are present (except __admin__), fail the job
if all_ok and len(globber(workdir)) < 2:
if nzo.precheck:
enough, ratio = nzo.check_quality()
req_ratio = float(cfg.req_completion_rate()) / 100.0
# Make sure that rounded ratio doesn't equal required ratio
# when it is actually below required
if (ratio < req_ratio) and (req_ratio - ratio) < 0.001:
ratio = req_ratio - 0.001
emsg = '%.1f%%' % (ratio * 100.0)
emsg2 = '%.1f%%' % float(cfg.req_completion_rate())
emsg = T(
'Download might fail, only %s of required %s available'
) % (emsg, emsg2)
else:
emsg = T('Download failed - Out of your server\'s retention?')
empty = True
nzo.fail_msg = emsg
nzo.set_unpack_info('Fail', emsg)
nzo.status = Status.FAILED
# do not run unpacking or parity verification
flag_repair = flag_unpack = False
all_ok = cfg.empty_postproc() and empty
if not all_ok:
par_error = unpack_error = True
script = nzo.script
cat = nzo.cat
logging.info('Starting PostProcessing on %s' + \
' => Repair:%s, Unpack:%s, Delete:%s, Script:%s, Cat:%s',
filename, flag_repair, flag_unpack, flag_delete, script, cat)
## Par processing, if enabled
if all_ok and flag_repair:
par_error, re_add = parring(nzo, workdir)
if re_add:
# Try to get more par files
return False
## Check if user allows unsafe post-processing
if flag_repair and cfg.safe_postproc():
all_ok = all_ok and not par_error
# Set complete dir to workdir in case we need to abort
workdir_complete = workdir
dirname = nzo.final_name
marker_file = None
if all_ok:
one_folder = False
## Determine class directory
if cfg.create_group_folders():
complete_dir = addPrefixes(cfg.complete_dir.get_path(),
nzo.dirprefix)
complete_dir = create_dirs(complete_dir)
else:
catdir = config.get_categories(cat).dir()
if catdir.endswith('*'):
catdir = catdir.strip('*')
one_folder = True
complete_dir = real_path(cfg.complete_dir.get_path(), catdir)
## TV/Movie/Date Renaming code part 1 - detect and construct paths
if cfg.enable_meta():
file_sorter = Sorter(nzo, cat)
else:
file_sorter = Sorter(None, cat)
complete_dir = file_sorter.detect(dirname, complete_dir)
if file_sorter.sort_file:
one_folder = False
if one_folder:
workdir_complete = create_dirs(complete_dir)
else:
workdir_complete = get_unique_path(os.path.join(
complete_dir, dirname),
create_dir=True)
marker_file = set_marker(workdir_complete)
if not workdir_complete or not os.path.exists(workdir_complete):
crash_msg = T('Cannot create final folder %s') % unicoder(
os.path.join(complete_dir, dirname))
raise IOError
if cfg.folder_rename() and not one_folder:
tmp_workdir_complete = prefix(workdir_complete, '_UNPACK_')
try:
renamer(workdir_complete, tmp_workdir_complete)
except:
pass # On failure, just use the original name
else:
tmp_workdir_complete = workdir_complete
newfiles = []
## Run Stage 2: Unpack
if flag_unpack:
if all_ok:
#set the current nzo status to "Extracting...". Used in History
nzo.status = Status.EXTRACTING
logging.info("Running unpack_magic on %s", filename)
unpack_error, newfiles = unpack_magic(
nzo, workdir, tmp_workdir_complete, flag_delete,
one_folder, (), (), (), ())
logging.info("unpack_magic finished on %s", filename)
else:
nzo.set_unpack_info(
'Unpack',
T('No post-processing because of failed verification'))
if cfg.safe_postproc():
all_ok = all_ok and not unpack_error
if all_ok:
## Move any (left-over) files to destination
nzo.status = Status.MOVING
nzo.set_action_line(T('Moving'), '...')
for root, dirs, files in os.walk(workdir):
if not root.endswith(JOB_ADMIN):
for file_ in files:
path = os.path.join(root, file_)
new_path = path.replace(workdir,
tmp_workdir_complete)
ok, new_path = move_to_path(path, new_path)
newfiles.append(new_path)
if not ok:
nzo.set_unpack_info(
'Unpack',
T('Failed moving %s to %s') %
(unicoder(path), unicoder(new_path)))
all_ok = False
break
## Set permissions right
set_permissions(tmp_workdir_complete)
if all_ok and marker_file:
del_marker(os.path.join(tmp_workdir_complete, marker_file))
remove_from_list(marker_file, newfiles)
if all_ok:
## Remove files matching the cleanup list
cleanup_list(tmp_workdir_complete, True)
## Check if this is an NZB-only download, if so redirect to queue
## except when PP was Download-only
if flag_repair:
nzb_list = nzb_redirect(tmp_workdir_complete,
nzo.final_name,
nzo.pp,
script,
cat,
priority=nzo.priority)
else:
nzb_list = None
if nzb_list:
nzo.set_unpack_info(
'Download',
T('Sent %s to queue') % unicoder(nzb_list))
cleanup_empty_directories(tmp_workdir_complete)
else:
cleanup_list(tmp_workdir_complete, False)
script_output = ''
script_ret = 0
if not nzb_list:
## Give destination its final name
if cfg.folder_rename() and tmp_workdir_complete and not one_folder:
if all_ok:
try:
newfiles = rename_and_collapse_folder(
tmp_workdir_complete, workdir_complete, newfiles)
except:
logging.error(Ta('Error renaming "%s" to "%s"'),
tmp_workdir_complete, workdir_complete)
logging.info('Traceback: ', exc_info=True)
# Better disable sorting because filenames are all off now
file_sorter.sort_file = None
else:
workdir_complete = tmp_workdir_complete.replace(
'_UNPACK_', '_FAILED_')
workdir_complete = get_unique_path(workdir_complete,
n=0,
create_dir=False)
if empty:
job_result = -1
else:
job_result = int(par_error) + int(unpack_error) * 2
if cfg.ignore_samples() > 0:
remove_samples(workdir_complete)
## TV/Movie/Date Renaming code part 2 - rename and move files to parent folder
if all_ok and file_sorter.sort_file:
if newfiles:
file_sorter.rename(newfiles, workdir_complete)
workdir_complete, ok = file_sorter.move(workdir_complete)
else:
workdir_complete, ok = file_sorter.rename_with_ext(
workdir_complete)
if not ok:
nzo.set_unpack_info('Unpack', T('Failed to move files'))
all_ok = False
## Run the user script
script_path = make_script_path(script)
if (all_ok or not cfg.safe_postproc()) and (
not nzb_list) and script_path:
#set the current nzo status to "Ext Script...". Used in History
nzo.status = Status.RUNNING
nzo.set_action_line(T('Running script'), unicoder(script))
nzo.set_unpack_info('Script',
T('Running user script %s') %
unicoder(script),
unique=True)
script_log, script_ret = external_processing(
script_path, workdir_complete, nzo.filename, msgid,
dirname, cat, nzo.group, job_result,
nzo.nzo_info.get('failure', ''))
script_line = get_last_line(script_log)
if script_log:
script_output = nzo.nzo_id
if script_line:
nzo.set_unpack_info('Script',
unicoder(script_line),
unique=True)
else:
nzo.set_unpack_info('Script',
T('Ran %s') % unicoder(script),
unique=True)
else:
script = ""
script_line = ""
script_ret = 0
## Email the results
if (not nzb_list) and cfg.email_endjob():
if (cfg.email_endjob() == 1) or (cfg.email_endjob() == 2 and
(unpack_error or par_error)):
emailer.endjob(dirname, msgid, cat, all_ok, workdir_complete,
nzo.bytes_downloaded,
nzo.fail_msg, nzo.unpack_info, script,
TRANS(script_log), script_ret)
if script_output:
# Can do this only now, otherwise it would show up in the email
if script_ret:
script_ret = 'Exit(%s) ' % script_ret
else:
script_ret = ''
if script_line:
nzo.set_unpack_info(
'Script',
u'%s%s <a href="./scriptlog?name=%s">(%s)</a>' %
(script_ret, unicoder(script_line),
urllib.quote(script_output), T('More')),
unique=True)
else:
nzo.set_unpack_info('Script',
u'%s<a href="./scriptlog?name=%s">%s</a>' %
(script_ret, urllib.quote(script_output),
T('View script output')),
unique=True)
## Cleanup again, including NZB files
if all_ok:
cleanup_list(workdir_complete, False)
## Remove newzbin bookmark, if any
if msgid and all_ok:
Bookmarks.do.del_bookmark(msgid)
elif all_ok and isinstance(nzo.url, str):
sabnzbd.proxy_rm_bookmark(nzo.url)
## Force error for empty result
all_ok = all_ok and not empty
## Update indexer with results
if nzo.encrypted > 0:
Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_ENCRYPTED)
if empty:
hosts = map(lambda s: s.host,
sabnzbd.downloader.Downloader.do.nzo_servers(nzo))
if not hosts: hosts = [None]
for host in hosts:
Rating.do.update_auto_flag(nzo.nzo_id, Rating.FLAG_EXPIRED,
host)
## Show final status in history
if all_ok:
growler.send_notification(T('Download Completed'), filename,
'complete')
nzo.status = Status.COMPLETED
else:
growler.send_notification(T('Download Failed'), filename,
'complete')
nzo.status = Status.FAILED
except:
logging.error(Ta('Post Processing Failed for %s (%s)'), filename,
crash_msg)
if not crash_msg:
logging.info("Traceback: ", exc_info=True)
crash_msg = T('see logfile')
nzo.fail_msg = T('PostProcessing was aborted (%s)') % unicoder(
crash_msg)
growler.send_notification(T('Download Failed'), filename, 'complete')
nzo.status = Status.FAILED
par_error = True
all_ok = False
if cfg.email_endjob():
emailer.endjob(dirname, msgid, cat, all_ok, workdir_complete,
nzo.bytes_downloaded, nzo.fail_msg, nzo.unpack_info,
'', '', 0)
if all_ok:
# If the folder only contains one file OR folder, have that as the path
# Be aware that series/generic/date sorting may move a single file into a folder containing other files
workdir_complete = one_file_or_folder(workdir_complete)
workdir_complete = os.path.normpath(workdir_complete)
# Log the overall time taken for postprocessing
postproc_time = int(time.time() - start)
# Create the history DB instance
history_db = database.get_history_handle()
# Add the nzo to the database. Only the path, script and time taken is passed
# Other information is obtained from the nzo
history_db.add_history_db(nzo, workdir_complete, nzo.downpath,
postproc_time, script_log, script_line)
# The connection is only used once, so close it here
history_db.close()
## Clean up the NZO
try:
logging.info('Cleaning up %s (keep_basic=%s)', filename,
str(not all_ok))
sabnzbd.nzbqueue.NzbQueue.do.cleanup_nzo(nzo, keep_basic=not all_ok)
except:
logging.error(Ta('Cleanup of %s failed.'), nzo.final_name)
logging.info("Traceback: ", exc_info=True)
## Remove download folder
if all_ok:
try:
if os.path.exists(workdir):
logging.debug('Removing workdir %s', workdir)
remove_all(workdir, recursive=True)
except:
logging.error(Ta('Error removing workdir (%s)'), workdir)
logging.info("Traceback: ", exc_info=True)
return True
def rename(self, _files, current_path):
""" Rename for Generic files """
logging.debug("Renaming Generic file")
def filter_files(_file, current_path):
if is_full_path(_file):
filepath = os.path.normpath(_file)
else:
filepath = os.path.normpath(os.path.join(current_path, _file))
if os.path.exists(filepath):
size = os.stat(filepath).st_size
if size >= cfg.movie_rename_limit.get_int() and not RE_SAMPLE.search(_file) \
and get_ext(_file) not in EXCLUDED_FILE_EXTS:
return True
return False
# remove any files below the limit from this list
files = [
_file for _file in _files if filter_files(_file, current_path)
]
length = len(files)
# Single File Handling
if length == 1:
file = files[0]
if is_full_path(file):
filepath = os.path.normpath(file)
else:
filepath = os.path.normpath(os.path.join(current_path, file))
if os.path.exists(filepath):
self.fname, ext = os.path.splitext(os.path.split(file)[1])
newname = "%s%s" % (self.filename_set, ext)
newname = newname.replace('%fn', self.fname)
newpath = os.path.join(current_path, newname)
try:
logging.debug("Rename: %s to %s", filepath, newpath)
renamer(filepath, newpath)
except:
logging.error(T('Failed to rename: %s to %s'),
clip_path(filepath), clip_path(newpath))
logging.info("Traceback: ", exc_info=True)
rename_similar(current_path, ext, self.filename_set, ())
# Sequence File Handling
# if there is more than one extracted file check for CD1/1/A in the title
elif self.extra:
matched_files = check_for_multiple(files)
# rename files marked as in a set
if matched_files:
logging.debug("Renaming a series of generic files (%s)",
matched_files)
renamed = matched_files.values()
for index, file in matched_files.iteritems():
filepath = os.path.join(current_path, file)
renamed.append(filepath)
self.fname, ext = os.path.splitext(os.path.split(file)[1])
name = '%s%s' % (self.filename_set, self.extra)
name = name.replace('%1',
str(index)).replace('%fn', self.fname)
name = name + ext
newpath = os.path.join(current_path, name)
try:
logging.debug("Rename: %s to %s", filepath, newpath)
renamer(filepath, newpath)
except:
logging.error(T('Failed to rename: %s to %s'),
clip_path(filepath), clip_path(newpath))
logging.info("Traceback: ", exc_info=True)
rename_similar(current_path, ext, self.filename_set, renamed)
else:
logging.debug("Movie files not in sequence %s", _files)
